diff --git a/.gitignore b/.gitignore index 233248dd..2f95285f 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,9 @@ test-ngtest-ut-trpc-agent-py.xml node_modules package-lock.json pyrightconfig.json + +# Generated artifacts for the evaluation/optimization loop example. +examples/optimization/eval_optimize_loop/runs/ + +# Local isolated development worktrees. +/.worktrees/ diff --git "a/analysis/evaluation optimization pipeline/\345\256\236\346\226\275\351\230\266\346\256\265\350\267\257\347\272\277\345\233\276.md" "b/analysis/evaluation optimization pipeline/\345\256\236\346\226\275\351\230\266\346\256\265\350\267\257\347\272\277\345\233\276.md" new file mode 100644 index 00000000..29d905d4 --- /dev/null +++ "b/analysis/evaluation optimization pipeline/\345\256\236\346\226\275\351\230\266\346\256\265\350\267\257\347\272\277\345\233\276.md" @@ -0,0 +1,125 @@ +# Evaluation + Optimization Pipeline 实施阶段路线图 + +本文记录完成 Evaluation + Optimization 自动回归与提示词优化闭环所需的实施阶段。每个阶段应形成一组可以独立测试、评审和提交的能力，规模大致与当前前两次实现提交相当。整体沿用 1–6 的主阶段编号，其中范围较大的阶段 3 拆成两个独立提交单元。 + +状态说明： + +- `[x]` 已完成 +- `[ ]` 待完成 + +## 阶段总览 + +| 阶段 | 状态 | 核心目标 | +|---|---|---| +| 1. 输入准备与 Prompt 工作区 | 已完成 | 校验配置和输入，创建可复现、隔离的 Prompt 工作副本 | +| 2. 确定性离线评测闭环 | 已完成 | 使用 fake agent/provider 完成 baseline 和 candidate 的四次完整评测 | +| 3a. 评测标准化、失败归因与 Case Diff | 已完成 | 统一评测数据，解释失败并比较候选变化 | +| 3b. 独立 Gate | 已完成 | 根据 diff、关键 case 和预算规则给出接受或拒绝决策 | +| 4. 真实优化器与安全写回 | 待完成 | 接入 AgentOptimizer，审计候选，并在 Gate 接受后安全更新源 Prompt | +| 5. 报告、产物与资源观测 | 待完成 | 输出结构化报告、可读报告和完整审计产物 | +| 6. 离线模式与端到端验收 | 待完成 | 补齐 fake judge/trace 场景、示例输出、文档和完整验收测试 | + +## 1. 输入准备与 Prompt 工作区 + +**状态：`[x]` 已完成** + +- 建立示例目录、`pipeline.json`、`optimizer.json`、train/validation evalset 和 baseline Prompt。 +- 校验配置、路径、评测集、metric、case 标签和运行参数。 +- 保存输入文件及 Prompt 的内容和哈希快照。 +- 将源 Prompt 复制到独立 run 工作区，区分 `source_target` 和 `working_target`。 +- 保证准备失败时不留下伪完整运行目录，且准备阶段不修改源 Prompt。 +- 提供阶段一单元测试和命令行入口。 + +对应提交：`6c47ddd feat: 新增评测优化闭环准备阶段` + +## 2. 确定性离线评测闭环 + +**状态：`[x]` 已完成** + +- 实现每次调用都重新读取工作 Prompt 的确定性 fake agent。 +- 实现 `improve`、`no_improvement`、`overfit` 三种 fake candidate。 +- 对 baseline 和 candidate 分别执行 train、validation 完整评测。 +- 保存 SDK 原始评测结果、通过数量、平均分和候选 Prompt 元数据。 +- 校验 evalset 和工作 Prompt 在准备后没有漂移，并保留 candidate 工作副本供审计。 +- 提供三种场景矩阵、异常路径和 CLI smoke test。 + +对应提交：`115c914 feat(evaluation): 实现确定性评测优化闭环第二阶段` + +## 3a. 评测标准化、失败归因与 Case Diff + +**状态：`[x]` 已完成** + +- 将 SDK 评测结果转换为稳定的逐 case、逐 metric 数据模型。 +- 根据 metric、预期/实际回复和调用轨迹执行确定性失败归因。 +- 比较 baseline 与 candidate，识别新增通过、新增失败、提升、退化和不变 case。 +- 标记 hard、critical 和 severe regression，并识别训练提升但验证退化的过拟合。 +- 保留归因依据和前后变化证据，供 Gate 与报告阶段直接消费。 + +阶段完成标准：可以仅根据四次评测结果生成可序列化、可解释的逐 case 差异，且不修改源 Prompt。 + +## 3b. 独立 Gate + +**状态：`[x]` 已完成** + +- 实现与优化器解耦的 Gate，消费阶段 3a 生成的评测与 diff 数据。 +- 执行验证集最小提升、通过率不得下降、hard/critical case、severe regression 和必需 metric 规则。 +- 纳入成本、token、耗时及不可观测数据策略，但不将未知数据误记为零。 +- 收集全部拒绝理由，不采用只返回第一个错误的方式。 +- 固定三种离线场景的决策：improve ACCEPT、no improvement REJECT、overfit REJECT。 +- 覆盖多规则同时失败、关键 case 退化、训练提升但验证下降等集成场景。 + +阶段完成标准：Gate 能稳定给出完整决策证据，且 ACCEPT/REJECT 本身不触发源 Prompt 写回。 + +## 4. 真实优化器与安全写回 + +**状态：`[ ]` 待完成** + +- 抽象统一 Candidate Provider 接口，接入 `AgentOptimizer` 真实候选生成。 +- 始终使用 `update_source=False`，让优化器只操作隔离工作区。 +- 保留优化器原生候选、轮次记录、分数和配置快照。 +- 对最终候选重新执行完整 train/validation 回归，并交给统一 diff 和 Gate。 +- 只有 Gate ACCEPT 且源 Prompt 哈希未变化时才允许写回；写回后必须回读校验。 +- REJECT、异常、源文件漂移或校验失败时保持源 Prompt 不变。 + +阶段完成标准：真实模式和 fake 模式共用同一条候选验证及 Gate 链路，写回行为具备明确的安全边界。 + +## 5. 报告、产物与资源观测 + +**状态：`[ ]` 待完成** + +- 生成 `optimization_report.json`，包含 baseline、candidate、归因、case diff、Gate 和写回状态。 +- 生成面向使用者的 `optimization_report.md`，解释候选是否值得接受及具体原因。 +- 建立 artifact index，索引输入快照、四次评测、候选 Prompt、优化器原生产物和报告。 +- 记录随机种子、配置、耗时以及可观测的 token、成本和调用信息。 +- 对无法可靠观测的资源数据明确记录为 `unavailable`，并按预算策略产生 reject 或 warning。 +- 使用原子写入，避免失败运行留下看似完整的报告和索引。 + +阶段完成标准：一次运行的输入、Prompt 变化、评测证据、决策和写回结果都可以从产物中复现和审计。 + +## 6. 离线模式与端到端验收 + +**状态：`[ ]` 待完成** + +必须完成： + +- 保持无 API Key 时可以运行 improve、no improvement 和 overfit 三个完整场景。 +- 如果保留 `use_fake_judge` 配置，则实现确定性 fake judge；如果保留 `trace` 模式，则让它能够驱动归因、diff、Gate 和报告链路。 +- 提供示例输出、完整 README、运行命令和各模式适用边界。 +- 覆盖 evaluator/optimizer 异常、Gate 拒绝、写回失败、输入漂移和产物不完整等端到端路径。 +- 验证离线完整 pipeline 在三分钟内完成，并核对 issue 要求的报告字段和交付物。 + +可选且默认跳过： + +- 通过显式环境变量启用真实 API 集成测试。 +- 真实 API 测试不作为普通 CI 或无 API Key 核心流程的必要条件。 + +阶段完成标准：公开样例能够稳定生成完整报告和正确决策，项目具备提交 issue 验收所需的文档、测试和审计产物。 + +## 全局实施约束 + +- Baseline 和最终 Candidate 都必须分别执行完整 train、validation 评测。 +- 真实优化器的内部 minibatch 或轮次分数不能替代 pipeline 的完整回归。 +- Gate 决策前不得修改源 Prompt，任何写回都必须经过源哈希校验和回读验证。 +- fake agent/provider/judge 必须保持确定性，不得读取 `eval_id`、期望答案或调用次数作弊。 +- 所有阶段优先提供无 API Key 的自动化测试，并保留后续真实模式的清晰接口。 +- 新增数据模型和产物需要保持可序列化、可解释和可审计。 diff --git a/examples/optimization/eval_optimize_loop/DESIGN.md b/examples/optimization/eval_optimize_loop/DESIGN.md new file mode 100644 index 00000000..b4a6f708 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/DESIGN.md @@ -0,0 +1,9 @@ +# 设计边界 + +本示例把运行环境与替代组件分开描述。`offline`、`real`、`trace` 是三种 Pipeline 运行模式。`offline` 仍使用 SDK 的 `LlmAgent`、Runner 和独立 Session，只把 Agent 内部模型替换成 `DeterministicFakeModel`，并用确定性 Candidate Provider 代替真实优化器。它用于验证 Prompt 改变是否经过真实 Agent 编排影响输出。 + +`real` 使用真实业务模型生成回复，并由 `AgentOptimizer` 调用真实反思模型产生 Prompt 候选；只有 Gate 接受、源 Prompt 哈希未漂移且显式启用写回时，才允许更新源文件。 + +`trace` 直接评测预录制的 `actual_conversation`，不再次运行 Agent、Model 或 Candidate Provider。它适合复现工具轨迹和生产故障，但只能证明候选版本与轨迹的关联，不能证明 Prompt 导致了该轨迹。因此 Trace 即使获得 ACCEPT，也固定跳过源 Prompt 写回。 + +确定性 metric 负责精确匹配等硬规则。LLM Judge 若需要，应作为带 rubric 的评测指标显式配置；本示例不提供容易混淆职责的 `use_fake_judge` 开关。 diff --git a/examples/optimization/eval_optimize_loop/README.md b/examples/optimization/eval_optimize_loop/README.md new file mode 100644 index 00000000..3fc547d1 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/README.md @@ -0,0 +1,162 @@ +# Evaluation + Optimization Loop — Stage 6 + +## 三种运行模式 + +- `offline`：使用真实 SDK `LlmAgent` 和 `Runner`，Agent 内部注入 `DeterministicFakeModel`，候选由确定性 Candidate Provider 生成。无需 API Key。 +- `trace`：使用 SDK 原生 `eval_mode="trace"` 回放 baseline/candidate 的 train 与 validation 轨迹，不运行 Agent、Model 或 Candidate Provider。即使 Gate 接受，也不会写回源 Prompt。 +- `real`：业务 Agent 和反思优化模型均使用真实模型，由显式入口 `run_real_integration.py --run-real` 启用。 + +Fake Model 是 `offline` 模式中的模型实现，不是独立运行模式；Fake Candidate Provider 只替代候选生成，不替代业务 Agent。精确匹配等硬规则由 `optimizer.json` 的确定性 metric 执行。需要语义判断时，应显式配置带 rubric 的 LLM Judge，而不是使用布尔 Fake Judge 开关。详细边界见 [DESIGN.md](DESIGN.md)。 + +This example provides an auditable evaluation and prompt-optimization loop. +Stages 1–3 prepare an isolated prompt workspace, run baseline and candidate +evaluations on both train and validation datasets, normalize failures, build +case diffs, detect overfitting, and apply an independent Gate. Stage 4 adds a +common Candidate Provider boundary, an `AgentOptimizer` adapter, and guarded +source-prompt writeback. Stage 5 publishes a complete JSON/Markdown report and +artifact index, or preserves a standalone failure report when a run fails. + +The deterministic offline mode runs through the SDK Agent without an API key, +judge, or real optimizer. Its built-in scenarios produce ACCEPT for `improve` and REJECT for +both `no_improvement` and `overfit`: + +```bash +python examples/optimization/eval_optimize_loop/run_pipeline.py \ + --run-id local_stage6 \ + --scenario improve +``` + +The same CLI also supports trace replay: + +```bash +python examples/optimization/eval_optimize_loop/run_pipeline.py \ + --config examples/optimization/eval_optimize_loop/pipeline.trace.json \ + --scenario overfit +``` + +For an +explicit real integration smoke run, configure the OpenAI-compatible business +model connection in the environment: + +```bash +export TRPC_AGENT_API_KEY=... +export TRPC_AGENT_BASE_URL=... +export TRPC_AGENT_MODEL_NAME=... + +.venv/bin/python examples/optimization/eval_optimize_loop/run_real_integration.py \ + --run-real \ + --optimizer-model-name mimo-v2.5 \ + --max-candidate-proposals 1 +``` + +The business model uses the three environment values above. The reflection +optimizer model is selected explicitly with CLI arguments; its endpoint and +credential reuse the environment without writing their resolved values to run +artifacts. `--run-real` is mandatory so an accidental command cannot spend API +quota. This entry always uses `pipeline.real.json`, where source writeback is +disabled, and reports ACCEPT or REJECT without treating REJECT as a process +failure. Both CLIs print the paths of the JSON report, Markdown report, and +artifact index after a completed run. + +Applications with a custom agent can still use the Python integration point. +Set `execution.mode` to `real`, prepare the run, and inject an async +SDK-compatible `call_agent`: + +```python +import asyncio + +from examples.optimization.eval_optimize_loop.pipeline import prepare_run +from examples.optimization.eval_optimize_loop.pipeline import run_real_stage + +prepared = prepare_run("examples/optimization/eval_optimize_loop/pipeline.json") +agent = MyBusinessAgent(target_prompt=prepared.working_target) +result = asyncio.run( + run_real_stage(prepared, call_agent=agent.call_agent) +) +``` + +The injected agent must reread `prepared.working_target` on every call. The +same callable is used for baseline regression, optimizer search, and final +candidate regression. Optimizer minibatches never replace the four complete +pipeline evaluations: baseline train/validation and candidate train/validation. + +`AgentOptimizer` always receives the isolated working target and +`update_source=False`. Its native `result.json`, round records, prompt snapshots, +scores, and configuration snapshot are retained under `runs//optimizer/` +when `artifacts.retain_optimizer_native_artifacts` is enabled. + +## Report artifacts + +A completed run atomically publishes the formal bundle at +`runs//report/`: + +```text +report/ +├── optimization_report.json +├── optimization_report.md +├── artifact_index.json +├── inputs/ +│ ├── pipeline_config.json +│ ├── optimizer_config.json +│ ├── train_evalset.json +│ └── validation_evalset.json +├── evaluations/ +│ ├── baseline_train.json +│ ├── baseline_validation.json +│ ├── candidate_train.json +│ └── candidate_validation.json +└── prompts/ + ├── baseline/ + └── candidate/ +``` + +The formal `report/` directory is visible only after all required artifacts +have been written and validated. If any post-preparation phase fails, the +pipeline does not leave a partial formal report; it atomically writes +`runs//failure_report.json` with the failed phase, completed phases, +sanitized error information, source Prompt hashes, and already existing +artifacts. A failure-report write error is surfaced together with the original +pipeline error. + +`artifact_index.json` records each artifact's relative path, SHA-256 hash, byte +size, producer phase, and availability. Consumers can use those fields to +verify that an artifact has not drifted since publication. Input copies are +validated against the preparation snapshot, and sensitive resolved credentials +are not accepted into the report bundle. + +Source prompts are updated only when all of these conditions hold: + +- Gate returns ACCEPT; +- `writeback.enabled` is true; +- every source Prompt still matches its preparation-time SHA-256 hash; +- the write succeeds and an exact readback matches the accepted candidate. + +Writeback returns `written`, `skipped`, `blocked`, or `failed` with an auditable +reason. Gate rejection and disabled writeback are skipped; concurrent source +edits are blocked; recoverable write/readback failures are rolled back and +returned as failed. If rollback integrity cannot be proven, the pipeline raises +an error instead of claiming the source is safe. The checked-in configuration +keeps writeback disabled by default. + +The report separates whole-pipeline resources from optimizer-only observations. +Pipeline duration is observable, while full pipeline monetary cost and token +usage remain `unavailable` because business-agent calls may not expose complete +telemetry. In real mode, optimizer rounds, reflection calls, duration, cost, and +token usage are reported independently from the native optimizer result; an +unreliable or incomplete field stays `unavailable` instead of being treated as +zero. In offline mode, optimizer-only fields are `not_applicable`. + +Run the Stage 1–5 tests with: + +```bash +.venv/bin/pytest -q \ + tests/evaluation/test_eval_optimize_loop_stage1.py \ + tests/evaluation/test_eval_optimize_loop_stage2.py \ + tests/evaluation/test_eval_optimize_loop_stage3a.py \ + tests/evaluation/test_eval_optimize_loop_stage3b.py \ + tests/evaluation/test_eval_optimize_loop_stage4.py \ + tests/evaluation/test_eval_optimize_loop_real_integration.py \ + tests/evaluation/test_eval_optimize_loop_stage5_report_builder.py \ + tests/evaluation/test_eval_optimize_loop_stage5_artifacts.py \ + tests/evaluation/test_eval_optimize_loop_stage5_pipeline.py +``` diff --git a/examples/optimization/eval_optimize_loop/__init__.py b/examples/optimization/eval_optimize_loop/__init__.py new file mode 100644 index 00000000..0f6ebd16 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/__init__.py @@ -0,0 +1,6 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under the Apache License, Version 2.0. +"""A safe, auditable evaluation and prompt-optimization pipeline example.""" diff --git a/examples/optimization/eval_optimize_loop/analysis.py b/examples/optimization/eval_optimize_loop/analysis.py new file mode 100644 index 00000000..9210ae3a --- /dev/null +++ b/examples/optimization/eval_optimize_loop/analysis.py @@ -0,0 +1,83 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under the Apache License, Version 2.0. +"""Stage 3a orchestration for normalization, attribution, and case diff.""" + +from __future__ import annotations + +from .attribution import attribute_evaluation +from .case_diff import compare_evaluations +from .evaluation_adapter import standardize_snapshot +from .schemas import EvaluationAnalysis +from .schemas import EvaluationSnapshot +from .schemas import ObservableValue +from .schemas import OverfitStatus + + +def _overfit_status( + train_delta: ObservableValue, + validation_delta: ObservableValue, +) -> tuple[OverfitStatus, str]: + if train_delta.status != "available" or validation_delta.status != "available": + return "unavailable", "Train or validation score delta is unavailable." + train_value = float(train_delta.value) + validation_value = float(validation_delta.value) + if train_value > 0.0 and validation_value < 0.0: + return ( + "detected", + f"Train score improved by {train_value:.6f} while validation regressed by " + f"{validation_value:.6f}.", + ) + return ( + "not_detected", + f"Train score delta is {train_value:.6f}; validation score delta is " + f"{validation_value:.6f}.", + ) + + +def build_evaluation_analysis( + *, + baseline_train: EvaluationSnapshot, + baseline_validation: EvaluationSnapshot, + candidate_train: EvaluationSnapshot, + candidate_validation: EvaluationSnapshot, + hard_case_ids: set[str], + critical_case_ids: set[str], + severe_case_score_drop: float, +) -> EvaluationAnalysis: + """Build stage 3a analysis from the four complete evaluation snapshots.""" + normalized_baseline_train = attribute_evaluation(standardize_snapshot(baseline_train)) + normalized_baseline_validation = attribute_evaluation(standardize_snapshot(baseline_validation)) + normalized_candidate_train = attribute_evaluation(standardize_snapshot(candidate_train)) + normalized_candidate_validation = attribute_evaluation(standardize_snapshot(candidate_validation)) + + train_diff = compare_evaluations( + normalized_baseline_train, + normalized_candidate_train, + hard_case_ids=hard_case_ids, + critical_case_ids=critical_case_ids, + severe_case_score_drop=severe_case_score_drop, + ) + validation_diff = compare_evaluations( + normalized_baseline_validation, + normalized_candidate_validation, + hard_case_ids=hard_case_ids, + critical_case_ids=critical_case_ids, + severe_case_score_drop=severe_case_score_drop, + ) + overfit_status, overfit_reason = _overfit_status( + train_diff.score_delta, + validation_diff.score_delta, + ) + return EvaluationAnalysis( + baseline_train=normalized_baseline_train, + baseline_validation=normalized_baseline_validation, + candidate_train=normalized_candidate_train, + candidate_validation=normalized_candidate_validation, + train_diff=train_diff, + validation_diff=validation_diff, + overfit_status=overfit_status, + overfit_reason=overfit_reason, + ) diff --git a/examples/optimization/eval_optimize_loop/artifact_writer.py b/examples/optimization/eval_optimize_loop/artifact_writer.py new file mode 100644 index 00000000..41e82169 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/artifact_writer.py @@ -0,0 +1,608 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under the Apache License, Version 2.0. +"""Safe and atomic materialization of optimization report artifacts.""" + +from __future__ import annotations + +import ctypes +import errno +import hashlib +import json +import os +from pathlib import Path +import shutil +import sys +from typing import Callable, Literal, TypeAlias +from uuid import uuid4 + +from pydantic import BaseModel + +from .report_builder import render_optimization_markdown +from .schemas import ArtifactIndex, ArtifactReference, FailureReport +from .schemas import OptimizationReport, ReportPhase +from .schemas import TraceCandidateProposal +from .sensitive_config import SensitiveConfigError +from .sensitive_config import validate_persisted_sensitive_values + + +ArtifactType: TypeAlias = Literal[ + "input", + "prompt", + "evaluation", + "candidate", + "optimizer_native", + "report", +] + +_INPUT_COPY_DISABLED = "artifacts.copy_input_files=false" +_AT_FDCWD = -100 +_RENAME_NOREPLACE = 1 +_RENAME_EXCL = 0x4 +_RENAMEAT2_UNAVAILABLE = { + errno.ENOSYS, + errno.EINVAL, + getattr(errno, "EOPNOTSUPP", errno.ENOTSUP), +} + + +class ArtifactWriteError(RuntimeError): + """Raised when an artifact cannot be safely materialized or discovered.""" + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _resolved_run_dir(run_dir: Path) -> Path: + if run_dir.is_symlink(): + raise ArtifactWriteError(f"run directory must not be a symbolic link: {run_dir}") + try: + root = run_dir.resolve(strict=True) + except OSError as exc: + raise ArtifactWriteError(f"run directory is unavailable: {run_dir}: {exc}") from exc + if not root.is_dir(): + raise ArtifactWriteError(f"run directory must be a directory: {run_dir}") + return root + + +def _inside_run(run_dir: Path, path: Path) -> Path: + root = run_dir.resolve(strict=True) + lexical = path if path.is_absolute() else root / path + try: + relative = lexical.relative_to(root) + except ValueError as exc: + raise ArtifactWriteError(f"artifact escapes run directory: {path}") from exc + + current = root + for component in relative.parts: + current /= component + if current.is_symlink(): + raise ArtifactWriteError(f"artifact must not be a symbolic link: {path}") + + try: + resolved = lexical.resolve(strict=True) + except OSError as exc: + raise ArtifactWriteError(f"artifact is unavailable: {path}: {exc}") from exc + if not resolved.is_relative_to(root): + raise ArtifactWriteError(f"artifact escapes run directory: {path}") + if not resolved.is_file(): + raise ArtifactWriteError(f"artifact must be a regular file: {path}") + return resolved + + +def discover_run_artifacts(run_dir: Path) -> list[str]: + """Return regular files below a run without ever accepting symlinks.""" + root = _resolved_run_dir(run_dir) + paths: list[str] = [] + for directory, directory_names, file_names in os.walk(root, followlinks=False): + current = Path(directory) + directory_names.sort() + file_names.sort() + + retained_directories = [] + for name in directory_names: + path = current / name + if path.is_symlink(): + raise ArtifactWriteError( + f"artifact must not be a symbolic link: {path}" + ) + relative = path.relative_to(root).as_posix() + if ".report.tmp-" not in relative: + retained_directories.append(name) + directory_names[:] = retained_directories + + for name in file_names: + path = current / name + if path.is_symlink(): + raise ArtifactWriteError( + f"artifact must not be a symbolic link: {path}" + ) + relative = path.relative_to(root).as_posix() + if name == "failure_report.json" or ".report.tmp-" in relative: + continue + if path.is_file(): + paths.append(relative) + return sorted(paths) + + +def _write_text(path: Path, content: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + + +def _json_text(model: BaseModel) -> str: + return model.model_dump_json(by_alias=False, indent=2) + "\n" + + +def _validate_optimizer_config_for_copy(path: Path) -> None: + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ArtifactWriteError( + f"failed to parse optimizer config snapshot: {path}: {exc}" + ) from exc + try: + validate_persisted_sensitive_values(payload) + except SensitiveConfigError as exc: + raise ArtifactWriteError(str(exc)) from exc + + +def _rename_directory_no_replace(source: Path, target: Path) -> None: + """Atomically publish a directory without replacing an existing target. + + Each supported platform uses an atomic no-replace primitive. Platforms + without that primitive fail closed rather than risking a replacement race. + """ + if sys.platform.startswith("linux"): + try: + libc = ctypes.CDLL(None, use_errno=True) + renameat2 = libc.renameat2 + except (AttributeError, OSError): + raise ArtifactWriteError( + "atomic no-replace unavailable: Linux renameat2 is unavailable" + ) + renameat2.argtypes = [ + ctypes.c_int, + ctypes.c_char_p, + ctypes.c_int, + ctypes.c_char_p, + ctypes.c_uint, + ] + renameat2.restype = ctypes.c_int + ctypes.set_errno(0) + result = renameat2( + _AT_FDCWD, + os.fsencode(source), + _AT_FDCWD, + os.fsencode(target), + _RENAME_NOREPLACE, + ) + if result == 0: + return + error_number = ctypes.get_errno() + if error_number == errno.EEXIST: + raise ArtifactWriteError(f"report directory already exists: {target}") + if error_number in _RENAMEAT2_UNAVAILABLE: + raise ArtifactWriteError( + "atomic no-replace unavailable: Linux renameat2 does not support " + f"RENAME_NOREPLACE ({os.strerror(error_number)})" + ) + raise OSError(error_number, os.strerror(error_number), target) + + if sys.platform.startswith("win"): + try: + os.rename(source, target) + except FileExistsError as exc: + raise ArtifactWriteError(f"report directory already exists: {target}") from exc + return + + if sys.platform == "darwin": + try: + libc = ctypes.CDLL(None, use_errno=True) + renamex_np = libc.renamex_np + except (AttributeError, OSError): + raise ArtifactWriteError( + "atomic no-replace unavailable: Darwin renamex_np is unavailable" + ) + renamex_np.argtypes = [ctypes.c_char_p, ctypes.c_char_p, ctypes.c_uint] + renamex_np.restype = ctypes.c_int + ctypes.set_errno(0) + result = renamex_np(os.fsencode(source), os.fsencode(target), _RENAME_EXCL) + if result == 0: + return + error_number = ctypes.get_errno() + if error_number == errno.EEXIST: + raise ArtifactWriteError(f"report directory already exists: {target}") + if error_number in _RENAMEAT2_UNAVAILABLE: + raise ArtifactWriteError( + "atomic no-replace unavailable: Darwin renamex_np does not support " + f"RENAME_EXCL ({os.strerror(error_number)})" + ) + raise OSError(error_number, os.strerror(error_number), target) + + raise ArtifactWriteError( + f"atomic no-replace unavailable: unsupported platform {sys.platform}" + ) + + +def _published_relative_path(root: Path, path: Path) -> str: + relative = path.relative_to(root) + if relative.parts and relative.parts[0].startswith(".report.tmp-"): + relative = Path("report", *relative.parts[1:]) + return relative.as_posix() + + +def _available_reference( + run_dir: Path, + path: Path, + *, + artifact_id: str, + artifact_type: ArtifactType, + required: bool, + produced_by: ReportPhase, +) -> ArtifactReference: + root = run_dir.resolve(strict=True) + resolved = _inside_run(root, path) + return ArtifactReference( + artifact_id=artifact_id, + artifact_type=artifact_type, + relative_path=_published_relative_path(root, path), + required=required, + produced_by=produced_by, + status="available", + size_bytes=resolved.stat().st_size, + sha256=_sha256(resolved), + ) + + +def _unavailable_input_reference( + *, artifact_id: str, produced_by: ReportPhase +) -> ArtifactReference: + return ArtifactReference( + artifact_id=artifact_id, + artifact_type="input", + required=True, + produced_by=produced_by, + status="unavailable", + unavailable_reason=_INPUT_COPY_DISABLED, + ) + + +def _safe_prompt_name(field_name: str) -> str: + safe = "".join( + character if character.isalnum() or character in "._-" else "_" + for character in field_name + ) + return safe if safe not in {"", ".", ".."} else "prompt" + + +def _validate_available_references( + root: Path, staging: Path, index: ArtifactIndex +) -> None: + for reference in index.artifacts: + if reference.status != "available": + continue + if reference.relative_path is None: + raise ArtifactWriteError( + f"available artifact has no relative path: {reference.artifact_id}" + ) + relative = Path(reference.relative_path) + if relative.is_absolute() or ".." in relative.parts: + raise ArtifactWriteError( + f"artifact path is not run-relative: {reference.relative_path}" + ) + if relative.parts and relative.parts[0] == "report": + path = staging.joinpath(*relative.parts[1:]) + else: + path = root / relative + resolved = _inside_run(root, path) + if resolved.stat().st_size != reference.size_bytes: + raise ArtifactWriteError( + f"artifact size changed during staging: {reference.relative_path}" + ) + if _sha256(resolved) != reference.sha256: + raise ArtifactWriteError( + f"artifact hash changed during staging: {reference.relative_path}" + ) + + +def _copy_input( + *, + root: Path, + staging: Path, + source: Path, + expected_sha256: str, + destination_name: str, + artifact_id: str, + produced_by: ReportPhase, + content_validator: Callable[[Path], None] | None = None, +) -> ArtifactReference: + if source.is_symlink(): + raise ArtifactWriteError(f"input must not be a symbolic link: {source}") + try: + actual_sha256 = _sha256(source) + except OSError as exc: + raise ArtifactWriteError(f"failed to read input {source}: {exc}") from exc + if actual_sha256 != expected_sha256: + raise ArtifactWriteError(f"input hash mismatch: {source}") + if content_validator is not None: + content_validator(source) + + destination = staging / "inputs" / destination_name + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source, destination) + if _sha256(destination) != expected_sha256: + raise ArtifactWriteError(f"input hash changed while copying: {source}") + return _available_reference( + root, + destination, + artifact_id=artifact_id, + artifact_type="input", + required=True, + produced_by=produced_by, + ) + + +def publish_report_bundle( + report: OptimizationReport, + *, + run_dir: Path, + copy_input_files: bool, +) -> ArtifactIndex: + """Build a complete report in staging and atomically publish its directory.""" + staging: Path | None = None + try: + root = _resolved_run_dir(run_dir) + target = root / "report" + if target.exists() or target.is_symlink(): + raise ArtifactWriteError(f"report directory already exists: {target}") + + existing_paths = discover_run_artifacts(root) + native_paths = [ + relative + for relative in existing_paths + if relative.startswith("optimizer/") + or ("/" not in relative and relative.endswith(".runtime.json")) + ] + + staging = root / f".report.tmp-{uuid4().hex}" + staging.mkdir() + references: list[ArtifactReference] = [] + + report_json = staging / "optimization_report.json" + _write_text(report_json, _json_text(report)) + references.append( + _available_reference( + root, + report_json, + artifact_id="report.optimization_json", + artifact_type="report", + required=True, + produced_by="reporting", + ) + ) + + report_markdown = staging / "optimization_report.md" + _write_text(report_markdown, render_optimization_markdown(report)) + references.append( + _available_reference( + root, + report_markdown, + artifact_id="report.optimization_markdown", + artifact_type="report", + required=True, + produced_by="reporting", + ) + ) + + evaluations = ( + ("baseline_train", report.baseline_train, "baseline_train"), + ("baseline_validation", report.baseline_validation, "baseline_validation"), + ("candidate_train", report.candidate_train, "candidate_train"), + ( + "candidate_validation", + report.candidate_validation, + "candidate_validation", + ), + ) + for name, evaluation, produced_by in evaluations: + path = staging / "evaluations" / f"{name}.json" + _write_text(path, _json_text(evaluation)) + references.append( + _available_reference( + root, + path, + artifact_id=f"evaluation.{name}", + artifact_type="evaluation", + required=True, + produced_by=produced_by, + ) + ) + + for index, snapshot in enumerate(report.input_snapshot.prompt_snapshots): + path = ( + staging + / "prompts" + / "baseline" + / f"{index:03d}-{_safe_prompt_name(snapshot.field_name)}.md" + ) + _write_text(path, snapshot.content) + references.append( + _available_reference( + root, + path, + artifact_id=f"prompt.baseline.{snapshot.field_name}", + artifact_type="prompt", + required=True, + produced_by="baseline_train", + ) + ) + + for index, (field_name, content) in enumerate(report.candidate.prompts.items()): + path = ( + staging + / "prompts" + / "candidate" + / f"{index:03d}-{_safe_prompt_name(field_name)}.md" + ) + _write_text(path, content) + references.append( + _available_reference( + root, + path, + artifact_id=f"prompt.candidate.{field_name}", + artifact_type="prompt", + required=True, + produced_by="candidate_generation", + ) + ) + + input_specs = [ + ( + "input.pipeline_config", + Path(report.input_snapshot.pipeline_config_path), + report.input_snapshot.pipeline_config_sha256, + "pipeline_config.json", + "baseline_train", + ), + ( + "input.optimizer_config", + Path(report.input_snapshot.optimizer_config_path), + report.input_snapshot.optimizer_config_sha256, + "optimizer_config.json", + "candidate_generation", + ), + ( + "input.train_evalset", + Path(report.input_snapshot.train_evalset_path), + report.input_snapshot.train_evalset_sha256, + "train_evalset.json", + "baseline_train", + ), + ( + "input.validation_evalset", + Path(report.input_snapshot.validation_evalset_path), + report.input_snapshot.validation_evalset_sha256, + "validation_evalset.json", + "baseline_validation", + ), + ] + if ( + isinstance(report.candidate, TraceCandidateProposal) + and report.input_snapshot.trace_inputs is not None + ): + trace = report.input_snapshot.trace_inputs.scenarios[ + report.candidate.scenario + ] + input_specs.extend( + [ + ( + "input.trace.candidate_train", + Path(trace.train_evalset_path), + trace.train_evalset_sha256, + "candidate_train_trace.json", + "candidate_train", + ), + ( + "input.trace.candidate_validation", + Path(trace.validation_evalset_path), + trace.validation_evalset_sha256, + "candidate_validation_trace.json", + "candidate_validation", + ), + ] + ) + for artifact_id, source, expected_hash, destination_name, produced_by in input_specs: + if copy_input_files: + content_validator = ( + _validate_optimizer_config_for_copy + if artifact_id == "input.optimizer_config" + else None + ) + references.append( + _copy_input( + root=root, + staging=staging, + source=source, + expected_sha256=expected_hash, + destination_name=destination_name, + artifact_id=artifact_id, + produced_by=produced_by, + content_validator=content_validator, + ) + ) + else: + references.append( + _unavailable_input_reference( + artifact_id=artifact_id, + produced_by=produced_by, + ) + ) + + for relative in native_paths: + native_path = root / relative + if native_path.name == "optimizer.runtime.json": + _validate_optimizer_config_for_copy(native_path) + references.append( + _available_reference( + root, + native_path, + artifact_id=f"optimizer_native.{relative}", + artifact_type="optimizer_native", + required=False, + produced_by="candidate_generation", + ) + ) + + index = ArtifactIndex( + run_id=report.run_id, + generated_at=report.finished_at, + artifacts=references, + ) + index_path = staging / "artifact_index.json" + _write_text(index_path, _json_text(index)) + + OptimizationReport.model_validate_json(report_json.read_text(encoding="utf-8")) + validated_index = ArtifactIndex.model_validate_json( + index_path.read_text(encoding="utf-8") + ) + _validate_available_references(root, staging, validated_index) + + _rename_directory_no_replace(staging, target) + staging = None + return validated_index + except Exception as exc: + if staging is not None: + shutil.rmtree(staging, ignore_errors=True) + raise ArtifactWriteError(f"failed to publish report bundle: {exc}") from exc + + +def write_failure_report(report: FailureReport, *, run_dir: Path) -> Path: + """Atomically write first-failure evidence without allowing replacement.""" + temporary: Path | None = None + try: + root = _resolved_run_dir(run_dir) + target = root / "failure_report.json" + if target.exists() or target.is_symlink(): + raise ArtifactWriteError(f"failure report already exists: {target}") + temporary = root / f".failure_report.tmp-{uuid4().hex}" + _write_text(temporary, _json_text(report)) + FailureReport.model_validate_json(temporary.read_text(encoding="utf-8")) + try: + os.link(temporary, target) + except FileExistsError as exc: + raise ArtifactWriteError(f"failure report already exists: {target}") from exc + temporary.unlink() + temporary = None + return target + except Exception as exc: + if temporary is not None: + temporary.unlink(missing_ok=True) + raise ArtifactWriteError(f"failed to write failure report: {exc}") from exc diff --git a/examples/optimization/eval_optimize_loop/attribution.py b/examples/optimization/eval_optimize_loop/attribution.py new file mode 100644 index 00000000..351ccd28 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/attribution.py @@ -0,0 +1,261 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under the Apache License, Version 2.0. +"""Deterministic, evidence-driven failure attribution for stage 3a.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass + +from .schemas import AttributionEvidence +from .schemas import CaseEvaluation +from .schemas import FailureAttribution +from .schemas import FailureCategory +from .schemas import StandardizedEvaluation + + +@dataclass(frozen=True) +class _CandidateReason: + priority: int + category: FailureCategory + summary: str + evidence: AttributionEvidence + + +def _json_object(text: str | None) -> dict | None: + if text is None: + return None + try: + value = json.loads(text) + except (TypeError, ValueError): + return None + return value if isinstance(value, dict) else None + + +def _case_attribution(case: CaseEvaluation) -> FailureAttribution | None: + if case.status == "passed": + return None + + reasons: list[_CandidateReason] = [] + for run in case.runs: + if run.status == "not_evaluated" or run.error_message: + summary = run.error_message or "Evaluation did not produce a usable result." + reasons.append( + _CandidateReason( + priority=10, + category="evaluation_error", + summary=summary, + evidence=AttributionEvidence( + evidence_type="execution_error", + message=summary, + run_id=run.run_id, + actual=run.error_message, + ), + ) + ) + for invocation in run.invocations: + expected_names = [tool.name for tool in invocation.expected_tools] + actual_names = [tool.name for tool in invocation.actual_tools] + if expected_names != actual_names and (expected_names or actual_names): + summary = f"Expected tool names {expected_names}, got {actual_names}." + reasons.append( + _CandidateReason( + priority=20, + category="tool_name_error", + summary=summary, + evidence=AttributionEvidence( + evidence_type="tool", + message=summary, + run_id=run.run_id, + invocation_id=invocation.invocation_id, + expected=expected_names, + actual=actual_names, + ), + ) + ) + + expected_arguments = [tool.arguments for tool in invocation.expected_tools] + actual_arguments = [tool.arguments for tool in invocation.actual_tools] + if ( + expected_names == actual_names + and (expected_names or actual_names) + and expected_arguments != actual_arguments + ): + summary = f"Expected tool arguments {expected_arguments}, got {actual_arguments}." + reasons.append( + _CandidateReason( + priority=30, + category="tool_argument_error", + summary=summary, + evidence=AttributionEvidence( + evidence_type="tool", + message=summary, + run_id=run.run_id, + invocation_id=invocation.invocation_id, + expected=expected_arguments, + actual=actual_arguments, + ), + ) + ) + + failed_knowledge_metrics = [ + metric + for metric in invocation.metrics + if metric.status == "failed" and metric.metric_name == "llm_rubric_knowledge_recall" + ] + if failed_knowledge_metrics: + metric = failed_knowledge_metrics[0] + summary = metric.reason or "Knowledge recall rubric was not satisfied." + reasons.append( + _CandidateReason( + priority=40, + category="knowledge_recall", + summary=summary, + evidence=AttributionEvidence( + evidence_type="metric", + message=summary, + run_id=run.run_id, + invocation_id=invocation.invocation_id, + metric_name=metric.metric_name, + actual=metric.reason, + ), + ) + ) + + expected_json = _json_object(invocation.expected_response) + actual_json = _json_object(invocation.actual_response) + if expected_json is not None and ( + actual_json is None or not set(expected_json).issubset(actual_json) + ): + summary = "Actual response is not valid JSON with the expected top-level fields." + reasons.append( + _CandidateReason( + priority=50, + category="format_error", + summary=summary, + evidence=AttributionEvidence( + evidence_type="response", + message=summary, + run_id=run.run_id, + invocation_id=invocation.invocation_id, + expected=invocation.expected_response, + actual=invocation.actual_response, + ), + ) + ) + + failed_rubric_metrics = [ + metric + for metric in invocation.metrics + if metric.status == "failed" + and metric.metric_name.startswith("llm_rubric_") + and metric.metric_name != "llm_rubric_knowledge_recall" + ] + if failed_rubric_metrics: + metric = failed_rubric_metrics[0] + summary = metric.reason or f"Rubric metric {metric.metric_name!r} was not satisfied." + reasons.append( + _CandidateReason( + priority=60, + category="rubric_failure", + summary=summary, + evidence=AttributionEvidence( + evidence_type="metric", + message=summary, + run_id=run.run_id, + invocation_id=invocation.invocation_id, + metric_name=metric.metric_name, + actual=metric.reason, + ), + ) + ) + + if ( + expected_json is not None + and actual_json is not None + and expected_json.get("route") != actual_json.get("route") + ): + summary = ( + f"Expected route {expected_json.get('route')!r}, " + f"got {actual_json.get('route')!r}." + ) + reasons.append( + _CandidateReason( + priority=70, + category="routing_error", + summary=summary, + evidence=AttributionEvidence( + evidence_type="response", + message=summary, + run_id=run.run_id, + invocation_id=invocation.invocation_id, + expected=expected_json.get("route"), + actual=actual_json.get("route"), + ), + ) + ) + + failed_final_metrics = [ + metric + for metric in invocation.metrics + if metric.status == "failed" + and metric.metric_name + in {"final_response_avg_score", "response_match_score", "llm_final_response"} + ] + if failed_final_metrics: + metric = failed_final_metrics[0] + summary = f"Final response did not satisfy metric {metric.metric_name!r}." + reasons.append( + _CandidateReason( + priority=80, + category="final_response_mismatch", + summary=summary, + evidence=AttributionEvidence( + evidence_type="response", + message=summary, + run_id=run.run_id, + invocation_id=invocation.invocation_id, + metric_name=metric.metric_name, + expected=invocation.expected_response, + actual=invocation.actual_response, + ), + ) + ) + + if not reasons: + summary = "Available evaluation evidence does not identify a specific failure category." + reasons.append( + _CandidateReason( + priority=90, + category="unknown", + summary=summary, + evidence=AttributionEvidence(evidence_type="metric", message=summary), + ) + ) + + reasons.sort(key=lambda reason: reason.priority) + categories: list[FailureCategory] = [] + for reason in reasons: + if reason.category not in categories: + categories.append(reason.category) + return FailureAttribution( + primary_category=categories[0], + secondary_categories=categories[1:], + summary=reasons[0].summary, + evidence=[reason.evidence for reason in reasons], + ) + + +def attribute_evaluation(evaluation: StandardizedEvaluation) -> StandardizedEvaluation: + """Return a copy with deterministic attribution attached to failed cases.""" + return evaluation.model_copy( + update={ + "cases": [ + case.model_copy(update={"attribution": _case_attribution(case)}) + for case in evaluation.cases + ] + } + ) diff --git a/examples/optimization/eval_optimize_loop/business_agent.py b/examples/optimization/eval_optimize_loop/business_agent.py new file mode 100644 index 00000000..e849e7e7 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/business_agent.py @@ -0,0 +1,102 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under the Apache License, Version 2.0. +"""共享的 SDK 业务 Agent：模型可注入，Prompt 每次重新读取。""" + +from __future__ import annotations + +from collections.abc import Callable +from uuid import uuid4 + +from trpc_agent_sdk.agents import LlmAgent +from trpc_agent_sdk.evaluation import TargetPrompt +from trpc_agent_sdk.models import LLMModel +from trpc_agent_sdk.runners import Runner +from trpc_agent_sdk.sessions import InMemorySessionService +from trpc_agent_sdk.types import Content +from trpc_agent_sdk.types import GenerateContentConfig +from trpc_agent_sdk.types import Part + + +ModelFactory = Callable[[], LLMModel] + + +def render_instruction(prompts: dict[str, str]) -> str: + """稳定拼接一个或多个工作 Prompt 字段。""" + if len(prompts) == 1: + return next(iter(prompts.values())) + return "\n\n".join( + f"## {name}\n{content}" for name, content in prompts.items() + ) + + +class BusinessAgent: + """以独立 SDK Session 执行一次 Prompt 敏感的业务请求。""" + + def __init__( + self, + target_prompt: TargetPrompt, + model_factory: ModelFactory, + *, + agent_name: str, + app_name: str, + user_id: str, + ) -> None: + self._target_prompt = target_prompt + self._model_factory = model_factory + self._agent_name = agent_name + self._app_name = app_name + self._user_id = user_id + + async def call_agent(self, query: str) -> str: + """重新读取 Prompt，运行独立 Session，并返回最终可见文本。""" + if not isinstance(query, str): + raise TypeError("query must be a string") + + prompts = await self._target_prompt.read_all() + model = self._model_factory() + root_agent = LlmAgent( + name=self._agent_name, + description="Evaluation and prompt optimization business agent.", + model=model, + instruction=render_instruction(prompts), + generate_content_config=GenerateContentConfig( + temperature=0.0, + max_output_tokens=512, + ), + ) + session_service = InMemorySessionService() + runner = Runner( + app_name=self._app_name, + agent=root_agent, + session_service=session_service, + ) + session_id = str(uuid4()) + await session_service.create_session( + app_name=self._app_name, + user_id=self._user_id, + session_id=session_id, + state={}, + ) + message = Content( + role="user", + parts=[Part.from_text(text=query)], + ) + final_text = "" + async for event in runner.run_async( + user_id=self._user_id, + session_id=session_id, + new_message=message, + ): + if ( + not event.is_final_response() + or not event.content + or not event.content.parts + ): + continue + for part in event.content.parts: + if not part.thought and part.text: + final_text += part.text + return final_text.strip() diff --git a/examples/optimization/eval_optimize_loop/candidate_provider.py b/examples/optimization/eval_optimize_loop/candidate_provider.py new file mode 100644 index 00000000..3b3882dc --- /dev/null +++ b/examples/optimization/eval_optimize_loop/candidate_provider.py @@ -0,0 +1,213 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under the Apache License, Version 2.0. +"""Candidate provider boundary for fake and AgentOptimizer-backed proposals.""" + +from __future__ import annotations + +import json +import shutil +from dataclasses import dataclass +from hashlib import sha256 +from pathlib import Path +from typing import Protocol + +from trpc_agent_sdk.evaluation import AgentOptimizer +from trpc_agent_sdk.evaluation import CallAgent +from trpc_agent_sdk.evaluation import OptimizeResult +from trpc_agent_sdk.evaluation import TargetPrompt + +from .fake.candidate_provider import DeterministicFakeCandidateProvider +from .schemas import CandidateProposal +from .schemas import CandidateScenario +from .schemas import OptimizerCandidateProposal +from .schemas import OptimizerRuntimeParameters +from .sensitive_config import replace_persisted_sensitive_values + + +class CandidateProviderError(RuntimeError): + """A provider could not produce a safe, complete candidate.""" + + +def prompt_mapping_sha256(prompts: dict[str, str]) -> str: + """Hash a complete prompt mapping using a stable JSON representation.""" + canonical = json.dumps( + prompts, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ) + return sha256(canonical.encode("utf-8")).hexdigest() + + +@dataclass(frozen=True) +class CandidateRequest: + """Validated inputs handed to one candidate provider.""" + + current_prompts: dict[str, str] + target_prompt: TargetPrompt + optimizer_config_path: Path + train_evalset_path: Path + validation_evalset_path: Path + output_dir: Path + seed: int + retain_native_artifacts: bool = True + runtime_parameters: OptimizerRuntimeParameters | None = None + expected_optimizer_sha256: str | None = None + + +@dataclass(frozen=True) +class CandidateGeneration: + """A normalized proposal plus an optional native optimizer result.""" + + proposal: CandidateProposal + optimize_result: OptimizeResult | None = None + + +class CandidateProvider(Protocol): + """Asynchronous candidate generation used by the pipeline orchestrator.""" + + async def propose(self, request: CandidateRequest) -> CandidateGeneration: + """Return one complete proposal without updating source prompts.""" + + +class FakeCandidateProviderAdapter: + """Lift the pure synchronous fake provider into the common async boundary.""" + + def __init__(self, scenario: CandidateScenario) -> None: + self._scenario = scenario + + async def propose(self, request: CandidateRequest) -> CandidateGeneration: + proposal = DeterministicFakeCandidateProvider().propose( + request.current_prompts, + scenario=self._scenario, + seed=request.seed, + ) + return CandidateGeneration(proposal=proposal) + + +class AgentOptimizerCandidateProvider: + """Adapt AgentOptimizer to the pipeline's review-before-write contract.""" + + def __init__(self, call_agent: CallAgent) -> None: + self._call_agent = call_agent + + @staticmethod + def _replace_persisted_connection_values(value: object) -> object: + """递归将可能被 SDK 复制到产物的连接值替换为环境占位符。""" + return replace_persisted_sensitive_values(value) + + @staticmethod + def _prepare_runtime_config(request: CandidateRequest) -> Path: + """由已校验模板生成无明文凭据的本次运行配置。""" + if request.runtime_parameters is None: + return request.optimizer_config_path + + try: + raw = request.optimizer_config_path.read_bytes() + if ( + request.expected_optimizer_sha256 is not None + and sha256(raw).hexdigest() != request.expected_optimizer_sha256 + ): + raise CandidateProviderError("optimizer config changed after preparation") + payload = AgentOptimizerCandidateProvider._replace_persisted_connection_values( + json.loads(raw.decode("utf-8")) + ) + algorithm = payload["optimize"]["algorithm"] + except CandidateProviderError: + raise + except (OSError, UnicodeDecodeError, json.JSONDecodeError, KeyError, TypeError) as exc: + raise CandidateProviderError(f"failed to prepare optimizer runtime config: {exc}") from exc + + parameters = request.runtime_parameters + reflection_lm: dict[str, object] = { + "provider_name": parameters.provider_name, + "model_name": parameters.model_name, + "variant": parameters.variant, + "base_url": "${TRPC_AGENT_BASE_URL}", + "api_key": "${TRPC_AGENT_API_KEY}", + "generation_config": { + "temperature": parameters.temperature, + "max_tokens": parameters.max_tokens, + }, + } + if parameters.think is not None: + reflection_lm["think"] = parameters.think + algorithm["reflection_lm"] = reflection_lm + algorithm["max_candidate_proposals"] = parameters.max_candidate_proposals + + runtime_path = request.output_dir.parent / "optimizer.runtime.json" + try: + runtime_path.parent.mkdir(parents=True, exist_ok=True) + runtime_path.write_text( + json.dumps(payload, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + except OSError as exc: + raise CandidateProviderError(f"failed to write optimizer runtime config: {exc}") from exc + return runtime_path + + async def propose(self, request: CandidateRequest) -> CandidateGeneration: + runtime_config_path = self._prepare_runtime_config(request) + try: + result = await AgentOptimizer.optimize( + config_path=str(runtime_config_path), + call_agent=self._call_agent, + target_prompt=request.target_prompt, + train_dataset_path=str(request.train_evalset_path), + validation_dataset_path=str(request.validation_evalset_path), + output_dir=str(request.output_dir), + update_source=False, + verbose=0, + ) + except Exception as exc: + raise CandidateProviderError(f"AgentOptimizer failed: {exc}") from exc + + if result.status != "SUCCEEDED": + raise CandidateProviderError( + f"AgentOptimizer returned {result.status}: {result.error_message or result.finish_reason}" + ) + expected_fields = set(request.current_prompts) + if set(result.baseline_prompts) != expected_fields: + raise CandidateProviderError("optimizer baseline prompt fields do not match the prepared target") + if result.baseline_prompts != request.current_prompts: + raise CandidateProviderError("optimizer baseline prompts do not match the prepared working prompts") + if set(result.best_prompts) != expected_fields: + raise CandidateProviderError("optimizer best prompt fields do not match the prepared target") + if any(not isinstance(value, str) for value in result.best_prompts.values()): + raise CandidateProviderError("optimizer best prompts must contain only strings") + + parent_hash = prompt_mapping_sha256(request.current_prompts) + candidate_hash = prompt_mapping_sha256(result.best_prompts) + changed_fields = [ + name + for name in request.current_prompts + if request.current_prompts[name] != result.best_prompts[name] + ] + retained_output_dir = str(request.output_dir) if request.retain_native_artifacts else None + proposal = OptimizerCandidateProposal( + prompts=dict(result.best_prompts), + changed_fields=changed_fields, + rationale=( + f"AgentOptimizer selected the best candidate after {result.total_rounds} rounds " + f"with finish_reason={result.finish_reason}." + ), + parent_prompt_sha256=parent_hash, + candidate_prompt_sha256=candidate_hash, + candidate_id=f"optimizer-{candidate_hash[:12]}", + finish_reason=result.finish_reason, + stop_reason=result.stop_reason, + baseline_pass_rate=result.baseline_pass_rate, + best_pass_rate=result.best_pass_rate, + optimizer_output_dir=retained_output_dir, + ) + if not request.retain_native_artifacts: + try: + shutil.rmtree(request.output_dir) + except OSError as exc: + raise CandidateProviderError( + f"failed to discard optimizer artifacts: {exc}" + ) from exc + return CandidateGeneration(proposal=proposal, optimize_result=result) diff --git a/examples/optimization/eval_optimize_loop/case_diff.py b/examples/optimization/eval_optimize_loop/case_diff.py new file mode 100644 index 00000000..ad7ba724 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/case_diff.py @@ -0,0 +1,181 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under the Apache License, Version 2.0. +"""Deterministic baseline/candidate case comparison for stage 3a.""" + +from __future__ import annotations + +import math +from typing import Literal + +from .evaluation_adapter import EvaluationAnalysisError +from .schemas import CaseDiff +from .schemas import CaseEvaluation +from .schemas import ChangeKind +from .schemas import DatasetDiff +from .schemas import EvaluationStatus +from .schemas import MetricDelta +from .schemas import MetricOutcome +from .schemas import ObservableValue +from .schemas import StandardizedEvaluation + + +def _unavailable(reason: str) -> ObservableValue: + return ObservableValue(status="unavailable", reason=reason) + + +def _delta(baseline: ObservableValue, candidate: ObservableValue, *, reason: str) -> ObservableValue: + if baseline.status != "available" or candidate.status != "available": + return _unavailable(reason) + return ObservableValue(status="available", value=float(candidate.value) - float(baseline.value)) + + +def _change( + baseline_status: EvaluationStatus, + candidate_status: EvaluationStatus, + score_delta: ObservableValue, +) -> ChangeKind: + if "not_evaluated" in {baseline_status, candidate_status}: + return "incomparable" + if baseline_status == "failed" and candidate_status == "passed": + return "newly_passed" + if baseline_status == "passed" and candidate_status == "failed": + return "newly_failed" + if score_delta.status != "available": + return "incomparable" + if float(score_delta.value) > 0.0 and not math.isclose(float(score_delta.value), 0.0, abs_tol=1e-12): + return "improved" + if float(score_delta.value) < 0.0 and not math.isclose(float(score_delta.value), 0.0, abs_tol=1e-12): + return "regressed" + return "unchanged" + + +def _metric_map(case: CaseEvaluation) -> dict[str, MetricOutcome]: + return {metric.metric_name: metric for metric in case.metrics} + + +def _metric_deltas(baseline: CaseEvaluation, candidate: CaseEvaluation) -> list[MetricDelta]: + baseline_metrics = _metric_map(baseline) + candidate_metrics = _metric_map(candidate) + if set(baseline_metrics) != set(candidate_metrics): + raise EvaluationAnalysisError( + f"case {baseline.eval_id!r} metric sets differ between baseline and candidate" + ) + deltas: list[MetricDelta] = [] + for name in sorted(baseline_metrics): + before = baseline_metrics[name] + after = candidate_metrics[name] + if before.threshold != after.threshold: + raise EvaluationAnalysisError( + f"case {baseline.eval_id!r} metric {name!r} threshold changed " + f"from {before.threshold} to {after.threshold}" + ) + score_delta = _delta( + before.score, + after.score, + reason=f"case {baseline.eval_id!r} metric {name!r} score delta is unavailable", + ) + deltas.append( + MetricDelta( + metric_name=name, + baseline_status=before.status, + candidate_status=after.status, + baseline_score=before.score, + candidate_score=after.score, + score_delta=score_delta, + change=_change(before.status, after.status, score_delta), + ) + ) + return deltas + + +def _case_diff( + baseline: CaseEvaluation, + candidate: CaseEvaluation, + *, + split: Literal["train", "validation"], + hard_case_ids: set[str], + critical_case_ids: set[str], + severe_case_score_drop: float, +) -> CaseDiff: + score_delta = _delta( + baseline.average_score, + candidate.average_score, + reason=f"case {baseline.eval_id!r} aggregate score delta is unavailable", + ) + severe = ( + score_delta.status == "available" + and float(score_delta.value) <= -severe_case_score_drop + and not math.isclose(float(score_delta.value), 0.0, abs_tol=1e-12) + ) + return CaseDiff( + eval_id=baseline.eval_id, + split=split, + baseline_status=baseline.status, + candidate_status=candidate.status, + baseline_score=baseline.average_score, + candidate_score=candidate.average_score, + score_delta=score_delta, + change=_change(baseline.status, candidate.status, score_delta), + metrics=_metric_deltas(baseline, candidate), + baseline_attribution=baseline.attribution, + candidate_attribution=candidate.attribution, + is_hard=baseline.eval_id in hard_case_ids, + is_critical=baseline.eval_id in critical_case_ids, + severe_regression=severe, + ) + + +def compare_evaluations( + baseline: StandardizedEvaluation, + candidate: StandardizedEvaluation, + *, + hard_case_ids: set[str], + critical_case_ids: set[str], + severe_case_score_drop: float, +) -> DatasetDiff: + """Compare matching baseline and candidate evaluations for one split.""" + if baseline.phase != "baseline" or candidate.phase != "candidate": + raise EvaluationAnalysisError("evaluation comparison requires baseline then candidate phases") + if baseline.split != candidate.split: + raise EvaluationAnalysisError("baseline and candidate splits do not match") + if baseline.eval_set_id != candidate.eval_set_id: + raise EvaluationAnalysisError("baseline and candidate eval_set_id values do not match") + + baseline_cases = {case.eval_id: case for case in baseline.cases} + candidate_cases = {case.eval_id: case for case in candidate.cases} + if set(baseline_cases) != set(candidate_cases): + raise EvaluationAnalysisError("baseline and candidate case ids do not match") + + cases = [ + _case_diff( + baseline_cases[eval_id], + candidate_cases[eval_id], + split=baseline.split, + hard_case_ids=hard_case_ids, + critical_case_ids=critical_case_ids, + severe_case_score_drop=severe_case_score_drop, + ) + for eval_id in sorted(baseline_cases) + ] + score_delta = _delta( + baseline.average_score, + candidate.average_score, + reason=f"{baseline.split} dataset score delta is unavailable", + ) + return DatasetDiff( + split=baseline.split, + eval_set_id=baseline.eval_set_id, + cases=cases, + baseline_average_score=baseline.average_score, + candidate_average_score=candidate.average_score, + score_delta=score_delta, + newly_passed_count=sum(case.change == "newly_passed" for case in cases), + newly_failed_count=sum(case.change == "newly_failed" for case in cases), + improved_count=sum(case.change == "improved" for case in cases), + regressed_count=sum(case.change == "regressed" for case in cases), + unchanged_count=sum(case.change == "unchanged" for case in cases), + incomparable_count=sum(case.change == "incomparable" for case in cases), + ) diff --git a/examples/optimization/eval_optimize_loop/config.py b/examples/optimization/eval_optimize_loop/config.py new file mode 100644 index 00000000..20536963 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/config.py @@ -0,0 +1,274 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under the Apache License, Version 2.0. +"""Pipeline-specific configuration. + +``optimizer.json`` deliberately remains an SDK ``OptimizeConfigFile``. The +configuration in this module contains only orchestration concerns that do not +belong in the SDK optimizer schema: isolated prompt sources, gate policy, +budgets, reporting, and artifact retention. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Literal +from typing import Optional +from typing import Union + +from pydantic import Field +from pydantic import field_validator +from pydantic import model_validator + +from trpc_agent_sdk.evaluation import EvalBaseModel + + +_PROMPT_NAME_PATTERN = r"^[A-Za-z][A-Za-z0-9_]*$" +_RUN_ID_PATTERN = r"^[A-Za-z0-9][A-Za-z0-9_-]*$" + + +class ExecutionConfig(EvalBaseModel): + """Pipeline execution mode and deterministic candidate scenario.""" + + mode: Literal["offline", "real", "trace"] = "offline" + candidate_scenario: Literal["improve", "no_improvement", "overfit"] = "improve" + + @model_validator(mode="before") + @classmethod + def _reject_removed_execution_options(cls, value: object) -> object: + if not isinstance(value, dict): + return value + if value.get("mode") == "fake": + raise ValueError("execution.mode='fake' was renamed to 'offline'") + if "use_fake_judge" in value: + raise ValueError( + "execution.use_fake_judge was removed; configure evaluation " + "metrics or rubric explicitly in optimizer.json" + ) + if "fake_candidate_scenario" in value: + raise ValueError( + "execution.fake_candidate_scenario was renamed to " + "execution.candidate_scenario" + ) + return value + + +class InputPathsConfig(EvalBaseModel): + """Files shared by the baseline, candidate, and optimizer runs.""" + + train_evalset: str + validation_evalset: str + optimizer_config: str + + @field_validator("train_evalset", "validation_evalset", "optimizer_config") + @classmethod + def _require_non_empty_relative_path(cls, value: str) -> str: + path = Path(value) + if not value.strip(): + raise ValueError("path must not be empty") + if path.is_absolute(): + raise ValueError("path must be relative to pipeline.json") + return value + + +class PromptFieldConfig(EvalBaseModel): + """One file-backed field that forms the pipeline TargetPrompt.""" + + name: str = Field(pattern=_PROMPT_NAME_PATTERN) + path: str + + @field_validator("path") + @classmethod + def _require_non_empty_relative_path(cls, value: str) -> str: + path = Path(value) + if not value.strip(): + raise ValueError("prompt path must not be empty") + if path.is_absolute(): + raise ValueError("prompt path must be relative to pipeline.json") + return value + + +class TraceCandidateInputsConfig(EvalBaseModel): + """一个 Trace 候选版本的评测集和 Prompt 快照路径。""" + + train_evalset: str + validation_evalset: str + prompts: list[PromptFieldConfig] = Field(min_length=1) + + @field_validator("train_evalset", "validation_evalset") + @classmethod + def _require_relative_trace_path(cls, value: str) -> str: + if not value.strip() or Path(value).is_absolute(): + raise ValueError("trace evalset path must be a non-empty relative path") + return value + + +class TraceInputsConfig(EvalBaseModel): + """三个确定性候选场景的 Trace 输入。""" + + candidates: dict[ + Literal["improve", "no_improvement", "overfit"], + TraceCandidateInputsConfig, + ] + + @model_validator(mode="after") + def _require_all_scenarios(self) -> "TraceInputsConfig": + required = {"improve", "no_improvement", "overfit"} + if set(self.candidates) != required: + raise ValueError("trace_inputs must define improve, no_improvement, and overfit") + return self + + +class RunConfig(EvalBaseModel): + """Reproducibility and workspace location settings.""" + + runs_dir: str = "runs" + run_id: Optional[str] = Field(default=None, pattern=_RUN_ID_PATTERN) + seed: int = 42 + + @field_validator("runs_dir") + @classmethod + def _require_non_empty_relative_path(cls, value: str) -> str: + path = Path(value) + if not value.strip(): + raise ValueError("runs_dir must not be empty") + if path.is_absolute(): + raise ValueError("runs_dir must be relative to pipeline.json") + return value + + +class CaseLabelsConfig(EvalBaseModel): + """Case identifiers with stronger gate guarantees.""" + + hard_case_ids: list[str] = Field(default_factory=list) + critical_case_ids: list[str] = Field(default_factory=list) + + @field_validator("hard_case_ids", "critical_case_ids") + @classmethod + def _require_unique_non_empty_ids(cls, values: list[str]) -> list[str]: + if any(not value.strip() for value in values): + raise ValueError("case labels must not contain empty IDs") + if len(values) != len(set(values)): + raise ValueError("case labels must not contain duplicate IDs") + return values + + +class GateConfig(EvalBaseModel): + """Acceptance policy consumed by the gate phase.""" + + min_validation_score_delta: float = Field(default=0.01, ge=0.0) + reject_on_validation_pass_rate_drop: bool = True + reject_new_hard_fail: bool = True + reject_critical_regression: bool = True + severe_case_score_drop: float = Field(default=0.20, ge=0.0, le=1.0) + required_metrics: Union[Literal["all"], list[str]] = "all" + + @field_validator("required_metrics") + @classmethod + def _require_unique_metric_names(cls, value: Union[str, list[str]]) -> Union[str, list[str]]: + if not isinstance(value, list): + return value + if any(not item.strip() for item in value): + raise ValueError("required_metrics must not contain empty metric names") + if len(value) != len(set(value)): + raise ValueError("required_metrics must not contain duplicates") + return value + + +class BudgetConfig(EvalBaseModel): + """Resource limits and the policy for measurements unavailable from the SDK.""" + + max_cost_usd: Optional[float] = Field(default=None, ge=0.0) + max_tokens: Optional[int] = Field(default=None, ge=0) + max_duration_seconds: Optional[float] = Field(default=None, gt=0.0) + on_unavailable: Literal["reject", "warning"] = "reject" + + +class ReportingConfig(EvalBaseModel): + """Report formats selected for a successful future pipeline run.""" + + write_json: bool = True + write_markdown: bool = True + include_case_evidence: bool = True + + @model_validator(mode="after") + def _require_report_format(self) -> "ReportingConfig": + if not self.write_json and not self.write_markdown: + raise ValueError("at least one of write_json or write_markdown must be enabled") + return self + + +class ArtifactConfig(EvalBaseModel): + """Which reproducibility artifacts future phases must retain.""" + + copy_input_files: bool = True + retain_optimizer_native_artifacts: bool = True + audit_all_candidates: bool = False + + +class WritebackConfig(EvalBaseModel): + """Safety settings used only after a future ACCEPT decision.""" + + enabled: bool = False + require_source_hash_match: bool = True + + @model_validator(mode="after") + def _require_hash_guard_when_enabled(self) -> "WritebackConfig": + if self.enabled and not self.require_source_hash_match: + raise ValueError("enabled writeback requires require_source_hash_match=true") + return self + + +class PipelineConfig(EvalBaseModel): + """The complete, example-local ``pipeline.json`` schema (version 1).""" + + config_version: Literal[1] = 1 + execution: ExecutionConfig = Field(default_factory=ExecutionConfig) + inputs: InputPathsConfig + prompts: list[PromptFieldConfig] = Field(min_length=1) + run: RunConfig = Field(default_factory=RunConfig) + case_labels: CaseLabelsConfig = Field(default_factory=CaseLabelsConfig) + gate: GateConfig = Field(default_factory=GateConfig) + budget: BudgetConfig = Field(default_factory=BudgetConfig) + reporting: ReportingConfig = Field(default_factory=ReportingConfig) + artifacts: ArtifactConfig = Field(default_factory=ArtifactConfig) + writeback: WritebackConfig = Field(default_factory=WritebackConfig) + trace_inputs: Optional[TraceInputsConfig] = None + + @model_validator(mode="after") + def _require_unique_prompt_names(self) -> "PipelineConfig": + names = [prompt.name for prompt in self.prompts] + if len(names) != len(set(names)): + raise ValueError("prompts must not contain duplicate field names") + if self.execution.mode == "trace": + if self.trace_inputs is None: + raise ValueError("trace mode requires trace_inputs") + if self.writeback.enabled: + raise ValueError("trace mode does not allow source Prompt writeback") + expected = set(names) + for scenario, inputs in self.trace_inputs.candidates.items(): + candidate_names = [prompt.name for prompt in inputs.prompts] + if len(candidate_names) != len(set(candidate_names)): + raise ValueError( + f"trace candidate {scenario} has duplicate prompt names" + ) + if set(candidate_names) != expected: + raise ValueError( + f"trace candidate {scenario} prompt fields must match baseline" + ) + elif self.trace_inputs is not None: + raise ValueError("trace_inputs is only allowed in trace mode") + return self + + +def load_pipeline_config(path: str | Path) -> PipelineConfig: + """Load a pipeline config while retaining path resolution at the caller. + + Paths intentionally remain relative strings in the model so a copied example + directory remains relocatable. ``prepare_run`` resolves and validates them + relative to this file. + """ + config_path = Path(path) + return PipelineConfig.model_validate_json(config_path.read_text(encoding="utf-8")) diff --git a/examples/optimization/eval_optimize_loop/data/traces/baseline.train.evalset.json b/examples/optimization/eval_optimize_loop/data/traces/baseline.train.evalset.json new file mode 100644 index 00000000..eef80beb --- /dev/null +++ b/examples/optimization/eval_optimize_loop/data/traces/baseline.train.evalset.json @@ -0,0 +1 @@ +{"eval_set_id":"trace_train","eval_cases":[{"eval_id":"train_output_format","eval_mode":"trace","conversation":[{"invocation_id":"train_output_format","user_content":{"role":"user","parts":[{"text":"How can I update my email address?"}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"account\",\"message\":\"Open profile settings to update your email.\"}"}]}}],"actual_conversation":[{"invocation_id":"train_output_format","user_content":{"role":"user","parts":[{"text":"How can I update my email address?"}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"account\",\"message\":\"Open profile settings to update your email.\"}"}]}}]},{"eval_id":"train_tool_choice","eval_mode":"trace","conversation":[{"invocation_id":"train_tool_choice","user_content":{"role":"user","parts":[{"text":"Check the status of order A100."}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"order_lookup\",\"message\":\"Checking order A100.\"}"}]}}],"actual_conversation":[{"invocation_id":"train_tool_choice","user_content":{"role":"user","parts":[{"text":"Check the status of order A100."}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"general_support\",\"message\":\"Please provide more details so I can route your request.\"}"}]}}]},{"eval_id":"train_tool_arguments","eval_mode":"trace","conversation":[{"invocation_id":"train_tool_arguments","user_content":{"role":"user","parts":[{"text":"Look up order B-204 for customer 17."}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"order_lookup\",\"message\":\"Checking order B-204 for customer 17.\"}"}]}}],"actual_conversation":[{"invocation_id":"train_tool_arguments","user_content":{"role":"user","parts":[{"text":"Look up order B-204 for customer 17."}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"general_support\",\"message\":\"Please provide more details so I can route your request.\"}"}]}}]}]} diff --git a/examples/optimization/eval_optimize_loop/data/traces/baseline.validation.evalset.json b/examples/optimization/eval_optimize_loop/data/traces/baseline.validation.evalset.json new file mode 100644 index 00000000..c3dfdf1f --- /dev/null +++ b/examples/optimization/eval_optimize_loop/data/traces/baseline.validation.evalset.json @@ -0,0 +1 @@ +{"eval_set_id":"trace_validation","eval_cases":[{"eval_id":"val_paraphrase","eval_mode":"trace","conversation":[{"invocation_id":"val_paraphrase","user_content":{"role":"user","parts":[{"text":"Where do I change the address tied to my account?"}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"account\",\"message\":\"Open profile settings to update your address.\"}"}]}}],"actual_conversation":[{"invocation_id":"val_paraphrase","user_content":{"role":"user","parts":[{"text":"Where do I change the address tied to my account?"}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"general_support\",\"message\":\"Please provide more details so I can route your request.\"}"}]}}]},{"eval_id":"val_knowledge_recall","eval_mode":"trace","conversation":[{"invocation_id":"val_knowledge_recall","user_content":{"role":"user","parts":[{"text":"How long does standard shipping usually take?"}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"shipping_policy\",\"message\":\"Standard shipping normally takes 3-5 business days.\"}"}]}}],"actual_conversation":[{"invocation_id":"val_knowledge_recall","user_content":{"role":"user","parts":[{"text":"How long does standard shipping usually take?"}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"general_support\",\"message\":\"Please provide more details so I can route your request.\"}"}]}}]},{"eval_id":"val_refund_route","eval_mode":"trace","conversation":[{"invocation_id":"val_refund_route","user_content":{"role":"user","parts":[{"text":"I was charged twice and need the duplicate payment refunded."}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"billing_refund\",\"message\":\"I will route this duplicate charge for refund review.\"}"}]}}],"actual_conversation":[{"invocation_id":"val_refund_route","user_content":{"role":"user","parts":[{"text":"I was charged twice and need the duplicate payment refunded."}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"billing_refund\",\"message\":\"I will route this duplicate charge for refund review.\"}"}]}}]}]} diff --git a/examples/optimization/eval_optimize_loop/data/traces/improve.train.evalset.json b/examples/optimization/eval_optimize_loop/data/traces/improve.train.evalset.json new file mode 100644 index 00000000..0ad61dae --- /dev/null +++ b/examples/optimization/eval_optimize_loop/data/traces/improve.train.evalset.json @@ -0,0 +1 @@ +{"eval_set_id":"trace_train","eval_cases":[{"eval_id":"train_output_format","eval_mode":"trace","conversation":[{"invocation_id":"train_output_format","user_content":{"role":"user","parts":[{"text":"How can I update my email address?"}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"account\",\"message\":\"Open profile settings to update your email.\"}"}]}}],"actual_conversation":[{"invocation_id":"train_output_format","user_content":{"role":"user","parts":[{"text":"How can I update my email address?"}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"account\",\"message\":\"Open profile settings to update your email.\"}"}]}}]},{"eval_id":"train_tool_choice","eval_mode":"trace","conversation":[{"invocation_id":"train_tool_choice","user_content":{"role":"user","parts":[{"text":"Check the status of order A100."}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"order_lookup\",\"message\":\"Checking order A100.\"}"}]}}],"actual_conversation":[{"invocation_id":"train_tool_choice","user_content":{"role":"user","parts":[{"text":"Check the status of order A100."}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"order_lookup\",\"message\":\"Checking order A100.\"}"}]}}]},{"eval_id":"train_tool_arguments","eval_mode":"trace","conversation":[{"invocation_id":"train_tool_arguments","user_content":{"role":"user","parts":[{"text":"Look up order B-204 for customer 17."}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"order_lookup\",\"message\":\"Checking order B-204 for customer 17.\"}"}]}}],"actual_conversation":[{"invocation_id":"train_tool_arguments","user_content":{"role":"user","parts":[{"text":"Look up order B-204 for customer 17."}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"order_lookup\",\"message\":\"Checking order B-204 for customer 17.\"}"}]}}]}]} diff --git a/examples/optimization/eval_optimize_loop/data/traces/improve.validation.evalset.json b/examples/optimization/eval_optimize_loop/data/traces/improve.validation.evalset.json new file mode 100644 index 00000000..4f9a9fde --- /dev/null +++ b/examples/optimization/eval_optimize_loop/data/traces/improve.validation.evalset.json @@ -0,0 +1 @@ +{"eval_set_id":"trace_validation","eval_cases":[{"eval_id":"val_paraphrase","eval_mode":"trace","conversation":[{"invocation_id":"val_paraphrase","user_content":{"role":"user","parts":[{"text":"Where do I change the address tied to my account?"}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"account\",\"message\":\"Open profile settings to update your address.\"}"}]}}],"actual_conversation":[{"invocation_id":"val_paraphrase","user_content":{"role":"user","parts":[{"text":"Where do I change the address tied to my account?"}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"account\",\"message\":\"Open profile settings to update your address.\"}"}]}}]},{"eval_id":"val_knowledge_recall","eval_mode":"trace","conversation":[{"invocation_id":"val_knowledge_recall","user_content":{"role":"user","parts":[{"text":"How long does standard shipping usually take?"}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"shipping_policy\",\"message\":\"Standard shipping normally takes 3-5 business days.\"}"}]}}],"actual_conversation":[{"invocation_id":"val_knowledge_recall","user_content":{"role":"user","parts":[{"text":"How long does standard shipping usually take?"}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"shipping_policy\",\"message\":\"Standard shipping normally takes 3-5 business days.\"}"}]}}]},{"eval_id":"val_refund_route","eval_mode":"trace","conversation":[{"invocation_id":"val_refund_route","user_content":{"role":"user","parts":[{"text":"I was charged twice and need the duplicate payment refunded."}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"billing_refund\",\"message\":\"I will route this duplicate charge for refund review.\"}"}]}}],"actual_conversation":[{"invocation_id":"val_refund_route","user_content":{"role":"user","parts":[{"text":"I was charged twice and need the duplicate payment refunded."}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"billing_refund\",\"message\":\"I will route this duplicate charge for refund review.\"}"}]}}]}]} diff --git a/examples/optimization/eval_optimize_loop/data/traces/no_improvement.train.evalset.json b/examples/optimization/eval_optimize_loop/data/traces/no_improvement.train.evalset.json new file mode 100644 index 00000000..eef80beb --- /dev/null +++ b/examples/optimization/eval_optimize_loop/data/traces/no_improvement.train.evalset.json @@ -0,0 +1 @@ +{"eval_set_id":"trace_train","eval_cases":[{"eval_id":"train_output_format","eval_mode":"trace","conversation":[{"invocation_id":"train_output_format","user_content":{"role":"user","parts":[{"text":"How can I update my email address?"}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"account\",\"message\":\"Open profile settings to update your email.\"}"}]}}],"actual_conversation":[{"invocation_id":"train_output_format","user_content":{"role":"user","parts":[{"text":"How can I update my email address?"}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"account\",\"message\":\"Open profile settings to update your email.\"}"}]}}]},{"eval_id":"train_tool_choice","eval_mode":"trace","conversation":[{"invocation_id":"train_tool_choice","user_content":{"role":"user","parts":[{"text":"Check the status of order A100."}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"order_lookup\",\"message\":\"Checking order A100.\"}"}]}}],"actual_conversation":[{"invocation_id":"train_tool_choice","user_content":{"role":"user","parts":[{"text":"Check the status of order A100."}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"general_support\",\"message\":\"Please provide more details so I can route your request.\"}"}]}}]},{"eval_id":"train_tool_arguments","eval_mode":"trace","conversation":[{"invocation_id":"train_tool_arguments","user_content":{"role":"user","parts":[{"text":"Look up order B-204 for customer 17."}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"order_lookup\",\"message\":\"Checking order B-204 for customer 17.\"}"}]}}],"actual_conversation":[{"invocation_id":"train_tool_arguments","user_content":{"role":"user","parts":[{"text":"Look up order B-204 for customer 17."}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"general_support\",\"message\":\"Please provide more details so I can route your request.\"}"}]}}]}]} diff --git a/examples/optimization/eval_optimize_loop/data/traces/no_improvement.validation.evalset.json b/examples/optimization/eval_optimize_loop/data/traces/no_improvement.validation.evalset.json new file mode 100644 index 00000000..c3dfdf1f --- /dev/null +++ b/examples/optimization/eval_optimize_loop/data/traces/no_improvement.validation.evalset.json @@ -0,0 +1 @@ +{"eval_set_id":"trace_validation","eval_cases":[{"eval_id":"val_paraphrase","eval_mode":"trace","conversation":[{"invocation_id":"val_paraphrase","user_content":{"role":"user","parts":[{"text":"Where do I change the address tied to my account?"}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"account\",\"message\":\"Open profile settings to update your address.\"}"}]}}],"actual_conversation":[{"invocation_id":"val_paraphrase","user_content":{"role":"user","parts":[{"text":"Where do I change the address tied to my account?"}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"general_support\",\"message\":\"Please provide more details so I can route your request.\"}"}]}}]},{"eval_id":"val_knowledge_recall","eval_mode":"trace","conversation":[{"invocation_id":"val_knowledge_recall","user_content":{"role":"user","parts":[{"text":"How long does standard shipping usually take?"}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"shipping_policy\",\"message\":\"Standard shipping normally takes 3-5 business days.\"}"}]}}],"actual_conversation":[{"invocation_id":"val_knowledge_recall","user_content":{"role":"user","parts":[{"text":"How long does standard shipping usually take?"}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"general_support\",\"message\":\"Please provide more details so I can route your request.\"}"}]}}]},{"eval_id":"val_refund_route","eval_mode":"trace","conversation":[{"invocation_id":"val_refund_route","user_content":{"role":"user","parts":[{"text":"I was charged twice and need the duplicate payment refunded."}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"billing_refund\",\"message\":\"I will route this duplicate charge for refund review.\"}"}]}}],"actual_conversation":[{"invocation_id":"val_refund_route","user_content":{"role":"user","parts":[{"text":"I was charged twice and need the duplicate payment refunded."}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"billing_refund\",\"message\":\"I will route this duplicate charge for refund review.\"}"}]}}]}]} diff --git a/examples/optimization/eval_optimize_loop/data/traces/overfit.train.evalset.json b/examples/optimization/eval_optimize_loop/data/traces/overfit.train.evalset.json new file mode 100644 index 00000000..0ad61dae --- /dev/null +++ b/examples/optimization/eval_optimize_loop/data/traces/overfit.train.evalset.json @@ -0,0 +1 @@ +{"eval_set_id":"trace_train","eval_cases":[{"eval_id":"train_output_format","eval_mode":"trace","conversation":[{"invocation_id":"train_output_format","user_content":{"role":"user","parts":[{"text":"How can I update my email address?"}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"account\",\"message\":\"Open profile settings to update your email.\"}"}]}}],"actual_conversation":[{"invocation_id":"train_output_format","user_content":{"role":"user","parts":[{"text":"How can I update my email address?"}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"account\",\"message\":\"Open profile settings to update your email.\"}"}]}}]},{"eval_id":"train_tool_choice","eval_mode":"trace","conversation":[{"invocation_id":"train_tool_choice","user_content":{"role":"user","parts":[{"text":"Check the status of order A100."}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"order_lookup\",\"message\":\"Checking order A100.\"}"}]}}],"actual_conversation":[{"invocation_id":"train_tool_choice","user_content":{"role":"user","parts":[{"text":"Check the status of order A100."}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"order_lookup\",\"message\":\"Checking order A100.\"}"}]}}]},{"eval_id":"train_tool_arguments","eval_mode":"trace","conversation":[{"invocation_id":"train_tool_arguments","user_content":{"role":"user","parts":[{"text":"Look up order B-204 for customer 17."}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"order_lookup\",\"message\":\"Checking order B-204 for customer 17.\"}"}]}}],"actual_conversation":[{"invocation_id":"train_tool_arguments","user_content":{"role":"user","parts":[{"text":"Look up order B-204 for customer 17."}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"order_lookup\",\"message\":\"Checking order B-204 for customer 17.\"}"}]}}]}]} diff --git a/examples/optimization/eval_optimize_loop/data/traces/overfit.validation.evalset.json b/examples/optimization/eval_optimize_loop/data/traces/overfit.validation.evalset.json new file mode 100644 index 00000000..2e7a3653 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/data/traces/overfit.validation.evalset.json @@ -0,0 +1 @@ +{"eval_set_id":"trace_validation","eval_cases":[{"eval_id":"val_paraphrase","eval_mode":"trace","conversation":[{"invocation_id":"val_paraphrase","user_content":{"role":"user","parts":[{"text":"Where do I change the address tied to my account?"}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"account\",\"message\":\"Open profile settings to update your address.\"}"}]}}],"actual_conversation":[{"invocation_id":"val_paraphrase","user_content":{"role":"user","parts":[{"text":"Where do I change the address tied to my account?"}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"general_support\",\"message\":\"Please provide more details so I can route your request.\"}"}]}}]},{"eval_id":"val_knowledge_recall","eval_mode":"trace","conversation":[{"invocation_id":"val_knowledge_recall","user_content":{"role":"user","parts":[{"text":"How long does standard shipping usually take?"}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"shipping_policy\",\"message\":\"Standard shipping normally takes 3-5 business days.\"}"}]}}],"actual_conversation":[{"invocation_id":"val_knowledge_recall","user_content":{"role":"user","parts":[{"text":"How long does standard shipping usually take?"}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"general_support\",\"message\":\"Please provide more details so I can route your request.\"}"}]}}]},{"eval_id":"val_refund_route","eval_mode":"trace","conversation":[{"invocation_id":"val_refund_route","user_content":{"role":"user","parts":[{"text":"I was charged twice and need the duplicate payment refunded."}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"billing_refund\",\"message\":\"I will route this duplicate charge for refund review.\"}"}]}}],"actual_conversation":[{"invocation_id":"val_refund_route","user_content":{"role":"user","parts":[{"text":"I was charged twice and need the duplicate payment refunded."}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"general_support\",\"message\":\"Please provide more details so I can route your request.\"}"}]}}]}]} diff --git a/examples/optimization/eval_optimize_loop/data/traces/prompts/improve.md b/examples/optimization/eval_optimize_loop/data/traces/prompts/improve.md new file mode 100644 index 00000000..602dee3b --- /dev/null +++ b/examples/optimization/eval_optimize_loop/data/traces/prompts/improve.md @@ -0,0 +1,5 @@ +Customer support routing candidate for trace replay. + + + + diff --git a/examples/optimization/eval_optimize_loop/data/traces/prompts/no_improvement.md b/examples/optimization/eval_optimize_loop/data/traces/prompts/no_improvement.md new file mode 100644 index 00000000..6fcc4d80 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/data/traces/prompts/no_improvement.md @@ -0,0 +1 @@ +Customer support routing candidate with wording-only changes for trace replay. diff --git a/examples/optimization/eval_optimize_loop/data/traces/prompts/overfit.md b/examples/optimization/eval_optimize_loop/data/traces/prompts/overfit.md new file mode 100644 index 00000000..56d2fe69 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/data/traces/prompts/overfit.md @@ -0,0 +1,5 @@ +Narrow customer support routing candidate for trace replay. + + + + diff --git a/examples/optimization/eval_optimize_loop/data/train.evalset.json b/examples/optimization/eval_optimize_loop/data/train.evalset.json new file mode 100644 index 00000000..6b41ca3a --- /dev/null +++ b/examples/optimization/eval_optimize_loop/data/train.evalset.json @@ -0,0 +1,40 @@ +{ + "eval_set_id": "eval_optimize_loop_train", + "name": "Evaluation optimization loop - train", + "description": "Three deterministic training cases for format, tool choice, and tool arguments.", + "eval_cases": [ + { + "eval_id": "train_output_format", + "conversation": [ + { + "invocation_id": "train-1", + "user_content": {"parts": [{"text": "How can I update my email address?"}], "role": "user"}, + "final_response": {"parts": [{"text": "{\"route\":\"account\",\"message\":\"Open profile settings to update your email.\"}"}], "role": "model"} + } + ], + "session_input": {"app_name": "eval_optimize_loop", "user_id": "demo", "state": {}} + }, + { + "eval_id": "train_tool_choice", + "conversation": [ + { + "invocation_id": "train-2", + "user_content": {"parts": [{"text": "Check the status of order A100."}], "role": "user"}, + "final_response": {"parts": [{"text": "{\"route\":\"order_lookup\",\"message\":\"Checking order A100.\"}"}], "role": "model"} + } + ], + "session_input": {"app_name": "eval_optimize_loop", "user_id": "demo", "state": {}} + }, + { + "eval_id": "train_tool_arguments", + "conversation": [ + { + "invocation_id": "train-3", + "user_content": {"parts": [{"text": "Look up order B-204 for customer 17."}], "role": "user"}, + "final_response": {"parts": [{"text": "{\"route\":\"order_lookup\",\"message\":\"Checking order B-204 for customer 17.\"}"}], "role": "model"} + } + ], + "session_input": {"app_name": "eval_optimize_loop", "user_id": "demo", "state": {}} + } + ] +} diff --git a/examples/optimization/eval_optimize_loop/data/val.evalset.json b/examples/optimization/eval_optimize_loop/data/val.evalset.json new file mode 100644 index 00000000..387e6de7 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/data/val.evalset.json @@ -0,0 +1,40 @@ +{ + "eval_set_id": "eval_optimize_loop_validation", + "name": "Evaluation optimization loop - validation", + "description": "Three deterministic validation cases for generalization, recall, and critical routing.", + "eval_cases": [ + { + "eval_id": "val_paraphrase", + "conversation": [ + { + "invocation_id": "val-1", + "user_content": {"parts": [{"text": "Where do I change the address tied to my account?"}], "role": "user"}, + "final_response": {"parts": [{"text": "{\"route\":\"account\",\"message\":\"Open profile settings to update your address.\"}"}], "role": "model"} + } + ], + "session_input": {"app_name": "eval_optimize_loop", "user_id": "demo", "state": {}} + }, + { + "eval_id": "val_knowledge_recall", + "conversation": [ + { + "invocation_id": "val-2", + "user_content": {"parts": [{"text": "How long does standard shipping usually take?"}], "role": "user"}, + "final_response": {"parts": [{"text": "{\"route\":\"shipping_policy\",\"message\":\"Standard shipping normally takes 3-5 business days.\"}"}], "role": "model"} + } + ], + "session_input": {"app_name": "eval_optimize_loop", "user_id": "demo", "state": {}} + }, + { + "eval_id": "val_refund_route", + "conversation": [ + { + "invocation_id": "val-3", + "user_content": {"parts": [{"text": "I was charged twice and need the duplicate payment refunded."}], "role": "user"}, + "final_response": {"parts": [{"text": "{\"route\":\"billing_refund\",\"message\":\"I will route this duplicate charge for refund review.\"}"}], "role": "model"} + } + ], + "session_input": {"app_name": "eval_optimize_loop", "user_id": "demo", "state": {}} + } + ] +} diff --git a/examples/optimization/eval_optimize_loop/evaluation_adapter.py b/examples/optimization/eval_optimize_loop/evaluation_adapter.py new file mode 100644 index 00000000..e32ee9b1 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/evaluation_adapter.py @@ -0,0 +1,227 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under the Apache License, Version 2.0. +"""Convert SDK evaluation results into stable, lightweight analysis data.""" + +from __future__ import annotations + +from collections.abc import Iterable +from statistics import mean + +from trpc_agent_sdk.evaluation import EvalCaseResult +from trpc_agent_sdk.evaluation import EvalMetricResult +from trpc_agent_sdk.evaluation import EvalMetricResultPerInvocation +from trpc_agent_sdk.evaluation import EvalStatus +from trpc_agent_sdk.evaluation import Invocation +from trpc_agent_sdk.evaluation import get_all_tool_calls + +from .schemas import CaseEvaluation +from .schemas import CaseRunOutcome +from .schemas import EvaluationStatus +from .schemas import EvaluationSnapshot +from .schemas import InvocationEvidence +from .schemas import MetricOutcome +from .schemas import ObservableValue +from .schemas import StandardizedEvaluation +from .schemas import ToolCallEvidence + + +class EvaluationAnalysisError(ValueError): + """Evaluation evidence is structurally inconsistent and unsafe to compare.""" + + +def _status(value: EvalStatus, *, error_message: str | None = None) -> EvaluationStatus: + if error_message or value == EvalStatus.NOT_EVALUATED: + return "not_evaluated" + if value == EvalStatus.PASSED: + return "passed" + return "failed" + + +def _content_text(content: object | None) -> str | None: + if content is None: + return None + parts = getattr(content, "parts", None) or [] + text = "\n".join(part.text for part in parts if getattr(part, "text", None)) + return text or None + + +def _tool_evidence(invocation: Invocation | None) -> list[ToolCallEvidence]: + if invocation is None: + return [] + return [ + ToolCallEvidence(name=call.name or "", arguments=dict(call.args or {})) + for call in get_all_tool_calls(invocation.intermediate_data) + ] + + +def _observable(scores: Iterable[float | None], *, reason: str) -> ObservableValue: + values = list(scores) + if not values or any(score is None for score in values): + return ObservableValue(status="unavailable", reason=reason) + return ObservableValue(status="available", value=mean(float(score) for score in values)) + + +def _metric_map(metrics: list[EvalMetricResult], *, context: str) -> dict[str, EvalMetricResult]: + result: dict[str, EvalMetricResult] = {} + for metric in metrics: + if metric.metric_name in result: + raise EvaluationAnalysisError(f"{context} contains duplicate metric {metric.metric_name!r}") + result[metric.metric_name] = metric + return result + + +def _metric_outcome(metric: EvalMetricResult, *, context: str) -> MetricOutcome: + reason = metric.details.reason if metric.details is not None else None + score = _observable([metric.score], reason=f"{context} metric score is unavailable") + return MetricOutcome( + metric_name=metric.metric_name, + threshold=metric.threshold, + status="not_evaluated" if metric.score is None else _status(metric.eval_status), + score=score, + reason=reason, + ) + + +def _invocation_evidence(result: EvalMetricResultPerInvocation, *, context: str) -> InvocationEvidence: + actual = result.actual_invocation + expected = result.expected_invocation + metrics = _metric_map(result.eval_metric_results, context=context) + return InvocationEvidence( + invocation_id=actual.invocation_id, + user_text=_content_text(actual.user_content) or "", + expected_response=_content_text(expected.final_response) if expected is not None else None, + actual_response=_content_text(actual.final_response), + expected_tools=_tool_evidence(expected), + actual_tools=_tool_evidence(actual), + metrics=[_metric_outcome(metrics[name], context=context) for name in sorted(metrics)], + ) + + +def _case_evaluation( + eval_id: str, + raw_runs: list[EvalCaseResult], + *, + eval_set_id: str, +) -> CaseEvaluation: + if not raw_runs: + raise EvaluationAnalysisError(f"case {eval_id!r} has no run results") + + ordered_runs = sorted(raw_runs, key=lambda run: run.run_id if run.run_id is not None else 0) + run_ids = [run.run_id if run.run_id is not None else index for index, run in enumerate(ordered_runs, 1)] + if len(run_ids) != len(set(run_ids)): + raise EvaluationAnalysisError(f"case {eval_id!r} contains duplicate run ids") + + metric_maps: list[dict[str, EvalMetricResult]] = [] + normalized_runs: list[CaseRunOutcome] = [] + for run_id, run in zip(run_ids, ordered_runs): + if run.eval_id != eval_id: + raise EvaluationAnalysisError( + f"case mapping key {eval_id!r} does not match result eval_id {run.eval_id!r}" + ) + if run.eval_set_id != eval_set_id: + raise EvaluationAnalysisError( + f"case {eval_id!r} run {run_id} has eval_set_id {run.eval_set_id!r}; " + f"expected {eval_set_id!r}" + ) + context = f"case {eval_id!r} run {run_id}" + metric_map = _metric_map(run.overall_eval_metric_results, context=context) + metric_maps.append(metric_map) + normalized_metrics = [ + _metric_outcome(metric_map[name], context=context) for name in sorted(metric_map) + ] + run_status = _status(run.final_eval_status, error_message=run.error_message) + if not normalized_metrics or any(metric.status == "not_evaluated" for metric in normalized_metrics): + run_status = "not_evaluated" + normalized_runs.append( + CaseRunOutcome( + run_id=run_id, + status=run_status, + error_message=run.error_message, + metrics=normalized_metrics, + invocations=[ + _invocation_evidence(invocation, context=f"{context} invocation {index}") + for index, invocation in enumerate(run.eval_metric_result_per_invocation, 1) + ], + ) + ) + + metric_names = sorted(set().union(*(metrics.keys() for metrics in metric_maps))) + aggregate_metrics: list[MetricOutcome] = [] + for name in metric_names: + present = [metrics.get(name) for metrics in metric_maps] + thresholds = {metric.threshold for metric in present if metric is not None} + if len(thresholds) > 1: + raise EvaluationAnalysisError(f"case {eval_id!r} metric {name!r} has inconsistent thresholds") + available_metrics = [metric for metric in present if metric is not None] + metric_status: EvaluationStatus + if len(available_metrics) != len(present) or any( + metric.eval_status == EvalStatus.NOT_EVALUATED or metric.score is None for metric in available_metrics + ): + metric_status = "not_evaluated" + elif all(metric.eval_status == EvalStatus.PASSED for metric in available_metrics): + metric_status = "passed" + else: + metric_status = "failed" + reasons = [ + metric.details.reason + for metric in available_metrics + if metric.details is not None and metric.details.reason + ] + aggregate_metrics.append( + MetricOutcome( + metric_name=name, + threshold=next(iter(thresholds), 0.0), + status=metric_status, + score=_observable( + [metric.score if metric is not None else None for metric in present], + reason=f"case {eval_id!r} metric {name!r} is unavailable in one or more runs", + ), + reason="; ".join(reasons) or None, + ) + ) + + statuses = [run.status for run in normalized_runs] + if "not_evaluated" in statuses or any(metric.status == "not_evaluated" for metric in aggregate_metrics): + case_status: EvaluationStatus = "not_evaluated" + elif all(status == "passed" for status in statuses): + case_status = "passed" + else: + case_status = "failed" + return CaseEvaluation( + eval_id=eval_id, + status=case_status, + average_score=_observable( + [metric.score.value if metric.score.status == "available" else None for metric in aggregate_metrics], + reason=f"case {eval_id!r} has unavailable metric scores", + ), + metrics=aggregate_metrics, + runs=normalized_runs, + ) + + +def standardize_snapshot(snapshot: EvaluationSnapshot) -> StandardizedEvaluation: + """Normalize one complete SDK snapshot without discarding raw evidence.""" + cases = [ + _case_evaluation( + eval_id, + snapshot.eval_results_by_eval_id[eval_id], + eval_set_id=snapshot.eval_set_id, + ) + for eval_id in sorted(snapshot.eval_results_by_eval_id) + ] + return StandardizedEvaluation( + phase=snapshot.phase, + split=snapshot.split, + eval_set_id=snapshot.eval_set_id, + cases=cases, + passed_case_count=sum(case.status == "passed" for case in cases), + failed_case_count=sum(case.status == "failed" for case in cases), + not_evaluated_case_count=sum(case.status == "not_evaluated" for case in cases), + average_score=_observable( + [case.average_score.value if case.average_score.status == "available" else None for case in cases], + reason="one or more case scores are unavailable", + ), + ) diff --git a/examples/optimization/eval_optimize_loop/fake/__init__.py b/examples/optimization/eval_optimize_loop/fake/__init__.py new file mode 100644 index 00000000..6524f8f7 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/fake/__init__.py @@ -0,0 +1,16 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under the Apache License, Version 2.0. +"""Deterministic offline components for the eval/optimization loop.""" + +from .candidate_provider import DeterministicFakeCandidateProvider +from .model import DeterministicFakeModel +from .model import deterministic_response + +__all__ = [ + "DeterministicFakeCandidateProvider", + "DeterministicFakeModel", + "deterministic_response", +] diff --git a/examples/optimization/eval_optimize_loop/fake/candidate_provider.py b/examples/optimization/eval_optimize_loop/fake/candidate_provider.py new file mode 100644 index 00000000..42fd054f --- /dev/null +++ b/examples/optimization/eval_optimize_loop/fake/candidate_provider.py @@ -0,0 +1,110 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under the Apache License, Version 2.0. +"""Pure deterministic candidate generation for the offline pipeline mode.""" + +from __future__ import annotations + +import json +from hashlib import sha256 +from typing import Mapping + +from ..schemas import FakeCandidateProposal +from ..schemas import CandidateScenario +from .model import RULE_PREFIX + + +_SCENARIO_BLOCKS: dict[CandidateScenario, tuple[str, str]] = { + "improve": ( + "Generalize routing across account synonyms, order lookup, shipping policy, and refunds.", + "\n".join( + [ + "", + "Apply general customer-support routing rules across equivalent user phrasings.", + f"", + f"", + f"", + f"", + "", + ] + ), + ), + "no_improvement": ( + "Add an auditable wording-only change that leaves routing behavior unchanged.", + "\n".join( + [ + "", + "Keep responses concise, direct, and suitable for customer support.", + "", + ] + ), + ), + "overfit": ( + "Narrow routing to email changes and order lookups while disabling unseen intents.", + "\n".join( + [ + "", + "Handle only email profile changes and order lookups; use general support otherwise.", + f"", + f"", + f"", + f"", + "", + ] + ), + ), +} + + +def _prompt_mapping_sha256(prompts: Mapping[str, str]) -> str: + canonical = json.dumps( + dict(prompts), + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ) + return sha256(canonical.encode("utf-8")).hexdigest() + + +class DeterministicFakeCandidateProvider: + """Generate one structured candidate without performing I/O or mutation.""" + + def __init__(self, target_field: str = "system_prompt") -> None: + if not target_field: + raise ValueError("target_field must not be empty") + self._target_field = target_field + + def propose( + self, + current_prompts: Mapping[str, str], + *, + scenario: CandidateScenario, + seed: int, + ) -> FakeCandidateProposal: + if self._target_field not in current_prompts: + raise ValueError(f"fake candidate target field is missing: {self._target_field}") + if scenario not in _SCENARIO_BLOCKS: + raise ValueError(f"unknown fake candidate scenario: {scenario}") + if any(not isinstance(name, str) or not isinstance(value, str) for name, value in current_prompts.items()): + raise TypeError("current_prompts must map string field names to string values") + + rationale, rule_block = _SCENARIO_BLOCKS[scenario] + prompts = dict(current_prompts) + baseline = prompts[self._target_field].rstrip() + prompts[self._target_field] = f"{baseline}\n\n{rule_block}\n" + + parent_hash = _prompt_mapping_sha256(current_prompts) + candidate_hash = _prompt_mapping_sha256(prompts) + changed_fields = [name for name in current_prompts if current_prompts[name] != prompts[name]] + return FakeCandidateProposal( + scenario=scenario, + prompts=prompts, + changed_fields=changed_fields, + rationale=rationale, + seed=seed, + parent_prompt_sha256=parent_hash, + candidate_prompt_sha256=candidate_hash, + candidate_id=f"fake-{scenario}-{candidate_hash[:12]}", + ) diff --git a/examples/optimization/eval_optimize_loop/fake/model.py b/examples/optimization/eval_optimize_loop/fake/model.py new file mode 100644 index 00000000..01460799 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/fake/model.py @@ -0,0 +1,189 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under the Apache License, Version 2.0. +"""由 Prompt 与用户输入驱动的确定性离线模型。""" + +from __future__ import annotations + +import json +import re +from collections.abc import AsyncGenerator +from dataclasses import dataclass + +from trpc_agent_sdk.context import InvocationContext +from trpc_agent_sdk.models import LLMModel +from trpc_agent_sdk.models import LlmRequest +from trpc_agent_sdk.models import LlmResponse +from trpc_agent_sdk.types import Content +from trpc_agent_sdk.types import Part + + +RULE_PREFIX = "deterministic-fake-rule" +_RULE_RE = re.compile( + rf"", + re.IGNORECASE, +) +_ORDER_ID_RE = re.compile( + r"\border\s+([A-Za-z0-9][A-Za-z0-9-]*)", + re.IGNORECASE, +) +_CUSTOMER_ID_RE = re.compile( + r"\bcustomer\s+([A-Za-z0-9][A-Za-z0-9-]*)", + re.IGNORECASE, +) + + +@dataclass(frozen=True) +class _RoutingPolicy: + account_terms: frozenset[str] = frozenset({"email"}) + order_lookup: bool = False + shipping_policy: bool = False + refund_route: bool = True + + +def _parse_bool(value: str, *, default: bool) -> bool: + normalized = value.strip().lower() + if normalized in {"true", "yes", "1", "enabled"}: + return True + if normalized in {"false", "no", "0", "disabled"}: + return False + return default + + +def _parse_policy(prompt_text: str) -> _RoutingPolicy: + values = { + key.lower(): value.strip() + for key, value in _RULE_RE.findall(prompt_text) + } + account_terms = frozenset( + term.strip().lower() + for term in values.get("account_terms", "email").split(",") + if term.strip() + ) + return _RoutingPolicy( + account_terms=account_terms, + order_lookup=_parse_bool( + values.get("order_lookup", "false"), + default=False, + ), + shipping_policy=_parse_bool( + values.get("shipping_policy", "false"), + default=False, + ), + refund_route=_parse_bool( + values.get("refund_route", "true"), + default=True, + ), + ) + + +def _compact_response(route: str, message: str) -> str: + return json.dumps( + {"route": route, "message": message}, + ensure_ascii=False, + separators=(",", ":"), + ) + + +def deterministic_response(instruction: str, user_text: str) -> str: + """仅根据 Prompt instruction 和用户文本生成稳定响应。""" + if not isinstance(instruction, str): + raise TypeError("instruction must be a string") + if not isinstance(user_text, str): + raise TypeError("user text must be a string") + + policy = _parse_policy(instruction) + normalized = " ".join(user_text.casefold().split()) + + if policy.refund_route and ( + "charged twice" in normalized + or ( + "duplicate" in normalized + and ("payment" in normalized or "charge" in normalized) + ) + ): + return _compact_response( + "billing_refund", + "I will route this duplicate charge for refund review.", + ) + + if policy.shipping_policy and "shipping" in normalized and ( + "standard" in normalized or "how long" in normalized + ): + return _compact_response( + "shipping_policy", + "Standard shipping normally takes 3-5 business days.", + ) + + order_match = _ORDER_ID_RE.search(user_text) + if policy.order_lookup and "order" in normalized and order_match is not None: + order_id = order_match.group(1) + customer_match = _CUSTOMER_ID_RE.search(user_text) + message = f"Checking order {order_id}." + if customer_match is not None: + message = ( + f"Checking order {order_id} for customer " + f"{customer_match.group(1)}." + ) + return _compact_response("order_lookup", message) + + account_term = next( + ( + term + for term in sorted(policy.account_terms) + if term in normalized + ), + None, + ) + if account_term and ("update" in normalized or "change" in normalized): + attribute = "email" if "email" in normalized else "address" + return _compact_response( + "account", + f"Open profile settings to update your {attribute}.", + ) + + return _compact_response( + "general_support", + "Please provide more details so I can route your request.", + ) + + +def _last_user_text(request: LlmRequest) -> str: + for content in reversed(request.contents): + if content.role != "user" or not content.parts: + continue + text = "".join(part.text or "" for part in content.parts).strip() + if text: + return text + raise ValueError("LLM request must contain non-empty user text") + + +class DeterministicFakeModel(LLMModel): + """通过 SDK Model 接口提供不访问网络的确定性响应。""" + + def __init__(self) -> None: + super().__init__(model_name="deterministic-fake-model") + + @classmethod + def supported_models(cls) -> list[str]: + return ["deterministic-fake-model"] + + async def _generate_async_impl( + self, + request: LlmRequest, + stream: bool = False, + ctx: InvocationContext | None = None, + ) -> AsyncGenerator[LlmResponse, None]: + del stream, ctx + instruction = "" + if request.config is not None and request.config.system_instruction: + instruction = str(request.config.system_instruction) + response = deterministic_response(instruction, _last_user_text(request)) + yield LlmResponse( + content=Content( + role="model", + parts=[Part.from_text(text=response)], + ) + ) diff --git a/examples/optimization/eval_optimize_loop/gate.py b/examples/optimization/eval_optimize_loop/gate.py new file mode 100644 index 00000000..1e4f7b49 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/gate.py @@ -0,0 +1,382 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under the Apache License, Version 2.0. +"""Deterministic acceptance Gate for stage 3b.""" + +from __future__ import annotations + +from collections.abc import Sequence +from typing import Literal + +from .config import BudgetConfig +from .config import GateConfig +from .schemas import CaseDiff +from .schemas import CaseEvaluation +from .schemas import EvaluationAnalysis +from .schemas import GateDecision +from .schemas import GateRuleId +from .schemas import GateRuleResult +from .schemas import ObservableValue +from .schemas import ResourceMeasurements + + +class GateEvaluationError(ValueError): + """Stage 3a analysis is structurally unsafe for Gate evaluation.""" + + +def _case_ids( + cases: Sequence[CaseEvaluation | CaseDiff], + *, + context: str, +) -> set[str]: + ids = [case.eval_id for case in cases] + if len(ids) != len(set(ids)): + raise GateEvaluationError(f"{context} contains duplicate case ids") + return set(ids) + + +def _validate_analysis(analysis: EvaluationAnalysis) -> None: + if analysis.train_diff.split != "train": + raise GateEvaluationError("train_diff.split must be 'train'") + if analysis.validation_diff.split != "validation": + raise GateEvaluationError("validation_diff.split must be 'validation'") + + evaluations = ( + ("baseline_train", analysis.baseline_train, "baseline", "train"), + ("baseline_validation", analysis.baseline_validation, "baseline", "validation"), + ("candidate_train", analysis.candidate_train, "candidate", "train"), + ("candidate_validation", analysis.candidate_validation, "candidate", "validation"), + ) + for label, evaluation, expected_phase, expected_split in evaluations: + if evaluation.phase != expected_phase or evaluation.split != expected_split: + raise GateEvaluationError(f"{label} has an unexpected phase or split") + _case_ids(evaluation.cases, context=label) + for case in evaluation.cases: + metric_names = [metric.metric_name for metric in case.metrics] + if len(metric_names) != len(set(metric_names)): + raise GateEvaluationError( + f"{label} case {case.eval_id!r} contains duplicate metric names" + ) + + train_diff_ids = _case_ids(analysis.train_diff.cases, context="train_diff") + validation_diff_ids = _case_ids( + analysis.validation_diff.cases, + context="validation_diff", + ) + candidate_train_ids = {case.eval_id for case in analysis.candidate_train.cases} + candidate_validation_ids = { + case.eval_id for case in analysis.candidate_validation.cases + } + if train_diff_ids != candidate_train_ids: + raise GateEvaluationError("train diff and candidate evaluation case ids do not match") + if validation_diff_ids != candidate_validation_ids: + raise GateEvaluationError( + "validation diff and candidate evaluation case ids do not match" + ) + + +def _evaluation_completeness(analysis: EvaluationAnalysis) -> GateRuleResult: + incomplete_case_ids: set[str] = set() + incomplete_metric_names: set[str] = set() + evaluations = ( + analysis.baseline_train, + analysis.baseline_validation, + analysis.candidate_train, + analysis.candidate_validation, + ) + complete = True + for evaluation in evaluations: + if not evaluation.cases or evaluation.average_score.status != "available": + complete = False + for case in evaluation.cases: + if ( + case.status == "not_evaluated" + or case.average_score.status != "available" + or not case.metrics + ): + complete = False + incomplete_case_ids.add(case.eval_id) + for metric in case.metrics: + if metric.status == "not_evaluated" or metric.score.status != "available": + complete = False + incomplete_case_ids.add(case.eval_id) + incomplete_metric_names.add(metric.metric_name) + return GateRuleResult( + rule_id="evaluation_completeness", + outcome="pass" if complete else "reject", + message=( + "All four evaluations contain complete case and metric results." + if complete + else "One or more evaluation cases or metrics are incomplete." + ), + case_ids=sorted(incomplete_case_ids), + metric_names=sorted(incomplete_metric_names), + ) + + +def _minimum_validation_score_delta( + analysis: EvaluationAnalysis, + config: GateConfig, +) -> GateRuleResult: + delta = analysis.validation_diff.score_delta + passed = ( + delta.status == "available" + and float(delta.value) >= config.min_validation_score_delta + ) + return GateRuleResult( + rule_id="minimum_validation_score_delta", + outcome="pass" if passed else "reject", + message=( + "Validation score improvement meets the configured minimum." + if passed + else "Validation score improvement is unavailable or below the configured minimum." + ), + observed={"validation_score_delta": delta}, + threshold=config.min_validation_score_delta, + ) + + +def _validation_pass_rate(analysis: EvaluationAnalysis, config: GateConfig) -> GateRuleResult: + if not config.reject_on_validation_pass_rate_drop: + return GateRuleResult( + rule_id="validation_pass_rate_non_decrease", + outcome="skipped", + message="Validation pass-rate protection is disabled.", + ) + baseline_total = len(analysis.baseline_validation.cases) + candidate_total = len(analysis.candidate_validation.cases) + if baseline_total == 0 or candidate_total == 0: + return GateRuleResult( + rule_id="validation_pass_rate_non_decrease", + outcome="reject", + message="Validation pass rate is unavailable because an evaluation has no cases.", + ) + baseline_rate = analysis.baseline_validation.passed_case_count / baseline_total + candidate_rate = analysis.candidate_validation.passed_case_count / candidate_total + passed = candidate_rate >= baseline_rate + return GateRuleResult( + rule_id="validation_pass_rate_non_decrease", + outcome="pass" if passed else "reject", + message=( + "Validation pass rate did not decrease." + if passed + else "Validation pass rate decreased from baseline." + ), + observed={ + "baseline_validation_pass_rate": ObservableValue( + status="available", value=baseline_rate, unit="ratio" + ), + "candidate_validation_pass_rate": ObservableValue( + status="available", value=candidate_rate, unit="ratio" + ), + }, + ) + + +def _all_case_diffs(analysis: EvaluationAnalysis) -> list[CaseDiff]: + return sorted( + [*analysis.train_diff.cases, *analysis.validation_diff.cases], + key=lambda case: (case.split, case.eval_id), + ) + + +def _new_hard_failures(analysis: EvaluationAnalysis, config: GateConfig) -> GateRuleResult: + if not config.reject_new_hard_fail: + return GateRuleResult( + rule_id="no_new_hard_fail", + outcome="skipped", + message="New hard-failure protection is disabled.", + ) + case_ids = sorted( + case.eval_id + for case in _all_case_diffs(analysis) + if case.is_hard and case.change == "newly_failed" + ) + return GateRuleResult( + rule_id="no_new_hard_fail", + outcome="reject" if case_ids else "pass", + message=( + "New hard failures were found." + if case_ids + else "No new hard failures were found." + ), + case_ids=case_ids, + ) + + +def _critical_regressions(analysis: EvaluationAnalysis, config: GateConfig) -> GateRuleResult: + if not config.reject_critical_regression: + return GateRuleResult( + rule_id="no_critical_regression", + outcome="skipped", + message="Critical-case regression protection is disabled.", + ) + case_ids = sorted( + case.eval_id + for case in _all_case_diffs(analysis) + if case.is_critical and case.change in {"newly_failed", "regressed"} + ) + return GateRuleResult( + rule_id="no_critical_regression", + outcome="reject" if case_ids else "pass", + message=( + "Critical-case regressions were found." + if case_ids + else "No critical-case regressions were found." + ), + case_ids=case_ids, + ) + + +def _severe_regressions(analysis: EvaluationAnalysis) -> GateRuleResult: + case_ids = sorted( + case.eval_id for case in _all_case_diffs(analysis) if case.severe_regression + ) + return GateRuleResult( + rule_id="no_severe_regression", + outcome="reject" if case_ids else "pass", + message=( + "Severe case regressions were found." + if case_ids + else "No severe case regressions were found." + ), + case_ids=case_ids, + ) + + +def _required_metrics(analysis: EvaluationAnalysis, config: GateConfig) -> GateRuleResult: + failed_case_ids: set[str] = set() + failed_metric_names: set[str] = set() + for evaluation in (analysis.candidate_train, analysis.candidate_validation): + for case in evaluation.cases: + metric_map = {metric.metric_name: metric for metric in case.metrics} + if config.required_metrics == "all": + required_names = sorted(metric_map) + if not required_names: + failed_case_ids.add(case.eval_id) + continue + else: + required_names = sorted(config.required_metrics) + for name in required_names: + metric = metric_map.get(name) + if ( + metric is None + or metric.status != "passed" + or metric.score.status != "available" + ): + failed_case_ids.add(case.eval_id) + failed_metric_names.add(name) + return GateRuleResult( + rule_id="required_metrics", + outcome="reject" if failed_case_ids else "pass", + message=( + "Required metrics are missing, unavailable, or below threshold." + if failed_case_ids + else "All required candidate metrics are available and passed." + ), + case_ids=sorted(failed_case_ids), + metric_names=sorted(failed_metric_names), + ) + + +def _overfitting(analysis: EvaluationAnalysis) -> GateRuleResult: + passed = analysis.overfit_status == "not_detected" + return GateRuleResult( + rule_id="no_overfitting", + outcome="pass" if passed else "reject", + message=( + "No train-improvement/validation-regression pattern was detected." + if passed + else f"Overfit status is {analysis.overfit_status!r}: {analysis.overfit_reason}" + ), + ) + + +def _budget_result( + rule_id: GateRuleId, + measurement_name: str, + measurement: ObservableValue, + limit: float | int | None, + on_unavailable: Literal["reject", "warning"], +) -> GateRuleResult: + if limit is None: + return GateRuleResult( + rule_id=rule_id, + outcome="skipped", + message=f"{measurement_name} budget is not configured.", + ) + if measurement.status != "available": + return GateRuleResult( + rule_id=rule_id, + outcome=on_unavailable, + message=( + f"{measurement_name} is unavailable; policy is {on_unavailable}." + ), + observed={measurement_name: measurement}, + threshold=float(limit), + ) + passed = float(measurement.value) <= float(limit) + return GateRuleResult( + rule_id=rule_id, + outcome="pass" if passed else "reject", + message=( + f"{measurement_name} is within the configured budget." + if passed + else f"{measurement_name} exceeds the configured budget." + ), + observed={measurement_name: measurement}, + threshold=float(limit), + ) + + +def evaluate_gate( + analysis: EvaluationAnalysis, + gate_config: GateConfig, + budget_config: BudgetConfig, + measurements: ResourceMeasurements, +) -> GateDecision: + """Evaluate every configured rule and return one complete decision.""" + _validate_analysis(analysis) + quality_results = [ + _evaluation_completeness(analysis), + _minimum_validation_score_delta(analysis, gate_config), + _validation_pass_rate(analysis, gate_config), + _new_hard_failures(analysis, gate_config), + _critical_regressions(analysis, gate_config), + _severe_regressions(analysis), + _required_metrics(analysis, gate_config), + _overfitting(analysis), + ] + results = quality_results + [ + _budget_result( + "cost_budget", + "cost_usd", + measurements.cost_usd, + budget_config.max_cost_usd, + budget_config.on_unavailable, + ), + _budget_result( + "token_budget", + "total_tokens", + measurements.total_tokens, + budget_config.max_tokens, + budget_config.on_unavailable, + ), + _budget_result( + "duration_budget", + "duration_seconds", + measurements.duration_seconds, + budget_config.max_duration_seconds, + budget_config.on_unavailable, + ), + ] + rejection_reasons = [result.message for result in results if result.outcome == "reject"] + warnings = [result.message for result in results if result.outcome == "warning"] + return GateDecision( + decision="reject" if rejection_reasons else "accept", + rule_results=results, + rejection_reasons=rejection_reasons, + warnings=warnings, + ) diff --git a/examples/optimization/eval_optimize_loop/optimizer.json b/examples/optimization/eval_optimize_loop/optimizer.json new file mode 100644 index 00000000..9efff8be --- /dev/null +++ b/examples/optimization/eval_optimize_loop/optimizer.json @@ -0,0 +1,36 @@ +{ + "evaluate": { + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + } + } + ], + "num_runs": 1 + }, + "optimize": { + "eval_case_parallelism": 1, + "stop": { + "required_metrics": "all" + }, + "algorithm": { + "name": "gepa_reflective", + "seed": 42, + "reflection_lm": { + "model_name": "fake-not-used-in-offline-mode", + "api_key": "fake-not-used-in-offline-mode" + }, + "reflection_minibatch_size": 3, + "skip_perfect_score": false, + "max_candidate_proposals": 3 + } + } +} diff --git a/examples/optimization/eval_optimize_loop/pipeline.json b/examples/optimization/eval_optimize_loop/pipeline.json new file mode 100644 index 00000000..1d88ff2a --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline.json @@ -0,0 +1,52 @@ +{ + "config_version": 1, + "execution": { + "mode": "offline", + "candidate_scenario": "improve" + }, + "inputs": { + "train_evalset": "data/train.evalset.json", + "validation_evalset": "data/val.evalset.json", + "optimizer_config": "optimizer.json" + }, + "prompts": [ + { + "name": "system_prompt", + "path": "prompts/system.md" + } + ], + "run": { + "runs_dir": "runs", + "seed": 42 + }, + "case_labels": { + "hard_case_ids": ["val_knowledge_recall"], + "critical_case_ids": ["val_refund_route"] + }, + "gate": { + "min_validation_score_delta": 0.05, + "reject_on_validation_pass_rate_drop": true, + "reject_new_hard_fail": true, + "reject_critical_regression": true, + "severe_case_score_drop": 0.2, + "required_metrics": ["final_response_avg_score"] + }, + "budget": { + "max_duration_seconds": 180, + "on_unavailable": "warning" + }, + "reporting": { + "write_json": true, + "write_markdown": true, + "include_case_evidence": true + }, + "artifacts": { + "copy_input_files": true, + "retain_optimizer_native_artifacts": true, + "audit_all_candidates": false + }, + "writeback": { + "enabled": false, + "require_source_hash_match": true + } +} diff --git a/examples/optimization/eval_optimize_loop/pipeline.py b/examples/optimization/eval_optimize_loop/pipeline.py new file mode 100644 index 00000000..9657f3de --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline.py @@ -0,0 +1,1169 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under the Apache License, Version 2.0. +"""Preparation, candidate regression, Gate, and guarded writeback orchestration.""" + +from __future__ import annotations + +import json +import re +import shutil +from dataclasses import dataclass +from dataclasses import field +from datetime import datetime +from datetime import timezone +from hashlib import sha256 +from pathlib import Path +from time import perf_counter +from typing import Literal +from uuid import uuid4 + +from trpc_agent_sdk.evaluation import AgentEvaluator +from trpc_agent_sdk.evaluation import CallAgent +from trpc_agent_sdk.evaluation import EvalCaseResult +from trpc_agent_sdk.evaluation import EvalSet +from trpc_agent_sdk.evaluation import OptimizeConfigFile +from trpc_agent_sdk.evaluation import TargetPrompt +from trpc_agent_sdk.evaluation import load_optimize_config + +from .analysis import build_evaluation_analysis +from .business_agent import BusinessAgent +from .artifact_writer import discover_run_artifacts +from .artifact_writer import publish_report_bundle +from .artifact_writer import write_failure_report +from .candidate_provider import AgentOptimizerCandidateProvider +from .candidate_provider import CandidateProviderError +from .candidate_provider import CandidateRequest +from .candidate_provider import FakeCandidateProviderAdapter +from .config import PipelineConfig +from .config import load_pipeline_config +from .evaluation_adapter import EvaluationAnalysisError +from .evaluation_adapter import standardize_snapshot +from .fake import DeterministicFakeModel +from .gate import evaluate_gate +from .gate import GateEvaluationError +from .prompt_workspace import PromptWorkspaceError +from .prompt_workspace import resolve_inside_example_root +from .prompt_workspace import stage_prompt_workspace +from .prompt_workspace import validate_prompt_sources +from .report_builder import build_failure_report +from .report_builder import build_optimization_report +from .schemas import InputSnapshot +from .schemas import CandidateScenario +from .schemas import EvaluationSnapshot +from .schemas import OfflineStageResult +from .schemas import ObservableValue +from .schemas import OptimizerRuntimeParameters +from .schemas import ResourceMeasurements +from .schemas import RealStageResult +from .schemas import ReportPhase +from .schemas import ReportProgress +from .schemas import TraceCandidateProposal +from .schemas import TraceInputSnapshot +from .schemas import TracePromptSnapshot +from .schemas import TraceScenarioInputSnapshot +from .schemas import TraceStageResult +from .schemas import WorkspaceSnapshot +from .schemas import WritebackResult +from .writeback import perform_writeback + + +class PipelinePreparationError(ValueError): + """The example cannot safely prepare an evaluation/optimization run.""" + + +class PipelineExecutionError(RuntimeError): + """A prepared pipeline run could not complete safely.""" + + +# Compatibility for callers and tests written before the real-mode stage. +PipelineStageExecutionError = PipelineExecutionError + + +@dataclass(frozen=True) +class PreparedRun: + """Validated inputs and isolated prompts handed to the next pipeline phase.""" + + config: PipelineConfig + optimizer_config: OptimizeConfigFile + input_snapshot: InputSnapshot + workspace: WorkspaceSnapshot + source_target: TargetPrompt + working_target: TargetPrompt + example_root: Path + + +@dataclass +class _MutableReportProgress: + """Track the active report phase without marking it complete too early.""" + + started_at: datetime + current_phase: ReportPhase = "baseline_train" + completed_phases: list[ReportPhase] = field(default_factory=list) + + def enter(self, phase: ReportPhase) -> None: + if self.current_phase not in self.completed_phases and self.current_phase != phase: + self.completed_phases.append(self.current_phase) + self.current_phase = phase + + def snapshot(self) -> ReportProgress: + return ReportProgress( + started_at=self.started_at, + current_phase=self.current_phase, + completed_phases=list(self.completed_phases), + ) + + +async def _source_prompt_hashes(prepared: PreparedRun) -> dict[str, str]: + try: + prompts = await prepared.source_target.read_all() + except Exception: + # Failure evidence must remain writable even when the source itself is + # unavailable. An empty mapping means the final source state could not + # be observed; it must never be replaced with stale snapshot hashes. + return {} + return { + name: sha256(value.encode("utf-8")).hexdigest() + for name, value in sorted(prompts.items()) + } + + +async def _record_failure( + prepared: PreparedRun, + progress: _MutableReportProgress, + error: Exception, +) -> None: + run_dir = Path(prepared.workspace.run_dir) + existing = discover_run_artifacts(run_dir) + report = build_failure_report( + prepared, + progress=progress.snapshot(), + error=error, + source_prompt_hashes=await _source_prompt_hashes(prepared), + existing_artifacts=existing, + generated_at=datetime.now(timezone.utc), + ) + write_failure_report(report, run_dir=run_dir) + + +async def _rollback_written_source( + prepared: PreparedRun, + result: OfflineStageResult | RealStageResult | TraceStageResult, +) -> None: + """Restore the prepared source Prompt if success reporting cannot publish.""" + if result.writeback.status != "written": + return + baseline = { + snapshot.field_name: snapshot.content + for snapshot in prepared.input_snapshot.prompt_snapshots + } + current = await prepared.source_target.read_all() + if current != result.candidate.prompts: + raise PipelineExecutionError( + "source Prompt changed after writeback; refusing reporting-failure rollback" + ) + # Path-backed TargetPrompt.write_all performs its atomic replacements + # synchronously, so this task does not yield between the adjacent check and + # write. Callback-backed sources retain the caller's documented atomicity + # responsibility, as they do for the normal writeback path. + await prepared.source_target.write_all(baseline) + restored = await prepared.source_target.read_all() + if restored != baseline: + raise PipelineExecutionError( + "source Prompt rollback after reporting failure could not be verified" + ) + + +async def _handle_stage_failure( + prepared: PreparedRun, + progress: _MutableReportProgress, + error: Exception, + result: OfflineStageResult | RealStageResult | TraceStageResult | None, +) -> None: + failure_error: Exception = error + if progress.current_phase == "reporting" and result is not None: + try: + await _rollback_written_source(prepared, result) + except Exception as rollback_exc: + failure_error = PipelineExecutionError( + f"{error}; additionally failed to roll back source Prompt: {rollback_exc}" + ) + try: + await _record_failure(prepared, progress, failure_error) + except Exception as report_exc: + raise PipelineExecutionError( + f"{failure_error}; additionally failed to write failure report: {report_exc}" + ) from error + if failure_error is not error: + raise failure_error from error + + +_RUN_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_-]*$") + + +def _load_evalset(path: Path, label: str) -> EvalSet: + if not path.is_file(): + raise PipelinePreparationError(f"{label} must be a file: {path}") + try: + return EvalSet.model_validate_json(path.read_text(encoding="utf-8")) + except UnicodeDecodeError as exc: + raise PipelinePreparationError(f"{label} is not UTF-8: {path}") from exc + except Exception as exc: + raise PipelinePreparationError(f"{label} is not a valid EvalSet: {path}: {exc}") from exc + + +def _validate_trace_evalset(eval_set: EvalSet, label: str) -> None: + invalid = [ + case.eval_id + for case in eval_set.eval_cases + if case.eval_mode != "trace" or not case.actual_conversation + ] + if invalid: + raise PipelinePreparationError( + f"{label} requires eval_mode='trace' and actual_conversation: {invalid}" + ) + + +def _validate_eval_case_ids(train: EvalSet, validation: EvalSet, config: PipelineConfig) -> None: + train_ids = [case.eval_id for case in train.eval_cases] + validation_ids = [case.eval_id for case in validation.eval_cases] + for label, ids in (("train", train_ids), ("validation", validation_ids)): + if len(ids) != len(set(ids)): + raise PipelinePreparationError(f"{label} evalset contains duplicate eval_id values") + if set(train_ids) & set(validation_ids): + raise PipelinePreparationError("train and validation evalsets must not share eval_id values") + + known_ids = set(train_ids) | set(validation_ids) + labels = set(config.case_labels.hard_case_ids) | set(config.case_labels.critical_case_ids) + unknown = sorted(labels - known_ids) + if unknown: + raise PipelinePreparationError(f"case_labels reference unknown eval_id values: {unknown}") + + +def _validate_gate_metrics(config: PipelineConfig, optimizer_config: object) -> None: + required = config.gate.required_metrics + if not isinstance(required, list): + return + available = {metric.metric_name for metric in optimizer_config.evaluate.get_eval_metrics()} + unknown = sorted(set(required) - available) + if unknown: + raise PipelinePreparationError( + f"gate.required_metrics references unknown metrics {unknown}; available metrics: {sorted(available)}") + + +def _resolve_inputs(example_root: Path, config: PipelineConfig) -> tuple[Path, Path, Path]: + train_path = resolve_inside_example_root(example_root, config.inputs.train_evalset, "train_evalset") + validation_path = resolve_inside_example_root(example_root, config.inputs.validation_evalset, "validation_evalset") + optimizer_path = resolve_inside_example_root(example_root, config.inputs.optimizer_config, "optimizer_config") + if train_path == validation_path: + raise PipelinePreparationError("train_evalset and validation_evalset must be different files") + if not optimizer_path.is_file(): + raise PipelinePreparationError(f"optimizer_config must be a file: {optimizer_path}") + return train_path, validation_path, optimizer_path + + +def _validate_run_id(run_id: str) -> str: + if not _RUN_ID_RE.fullmatch(run_id): + raise PipelinePreparationError("run_id may contain only letters, numbers, underscores, and hyphens") + return run_id + + +def _new_run_id() -> str: + return datetime.now(timezone.utc).strftime("run_%Y%m%dT%H%M%S_%fZ") + + +def _file_sha256(path: Path) -> str: + return sha256(path.read_bytes()).hexdigest() + + +def _verify_prepared_file(path: Path, *, label: str, expected_sha256: str) -> None: + """Reject an input whose bytes changed after ``prepare_run``.""" + try: + actual_sha256 = _file_sha256(path) + except OSError as exc: + raise PipelineStageExecutionError(f"failed to reload prepared {label}: {path}: {exc}") from exc + if actual_sha256 != expected_sha256: + raise PipelineStageExecutionError( + f"{label} changed after prepare_run: {path}; " + f"expected sha256 {expected_sha256}, got {actual_sha256}" + ) + + +def _reload_prepared_evalset( + path: Path, + *, + label: str, + expected_sha256: str, +) -> EvalSet: + """Reload exactly the evalset bytes whose identity was prepared.""" + try: + payload = path.read_bytes() + except OSError as exc: + raise PipelineStageExecutionError(f"failed to reload prepared {label}: {path}: {exc}") from exc + + actual_sha256 = sha256(payload).hexdigest() + if actual_sha256 != expected_sha256: + raise PipelineStageExecutionError( + f"{label} changed after prepare_run: {path}; " + f"expected sha256 {expected_sha256}, got {actual_sha256}" + ) + + try: + return EvalSet.model_validate_json(payload) + except Exception as exc: + raise PipelineStageExecutionError(f"prepared {label} is no longer a valid EvalSet: {path}: {exc}") from exc + + +def _prepare_trace_inputs( + example_root: Path, + config: PipelineConfig, + baseline_train: EvalSet, + baseline_validation: EvalSet, +) -> TraceInputSnapshot | None: + if config.execution.mode != "trace": + return None + _validate_trace_evalset(baseline_train, "baseline train trace") + _validate_trace_evalset(baseline_validation, "baseline validation trace") + assert config.trace_inputs is not None + train_ids = {case.eval_id for case in baseline_train.eval_cases} + validation_ids = {case.eval_id for case in baseline_validation.eval_cases} + scenarios: dict[str, TraceScenarioInputSnapshot] = {} + for scenario, inputs in config.trace_inputs.candidates.items(): + train_path = resolve_inside_example_root( + example_root, inputs.train_evalset, f"trace {scenario} train" + ) + validation_path = resolve_inside_example_root( + example_root, + inputs.validation_evalset, + f"trace {scenario} validation", + ) + train = _load_evalset(train_path, f"trace {scenario} train") + validation = _load_evalset( + validation_path, f"trace {scenario} validation" + ) + _validate_trace_evalset(train, f"trace {scenario} train") + _validate_trace_evalset(validation, f"trace {scenario} validation") + if {case.eval_id for case in train.eval_cases} != train_ids: + raise PipelinePreparationError( + f"trace {scenario} train eval IDs must match baseline" + ) + if {case.eval_id for case in validation.eval_cases} != validation_ids: + raise PipelinePreparationError( + f"trace {scenario} validation eval IDs must match baseline" + ) + prompt_snapshots: list[TracePromptSnapshot] = [] + for prompt in inputs.prompts: + path = resolve_inside_example_root( + example_root, prompt.path, f"trace {scenario} prompt" + ) + try: + content = path.read_text(encoding="utf-8") + except (OSError, UnicodeError) as exc: + raise PipelinePreparationError( + f"trace {scenario} prompt is invalid: {path}: {exc}" + ) from exc + prompt_snapshots.append( + TracePromptSnapshot( + field_name=prompt.name, + path=str(path), + content=content, + sha256=_file_sha256(path), + ) + ) + scenarios[scenario] = TraceScenarioInputSnapshot( + train_evalset_path=str(train_path), + train_evalset_sha256=_file_sha256(train_path), + validation_evalset_path=str(validation_path), + validation_evalset_sha256=_file_sha256(validation_path), + prompt_snapshots=prompt_snapshots, + ) + return TraceInputSnapshot(scenarios=scenarios) + + +def prepare_run(pipeline_config_path: str | Path, *, run_id: str | None = None) -> PreparedRun: + """Prepare a run without evaluating, optimizing, reporting, or writing a source prompt. + + All configuration and input validation completes before a staging directory + is created. The final run directory appears only through an atomic rename, + and an exception removes the staging directory. This keeps failed setup + from looking like a runnable or audited pipeline result. + """ + config_path = Path(pipeline_config_path).resolve() + config = load_pipeline_config(config_path) + example_root = config_path.parent + + train_path, validation_path, optimizer_path = _resolve_inputs(example_root, config) + train_evalset = _load_evalset(train_path, "train_evalset") + validation_evalset = _load_evalset(validation_path, "validation_evalset") + _validate_eval_case_ids(train_evalset, validation_evalset, config) + trace_inputs = _prepare_trace_inputs( + example_root, config, train_evalset, validation_evalset + ) + + try: + optimizer_config = load_optimize_config(str(optimizer_path)) + except Exception as exc: + raise PipelinePreparationError(f"optimizer_config is invalid: {optimizer_path}: {exc}") from exc + if not optimizer_config.evaluate.get_eval_metrics(): + raise PipelinePreparationError("optimizer_config must define at least one evaluation metric") + if optimizer_config.evaluate.num_runs < 1: + raise PipelinePreparationError("optimizer_config evaluate.num_runs must be at least 1") + if optimizer_config.optimize.eval_case_parallelism < 1: + raise PipelinePreparationError("optimizer_config optimize.eval_case_parallelism must be at least 1") + _validate_gate_metrics(config, optimizer_config) + + try: + prompt_sources = validate_prompt_sources(example_root, config.prompts) + runs_dir = resolve_inside_example_root(example_root, config.run.runs_dir, "runs_dir") + except PromptWorkspaceError as exc: + raise PipelinePreparationError(str(exc)) from exc + + configured_run_id = run_id if run_id is not None else config.run.run_id + selected_run_id = _validate_run_id(configured_run_id or _new_run_id()) + runs_dir.mkdir(parents=True, exist_ok=True) + final_run_dir = runs_dir / selected_run_id + if final_run_dir.exists(): + raise FileExistsError(f"run directory already exists: {final_run_dir}") + + staging_run_dir = runs_dir / f".{selected_run_id}.tmp-{uuid4().hex}" + try: + staging_run_dir.mkdir() + prompt_snapshots, source_target, working_target = stage_prompt_workspace( + example_root=example_root, + staging_run_dir=staging_run_dir, + final_run_dir=final_run_dir, + prompts=config.prompts, + sources=prompt_sources, + ) + workspace_dir = final_run_dir / "workspace" + workspace = WorkspaceSnapshot( + run_id=selected_run_id, + run_dir=str(final_run_dir), + workspace_dir=str(workspace_dir), + prompts_dir=str(workspace_dir / "prompts"), + ) + input_snapshot = InputSnapshot( + pipeline_config_path=str(config_path), + pipeline_config_sha256=_file_sha256(config_path), + optimizer_config_path=str(optimizer_path), + optimizer_config_sha256=_file_sha256(optimizer_path), + train_evalset_path=str(train_path), + train_evalset_sha256=_file_sha256(train_path), + validation_evalset_path=str(validation_path), + validation_evalset_sha256=_file_sha256(validation_path), + prompt_snapshots=prompt_snapshots, + seed=config.run.seed, + trace_inputs=trace_inputs, + ) + prepared = PreparedRun( + config=config, + optimizer_config=optimizer_config, + input_snapshot=input_snapshot, + workspace=workspace, + source_target=source_target, + working_target=working_target, + example_root=example_root, + ) + staging_run_dir.replace(final_run_dir) + return prepared + except BaseException: + shutil.rmtree(staging_run_dir, ignore_errors=True) + raise + + +def _validate_results( + *, + eval_set: EvalSet, + eval_results_by_eval_id: dict[str, list[EvalCaseResult]], + num_runs: int, + phase: Literal["baseline", "candidate"], + split: Literal["train", "validation"], +) -> None: + expected_ids = {case.eval_id for case in eval_set.eval_cases} + actual_ids = set(eval_results_by_eval_id) + if actual_ids != expected_ids: + raise PipelineStageExecutionError( + f"{phase} {split} evaluation returned case ids {sorted(actual_ids)}; " + f"expected {sorted(expected_ids)}" + ) + wrong_run_counts = { + eval_id: len(results) + for eval_id, results in eval_results_by_eval_id.items() + if len(results) != num_runs + } + if wrong_run_counts: + raise PipelineStageExecutionError( + f"{phase} {split} evaluation returned unexpected run counts: {wrong_run_counts}; " + f"expected {num_runs}" + ) + + +async def _evaluate_split( + *, + prepared: PreparedRun, + eval_set: EvalSet, + call_agent: CallAgent | None, + phase: Literal["baseline", "candidate"], + split: Literal["train", "validation"], +) -> EvaluationSnapshot: + num_runs = prepared.optimizer_config.evaluate.num_runs + try: + failed_summary, details_lines, result_lines, eval_results_by_eval_id = ( + await AgentEvaluator.evaluate_eval_set( + eval_set, + call_agent=call_agent, + eval_config=prepared.optimizer_config.evaluate, + num_runs=num_runs, + print_detailed_results=False, + case_parallelism=prepared.optimizer_config.optimize.eval_case_parallelism, + case_eval_parallelism=prepared.optimizer_config.optimize.eval_case_parallelism, + ) + ) + except Exception as exc: + raise PipelineStageExecutionError(f"{phase} {split} evaluation failed: {exc}") from exc + + _validate_results( + eval_set=eval_set, + eval_results_by_eval_id=eval_results_by_eval_id, + num_runs=num_runs, + phase=phase, + split=split, + ) + snapshot = EvaluationSnapshot( + phase=phase, + split=split, + eval_set_id=eval_set.eval_set_id, + failed_summary=failed_summary, + details_lines=details_lines, + result_lines=result_lines, + eval_results_by_eval_id=eval_results_by_eval_id, + passed_case_count=0, + total_case_count=len(eval_results_by_eval_id), + average_score=None, + ) + try: + standardized = standardize_snapshot(snapshot) + except EvaluationAnalysisError as exc: + raise PipelineStageExecutionError( + f"{phase} {split} evaluation result standardization failed: {exc}" + ) from exc + return snapshot.model_copy( + update={ + "passed_case_count": standardized.passed_case_count, + "total_case_count": len(standardized.cases), + "average_score": ( + standardized.average_score.value + if standardized.average_score.status == "available" + else None + ), + } + ) + + +async def _restore_working_baseline( + prepared: PreparedRun, + baseline_prompts: dict[str, str], +) -> bool: + """Restore optimizer leftovers and prove the isolated baseline is present.""" + initial_read_error: Exception | None = None + was_modified = True + try: + current = await prepared.working_target.read_all() + except Exception as exc: + initial_read_error = exc + else: + was_modified = current != baseline_prompts + + if was_modified: + try: + await prepared.working_target.write_all(baseline_prompts) + except Exception as exc: + if initial_read_error is not None: + raise PipelineStageExecutionError( + "failed to restore optimizer working prompts after initial " + f"read failed ({initial_read_error}): {exc}" + ) from exc + raise PipelineStageExecutionError( + f"failed to restore optimizer working prompts: {exc}" + ) from exc + + try: + restored = await prepared.working_target.read_all() + except Exception as exc: + raise PipelineStageExecutionError( + f"failed to verify restored optimizer working prompts: {exc}" + ) from exc + if restored != baseline_prompts: + raise PipelineStageExecutionError("optimizer working prompts did not match baseline after restoration") + return was_modified + + +async def _execute_offline_stage( + prepared: PreparedRun, + *, + scenario: CandidateScenario | None = None, + progress: _MutableReportProgress, +) -> OfflineStageResult: + """Run four evaluations through SDK LlmAgent and a deterministic model. + + Source prompts are never written. Once generated, the candidate remains in + the isolated working target on success or candidate-evaluation failure so + the run can be inspected later. + """ + if prepared.config.execution.mode != "offline": + raise PipelineStageExecutionError( + "run_offline_stage requires execution.mode='offline', got " + f"{prepared.config.execution.mode!r}" + ) + + started_at = perf_counter() + selected_scenario = scenario or prepared.config.execution.candidate_scenario + train_evalset = _reload_prepared_evalset( + Path(prepared.input_snapshot.train_evalset_path), + label="train_evalset", + expected_sha256=prepared.input_snapshot.train_evalset_sha256, + ) + validation_evalset = _reload_prepared_evalset( + Path(prepared.input_snapshot.validation_evalset_path), + label="validation_evalset", + expected_sha256=prepared.input_snapshot.validation_evalset_sha256, + ) + + try: + baseline_prompts = await prepared.working_target.read_all() + except Exception as exc: + raise PipelineStageExecutionError(f"failed to read prepared working prompts: {exc}") from exc + expected_baseline = { + snapshot.field_name: snapshot.content for snapshot in prepared.input_snapshot.prompt_snapshots + } + if baseline_prompts != expected_baseline: + raise PipelineStageExecutionError("working prompts no longer match the prepared baseline snapshot") + + agent = BusinessAgent( + prepared.working_target, + DeterministicFakeModel, + agent_name="eval_optimize_offline_agent", + app_name="eval_optimize_offline", + user_id="offline-evaluation", + ) + progress.enter("baseline_train") + baseline_train = await _evaluate_split( + prepared=prepared, + eval_set=train_evalset, + call_agent=agent.call_agent, + phase="baseline", + split="train", + ) + progress.enter("baseline_validation") + baseline_validation = await _evaluate_split( + prepared=prepared, + eval_set=validation_evalset, + call_agent=agent.call_agent, + phase="baseline", + split="validation", + ) + + progress.enter("candidate_generation") + request = CandidateRequest( + current_prompts=baseline_prompts, + target_prompt=prepared.working_target, + optimizer_config_path=Path(prepared.input_snapshot.optimizer_config_path), + train_evalset_path=Path(prepared.input_snapshot.train_evalset_path), + validation_evalset_path=Path(prepared.input_snapshot.validation_evalset_path), + output_dir=Path(prepared.workspace.run_dir) / "fake_provider", + seed=prepared.input_snapshot.seed, + ) + try: + generated = await FakeCandidateProviderAdapter(selected_scenario).propose(request) + candidate = generated.proposal + except Exception as exc: + raise PipelineStageExecutionError(f"fake candidate generation failed: {exc}") from exc + + try: + await prepared.working_target.write_all(candidate.prompts) + written_prompts = await prepared.working_target.read_all() + except Exception as exc: + raise PipelineStageExecutionError(f"candidate prompt write failed: {exc}") from exc + if written_prompts != candidate.prompts: + raise PipelineStageExecutionError("candidate prompt readback did not match the generated proposal") + + progress.enter("candidate_train") + candidate_train = await _evaluate_split( + prepared=prepared, + eval_set=train_evalset, + call_agent=agent.call_agent, + phase="candidate", + split="train", + ) + progress.enter("candidate_validation") + candidate_validation = await _evaluate_split( + prepared=prepared, + eval_set=validation_evalset, + call_agent=agent.call_agent, + phase="candidate", + split="validation", + ) + progress.enter("analysis") + try: + analysis = build_evaluation_analysis( + baseline_train=baseline_train, + baseline_validation=baseline_validation, + candidate_train=candidate_train, + candidate_validation=candidate_validation, + hard_case_ids=set(prepared.config.case_labels.hard_case_ids), + critical_case_ids=set(prepared.config.case_labels.critical_case_ids), + severe_case_score_drop=prepared.config.gate.severe_case_score_drop, + ) + except EvaluationAnalysisError as exc: + raise PipelineStageExecutionError(f"stage 3a analysis failed: {exc}") from exc + measurements = ResourceMeasurements( + cost_usd=ObservableValue( + status="unavailable", + unit="USD", + reason="Offline deterministic model does not report monetary cost.", + ), + total_tokens=ObservableValue( + status="unavailable", + unit="tokens", + reason="Offline deterministic model does not report token usage.", + ), + duration_seconds=ObservableValue( + status="available", + value=perf_counter() - started_at, + unit="seconds", + ), + ) + progress.enter("gate") + try: + gate_decision = evaluate_gate( + analysis, + prepared.config.gate, + prepared.config.budget, + measurements, + ) + except GateEvaluationError as exc: + raise PipelineStageExecutionError(f"stage 3b gate failed: {exc}") from exc + progress.enter("writeback") + writeback = await perform_writeback( + decision=gate_decision, + config=prepared.config.writeback, + snapshots=prepared.input_snapshot.prompt_snapshots, + source_target=prepared.source_target, + candidate=candidate, + ) + return OfflineStageResult( + scenario=selected_scenario, + candidate=candidate, + baseline_train=baseline_train, + baseline_validation=baseline_validation, + candidate_train=candidate_train, + candidate_validation=candidate_validation, + analysis=analysis, + measurements=measurements, + gate_decision=gate_decision, + writeback=writeback, + ) + + +async def _execute_real_stage( + prepared: PreparedRun, + *, + call_agent: CallAgent, + optimizer_parameters: OptimizerRuntimeParameters | None = None, + progress: _MutableReportProgress, +) -> RealStageResult: + """Generate a real optimizer candidate and run the full guarded regression.""" + if prepared.config.execution.mode != "real": + raise PipelineStageExecutionError( + f"run_real_stage requires execution.mode='real', got {prepared.config.execution.mode!r}" + ) + started_at = perf_counter() + _verify_prepared_file( + Path(prepared.input_snapshot.optimizer_config_path), + label="optimizer_config", + expected_sha256=prepared.input_snapshot.optimizer_config_sha256, + ) + train_evalset = _reload_prepared_evalset( + Path(prepared.input_snapshot.train_evalset_path), + label="train_evalset", + expected_sha256=prepared.input_snapshot.train_evalset_sha256, + ) + validation_evalset = _reload_prepared_evalset( + Path(prepared.input_snapshot.validation_evalset_path), + label="validation_evalset", + expected_sha256=prepared.input_snapshot.validation_evalset_sha256, + ) + try: + baseline_prompts = await prepared.working_target.read_all() + except Exception as exc: + raise PipelineStageExecutionError(f"failed to read prepared working prompts: {exc}") from exc + expected_baseline = { + snapshot.field_name: snapshot.content for snapshot in prepared.input_snapshot.prompt_snapshots + } + if baseline_prompts != expected_baseline: + raise PipelineStageExecutionError("working prompts no longer match the prepared baseline snapshot") + + progress.enter("baseline_train") + baseline_train = await _evaluate_split( + prepared=prepared, + eval_set=train_evalset, + call_agent=call_agent, + phase="baseline", + split="train", + ) + progress.enter("baseline_validation") + baseline_validation = await _evaluate_split( + prepared=prepared, + eval_set=validation_evalset, + call_agent=call_agent, + phase="baseline", + split="validation", + ) + + progress.enter("candidate_generation") + request = CandidateRequest( + current_prompts=baseline_prompts, + target_prompt=prepared.working_target, + optimizer_config_path=Path(prepared.input_snapshot.optimizer_config_path), + train_evalset_path=Path(prepared.input_snapshot.train_evalset_path), + validation_evalset_path=Path(prepared.input_snapshot.validation_evalset_path), + output_dir=Path(prepared.workspace.run_dir) / "optimizer", + seed=prepared.input_snapshot.seed, + retain_native_artifacts=prepared.config.artifacts.retain_optimizer_native_artifacts, + runtime_parameters=optimizer_parameters, + expected_optimizer_sha256=prepared.input_snapshot.optimizer_config_sha256, + ) + try: + generated = await AgentOptimizerCandidateProvider(call_agent).propose(request) + except CandidateProviderError as exc: + await _restore_working_baseline(prepared, baseline_prompts) + raise PipelineStageExecutionError(f"real candidate generation failed: {exc}") from exc + + if await _restore_working_baseline(prepared, baseline_prompts): + raise PipelineStageExecutionError("optimizer did not restore working prompts after update_source=False") + + candidate = generated.proposal + try: + await prepared.working_target.write_all(candidate.prompts) + written_prompts = await prepared.working_target.read_all() + except Exception as exc: + raise PipelineStageExecutionError(f"candidate prompt write failed: {exc}") from exc + if written_prompts != candidate.prompts: + raise PipelineStageExecutionError("candidate prompt readback did not match the generated proposal") + + progress.enter("candidate_train") + candidate_train = await _evaluate_split( + prepared=prepared, + eval_set=train_evalset, + call_agent=call_agent, + phase="candidate", + split="train", + ) + progress.enter("candidate_validation") + candidate_validation = await _evaluate_split( + prepared=prepared, + eval_set=validation_evalset, + call_agent=call_agent, + phase="candidate", + split="validation", + ) + progress.enter("analysis") + try: + analysis = build_evaluation_analysis( + baseline_train=baseline_train, + baseline_validation=baseline_validation, + candidate_train=candidate_train, + candidate_validation=candidate_validation, + hard_case_ids=set(prepared.config.case_labels.hard_case_ids), + critical_case_ids=set(prepared.config.case_labels.critical_case_ids), + severe_case_score_drop=prepared.config.gate.severe_case_score_drop, + ) + except EvaluationAnalysisError as exc: + raise PipelineStageExecutionError(f"stage 3a analysis failed: {exc}") from exc + measurements = ResourceMeasurements( + cost_usd=ObservableValue( + status="unavailable", + unit="USD", + reason="The injected agent's full pipeline cost is not observable.", + ), + total_tokens=ObservableValue( + status="unavailable", + unit="tokens", + reason="The injected agent's full pipeline token usage is not observable.", + ), + duration_seconds=ObservableValue( + status="available", + value=perf_counter() - started_at, + unit="seconds", + ), + ) + progress.enter("gate") + try: + gate_decision = evaluate_gate( + analysis, + prepared.config.gate, + prepared.config.budget, + measurements, + ) + except GateEvaluationError as exc: + raise PipelineStageExecutionError(f"stage 3b gate failed: {exc}") from exc + + progress.enter("writeback") + writeback = await perform_writeback( + decision=gate_decision, + config=prepared.config.writeback, + snapshots=prepared.input_snapshot.prompt_snapshots, + source_target=prepared.source_target, + candidate=candidate, + ) + + assert generated.optimize_result is not None + return RealStageResult( + candidate=candidate, + optimize_result=generated.optimize_result, + baseline_train=baseline_train, + baseline_validation=baseline_validation, + candidate_train=candidate_train, + candidate_validation=candidate_validation, + analysis=analysis, + measurements=measurements, + gate_decision=gate_decision, + writeback=writeback, + ) + + +async def run_offline_stage( + prepared: PreparedRun, + *, + scenario: CandidateScenario | None = None, +) -> OfflineStageResult: + """Run offline SDK-Agent regression and publish its audit report.""" + progress = _MutableReportProgress(started_at=datetime.now(timezone.utc)) + result: OfflineStageResult | None = None + try: + result = await _execute_offline_stage( + prepared, + scenario=scenario, + progress=progress, + ) + progress.enter("reporting") + report = build_optimization_report( + prepared, + result, + progress=progress.snapshot(), + finished_at=datetime.now(timezone.utc), + ) + publish_report_bundle( + report, + run_dir=Path(prepared.workspace.run_dir), + copy_input_files=prepared.config.artifacts.copy_input_files, + ) + return result + except Exception as exc: + await _handle_stage_failure(prepared, progress, exc, result) + raise + + +async def _execute_trace_stage( + prepared: PreparedRun, + *, + scenario: CandidateScenario | None, + progress: _MutableReportProgress, +) -> TraceStageResult: + if prepared.config.execution.mode != "trace": + raise PipelineExecutionError( + "run_trace_stage requires execution.mode='trace', got " + f"{prepared.config.execution.mode!r}" + ) + trace_inputs = prepared.input_snapshot.trace_inputs + if trace_inputs is None: + raise PipelineExecutionError("prepared trace inputs are missing") + selected = scenario or prepared.config.execution.candidate_scenario + candidate_inputs = trace_inputs.scenarios[selected] + started_at = perf_counter() + + baseline_train_set = _reload_prepared_evalset( + Path(prepared.input_snapshot.train_evalset_path), + label="baseline train trace", + expected_sha256=prepared.input_snapshot.train_evalset_sha256, + ) + baseline_validation_set = _reload_prepared_evalset( + Path(prepared.input_snapshot.validation_evalset_path), + label="baseline validation trace", + expected_sha256=prepared.input_snapshot.validation_evalset_sha256, + ) + candidate_train_set = _reload_prepared_evalset( + Path(candidate_inputs.train_evalset_path), + label=f"candidate {selected} train trace", + expected_sha256=candidate_inputs.train_evalset_sha256, + ) + candidate_validation_set = _reload_prepared_evalset( + Path(candidate_inputs.validation_evalset_path), + label=f"candidate {selected} validation trace", + expected_sha256=candidate_inputs.validation_evalset_sha256, + ) + + progress.enter("baseline_train") + baseline_train = await _evaluate_split( + prepared=prepared, eval_set=baseline_train_set, call_agent=None, + phase="baseline", split="train", + ) + progress.enter("baseline_validation") + baseline_validation = await _evaluate_split( + prepared=prepared, eval_set=baseline_validation_set, call_agent=None, + phase="baseline", split="validation", + ) + progress.enter("candidate_generation") + prompts = { + snapshot.field_name: snapshot.content + for snapshot in candidate_inputs.prompt_snapshots + } + baseline_prompts = { + snapshot.field_name: snapshot.content + for snapshot in prepared.input_snapshot.prompt_snapshots + } + canonical = json.dumps( + prompts, ensure_ascii=False, sort_keys=True, separators=(",", ":") + ) + candidate_hash = sha256(canonical.encode("utf-8")).hexdigest() + parent_canonical = json.dumps( + baseline_prompts, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ) + parent_hash = sha256(parent_canonical.encode("utf-8")).hexdigest() + candidate = TraceCandidateProposal( + scenario=selected, + prompts=prompts, + changed_fields=[ + name for name in baseline_prompts if baseline_prompts[name] != prompts[name] + ], + rationale="Replay the selected pre-recorded candidate trace.", + parent_prompt_sha256=parent_hash, + candidate_prompt_sha256=candidate_hash, + candidate_id=f"trace-{selected}-{candidate_hash[:12]}", + source_trace_sha256={ + "train": candidate_inputs.train_evalset_sha256, + "validation": candidate_inputs.validation_evalset_sha256, + }, + ) + progress.enter("candidate_train") + candidate_train = await _evaluate_split( + prepared=prepared, eval_set=candidate_train_set, call_agent=None, + phase="candidate", split="train", + ) + progress.enter("candidate_validation") + candidate_validation = await _evaluate_split( + prepared=prepared, eval_set=candidate_validation_set, call_agent=None, + phase="candidate", split="validation", + ) + progress.enter("analysis") + try: + analysis = build_evaluation_analysis( + baseline_train=baseline_train, + baseline_validation=baseline_validation, + candidate_train=candidate_train, + candidate_validation=candidate_validation, + hard_case_ids=set(prepared.config.case_labels.hard_case_ids), + critical_case_ids=set(prepared.config.case_labels.critical_case_ids), + severe_case_score_drop=prepared.config.gate.severe_case_score_drop, + ) + except EvaluationAnalysisError as exc: + raise PipelineStageExecutionError( + f"stage 3a analysis failed: {exc}" + ) from exc + measurements = ResourceMeasurements( + cost_usd=ObservableValue(status="unavailable", unit="USD", reason="Trace replay does not call a model."), + total_tokens=ObservableValue(status="unavailable", unit="tokens", reason="Trace replay does not call a model."), + duration_seconds=ObservableValue(status="available", value=perf_counter() - started_at, unit="seconds"), + ) + progress.enter("gate") + try: + gate_decision = evaluate_gate( + analysis, + prepared.config.gate, + prepared.config.budget, + measurements, + ) + except GateEvaluationError as exc: + raise PipelineStageExecutionError( + f"stage 3b gate failed: {exc}" + ) from exc + progress.enter("writeback") + writeback = WritebackResult( + status="skipped", reason="trace_replay", attempted=False + ) + return TraceStageResult( + scenario=selected, candidate=candidate, + baseline_train=baseline_train, + baseline_validation=baseline_validation, + candidate_train=candidate_train, + candidate_validation=candidate_validation, + analysis=analysis, measurements=measurements, + gate_decision=gate_decision, writeback=writeback, + ) + + +async def run_trace_stage( + prepared: PreparedRun, + *, + scenario: CandidateScenario | None = None, +) -> TraceStageResult: + """回放四个 Trace EvalSet，并发布分析与 Gate 报告。""" + progress = _MutableReportProgress(started_at=datetime.now(timezone.utc)) + result: TraceStageResult | None = None + try: + result = await _execute_trace_stage( + prepared, scenario=scenario, progress=progress + ) + progress.enter("reporting") + report = build_optimization_report( + prepared, result, progress=progress.snapshot(), + finished_at=datetime.now(timezone.utc), + ) + publish_report_bundle( + report, + run_dir=Path(prepared.workspace.run_dir), + copy_input_files=prepared.config.artifacts.copy_input_files, + ) + return result + except Exception as exc: + await _handle_stage_failure(prepared, progress, exc, result) + raise + + +async def run_real_stage( + prepared: PreparedRun, + *, + call_agent: CallAgent, + optimizer_parameters: OptimizerRuntimeParameters | None = None, +) -> RealStageResult: + """Run real optimization and atomically publish its audit report.""" + progress = _MutableReportProgress(started_at=datetime.now(timezone.utc)) + result: RealStageResult | None = None + try: + result = await _execute_real_stage( + prepared, + call_agent=call_agent, + optimizer_parameters=optimizer_parameters, + progress=progress, + ) + progress.enter("reporting") + report = build_optimization_report( + prepared, + result, + progress=progress.snapshot(), + finished_at=datetime.now(timezone.utc), + ) + publish_report_bundle( + report, + run_dir=Path(prepared.workspace.run_dir), + copy_input_files=prepared.config.artifacts.copy_input_files, + ) + return result + except Exception as exc: + await _handle_stage_failure(prepared, progress, exc, result) + raise diff --git a/examples/optimization/eval_optimize_loop/pipeline.real.json b/examples/optimization/eval_optimize_loop/pipeline.real.json new file mode 100644 index 00000000..1ac989dd --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline.real.json @@ -0,0 +1,52 @@ +{ + "config_version": 1, + "execution": { + "mode": "real", + "candidate_scenario": "improve" + }, + "inputs": { + "train_evalset": "data/train.evalset.json", + "validation_evalset": "data/val.evalset.json", + "optimizer_config": "optimizer.json" + }, + "prompts": [ + { + "name": "system_prompt", + "path": "prompts/system.md" + } + ], + "run": { + "runs_dir": "runs", + "seed": 42 + }, + "case_labels": { + "hard_case_ids": ["val_knowledge_recall"], + "critical_case_ids": ["val_refund_route"] + }, + "gate": { + "min_validation_score_delta": 0.05, + "reject_on_validation_pass_rate_drop": true, + "reject_new_hard_fail": true, + "reject_critical_regression": true, + "severe_case_score_drop": 0.2, + "required_metrics": ["final_response_avg_score"] + }, + "budget": { + "max_duration_seconds": 180, + "on_unavailable": "warning" + }, + "reporting": { + "write_json": true, + "write_markdown": true, + "include_case_evidence": true + }, + "artifacts": { + "copy_input_files": true, + "retain_optimizer_native_artifacts": true, + "audit_all_candidates": false + }, + "writeback": { + "enabled": false, + "require_source_hash_match": true + } +} diff --git a/examples/optimization/eval_optimize_loop/pipeline.trace.json b/examples/optimization/eval_optimize_loop/pipeline.trace.json new file mode 100644 index 00000000..56946e7b --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline.trace.json @@ -0,0 +1,46 @@ +{ + "config_version": 1, + "execution": {"mode": "trace", "candidate_scenario": "improve"}, + "inputs": { + "train_evalset": "data/traces/baseline.train.evalset.json", + "validation_evalset": "data/traces/baseline.validation.evalset.json", + "optimizer_config": "optimizer.json" + }, + "prompts": [{"name": "system_prompt", "path": "prompts/system.md"}], + "trace_inputs": { + "candidates": { + "improve": { + "train_evalset": "data/traces/improve.train.evalset.json", + "validation_evalset": "data/traces/improve.validation.evalset.json", + "prompts": [{"name": "system_prompt", "path": "data/traces/prompts/improve.md"}] + }, + "no_improvement": { + "train_evalset": "data/traces/no_improvement.train.evalset.json", + "validation_evalset": "data/traces/no_improvement.validation.evalset.json", + "prompts": [{"name": "system_prompt", "path": "data/traces/prompts/no_improvement.md"}] + }, + "overfit": { + "train_evalset": "data/traces/overfit.train.evalset.json", + "validation_evalset": "data/traces/overfit.validation.evalset.json", + "prompts": [{"name": "system_prompt", "path": "data/traces/prompts/overfit.md"}] + } + } + }, + "run": {"runs_dir": "runs", "seed": 42}, + "case_labels": { + "hard_case_ids": ["val_knowledge_recall"], + "critical_case_ids": ["val_refund_route"] + }, + "gate": { + "min_validation_score_delta": 0.05, + "reject_on_validation_pass_rate_drop": true, + "reject_new_hard_fail": true, + "reject_critical_regression": true, + "severe_case_score_drop": 0.2, + "required_metrics": ["final_response_avg_score"] + }, + "budget": {"max_duration_seconds": 180, "on_unavailable": "warning"}, + "reporting": {"write_json": true, "write_markdown": true, "include_case_evidence": true}, + "artifacts": {"copy_input_files": true, "retain_optimizer_native_artifacts": true, "audit_all_candidates": false}, + "writeback": {"enabled": false, "require_source_hash_match": true} +} diff --git a/examples/optimization/eval_optimize_loop/prompt_workspace.py b/examples/optimization/eval_optimize_loop/prompt_workspace.py new file mode 100644 index 00000000..1e30a6e4 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/prompt_workspace.py @@ -0,0 +1,123 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under the Apache License, Version 2.0. +"""Prompt snapshots and isolated path-backed ``TargetPrompt`` instances.""" + +from __future__ import annotations + +from hashlib import sha256 +from pathlib import Path + +from trpc_agent_sdk.evaluation import TargetPrompt + +from .config import PromptFieldConfig +from .schemas import PromptSnapshot + + +class PromptWorkspaceError(ValueError): + """A prompt source cannot safely participate in an isolated run.""" + + +class SourcePromptDriftError(RuntimeError): + """One or more source prompts changed after the baseline snapshot.""" + + +def resolve_inside_example_root(example_root: Path, relative_path: str, label: str) -> Path: + """Resolve a configured path and reject traversal or symlink escape.""" + root = example_root.resolve() + candidate = (root / relative_path).resolve() + try: + candidate.relative_to(root) + except ValueError as exc: + raise PromptWorkspaceError(f"{label} escapes the example root: {relative_path}") from exc + return candidate + + +def validate_prompt_sources(example_root: Path, prompts: list[PromptFieldConfig]) -> list[Path]: + """Validate path-backed, UTF-8 prompt files and return resolved sources.""" + sources: list[Path] = [] + seen_paths: set[Path] = set() + for prompt in prompts: + source = resolve_inside_example_root(example_root, prompt.path, f"prompt {prompt.name!r}") + raw_source = example_root.resolve() / prompt.path + if raw_source.is_symlink(): + raise PromptWorkspaceError(f"prompt {prompt.name!r} must not be a symlink") + if not source.is_file(): + raise PromptWorkspaceError(f"prompt {prompt.name!r} is not a regular file: {prompt.path}") + try: + source.read_text(encoding="utf-8") + except UnicodeDecodeError as exc: + raise PromptWorkspaceError(f"prompt {prompt.name!r} is not UTF-8: {prompt.path}") from exc + if source in seen_paths: + raise PromptWorkspaceError(f"multiple prompt fields reference {prompt.path}") + seen_paths.add(source) + sources.append(source) + return sources + + +def verify_source_hashes(snapshots: list[PromptSnapshot]) -> None: + """Fail if a source prompt no longer matches its preparation snapshot. + + Later writeback code must call this immediately before an ACCEPT write. It + is useful in stage one as a read-only concurrency guard; this module does + not expose a source-writing operation. + """ + drifted: list[str] = [] + for snapshot in snapshots: + source = Path(snapshot.source_path) + try: + content = source.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError): + drifted.append(snapshot.field_name) + continue + digest = sha256(content.encode("utf-8")).hexdigest() + if digest != snapshot.sha256: + drifted.append(snapshot.field_name) + if drifted: + raise SourcePromptDriftError(f"source prompt hash changed for fields: {sorted(drifted)}") + + +def stage_prompt_workspace( + *, + example_root: Path, + staging_run_dir: Path, + final_run_dir: Path, + prompts: list[PromptFieldConfig], + sources: list[Path], +) -> tuple[list[PromptSnapshot], TargetPrompt, TargetPrompt]: + """Copy prompt sources into a staging run and build source/working targets. + + The returned working target intentionally points at *final* paths. The + caller atomically renames ``staging_run_dir`` into ``final_run_dir`` only + once every source has been copied, so no later phase can observe a partial + prompt workspace. + """ + prompts_dir = staging_run_dir / "workspace" / "prompts" + prompts_dir.mkdir(parents=True) + + source_target = TargetPrompt() + working_target = TargetPrompt() + snapshots: list[PromptSnapshot] = [] + + for index, (prompt, source) in enumerate(zip(prompts, sources, strict=True), start=1): + content = source.read_text(encoding="utf-8") + suffix = source.suffix or ".txt" + working_name = f"{index:02d}_{prompt.name}{suffix}" + staged_path = prompts_dir / working_name + final_path = final_run_dir / "workspace" / "prompts" / working_name + staged_path.write_text(content, encoding="utf-8") + + source_target.add_path(prompt.name, str(source)) + working_target.add_path(prompt.name, str(final_path)) + snapshots.append( + PromptSnapshot( + field_name=prompt.name, + source_path=str(source), + working_path=str(final_path), + content=content, + sha256=sha256(content.encode("utf-8")).hexdigest(), + )) + + return snapshots, source_target, working_target diff --git a/examples/optimization/eval_optimize_loop/prompts/system.md b/examples/optimization/eval_optimize_loop/prompts/system.md new file mode 100644 index 00000000..345c342d --- /dev/null +++ b/examples/optimization/eval_optimize_loop/prompts/system.md @@ -0,0 +1,5 @@ +You are a customer-support routing assistant. + +Answer with a compact JSON object containing `route` and `message`. Use the +available account tool for account-specific requests. Never invent account +facts. diff --git a/examples/optimization/eval_optimize_loop/real_agent.py b/examples/optimization/eval_optimize_loop/real_agent.py new file mode 100644 index 00000000..1aa0a554 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/real_agent.py @@ -0,0 +1,67 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under the Apache License, Version 2.0. +"""真实业务模型适配器：每个评测 case 都读取当前工作 Prompt。""" + +from __future__ import annotations + +import os +from dataclasses import dataclass +from typing import Mapping + +from trpc_agent_sdk.evaluation import TargetPrompt +from trpc_agent_sdk.models import OpenAIModel + +from .business_agent import BusinessAgent + + +@dataclass(frozen=True) +class BusinessModelConfig: + """来自环境变量的业务模型连接信息。""" + + api_key: str + base_url: str + model_name: str + + +def load_business_model_config( + environ: Mapping[str, str] | None = None, +) -> BusinessModelConfig: + """读取业务模型环境变量，缺失时一次性报告全部字段。""" + values = os.environ if environ is None else environ + names = ( + "TRPC_AGENT_API_KEY", + "TRPC_AGENT_BASE_URL", + "TRPC_AGENT_MODEL_NAME", + ) + missing = [name for name in names if not values.get(name, "").strip()] + if missing: + raise ValueError(f"missing required environment variables: {', '.join(missing)}") + return BusinessModelConfig( + api_key=values["TRPC_AGENT_API_KEY"].strip(), + base_url=values["TRPC_AGENT_BASE_URL"].strip(), + model_name=values["TRPC_AGENT_MODEL_NAME"].strip(), + ) + + +class RealBusinessAgent: + """以真实模型执行评测，并确保 case 与 Prompt 版本相互隔离。""" + + def __init__(self, target_prompt: TargetPrompt, config: BusinessModelConfig) -> None: + self._delegate = BusinessAgent( + target_prompt, + lambda: OpenAIModel( + model_name=config.model_name, + api_key=config.api_key, + base_url=config.base_url, + ), + agent_name="eval_optimize_real_agent", + app_name="eval_optimize_real_integration", + user_id="real-integration", + ) + + async def call_agent(self, query: str) -> str: + """重新读取工作 Prompt，运行独立 session，只返回正式最终文本。""" + return await self._delegate.call_agent(query) diff --git a/examples/optimization/eval_optimize_loop/report_builder.py b/examples/optimization/eval_optimize_loop/report_builder.py new file mode 100644 index 00000000..d5cc3182 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/report_builder.py @@ -0,0 +1,222 @@ +"""Pure builders for serializable optimization reports.""" + +from __future__ import annotations + +from datetime import datetime +import os +import re +from typing import TYPE_CHECKING + +from .schemas import FailureReport, OptimizerResourceObservation, OptimizerResourceValue +from .schemas import OptimizationReport +from .schemas import PipelineStageResult, RealStageResult, ReportProgress, TraceStageResult + +if TYPE_CHECKING: + from .pipeline import PreparedRun + +_OPTIMIZER_SCOPE = ( + "Optimizer-only observation; excludes complete business Agent evaluation usage." +) +_OFFLINE_OPTIMIZER_REASON = "Offline mode uses a deterministic candidate provider." +_TRACE_OPTIMIZER_REASON = "Trace replay does not run a candidate provider or AgentOptimizer." +_MISSING_COST_REASON = ( + "Reflection LM calls were observed but optimizer cost was not reported." +) +_MISSING_TOKEN_REASON = ( + "Reflection LM calls were observed but optimizer token usage was not reported." +) +_INVALID_TOKEN_REASON = "Optimizer token usage was malformed or inconsistent." +_REDACTED = "[REDACTED]" +_SENSITIVE_ENV_NAMES = ("TRPC_AGENT_API_KEY", "TRPC_AGENT_BASE_URL") +_SENSITIVE_KEY_VALUE = re.compile( + r"(?P[\"']?(?:api[_-]?key|base[_-]?url|authorization)[\"']?\s*[:=]\s*)" + r"(?P[\"'][^\"']*[\"']|(?:(?:bearer|basic|token)\s+)?[^\s,;}\]]+)", + re.IGNORECASE, +) +_HTTP_URL = re.compile(r"https?://[^\s,;}\]<>\"']+", re.IGNORECASE) +_BEARER_VALUE = re.compile( + r"\bbearer(?:\s+|\s*[:=]\s*)[\"']?[^\s,;}\]\"']+[\"']?", + re.IGNORECASE, +) + + +def _not_applicable_optimizer_value( + unit: str, reason: str, +) -> OptimizerResourceValue[object]: + return OptimizerResourceValue[object]( + status="not_applicable", + unit=unit, + reason=reason, + ) + + +def redact_error_message(error: Exception) -> str: + """移除异常文本中的环境凭据、认证字段和连接地址。""" + message = str(error) + environment_values = { + os.environ.get(name, "") + for name in _SENSITIVE_ENV_NAMES + if os.environ.get(name, "") + } + for sensitive_value in sorted(environment_values, key=len, reverse=True): + message = message.replace(sensitive_value, _REDACTED) + message = _SENSITIVE_KEY_VALUE.sub( + lambda match: f"{match.group('prefix')}{_REDACTED}", + message, + ) + message = _BEARER_VALUE.sub(f"Bearer {_REDACTED}", message) + return _HTTP_URL.sub(_REDACTED, message) + + +def _is_complete_token_usage(value: object) -> bool: + if not isinstance(value, dict): + return False + required = ("prompt", "completion", "total") + if not all(key in value for key in required): + return False + if not all(type(value[key]) is int and value[key] >= 0 for key in required): + return False + return value["total"] == value["prompt"] + value["completion"] + + +def _optimizer_resources(result: PipelineStageResult) -> OptimizerResourceObservation: + if not isinstance(result, RealStageResult): + reason = ( + _TRACE_OPTIMIZER_REASON + if isinstance(result, TraceStageResult) + else _OFFLINE_OPTIMIZER_REASON + ) + return OptimizerResourceObservation( + scope_note=reason, + total_rounds=_not_applicable_optimizer_value("rounds", reason), + reflection_lm_calls=_not_applicable_optimizer_value("calls", reason), + cost_usd=_not_applicable_optimizer_value("USD", reason), + token_usage=_not_applicable_optimizer_value("tokens", reason), + duration_seconds=_not_applicable_optimizer_value("seconds", reason), + ) + native = result.optimize_result + reflection_calls = native.total_reflection_lm_calls + cost_missing = reflection_calls > 0 and native.total_llm_cost <= 0 + token_usage = native.total_token_usage + token_usage_valid = _is_complete_token_usage(token_usage) + tokens_missing = ( + not token_usage_valid + or (reflection_calls > 0 and token_usage["total"] <= 0) + ) + return OptimizerResourceObservation( + scope_note=_OPTIMIZER_SCOPE, + total_rounds=OptimizerResourceValue[int]( + status="available", value=native.total_rounds, unit="rounds", + ), + reflection_lm_calls=OptimizerResourceValue[int]( + status="available", value=reflection_calls, unit="calls", + ), + cost_usd=OptimizerResourceValue[float]( + status="unavailable" if cost_missing else "available", + value=None if cost_missing else native.total_llm_cost, + unit="USD", + reason=_MISSING_COST_REASON if cost_missing else None, + ), + token_usage=OptimizerResourceValue[dict[str, int]]( + status="unavailable" if tokens_missing else "available", + value=None if tokens_missing else token_usage, + unit="tokens", + reason=( + _INVALID_TOKEN_REASON + if tokens_missing and not token_usage_valid + else _MISSING_TOKEN_REASON if tokens_missing else None + ), + ), + duration_seconds=OptimizerResourceValue[float]( + status="available", value=native.duration_seconds, unit="seconds", + ), + ) + +def build_optimization_report( + prepared: PreparedRun, result: PipelineStageResult, *, progress: ReportProgress, finished_at: datetime, +) -> OptimizationReport: + return OptimizationReport( + run_id=prepared.workspace.run_id, execution_mode=prepared.config.execution.mode, + seed=prepared.input_snapshot.seed, started_at=progress.started_at, finished_at=finished_at, + input_snapshot=prepared.input_snapshot, candidate=result.candidate, + baseline_train=result.baseline_train, baseline_validation=result.baseline_validation, + candidate_train=result.candidate_train, candidate_validation=result.candidate_validation, + analysis=result.analysis, pipeline_resources=result.measurements, + optimizer_resources=_optimizer_resources(result), gate_decision=result.gate_decision, + writeback=result.writeback, + ) + +def build_failure_report( + prepared: PreparedRun, *, progress: ReportProgress, error: Exception, + source_prompt_hashes: dict[str, str], existing_artifacts: list[str], generated_at: datetime, +) -> FailureReport: + return FailureReport( + run_id=prepared.workspace.run_id, execution_mode=prepared.config.execution.mode, + failed_phase=progress.current_phase, exception_type=type(error).__name__, + error_message=redact_error_message(error), generated_at=generated_at, + input_snapshot=prepared.input_snapshot, + source_prompt_hashes=dict(sorted(source_prompt_hashes.items())), + completed_phases=progress.completed_phases, existing_artifacts=sorted(existing_artifacts), + ) + + +def render_optimization_markdown(report: OptimizationReport) -> str: + decision = report.gate_decision.decision.upper() + lines = [ + "# Optimization Report", + "", + f"- Run: `{report.run_id}`", + f"- Mode: `{report.execution_mode}`", + f"- Gate decision: {decision}", + f"- Candidate: `{report.candidate.candidate_id}`", + "", + "## Full Evaluations", + "", + ] + for label, snapshot in ( + ("Baseline train", report.baseline_train), + ("Baseline validation", report.baseline_validation), + ("Candidate train", report.candidate_train), + ("Candidate validation", report.candidate_validation), + ): + score = snapshot.average_score if snapshot.average_score is not None else "unavailable" + lines.append( + f"- {label}: {snapshot.passed_case_count}/{snapshot.total_case_count} passed; " + f"average score={score}" + ) + lines.extend(["", "## Gate", ""]) + lines.extend(f"- Rejection: {reason}" for reason in report.gate_decision.rejection_reasons) + lines.extend(f"- Warning: {warning}" for warning in report.gate_decision.warnings) + if not report.gate_decision.rejection_reasons and not report.gate_decision.warnings: + lines.append("- No rejection reasons or warnings.") + lines.extend(["", "## Candidate Changes", ""]) + changed = report.candidate.changed_fields or ["none"] + lines.extend(f"- {field}" for field in changed) + lines.extend(["", "## Overfit", f"- Status: {report.analysis.overfit_status}", + f"- Reason: {report.analysis.overfit_reason}", "", "## Writeback", + f"- Status: {report.writeback.status}", f"- Reason: {report.writeback.reason}", + "", "## Pipeline Observations", + f"- Cost: {report.pipeline_resources.cost_usd.status}", + f"- Tokens: {report.pipeline_resources.total_tokens.status}", + f"- Duration: {report.pipeline_resources.duration_seconds.status}", + "", "## Optimizer Resources"]) + for label, observation in ( + ("Rounds", report.optimizer_resources.total_rounds), + ("Reflection calls", report.optimizer_resources.reflection_lm_calls), + ("Cost", report.optimizer_resources.cost_usd), + ("Token usage", report.optimizer_resources.token_usage), + ("Duration", report.optimizer_resources.duration_seconds), + ): + line = f"- {label}: {observation.status}; unit={observation.unit}" + if observation.value is not None: + value = observation.value + if isinstance(value, dict): + value = ", ".join( + f"{key}={item}" for key, item in sorted(value.items()) + ) + line += f"; value={value}" + if observation.reason is not None: + line += f"; reason={observation.reason}" + lines.append(line) + lines.extend(["", "## Optimizer Scope", f"- {report.optimizer_resources.scope_note}"]) + return "\n".join(lines) + "\n" diff --git a/examples/optimization/eval_optimize_loop/run_pipeline.py b/examples/optimization/eval_optimize_loop/run_pipeline.py new file mode 100644 index 00000000..b18801f6 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/run_pipeline.py @@ -0,0 +1,96 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under the Apache License, Version 2.0. +"""Run the deterministic offline evaluation, candidate, analysis, and Gate stage.""" + +from __future__ import annotations + +import argparse +import asyncio +import sys +from pathlib import Path + +_HERE = Path(__file__).resolve().parent +if __package__ in (None, ""): + _REPO_ROOT = _HERE.parents[2] + if str(_REPO_ROOT) not in sys.path: + sys.path.insert(0, str(_REPO_ROOT)) + from examples.optimization.eval_optimize_loop.pipeline import prepare_run + from examples.optimization.eval_optimize_loop.pipeline import run_offline_stage + from examples.optimization.eval_optimize_loop.pipeline import run_trace_stage + from examples.optimization.eval_optimize_loop.config import load_pipeline_config +else: + from .config import load_pipeline_config + from .pipeline import prepare_run + from .pipeline import run_offline_stage + from .pipeline import run_trace_stage + + +def _format_snapshot(label: str, snapshot: object) -> str: + score = getattr(snapshot, "average_score", None) + score_text = "unavailable" if score is None else f"{score:.3f}" + return ( + f"{label}: {snapshot.passed_case_count}/{snapshot.total_case_count} passed, " + f"average score={score_text}" + ) + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Run the deterministic offline evaluation and Gate stage." + ) + parser.add_argument("--config", type=Path, default=_HERE / "pipeline.json") + parser.add_argument("--run-id", help="Optional reproducible run identifier.") + parser.add_argument( + "--scenario", + choices=("improve", "no_improvement", "overfit"), + help="Override execution.candidate_scenario for this run.", + ) + args = parser.parse_args() + + config = load_pipeline_config(args.config) + if config.execution.mode == "real": + parser.error( + "real mode requires an injected business agent; use the Python API " + "run_real_stage(prepared, call_agent=...)" + ) + + prepared = prepare_run(args.config, run_id=args.run_id) + if config.execution.mode == "offline": + result = asyncio.run(run_offline_stage(prepared, scenario=args.scenario)) + elif config.execution.mode == "trace": + result = asyncio.run(run_trace_stage(prepared, scenario=args.scenario)) + else: + parser.error(f"unsupported execution mode: {config.execution.mode}") + print(f"Completed deterministic pipeline stage: {prepared.workspace.run_dir}") + print(f"Candidate: {result.candidate.candidate_id} ({result.scenario})") + print(_format_snapshot("Baseline train", result.baseline_train)) + print(_format_snapshot("Baseline validation", result.baseline_validation)) + print(_format_snapshot("Candidate train", result.candidate_train)) + print(_format_snapshot("Candidate validation", result.candidate_validation)) + print(f"Gate decision: {result.gate_decision.decision.upper()}") + rejected_rules = [ + rule for rule in result.gate_decision.rule_results if rule.outcome == "reject" + ] + if rejected_rules: + print("Rejection reasons:") + for rule in rejected_rules: + print(f"- [{rule.rule_id}] {rule.message}") + if result.gate_decision.warnings: + print("Warnings:") + for warning in result.gate_decision.warnings: + print(f"- {warning}") + print( + f"Writeback: {result.writeback.status.upper()} " + f"({result.writeback.reason})" + ) + report_dir = Path(prepared.workspace.run_dir) / "report" + print(f"JSON report: {report_dir / 'optimization_report.json'}") + print(f"Markdown report: {report_dir / 'optimization_report.md'}") + print(f"Artifact index: {report_dir / 'artifact_index.json'}") + + +if __name__ == "__main__": + main() diff --git a/examples/optimization/eval_optimize_loop/run_real_integration.py b/examples/optimization/eval_optimize_loop/run_real_integration.py new file mode 100644 index 00000000..c326e5ea --- /dev/null +++ b/examples/optimization/eval_optimize_loop/run_real_integration.py @@ -0,0 +1,149 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under the Apache License, Version 2.0. +"""显式运行真实业务模型与真实反思模型的完整回归闭环。""" + +from __future__ import annotations + +import argparse +import asyncio +import sys +from pathlib import Path + +from pydantic import ValidationError + +_HERE = Path(__file__).resolve().parent +if __package__ in (None, ""): + _REPO_ROOT = _HERE.parents[2] + if str(_REPO_ROOT) not in sys.path: + sys.path.insert(0, str(_REPO_ROOT)) + from examples.optimization.eval_optimize_loop.config import load_pipeline_config + from examples.optimization.eval_optimize_loop.pipeline import prepare_run + from examples.optimization.eval_optimize_loop.pipeline import run_real_stage + from examples.optimization.eval_optimize_loop.real_agent import BusinessModelConfig + from examples.optimization.eval_optimize_loop.real_agent import RealBusinessAgent + from examples.optimization.eval_optimize_loop.real_agent import load_business_model_config + from examples.optimization.eval_optimize_loop.report_builder import redact_error_message + from examples.optimization.eval_optimize_loop.schemas import OptimizerRuntimeParameters +else: + from .config import load_pipeline_config + from .pipeline import prepare_run + from .pipeline import run_real_stage + from .real_agent import BusinessModelConfig + from .real_agent import RealBusinessAgent + from .real_agent import load_business_model_config + from .report_builder import redact_error_message + from .schemas import OptimizerRuntimeParameters + + +def _think_value(value: str) -> bool | None: + return {"auto": None, "on": True, "off": False}[value] + + +def _format_snapshot(label: str, snapshot: object) -> str: + score = getattr(snapshot, "average_score", None) + score_text = "unavailable" if score is None else f"{score:.3f}" + return ( + f"{label}: {snapshot.passed_case_count}/{snapshot.total_case_count} passed, " + f"average score={score_text}" + ) + + +def _build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Run the explicitly enabled real-model pipeline.") + parser.add_argument("--run-real", action="store_true", help="Confirm that real API calls are intended.") + parser.add_argument("--config", type=Path, default=_HERE / "pipeline.real.json") + parser.add_argument("--run-id") + parser.add_argument("--optimizer-model-name", required=True) + parser.add_argument("--optimizer-provider-name", default="openai") + parser.add_argument("--optimizer-temperature", type=float, default=0.8) + parser.add_argument("--optimizer-max-tokens", type=int, default=4096) + parser.add_argument("--optimizer-think", choices=("auto", "on", "off"), default="auto") + parser.add_argument("--max-candidate-proposals", type=int, default=1) + return parser + + +async def _run( + args: argparse.Namespace, + business_config: BusinessModelConfig, + parameters: OptimizerRuntimeParameters, +): + prepared = prepare_run(args.config, run_id=args.run_id) + source_before = await prepared.source_target.read_all() + agent = RealBusinessAgent(prepared.working_target, business_config) + try: + result = await run_real_stage( + prepared, + call_agent=agent.call_agent, + optimizer_parameters=parameters, + ) + except Exception as exc: + source_after = await prepared.source_target.read_all() + if source_after != source_before: + raise RuntimeError( + "source Prompt changed during a failed real integration run" + ) from exc + raise + source_after = await prepared.source_target.read_all() + if source_after != source_before: + raise RuntimeError("source Prompt changed even though real integration writeback is disabled") + return prepared, result + + +def main() -> int: + parser = _build_parser() + args = parser.parse_args() + if not args.run_real: + parser.error("real API calls require explicit --run-real confirmation") + try: + business_config = load_business_model_config() + parameters = OptimizerRuntimeParameters( + provider_name=args.optimizer_provider_name, + model_name=args.optimizer_model_name, + temperature=args.optimizer_temperature, + max_tokens=args.optimizer_max_tokens, + think=_think_value(args.optimizer_think), + max_candidate_proposals=args.max_candidate_proposals, + ) + config = load_pipeline_config(args.config) + except (OSError, ValueError, ValidationError) as exc: + parser.error(str(exc)) + if config.execution.mode != "real": + parser.error("real integration requires execution.mode='real'") + if config.writeback.enabled: + parser.error("real integration requires writeback.enabled=false") + + try: + prepared, result = asyncio.run(_run(args, business_config, parameters)) + except Exception as exc: + print( + f"Real integration failed: {redact_error_message(exc)}", + file=sys.stderr, + ) + return 1 + + print(f"Completed real-model pipeline: {prepared.workspace.run_dir}") + print(_format_snapshot("Baseline train", result.baseline_train)) + print(_format_snapshot("Baseline validation", result.baseline_validation)) + print(_format_snapshot("Candidate train", result.candidate_train)) + print(_format_snapshot("Candidate validation", result.candidate_validation)) + print( + f"Optimizer: {result.optimize_result.status}, " + f"rounds={result.optimize_result.total_rounds}, candidate={result.candidate.candidate_id}" + ) + print(f"Gate decision: {result.gate_decision.decision.upper()}") + for reason in result.gate_decision.rejection_reasons: + print(f"- {reason}") + print(f"Writeback: {result.writeback.status.upper()} ({result.writeback.reason})") + print("Source Prompt unchanged: yes") + report_dir = Path(prepared.workspace.run_dir) / "report" + print(f"JSON report: {report_dir / 'optimization_report.json'}") + print(f"Markdown report: {report_dir / 'optimization_report.md'}") + print(f"Artifact index: {report_dir / 'artifact_index.json'}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/optimization/eval_optimize_loop/sample_output/artifact_index.json b/examples/optimization/eval_optimize_loop/sample_output/artifact_index.json new file mode 100644 index 00000000..640ae07a --- /dev/null +++ b/examples/optimization/eval_optimize_loop/sample_output/artifact_index.json @@ -0,0 +1,139 @@ +{ + "schema_version": 1, + "run_id": "stage6_sample", + "generated_at": "2026-07-21T07:36:54.029902Z", + "artifacts": [ + { + "artifact_id": "report.optimization_json", + "artifact_type": "report", + "relative_path": "optimization_report.json", + "required": true, + "produced_by": "reporting", + "status": "available", + "size_bytes": 128820, + "sha256": "451fc8b7a32552f52075dd7fbb6bf4b4014d0477d351c172e2ef21785a323ab0", + "unavailable_reason": null + }, + { + "artifact_id": "report.optimization_markdown", + "artifact_type": "report", + "relative_path": "optimization_report.md", + "required": true, + "produced_by": "reporting", + "status": "available", + "size_bytes": 1335, + "sha256": "ee44b5e042e554bce89662d9d85c64ca861984ce2e69e0e32d5e5ddb7a26570e", + "unavailable_reason": null + }, + { + "artifact_id": "evaluation.baseline_train", + "artifact_type": "evaluation", + "relative_path": "evaluations/baseline_train.json", + "required": true, + "produced_by": "baseline_train", + "status": "available", + "size_bytes": 17082, + "sha256": "e3763b13c8e2909185c91b61b93266757a3fdabe67ee78be134d171fb12d2601", + "unavailable_reason": null + }, + { + "artifact_id": "evaluation.baseline_validation", + "artifact_type": "evaluation", + "relative_path": "evaluations/baseline_validation.json", + "required": true, + "produced_by": "baseline_validation", + "status": "available", + "size_bytes": 17257, + "sha256": "9032bf067f4e0dc129bc63bdd862ab740cb9552960a72145b5b8d24c3b5ca2b3", + "unavailable_reason": null + }, + { + "artifact_id": "evaluation.candidate_train", + "artifact_type": "evaluation", + "relative_path": "evaluations/candidate_train.json", + "required": true, + "produced_by": "candidate_train", + "status": "available", + "size_bytes": 15950, + "sha256": "606d512943a332fcc159fbfa9c35c8cf0d51c4c545ff74067fc1ad7ed3201608", + "unavailable_reason": null + }, + { + "artifact_id": "evaluation.candidate_validation", + "artifact_type": "evaluation", + "relative_path": "evaluations/candidate_validation.json", + "required": true, + "produced_by": "candidate_validation", + "status": "available", + "size_bytes": 16165, + "sha256": "9a8bef5b2d7beedcd9535ab0844af47a91ce9dd3e8031801c375c17308a8bd06", + "unavailable_reason": null + }, + { + "artifact_id": "prompt.baseline.system_prompt", + "artifact_type": "prompt", + "relative_path": "prompts/baseline/000-system_prompt.md", + "required": true, + "produced_by": "baseline_train", + "status": "available", + "size_bytes": 205, + "sha256": "ded1ce1a1b01fa5f42e8a98ff81c8f816452f77b1ec510ce4b951769420e9871", + "unavailable_reason": null + }, + { + "artifact_id": "prompt.candidate.system_prompt", + "artifact_type": "prompt", + "relative_path": "prompts/candidate/000-system_prompt.md", + "required": true, + "produced_by": "candidate_generation", + "status": "available", + "size_bytes": 588, + "sha256": "70613f68877e0d15f123648ee80513dc8ded4ecbc685fad1f040bd901267252c", + "unavailable_reason": null + }, + { + "artifact_id": "input.pipeline_config", + "artifact_type": "input", + "relative_path": "inputs/pipeline_config.json", + "required": true, + "produced_by": "baseline_train", + "status": "available", + "size_bytes": 1210, + "sha256": "6bc16344d7a0691fd2aec05ae29a19fcd1507c81f7eaf5e50535c27a07775fcb", + "unavailable_reason": null + }, + { + "artifact_id": "input.optimizer_config", + "artifact_type": "input", + "relative_path": "inputs/optimizer_config.json", + "required": true, + "produced_by": "candidate_generation", + "status": "available", + "size_bytes": 772, + "sha256": "5f936b9abd6fbd0cfb8e72c5ee8783c3001791434f5881a2196ac3ce65145c49", + "unavailable_reason": null + }, + { + "artifact_id": "input.train_evalset", + "artifact_type": "input", + "relative_path": "inputs/train_evalset.json", + "required": true, + "produced_by": "baseline_train", + "status": "available", + "size_bytes": 1672, + "sha256": "d2849889340afd9b624583d75caccc96c1657281c90a8a3373b60563e332b8a9", + "unavailable_reason": null + }, + { + "artifact_id": "input.validation_evalset", + "artifact_type": "input", + "relative_path": "inputs/validation_evalset.json", + "required": true, + "produced_by": "baseline_validation", + "status": "available", + "size_bytes": 1784, + "sha256": "719cf1c6c95dd2d1e164e3383bc0c28ba9405fc438b6b4f2d34195db51028341", + "unavailable_reason": null + } + ] +} diff --git a/examples/optimization/eval_optimize_loop/sample_output/evaluations/baseline_train.json b/examples/optimization/eval_optimize_loop/sample_output/evaluations/baseline_train.json new file mode 100644 index 00000000..6a031dbd --- /dev/null +++ b/examples/optimization/eval_optimize_loop/sample_output/evaluations/baseline_train.json @@ -0,0 +1,500 @@ +{ + "phase": "baseline", + "split": "train", + "eval_set_id": "eval_optimize_loop_train", + "failed_summary": { + "agentName": "call-agent", + "evalSetId": "eval_optimize_loop_train", + "overallStatus": "failed", + "runs": 1, + "evalCases": [ + { + "evalCaseId": "train_output_format", + "overallStatus": "passed", + "metricResults": [ + { + "metricName": "final_response_avg_score", + "score": 1.0, + "threshold": 1.0, + "evalStatus": "passed" + } + ] + }, + { + "evalCaseId": "train_tool_arguments", + "overallStatus": "failed", + "metricResults": [ + { + "metricName": "final_response_avg_score", + "score": 0.0, + "threshold": 1.0, + "evalStatus": "failed" + } + ] + }, + { + "evalCaseId": "train_tool_choice", + "overallStatus": "failed", + "metricResults": [ + { + "metricName": "final_response_avg_score", + "score": 0.0, + "threshold": 1.0, + "evalStatus": "failed" + } + ] + } + ] + }, + "details_lines": [], + "result_lines": [ + "Eval Set : eval_optimize_loop_train", + "Overall Status: failed", + "Case train_output_format -> passed", + " Metric final_response_avg_score: score 1.0 (threshold 1.0) => passed", + "", + "Case train_tool_arguments -> failed", + " Metric final_response_avg_score: score 0.0 (threshold 1.0) => failed", + "", + "Case train_tool_choice -> failed", + " Metric final_response_avg_score: score 0.0 (threshold 1.0) => failed", + "" + ], + "eval_results_by_eval_id": { + "train_tool_choice": [ + { + "eval_set_id": "eval_optimize_loop_train", + "eval_id": "train_tool_choice", + "run_id": 1, + "final_eval_status": 2, + "error_message": null, + "overall_eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 0.0, + "eval_status": 2, + "details": null + } + ], + "eval_metric_result_per_invocation": [ + { + "actual_invocation": { + "invocation_id": "train-2", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "Check the status of order A100.", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"general_support\",\"message\":\"Please provide more details so I can route your request.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": null + }, + "intermediate_data": null, + "creation_timestamp": 1784619414.0021462 + }, + "expected_invocation": { + "invocation_id": "train-2", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "Check the status of order A100.", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"order_lookup\",\"message\":\"Checking order A100.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "model" + }, + "intermediate_data": null, + "creation_timestamp": 0.0 + }, + "eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 0.0, + "eval_status": 2, + "details": null + } + ] + } + ], + "session_id": "___remote_eval___session___fe46fce6-d440-432c-ac3d-3a1a03db01f1", + "user_id": null, + "session_details": null + } + ], + "train_tool_arguments": [ + { + "eval_set_id": "eval_optimize_loop_train", + "eval_id": "train_tool_arguments", + "run_id": 1, + "final_eval_status": 2, + "error_message": null, + "overall_eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 0.0, + "eval_status": 2, + "details": null + } + ], + "eval_metric_result_per_invocation": [ + { + "actual_invocation": { + "invocation_id": "train-3", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "Look up order B-204 for customer 17.", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"general_support\",\"message\":\"Please provide more details so I can route your request.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": null + }, + "intermediate_data": null, + "creation_timestamp": 1784619413.9986317 + }, + "expected_invocation": { + "invocation_id": "train-3", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "Look up order B-204 for customer 17.", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"order_lookup\",\"message\":\"Checking order B-204 for customer 17.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "model" + }, + "intermediate_data": null, + "creation_timestamp": 0.0 + }, + "eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 0.0, + "eval_status": 2, + "details": null + } + ] + } + ], + "session_id": "___remote_eval___session___a6cd661a-f2ca-4f9b-8c5f-e9d875e61f41", + "user_id": null, + "session_details": null + } + ], + "train_output_format": [ + { + "eval_set_id": "eval_optimize_loop_train", + "eval_id": "train_output_format", + "run_id": 1, + "final_eval_status": 1, + "error_message": null, + "overall_eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 1.0, + "eval_status": 1, + "details": null + } + ], + "eval_metric_result_per_invocation": [ + { + "actual_invocation": { + "invocation_id": "train-1", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "How can I update my email address?", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"account\",\"message\":\"Open profile settings to update your email.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": null + }, + "intermediate_data": null, + "creation_timestamp": 1784619414.0005472 + }, + "expected_invocation": { + "invocation_id": "train-1", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "How can I update my email address?", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"account\",\"message\":\"Open profile settings to update your email.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "model" + }, + "intermediate_data": null, + "creation_timestamp": 0.0 + }, + "eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 1.0, + "eval_status": 1, + "details": null + } + ] + } + ], + "session_id": "___remote_eval___session___168d89d2-fc32-4a89-ae7d-49993d76be8f", + "user_id": null, + "session_details": null + } + ] + }, + "passed_case_count": 1, + "total_case_count": 3, + "average_score": 0.3333333333333333 +} diff --git a/examples/optimization/eval_optimize_loop/sample_output/evaluations/baseline_validation.json b/examples/optimization/eval_optimize_loop/sample_output/evaluations/baseline_validation.json new file mode 100644 index 00000000..e6307421 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/sample_output/evaluations/baseline_validation.json @@ -0,0 +1,500 @@ +{ + "phase": "baseline", + "split": "validation", + "eval_set_id": "eval_optimize_loop_validation", + "failed_summary": { + "agentName": "call-agent", + "evalSetId": "eval_optimize_loop_validation", + "overallStatus": "failed", + "runs": 1, + "evalCases": [ + { + "evalCaseId": "val_knowledge_recall", + "overallStatus": "failed", + "metricResults": [ + { + "metricName": "final_response_avg_score", + "score": 0.0, + "threshold": 1.0, + "evalStatus": "failed" + } + ] + }, + { + "evalCaseId": "val_paraphrase", + "overallStatus": "failed", + "metricResults": [ + { + "metricName": "final_response_avg_score", + "score": 0.0, + "threshold": 1.0, + "evalStatus": "failed" + } + ] + }, + { + "evalCaseId": "val_refund_route", + "overallStatus": "passed", + "metricResults": [ + { + "metricName": "final_response_avg_score", + "score": 1.0, + "threshold": 1.0, + "evalStatus": "passed" + } + ] + } + ] + }, + "details_lines": [], + "result_lines": [ + "Eval Set : eval_optimize_loop_validation", + "Overall Status: failed", + "Case val_knowledge_recall -> failed", + " Metric final_response_avg_score: score 0.0 (threshold 1.0) => failed", + "", + "Case val_paraphrase -> failed", + " Metric final_response_avg_score: score 0.0 (threshold 1.0) => failed", + "", + "Case val_refund_route -> passed", + " Metric final_response_avg_score: score 1.0 (threshold 1.0) => passed", + "" + ], + "eval_results_by_eval_id": { + "val_knowledge_recall": [ + { + "eval_set_id": "eval_optimize_loop_validation", + "eval_id": "val_knowledge_recall", + "run_id": 1, + "final_eval_status": 2, + "error_message": null, + "overall_eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 0.0, + "eval_status": 2, + "details": null + } + ], + "eval_metric_result_per_invocation": [ + { + "actual_invocation": { + "invocation_id": "val-2", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "How long does standard shipping usually take?", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"general_support\",\"message\":\"Please provide more details so I can route your request.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": null + }, + "intermediate_data": null, + "creation_timestamp": 1784619414.0093913 + }, + "expected_invocation": { + "invocation_id": "val-2", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "How long does standard shipping usually take?", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"shipping_policy\",\"message\":\"Standard shipping normally takes 3-5 business days.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "model" + }, + "intermediate_data": null, + "creation_timestamp": 0.0 + }, + "eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 0.0, + "eval_status": 2, + "details": null + } + ] + } + ], + "session_id": "___remote_eval___session___45c563f4-d805-4a5c-82c4-77f2237e4d20", + "user_id": null, + "session_details": null + } + ], + "val_refund_route": [ + { + "eval_set_id": "eval_optimize_loop_validation", + "eval_id": "val_refund_route", + "run_id": 1, + "final_eval_status": 1, + "error_message": null, + "overall_eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 1.0, + "eval_status": 1, + "details": null + } + ], + "eval_metric_result_per_invocation": [ + { + "actual_invocation": { + "invocation_id": "val-3", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "I was charged twice and need the duplicate payment refunded.", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"billing_refund\",\"message\":\"I will route this duplicate charge for refund review.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": null + }, + "intermediate_data": null, + "creation_timestamp": 1784619414.0046532 + }, + "expected_invocation": { + "invocation_id": "val-3", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "I was charged twice and need the duplicate payment refunded.", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"billing_refund\",\"message\":\"I will route this duplicate charge for refund review.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "model" + }, + "intermediate_data": null, + "creation_timestamp": 0.0 + }, + "eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 1.0, + "eval_status": 1, + "details": null + } + ] + } + ], + "session_id": "___remote_eval___session___5a98aee1-19fd-4665-9259-1b1fe3aaf23f", + "user_id": null, + "session_details": null + } + ], + "val_paraphrase": [ + { + "eval_set_id": "eval_optimize_loop_validation", + "eval_id": "val_paraphrase", + "run_id": 1, + "final_eval_status": 2, + "error_message": null, + "overall_eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 0.0, + "eval_status": 2, + "details": null + } + ], + "eval_metric_result_per_invocation": [ + { + "actual_invocation": { + "invocation_id": "val-1", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "Where do I change the address tied to my account?", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"general_support\",\"message\":\"Please provide more details so I can route your request.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": null + }, + "intermediate_data": null, + "creation_timestamp": 1784619414.006062 + }, + "expected_invocation": { + "invocation_id": "val-1", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "Where do I change the address tied to my account?", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"account\",\"message\":\"Open profile settings to update your address.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "model" + }, + "intermediate_data": null, + "creation_timestamp": 0.0 + }, + "eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 0.0, + "eval_status": 2, + "details": null + } + ] + } + ], + "session_id": "___remote_eval___session___3ac5ca32-1910-4b66-b165-9134276d069c", + "user_id": null, + "session_details": null + } + ] + }, + "passed_case_count": 1, + "total_case_count": 3, + "average_score": 0.3333333333333333 +} diff --git a/examples/optimization/eval_optimize_loop/sample_output/evaluations/candidate_train.json b/examples/optimization/eval_optimize_loop/sample_output/evaluations/candidate_train.json new file mode 100644 index 00000000..3a5d2561 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/sample_output/evaluations/candidate_train.json @@ -0,0 +1,457 @@ +{ + "phase": "candidate", + "split": "train", + "eval_set_id": "eval_optimize_loop_train", + "failed_summary": null, + "details_lines": [], + "result_lines": [ + "Eval Set : eval_optimize_loop_train", + "Overall Status: passed", + "Case train_output_format -> passed", + " Metric final_response_avg_score: score 1.0 (threshold 1.0) => passed", + "", + "Case train_tool_arguments -> passed", + " Metric final_response_avg_score: score 1.0 (threshold 1.0) => passed", + "", + "Case train_tool_choice -> passed", + " Metric final_response_avg_score: score 1.0 (threshold 1.0) => passed", + "" + ], + "eval_results_by_eval_id": { + "train_tool_choice": [ + { + "eval_set_id": "eval_optimize_loop_train", + "eval_id": "train_tool_choice", + "run_id": 1, + "final_eval_status": 1, + "error_message": null, + "overall_eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 1.0, + "eval_status": 1, + "details": null + } + ], + "eval_metric_result_per_invocation": [ + { + "actual_invocation": { + "invocation_id": "train-2", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "Check the status of order A100.", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"order_lookup\",\"message\":\"Checking order A100.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": null + }, + "intermediate_data": null, + "creation_timestamp": 1784619414.017015 + }, + "expected_invocation": { + "invocation_id": "train-2", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "Check the status of order A100.", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"order_lookup\",\"message\":\"Checking order A100.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "model" + }, + "intermediate_data": null, + "creation_timestamp": 0.0 + }, + "eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 1.0, + "eval_status": 1, + "details": null + } + ] + } + ], + "session_id": "___remote_eval___session___640410c0-cbbc-4cad-aa4a-d4b169b4fac7", + "user_id": null, + "session_details": null + } + ], + "train_tool_arguments": [ + { + "eval_set_id": "eval_optimize_loop_train", + "eval_id": "train_tool_arguments", + "run_id": 1, + "final_eval_status": 1, + "error_message": null, + "overall_eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 1.0, + "eval_status": 1, + "details": null + } + ], + "eval_metric_result_per_invocation": [ + { + "actual_invocation": { + "invocation_id": "train-3", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "Look up order B-204 for customer 17.", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"order_lookup\",\"message\":\"Checking order B-204 for customer 17.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": null + }, + "intermediate_data": null, + "creation_timestamp": 1784619414.012829 + }, + "expected_invocation": { + "invocation_id": "train-3", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "Look up order B-204 for customer 17.", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"order_lookup\",\"message\":\"Checking order B-204 for customer 17.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "model" + }, + "intermediate_data": null, + "creation_timestamp": 0.0 + }, + "eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 1.0, + "eval_status": 1, + "details": null + } + ] + } + ], + "session_id": "___remote_eval___session___785bd08e-706e-4f83-9c44-ac03997c67c3", + "user_id": null, + "session_details": null + } + ], + "train_output_format": [ + { + "eval_set_id": "eval_optimize_loop_train", + "eval_id": "train_output_format", + "run_id": 1, + "final_eval_status": 1, + "error_message": null, + "overall_eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 1.0, + "eval_status": 1, + "details": null + } + ], + "eval_metric_result_per_invocation": [ + { + "actual_invocation": { + "invocation_id": "train-1", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "How can I update my email address?", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"account\",\"message\":\"Open profile settings to update your email.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": null + }, + "intermediate_data": null, + "creation_timestamp": 1784619414.0143127 + }, + "expected_invocation": { + "invocation_id": "train-1", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "How can I update my email address?", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"account\",\"message\":\"Open profile settings to update your email.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "model" + }, + "intermediate_data": null, + "creation_timestamp": 0.0 + }, + "eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 1.0, + "eval_status": 1, + "details": null + } + ] + } + ], + "session_id": "___remote_eval___session___4c9ebf1d-9e6f-4488-a64c-ca2a0c01f40a", + "user_id": null, + "session_details": null + } + ] + }, + "passed_case_count": 3, + "total_case_count": 3, + "average_score": 1.0 +} diff --git a/examples/optimization/eval_optimize_loop/sample_output/evaluations/candidate_validation.json b/examples/optimization/eval_optimize_loop/sample_output/evaluations/candidate_validation.json new file mode 100644 index 00000000..0a379a3f --- /dev/null +++ b/examples/optimization/eval_optimize_loop/sample_output/evaluations/candidate_validation.json @@ -0,0 +1,457 @@ +{ + "phase": "candidate", + "split": "validation", + "eval_set_id": "eval_optimize_loop_validation", + "failed_summary": null, + "details_lines": [], + "result_lines": [ + "Eval Set : eval_optimize_loop_validation", + "Overall Status: passed", + "Case val_knowledge_recall -> passed", + " Metric final_response_avg_score: score 1.0 (threshold 1.0) => passed", + "", + "Case val_paraphrase -> passed", + " Metric final_response_avg_score: score 1.0 (threshold 1.0) => passed", + "", + "Case val_refund_route -> passed", + " Metric final_response_avg_score: score 1.0 (threshold 1.0) => passed", + "" + ], + "eval_results_by_eval_id": { + "val_knowledge_recall": [ + { + "eval_set_id": "eval_optimize_loop_validation", + "eval_id": "val_knowledge_recall", + "run_id": 1, + "final_eval_status": 1, + "error_message": null, + "overall_eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 1.0, + "eval_status": 1, + "details": null + } + ], + "eval_metric_result_per_invocation": [ + { + "actual_invocation": { + "invocation_id": "val-2", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "How long does standard shipping usually take?", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"shipping_policy\",\"message\":\"Standard shipping normally takes 3-5 business days.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": null + }, + "intermediate_data": null, + "creation_timestamp": 1784619414.024159 + }, + "expected_invocation": { + "invocation_id": "val-2", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "How long does standard shipping usually take?", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"shipping_policy\",\"message\":\"Standard shipping normally takes 3-5 business days.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "model" + }, + "intermediate_data": null, + "creation_timestamp": 0.0 + }, + "eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 1.0, + "eval_status": 1, + "details": null + } + ] + } + ], + "session_id": "___remote_eval___session___426070b6-19e9-4441-941e-321dcbb6b443", + "user_id": null, + "session_details": null + } + ], + "val_refund_route": [ + { + "eval_set_id": "eval_optimize_loop_validation", + "eval_id": "val_refund_route", + "run_id": 1, + "final_eval_status": 1, + "error_message": null, + "overall_eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 1.0, + "eval_status": 1, + "details": null + } + ], + "eval_metric_result_per_invocation": [ + { + "actual_invocation": { + "invocation_id": "val-3", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "I was charged twice and need the duplicate payment refunded.", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"billing_refund\",\"message\":\"I will route this duplicate charge for refund review.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": null + }, + "intermediate_data": null, + "creation_timestamp": 1784619414.0204713 + }, + "expected_invocation": { + "invocation_id": "val-3", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "I was charged twice and need the duplicate payment refunded.", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"billing_refund\",\"message\":\"I will route this duplicate charge for refund review.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "model" + }, + "intermediate_data": null, + "creation_timestamp": 0.0 + }, + "eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 1.0, + "eval_status": 1, + "details": null + } + ] + } + ], + "session_id": "___remote_eval___session___3439f8ac-062e-4b85-b8e3-b13ef260efe3", + "user_id": null, + "session_details": null + } + ], + "val_paraphrase": [ + { + "eval_set_id": "eval_optimize_loop_validation", + "eval_id": "val_paraphrase", + "run_id": 1, + "final_eval_status": 1, + "error_message": null, + "overall_eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 1.0, + "eval_status": 1, + "details": null + } + ], + "eval_metric_result_per_invocation": [ + { + "actual_invocation": { + "invocation_id": "val-1", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "Where do I change the address tied to my account?", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"account\",\"message\":\"Open profile settings to update your address.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": null + }, + "intermediate_data": null, + "creation_timestamp": 1784619414.0225098 + }, + "expected_invocation": { + "invocation_id": "val-1", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "Where do I change the address tied to my account?", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"account\",\"message\":\"Open profile settings to update your address.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "model" + }, + "intermediate_data": null, + "creation_timestamp": 0.0 + }, + "eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 1.0, + "eval_status": 1, + "details": null + } + ] + } + ], + "session_id": "___remote_eval___session___bad8b5f6-d2d4-4512-8536-786f53cd2f5a", + "user_id": null, + "session_details": null + } + ] + }, + "passed_case_count": 3, + "total_case_count": 3, + "average_score": 1.0 +} diff --git a/examples/optimization/eval_optimize_loop/sample_output/inputs/optimizer_config.json b/examples/optimization/eval_optimize_loop/sample_output/inputs/optimizer_config.json new file mode 100644 index 00000000..9efff8be --- /dev/null +++ b/examples/optimization/eval_optimize_loop/sample_output/inputs/optimizer_config.json @@ -0,0 +1,36 @@ +{ + "evaluate": { + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + } + } + ], + "num_runs": 1 + }, + "optimize": { + "eval_case_parallelism": 1, + "stop": { + "required_metrics": "all" + }, + "algorithm": { + "name": "gepa_reflective", + "seed": 42, + "reflection_lm": { + "model_name": "fake-not-used-in-offline-mode", + "api_key": "fake-not-used-in-offline-mode" + }, + "reflection_minibatch_size": 3, + "skip_perfect_score": false, + "max_candidate_proposals": 3 + } + } +} diff --git a/examples/optimization/eval_optimize_loop/sample_output/inputs/pipeline_config.json b/examples/optimization/eval_optimize_loop/sample_output/inputs/pipeline_config.json new file mode 100644 index 00000000..1d88ff2a --- /dev/null +++ b/examples/optimization/eval_optimize_loop/sample_output/inputs/pipeline_config.json @@ -0,0 +1,52 @@ +{ + "config_version": 1, + "execution": { + "mode": "offline", + "candidate_scenario": "improve" + }, + "inputs": { + "train_evalset": "data/train.evalset.json", + "validation_evalset": "data/val.evalset.json", + "optimizer_config": "optimizer.json" + }, + "prompts": [ + { + "name": "system_prompt", + "path": "prompts/system.md" + } + ], + "run": { + "runs_dir": "runs", + "seed": 42 + }, + "case_labels": { + "hard_case_ids": ["val_knowledge_recall"], + "critical_case_ids": ["val_refund_route"] + }, + "gate": { + "min_validation_score_delta": 0.05, + "reject_on_validation_pass_rate_drop": true, + "reject_new_hard_fail": true, + "reject_critical_regression": true, + "severe_case_score_drop": 0.2, + "required_metrics": ["final_response_avg_score"] + }, + "budget": { + "max_duration_seconds": 180, + "on_unavailable": "warning" + }, + "reporting": { + "write_json": true, + "write_markdown": true, + "include_case_evidence": true + }, + "artifacts": { + "copy_input_files": true, + "retain_optimizer_native_artifacts": true, + "audit_all_candidates": false + }, + "writeback": { + "enabled": false, + "require_source_hash_match": true + } +} diff --git a/examples/optimization/eval_optimize_loop/sample_output/inputs/train_evalset.json b/examples/optimization/eval_optimize_loop/sample_output/inputs/train_evalset.json new file mode 100644 index 00000000..6b41ca3a --- /dev/null +++ b/examples/optimization/eval_optimize_loop/sample_output/inputs/train_evalset.json @@ -0,0 +1,40 @@ +{ + "eval_set_id": "eval_optimize_loop_train", + "name": "Evaluation optimization loop - train", + "description": "Three deterministic training cases for format, tool choice, and tool arguments.", + "eval_cases": [ + { + "eval_id": "train_output_format", + "conversation": [ + { + "invocation_id": "train-1", + "user_content": {"parts": [{"text": "How can I update my email address?"}], "role": "user"}, + "final_response": {"parts": [{"text": "{\"route\":\"account\",\"message\":\"Open profile settings to update your email.\"}"}], "role": "model"} + } + ], + "session_input": {"app_name": "eval_optimize_loop", "user_id": "demo", "state": {}} + }, + { + "eval_id": "train_tool_choice", + "conversation": [ + { + "invocation_id": "train-2", + "user_content": {"parts": [{"text": "Check the status of order A100."}], "role": "user"}, + "final_response": {"parts": [{"text": "{\"route\":\"order_lookup\",\"message\":\"Checking order A100.\"}"}], "role": "model"} + } + ], + "session_input": {"app_name": "eval_optimize_loop", "user_id": "demo", "state": {}} + }, + { + "eval_id": "train_tool_arguments", + "conversation": [ + { + "invocation_id": "train-3", + "user_content": {"parts": [{"text": "Look up order B-204 for customer 17."}], "role": "user"}, + "final_response": {"parts": [{"text": "{\"route\":\"order_lookup\",\"message\":\"Checking order B-204 for customer 17.\"}"}], "role": "model"} + } + ], + "session_input": {"app_name": "eval_optimize_loop", "user_id": "demo", "state": {}} + } + ] +} diff --git a/examples/optimization/eval_optimize_loop/sample_output/inputs/validation_evalset.json b/examples/optimization/eval_optimize_loop/sample_output/inputs/validation_evalset.json new file mode 100644 index 00000000..387e6de7 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/sample_output/inputs/validation_evalset.json @@ -0,0 +1,40 @@ +{ + "eval_set_id": "eval_optimize_loop_validation", + "name": "Evaluation optimization loop - validation", + "description": "Three deterministic validation cases for generalization, recall, and critical routing.", + "eval_cases": [ + { + "eval_id": "val_paraphrase", + "conversation": [ + { + "invocation_id": "val-1", + "user_content": {"parts": [{"text": "Where do I change the address tied to my account?"}], "role": "user"}, + "final_response": {"parts": [{"text": "{\"route\":\"account\",\"message\":\"Open profile settings to update your address.\"}"}], "role": "model"} + } + ], + "session_input": {"app_name": "eval_optimize_loop", "user_id": "demo", "state": {}} + }, + { + "eval_id": "val_knowledge_recall", + "conversation": [ + { + "invocation_id": "val-2", + "user_content": {"parts": [{"text": "How long does standard shipping usually take?"}], "role": "user"}, + "final_response": {"parts": [{"text": "{\"route\":\"shipping_policy\",\"message\":\"Standard shipping normally takes 3-5 business days.\"}"}], "role": "model"} + } + ], + "session_input": {"app_name": "eval_optimize_loop", "user_id": "demo", "state": {}} + }, + { + "eval_id": "val_refund_route", + "conversation": [ + { + "invocation_id": "val-3", + "user_content": {"parts": [{"text": "I was charged twice and need the duplicate payment refunded."}], "role": "user"}, + "final_response": {"parts": [{"text": "{\"route\":\"billing_refund\",\"message\":\"I will route this duplicate charge for refund review.\"}"}], "role": "model"} + } + ], + "session_input": {"app_name": "eval_optimize_loop", "user_id": "demo", "state": {}} + } + ] +} diff --git a/examples/optimization/eval_optimize_loop/sample_output/optimization_report.json b/examples/optimization/eval_optimize_loop/sample_output/optimization_report.json new file mode 100644 index 00000000..df7830ef --- /dev/null +++ b/examples/optimization/eval_optimize_loop/sample_output/optimization_report.json @@ -0,0 +1,3666 @@ +{ + "schema_version": 1, + "status": "completed", + "run_id": "stage6_sample", + "execution_mode": "offline", + "seed": 42, + "started_at": "2026-07-21T07:36:53.995239Z", + "finished_at": "2026-07-21T07:36:54.029902Z", + "input_snapshot": { + "pipeline_config_path": "/pipeline.json", + "pipeline_config_sha256": "6bc16344d7a0691fd2aec05ae29a19fcd1507c81f7eaf5e50535c27a07775fcb", + "optimizer_config_path": "/optimizer.json", + "optimizer_config_sha256": "5f936b9abd6fbd0cfb8e72c5ee8783c3001791434f5881a2196ac3ce65145c49", + "train_evalset_path": "/data/train.evalset.json", + "train_evalset_sha256": "d2849889340afd9b624583d75caccc96c1657281c90a8a3373b60563e332b8a9", + "validation_evalset_path": "/data/val.evalset.json", + "validation_evalset_sha256": "719cf1c6c95dd2d1e164e3383bc0c28ba9405fc438b6b4f2d34195db51028341", + "prompt_snapshots": [ + { + "field_name": "system_prompt", + "source_path": "/prompts/system.md", + "working_path": "/runs/stage6_sample/workspace/prompts/01_system_prompt.md", + "content": "You are a customer-support routing assistant.\n\nAnswer with a compact JSON object containing `route` and `message`. Use the\navailable account tool for account-specific requests. Never invent account\nfacts.\n", + "sha256": "ded1ce1a1b01fa5f42e8a98ff81c8f816452f77b1ec510ce4b951769420e9871" + } + ], + "seed": 42, + "trace_inputs": null + }, + "candidate": { + "provider": "fake", + "prompts": { + "system_prompt": "You are a customer-support routing assistant.\n\nAnswer with a compact JSON object containing `route` and `message`. Use the\navailable account tool for account-specific requests. Never invent account\nfacts.\n\n\nApply general customer-support routing rules across equivalent user phrasings.\n\n\n\n\n\n" + }, + "changed_fields": [ + "system_prompt" + ], + "rationale": "Generalize routing across account synonyms, order lookup, shipping policy, and refunds.", + "parent_prompt_sha256": "85420c1a6ffbfc6681d7d4439154e798c02a173b5e336896c4d6046b98c83f3a", + "candidate_prompt_sha256": "60cc05b773e8ba70320c6789ccadcdc0b28da43ed1904743580791601cd6d901", + "candidate_id": "fake-improve-60cc05b773e8", + "scenario": "improve", + "seed": 42 + }, + "baseline_train": { + "phase": "baseline", + "split": "train", + "eval_set_id": "eval_optimize_loop_train", + "failed_summary": { + "agentName": "call-agent", + "evalSetId": "eval_optimize_loop_train", + "overallStatus": "failed", + "runs": 1, + "evalCases": [ + { + "evalCaseId": "train_output_format", + "overallStatus": "passed", + "metricResults": [ + { + "metricName": "final_response_avg_score", + "score": 1.0, + "threshold": 1.0, + "evalStatus": "passed" + } + ] + }, + { + "evalCaseId": "train_tool_arguments", + "overallStatus": "failed", + "metricResults": [ + { + "metricName": "final_response_avg_score", + "score": 0.0, + "threshold": 1.0, + "evalStatus": "failed" + } + ] + }, + { + "evalCaseId": "train_tool_choice", + "overallStatus": "failed", + "metricResults": [ + { + "metricName": "final_response_avg_score", + "score": 0.0, + "threshold": 1.0, + "evalStatus": "failed" + } + ] + } + ] + }, + "details_lines": [], + "result_lines": [ + "Eval Set : eval_optimize_loop_train", + "Overall Status: failed", + "Case train_output_format -> passed", + " Metric final_response_avg_score: score 1.0 (threshold 1.0) => passed", + "", + "Case train_tool_arguments -> failed", + " Metric final_response_avg_score: score 0.0 (threshold 1.0) => failed", + "", + "Case train_tool_choice -> failed", + " Metric final_response_avg_score: score 0.0 (threshold 1.0) => failed", + "" + ], + "eval_results_by_eval_id": { + "train_tool_choice": [ + { + "eval_set_id": "eval_optimize_loop_train", + "eval_id": "train_tool_choice", + "run_id": 1, + "final_eval_status": 2, + "error_message": null, + "overall_eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 0.0, + "eval_status": 2, + "details": null + } + ], + "eval_metric_result_per_invocation": [ + { + "actual_invocation": { + "invocation_id": "train-2", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "Check the status of order A100.", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"general_support\",\"message\":\"Please provide more details so I can route your request.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": null + }, + "intermediate_data": null, + "creation_timestamp": 1784619414.0021462 + }, + "expected_invocation": { + "invocation_id": "train-2", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "Check the status of order A100.", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"order_lookup\",\"message\":\"Checking order A100.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "model" + }, + "intermediate_data": null, + "creation_timestamp": 0.0 + }, + "eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 0.0, + "eval_status": 2, + "details": null + } + ] + } + ], + "session_id": "___remote_eval___session___fe46fce6-d440-432c-ac3d-3a1a03db01f1", + "user_id": null, + "session_details": null + } + ], + "train_tool_arguments": [ + { + "eval_set_id": "eval_optimize_loop_train", + "eval_id": "train_tool_arguments", + "run_id": 1, + "final_eval_status": 2, + "error_message": null, + "overall_eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 0.0, + "eval_status": 2, + "details": null + } + ], + "eval_metric_result_per_invocation": [ + { + "actual_invocation": { + "invocation_id": "train-3", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "Look up order B-204 for customer 17.", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"general_support\",\"message\":\"Please provide more details so I can route your request.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": null + }, + "intermediate_data": null, + "creation_timestamp": 1784619413.9986317 + }, + "expected_invocation": { + "invocation_id": "train-3", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "Look up order B-204 for customer 17.", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"order_lookup\",\"message\":\"Checking order B-204 for customer 17.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "model" + }, + "intermediate_data": null, + "creation_timestamp": 0.0 + }, + "eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 0.0, + "eval_status": 2, + "details": null + } + ] + } + ], + "session_id": "___remote_eval___session___a6cd661a-f2ca-4f9b-8c5f-e9d875e61f41", + "user_id": null, + "session_details": null + } + ], + "train_output_format": [ + { + "eval_set_id": "eval_optimize_loop_train", + "eval_id": "train_output_format", + "run_id": 1, + "final_eval_status": 1, + "error_message": null, + "overall_eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 1.0, + "eval_status": 1, + "details": null + } + ], + "eval_metric_result_per_invocation": [ + { + "actual_invocation": { + "invocation_id": "train-1", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "How can I update my email address?", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"account\",\"message\":\"Open profile settings to update your email.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": null + }, + "intermediate_data": null, + "creation_timestamp": 1784619414.0005472 + }, + "expected_invocation": { + "invocation_id": "train-1", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "How can I update my email address?", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"account\",\"message\":\"Open profile settings to update your email.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "model" + }, + "intermediate_data": null, + "creation_timestamp": 0.0 + }, + "eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 1.0, + "eval_status": 1, + "details": null + } + ] + } + ], + "session_id": "___remote_eval___session___168d89d2-fc32-4a89-ae7d-49993d76be8f", + "user_id": null, + "session_details": null + } + ] + }, + "passed_case_count": 1, + "total_case_count": 3, + "average_score": 0.3333333333333333 + }, + "baseline_validation": { + "phase": "baseline", + "split": "validation", + "eval_set_id": "eval_optimize_loop_validation", + "failed_summary": { + "agentName": "call-agent", + "evalSetId": "eval_optimize_loop_validation", + "overallStatus": "failed", + "runs": 1, + "evalCases": [ + { + "evalCaseId": "val_knowledge_recall", + "overallStatus": "failed", + "metricResults": [ + { + "metricName": "final_response_avg_score", + "score": 0.0, + "threshold": 1.0, + "evalStatus": "failed" + } + ] + }, + { + "evalCaseId": "val_paraphrase", + "overallStatus": "failed", + "metricResults": [ + { + "metricName": "final_response_avg_score", + "score": 0.0, + "threshold": 1.0, + "evalStatus": "failed" + } + ] + }, + { + "evalCaseId": "val_refund_route", + "overallStatus": "passed", + "metricResults": [ + { + "metricName": "final_response_avg_score", + "score": 1.0, + "threshold": 1.0, + "evalStatus": "passed" + } + ] + } + ] + }, + "details_lines": [], + "result_lines": [ + "Eval Set : eval_optimize_loop_validation", + "Overall Status: failed", + "Case val_knowledge_recall -> failed", + " Metric final_response_avg_score: score 0.0 (threshold 1.0) => failed", + "", + "Case val_paraphrase -> failed", + " Metric final_response_avg_score: score 0.0 (threshold 1.0) => failed", + "", + "Case val_refund_route -> passed", + " Metric final_response_avg_score: score 1.0 (threshold 1.0) => passed", + "" + ], + "eval_results_by_eval_id": { + "val_knowledge_recall": [ + { + "eval_set_id": "eval_optimize_loop_validation", + "eval_id": "val_knowledge_recall", + "run_id": 1, + "final_eval_status": 2, + "error_message": null, + "overall_eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 0.0, + "eval_status": 2, + "details": null + } + ], + "eval_metric_result_per_invocation": [ + { + "actual_invocation": { + "invocation_id": "val-2", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "How long does standard shipping usually take?", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"general_support\",\"message\":\"Please provide more details so I can route your request.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": null + }, + "intermediate_data": null, + "creation_timestamp": 1784619414.0093913 + }, + "expected_invocation": { + "invocation_id": "val-2", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "How long does standard shipping usually take?", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"shipping_policy\",\"message\":\"Standard shipping normally takes 3-5 business days.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "model" + }, + "intermediate_data": null, + "creation_timestamp": 0.0 + }, + "eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 0.0, + "eval_status": 2, + "details": null + } + ] + } + ], + "session_id": "___remote_eval___session___45c563f4-d805-4a5c-82c4-77f2237e4d20", + "user_id": null, + "session_details": null + } + ], + "val_refund_route": [ + { + "eval_set_id": "eval_optimize_loop_validation", + "eval_id": "val_refund_route", + "run_id": 1, + "final_eval_status": 1, + "error_message": null, + "overall_eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 1.0, + "eval_status": 1, + "details": null + } + ], + "eval_metric_result_per_invocation": [ + { + "actual_invocation": { + "invocation_id": "val-3", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "I was charged twice and need the duplicate payment refunded.", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"billing_refund\",\"message\":\"I will route this duplicate charge for refund review.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": null + }, + "intermediate_data": null, + "creation_timestamp": 1784619414.0046532 + }, + "expected_invocation": { + "invocation_id": "val-3", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "I was charged twice and need the duplicate payment refunded.", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"billing_refund\",\"message\":\"I will route this duplicate charge for refund review.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "model" + }, + "intermediate_data": null, + "creation_timestamp": 0.0 + }, + "eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 1.0, + "eval_status": 1, + "details": null + } + ] + } + ], + "session_id": "___remote_eval___session___5a98aee1-19fd-4665-9259-1b1fe3aaf23f", + "user_id": null, + "session_details": null + } + ], + "val_paraphrase": [ + { + "eval_set_id": "eval_optimize_loop_validation", + "eval_id": "val_paraphrase", + "run_id": 1, + "final_eval_status": 2, + "error_message": null, + "overall_eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 0.0, + "eval_status": 2, + "details": null + } + ], + "eval_metric_result_per_invocation": [ + { + "actual_invocation": { + "invocation_id": "val-1", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "Where do I change the address tied to my account?", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"general_support\",\"message\":\"Please provide more details so I can route your request.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": null + }, + "intermediate_data": null, + "creation_timestamp": 1784619414.006062 + }, + "expected_invocation": { + "invocation_id": "val-1", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "Where do I change the address tied to my account?", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"account\",\"message\":\"Open profile settings to update your address.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "model" + }, + "intermediate_data": null, + "creation_timestamp": 0.0 + }, + "eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 0.0, + "eval_status": 2, + "details": null + } + ] + } + ], + "session_id": "___remote_eval___session___3ac5ca32-1910-4b66-b165-9134276d069c", + "user_id": null, + "session_details": null + } + ] + }, + "passed_case_count": 1, + "total_case_count": 3, + "average_score": 0.3333333333333333 + }, + "candidate_train": { + "phase": "candidate", + "split": "train", + "eval_set_id": "eval_optimize_loop_train", + "failed_summary": null, + "details_lines": [], + "result_lines": [ + "Eval Set : eval_optimize_loop_train", + "Overall Status: passed", + "Case train_output_format -> passed", + " Metric final_response_avg_score: score 1.0 (threshold 1.0) => passed", + "", + "Case train_tool_arguments -> passed", + " Metric final_response_avg_score: score 1.0 (threshold 1.0) => passed", + "", + "Case train_tool_choice -> passed", + " Metric final_response_avg_score: score 1.0 (threshold 1.0) => passed", + "" + ], + "eval_results_by_eval_id": { + "train_tool_choice": [ + { + "eval_set_id": "eval_optimize_loop_train", + "eval_id": "train_tool_choice", + "run_id": 1, + "final_eval_status": 1, + "error_message": null, + "overall_eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 1.0, + "eval_status": 1, + "details": null + } + ], + "eval_metric_result_per_invocation": [ + { + "actual_invocation": { + "invocation_id": "train-2", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "Check the status of order A100.", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"order_lookup\",\"message\":\"Checking order A100.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": null + }, + "intermediate_data": null, + "creation_timestamp": 1784619414.017015 + }, + "expected_invocation": { + "invocation_id": "train-2", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "Check the status of order A100.", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"order_lookup\",\"message\":\"Checking order A100.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "model" + }, + "intermediate_data": null, + "creation_timestamp": 0.0 + }, + "eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 1.0, + "eval_status": 1, + "details": null + } + ] + } + ], + "session_id": "___remote_eval___session___640410c0-cbbc-4cad-aa4a-d4b169b4fac7", + "user_id": null, + "session_details": null + } + ], + "train_tool_arguments": [ + { + "eval_set_id": "eval_optimize_loop_train", + "eval_id": "train_tool_arguments", + "run_id": 1, + "final_eval_status": 1, + "error_message": null, + "overall_eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 1.0, + "eval_status": 1, + "details": null + } + ], + "eval_metric_result_per_invocation": [ + { + "actual_invocation": { + "invocation_id": "train-3", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "Look up order B-204 for customer 17.", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"order_lookup\",\"message\":\"Checking order B-204 for customer 17.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": null + }, + "intermediate_data": null, + "creation_timestamp": 1784619414.012829 + }, + "expected_invocation": { + "invocation_id": "train-3", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "Look up order B-204 for customer 17.", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"order_lookup\",\"message\":\"Checking order B-204 for customer 17.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "model" + }, + "intermediate_data": null, + "creation_timestamp": 0.0 + }, + "eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 1.0, + "eval_status": 1, + "details": null + } + ] + } + ], + "session_id": "___remote_eval___session___785bd08e-706e-4f83-9c44-ac03997c67c3", + "user_id": null, + "session_details": null + } + ], + "train_output_format": [ + { + "eval_set_id": "eval_optimize_loop_train", + "eval_id": "train_output_format", + "run_id": 1, + "final_eval_status": 1, + "error_message": null, + "overall_eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 1.0, + "eval_status": 1, + "details": null + } + ], + "eval_metric_result_per_invocation": [ + { + "actual_invocation": { + "invocation_id": "train-1", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "How can I update my email address?", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"account\",\"message\":\"Open profile settings to update your email.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": null + }, + "intermediate_data": null, + "creation_timestamp": 1784619414.0143127 + }, + "expected_invocation": { + "invocation_id": "train-1", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "How can I update my email address?", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"account\",\"message\":\"Open profile settings to update your email.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "model" + }, + "intermediate_data": null, + "creation_timestamp": 0.0 + }, + "eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 1.0, + "eval_status": 1, + "details": null + } + ] + } + ], + "session_id": "___remote_eval___session___4c9ebf1d-9e6f-4488-a64c-ca2a0c01f40a", + "user_id": null, + "session_details": null + } + ] + }, + "passed_case_count": 3, + "total_case_count": 3, + "average_score": 1.0 + }, + "candidate_validation": { + "phase": "candidate", + "split": "validation", + "eval_set_id": "eval_optimize_loop_validation", + "failed_summary": null, + "details_lines": [], + "result_lines": [ + "Eval Set : eval_optimize_loop_validation", + "Overall Status: passed", + "Case val_knowledge_recall -> passed", + " Metric final_response_avg_score: score 1.0 (threshold 1.0) => passed", + "", + "Case val_paraphrase -> passed", + " Metric final_response_avg_score: score 1.0 (threshold 1.0) => passed", + "", + "Case val_refund_route -> passed", + " Metric final_response_avg_score: score 1.0 (threshold 1.0) => passed", + "" + ], + "eval_results_by_eval_id": { + "val_knowledge_recall": [ + { + "eval_set_id": "eval_optimize_loop_validation", + "eval_id": "val_knowledge_recall", + "run_id": 1, + "final_eval_status": 1, + "error_message": null, + "overall_eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 1.0, + "eval_status": 1, + "details": null + } + ], + "eval_metric_result_per_invocation": [ + { + "actual_invocation": { + "invocation_id": "val-2", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "How long does standard shipping usually take?", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"shipping_policy\",\"message\":\"Standard shipping normally takes 3-5 business days.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": null + }, + "intermediate_data": null, + "creation_timestamp": 1784619414.024159 + }, + "expected_invocation": { + "invocation_id": "val-2", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "How long does standard shipping usually take?", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"shipping_policy\",\"message\":\"Standard shipping normally takes 3-5 business days.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "model" + }, + "intermediate_data": null, + "creation_timestamp": 0.0 + }, + "eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 1.0, + "eval_status": 1, + "details": null + } + ] + } + ], + "session_id": "___remote_eval___session___426070b6-19e9-4441-941e-321dcbb6b443", + "user_id": null, + "session_details": null + } + ], + "val_refund_route": [ + { + "eval_set_id": "eval_optimize_loop_validation", + "eval_id": "val_refund_route", + "run_id": 1, + "final_eval_status": 1, + "error_message": null, + "overall_eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 1.0, + "eval_status": 1, + "details": null + } + ], + "eval_metric_result_per_invocation": [ + { + "actual_invocation": { + "invocation_id": "val-3", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "I was charged twice and need the duplicate payment refunded.", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"billing_refund\",\"message\":\"I will route this duplicate charge for refund review.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": null + }, + "intermediate_data": null, + "creation_timestamp": 1784619414.0204713 + }, + "expected_invocation": { + "invocation_id": "val-3", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "I was charged twice and need the duplicate payment refunded.", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"billing_refund\",\"message\":\"I will route this duplicate charge for refund review.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "model" + }, + "intermediate_data": null, + "creation_timestamp": 0.0 + }, + "eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 1.0, + "eval_status": 1, + "details": null + } + ] + } + ], + "session_id": "___remote_eval___session___3439f8ac-062e-4b85-b8e3-b13ef260efe3", + "user_id": null, + "session_details": null + } + ], + "val_paraphrase": [ + { + "eval_set_id": "eval_optimize_loop_validation", + "eval_id": "val_paraphrase", + "run_id": 1, + "final_eval_status": 1, + "error_message": null, + "overall_eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 1.0, + "eval_status": 1, + "details": null + } + ], + "eval_metric_result_per_invocation": [ + { + "actual_invocation": { + "invocation_id": "val-1", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "Where do I change the address tied to my account?", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"account\",\"message\":\"Open profile settings to update your address.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": null + }, + "intermediate_data": null, + "creation_timestamp": 1784619414.0225098 + }, + "expected_invocation": { + "invocation_id": "val-1", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "Where do I change the address tied to my account?", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"account\",\"message\":\"Open profile settings to update your address.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "model" + }, + "intermediate_data": null, + "creation_timestamp": 0.0 + }, + "eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 1.0, + "eval_status": 1, + "details": null + } + ] + } + ], + "session_id": "___remote_eval___session___bad8b5f6-d2d4-4512-8536-786f53cd2f5a", + "user_id": null, + "session_details": null + } + ] + }, + "passed_case_count": 3, + "total_case_count": 3, + "average_score": 1.0 + }, + "analysis": { + "baseline_train": { + "phase": "baseline", + "split": "train", + "eval_set_id": "eval_optimize_loop_train", + "cases": [ + { + "eval_id": "train_output_format", + "status": "passed", + "average_score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "status": "passed", + "score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "reason": null + } + ], + "runs": [ + { + "run_id": 1, + "status": "passed", + "error_message": null, + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "status": "passed", + "score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "reason": null + } + ], + "invocations": [ + { + "invocation_id": "train-1", + "user_text": "How can I update my email address?", + "expected_response": "{\"route\":\"account\",\"message\":\"Open profile settings to update your email.\"}", + "actual_response": "{\"route\":\"account\",\"message\":\"Open profile settings to update your email.\"}", + "expected_tools": [], + "actual_tools": [], + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "status": "passed", + "score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "reason": null + } + ] + } + ] + } + ], + "attribution": null + }, + { + "eval_id": "train_tool_arguments", + "status": "failed", + "average_score": { + "status": "available", + "value": 0.0, + "unit": null, + "reason": null + }, + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "status": "failed", + "score": { + "status": "available", + "value": 0.0, + "unit": null, + "reason": null + }, + "reason": null + } + ], + "runs": [ + { + "run_id": 1, + "status": "failed", + "error_message": null, + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "status": "failed", + "score": { + "status": "available", + "value": 0.0, + "unit": null, + "reason": null + }, + "reason": null + } + ], + "invocations": [ + { + "invocation_id": "train-3", + "user_text": "Look up order B-204 for customer 17.", + "expected_response": "{\"route\":\"order_lookup\",\"message\":\"Checking order B-204 for customer 17.\"}", + "actual_response": "{\"route\":\"general_support\",\"message\":\"Please provide more details so I can route your request.\"}", + "expected_tools": [], + "actual_tools": [], + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "status": "failed", + "score": { + "status": "available", + "value": 0.0, + "unit": null, + "reason": null + }, + "reason": null + } + ] + } + ] + } + ], + "attribution": { + "primary_category": "routing_error", + "secondary_categories": [ + "final_response_mismatch" + ], + "summary": "Expected route 'order_lookup', got 'general_support'.", + "evidence": [ + { + "evidence_type": "response", + "message": "Expected route 'order_lookup', got 'general_support'.", + "run_id": 1, + "invocation_id": "train-3", + "metric_name": null, + "expected": "order_lookup", + "actual": "general_support" + }, + { + "evidence_type": "response", + "message": "Final response did not satisfy metric 'final_response_avg_score'.", + "run_id": 1, + "invocation_id": "train-3", + "metric_name": "final_response_avg_score", + "expected": "{\"route\":\"order_lookup\",\"message\":\"Checking order B-204 for customer 17.\"}", + "actual": "{\"route\":\"general_support\",\"message\":\"Please provide more details so I can route your request.\"}" + } + ] + } + }, + { + "eval_id": "train_tool_choice", + "status": "failed", + "average_score": { + "status": "available", + "value": 0.0, + "unit": null, + "reason": null + }, + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "status": "failed", + "score": { + "status": "available", + "value": 0.0, + "unit": null, + "reason": null + }, + "reason": null + } + ], + "runs": [ + { + "run_id": 1, + "status": "failed", + "error_message": null, + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "status": "failed", + "score": { + "status": "available", + "value": 0.0, + "unit": null, + "reason": null + }, + "reason": null + } + ], + "invocations": [ + { + "invocation_id": "train-2", + "user_text": "Check the status of order A100.", + "expected_response": "{\"route\":\"order_lookup\",\"message\":\"Checking order A100.\"}", + "actual_response": "{\"route\":\"general_support\",\"message\":\"Please provide more details so I can route your request.\"}", + "expected_tools": [], + "actual_tools": [], + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "status": "failed", + "score": { + "status": "available", + "value": 0.0, + "unit": null, + "reason": null + }, + "reason": null + } + ] + } + ] + } + ], + "attribution": { + "primary_category": "routing_error", + "secondary_categories": [ + "final_response_mismatch" + ], + "summary": "Expected route 'order_lookup', got 'general_support'.", + "evidence": [ + { + "evidence_type": "response", + "message": "Expected route 'order_lookup', got 'general_support'.", + "run_id": 1, + "invocation_id": "train-2", + "metric_name": null, + "expected": "order_lookup", + "actual": "general_support" + }, + { + "evidence_type": "response", + "message": "Final response did not satisfy metric 'final_response_avg_score'.", + "run_id": 1, + "invocation_id": "train-2", + "metric_name": "final_response_avg_score", + "expected": "{\"route\":\"order_lookup\",\"message\":\"Checking order A100.\"}", + "actual": "{\"route\":\"general_support\",\"message\":\"Please provide more details so I can route your request.\"}" + } + ] + } + } + ], + "passed_case_count": 1, + "failed_case_count": 2, + "not_evaluated_case_count": 0, + "average_score": { + "status": "available", + "value": 0.3333333333333333, + "unit": null, + "reason": null + } + }, + "baseline_validation": { + "phase": "baseline", + "split": "validation", + "eval_set_id": "eval_optimize_loop_validation", + "cases": [ + { + "eval_id": "val_knowledge_recall", + "status": "failed", + "average_score": { + "status": "available", + "value": 0.0, + "unit": null, + "reason": null + }, + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "status": "failed", + "score": { + "status": "available", + "value": 0.0, + "unit": null, + "reason": null + }, + "reason": null + } + ], + "runs": [ + { + "run_id": 1, + "status": "failed", + "error_message": null, + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "status": "failed", + "score": { + "status": "available", + "value": 0.0, + "unit": null, + "reason": null + }, + "reason": null + } + ], + "invocations": [ + { + "invocation_id": "val-2", + "user_text": "How long does standard shipping usually take?", + "expected_response": "{\"route\":\"shipping_policy\",\"message\":\"Standard shipping normally takes 3-5 business days.\"}", + "actual_response": "{\"route\":\"general_support\",\"message\":\"Please provide more details so I can route your request.\"}", + "expected_tools": [], + "actual_tools": [], + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "status": "failed", + "score": { + "status": "available", + "value": 0.0, + "unit": null, + "reason": null + }, + "reason": null + } + ] + } + ] + } + ], + "attribution": { + "primary_category": "routing_error", + "secondary_categories": [ + "final_response_mismatch" + ], + "summary": "Expected route 'shipping_policy', got 'general_support'.", + "evidence": [ + { + "evidence_type": "response", + "message": "Expected route 'shipping_policy', got 'general_support'.", + "run_id": 1, + "invocation_id": "val-2", + "metric_name": null, + "expected": "shipping_policy", + "actual": "general_support" + }, + { + "evidence_type": "response", + "message": "Final response did not satisfy metric 'final_response_avg_score'.", + "run_id": 1, + "invocation_id": "val-2", + "metric_name": "final_response_avg_score", + "expected": "{\"route\":\"shipping_policy\",\"message\":\"Standard shipping normally takes 3-5 business days.\"}", + "actual": "{\"route\":\"general_support\",\"message\":\"Please provide more details so I can route your request.\"}" + } + ] + } + }, + { + "eval_id": "val_paraphrase", + "status": "failed", + "average_score": { + "status": "available", + "value": 0.0, + "unit": null, + "reason": null + }, + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "status": "failed", + "score": { + "status": "available", + "value": 0.0, + "unit": null, + "reason": null + }, + "reason": null + } + ], + "runs": [ + { + "run_id": 1, + "status": "failed", + "error_message": null, + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "status": "failed", + "score": { + "status": "available", + "value": 0.0, + "unit": null, + "reason": null + }, + "reason": null + } + ], + "invocations": [ + { + "invocation_id": "val-1", + "user_text": "Where do I change the address tied to my account?", + "expected_response": "{\"route\":\"account\",\"message\":\"Open profile settings to update your address.\"}", + "actual_response": "{\"route\":\"general_support\",\"message\":\"Please provide more details so I can route your request.\"}", + "expected_tools": [], + "actual_tools": [], + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "status": "failed", + "score": { + "status": "available", + "value": 0.0, + "unit": null, + "reason": null + }, + "reason": null + } + ] + } + ] + } + ], + "attribution": { + "primary_category": "routing_error", + "secondary_categories": [ + "final_response_mismatch" + ], + "summary": "Expected route 'account', got 'general_support'.", + "evidence": [ + { + "evidence_type": "response", + "message": "Expected route 'account', got 'general_support'.", + "run_id": 1, + "invocation_id": "val-1", + "metric_name": null, + "expected": "account", + "actual": "general_support" + }, + { + "evidence_type": "response", + "message": "Final response did not satisfy metric 'final_response_avg_score'.", + "run_id": 1, + "invocation_id": "val-1", + "metric_name": "final_response_avg_score", + "expected": "{\"route\":\"account\",\"message\":\"Open profile settings to update your address.\"}", + "actual": "{\"route\":\"general_support\",\"message\":\"Please provide more details so I can route your request.\"}" + } + ] + } + }, + { + "eval_id": "val_refund_route", + "status": "passed", + "average_score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "status": "passed", + "score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "reason": null + } + ], + "runs": [ + { + "run_id": 1, + "status": "passed", + "error_message": null, + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "status": "passed", + "score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "reason": null + } + ], + "invocations": [ + { + "invocation_id": "val-3", + "user_text": "I was charged twice and need the duplicate payment refunded.", + "expected_response": "{\"route\":\"billing_refund\",\"message\":\"I will route this duplicate charge for refund review.\"}", + "actual_response": "{\"route\":\"billing_refund\",\"message\":\"I will route this duplicate charge for refund review.\"}", + "expected_tools": [], + "actual_tools": [], + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "status": "passed", + "score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "reason": null + } + ] + } + ] + } + ], + "attribution": null + } + ], + "passed_case_count": 1, + "failed_case_count": 2, + "not_evaluated_case_count": 0, + "average_score": { + "status": "available", + "value": 0.3333333333333333, + "unit": null, + "reason": null + } + }, + "candidate_train": { + "phase": "candidate", + "split": "train", + "eval_set_id": "eval_optimize_loop_train", + "cases": [ + { + "eval_id": "train_output_format", + "status": "passed", + "average_score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "status": "passed", + "score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "reason": null + } + ], + "runs": [ + { + "run_id": 1, + "status": "passed", + "error_message": null, + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "status": "passed", + "score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "reason": null + } + ], + "invocations": [ + { + "invocation_id": "train-1", + "user_text": "How can I update my email address?", + "expected_response": "{\"route\":\"account\",\"message\":\"Open profile settings to update your email.\"}", + "actual_response": "{\"route\":\"account\",\"message\":\"Open profile settings to update your email.\"}", + "expected_tools": [], + "actual_tools": [], + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "status": "passed", + "score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "reason": null + } + ] + } + ] + } + ], + "attribution": null + }, + { + "eval_id": "train_tool_arguments", + "status": "passed", + "average_score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "status": "passed", + "score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "reason": null + } + ], + "runs": [ + { + "run_id": 1, + "status": "passed", + "error_message": null, + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "status": "passed", + "score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "reason": null + } + ], + "invocations": [ + { + "invocation_id": "train-3", + "user_text": "Look up order B-204 for customer 17.", + "expected_response": "{\"route\":\"order_lookup\",\"message\":\"Checking order B-204 for customer 17.\"}", + "actual_response": "{\"route\":\"order_lookup\",\"message\":\"Checking order B-204 for customer 17.\"}", + "expected_tools": [], + "actual_tools": [], + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "status": "passed", + "score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "reason": null + } + ] + } + ] + } + ], + "attribution": null + }, + { + "eval_id": "train_tool_choice", + "status": "passed", + "average_score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "status": "passed", + "score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "reason": null + } + ], + "runs": [ + { + "run_id": 1, + "status": "passed", + "error_message": null, + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "status": "passed", + "score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "reason": null + } + ], + "invocations": [ + { + "invocation_id": "train-2", + "user_text": "Check the status of order A100.", + "expected_response": "{\"route\":\"order_lookup\",\"message\":\"Checking order A100.\"}", + "actual_response": "{\"route\":\"order_lookup\",\"message\":\"Checking order A100.\"}", + "expected_tools": [], + "actual_tools": [], + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "status": "passed", + "score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "reason": null + } + ] + } + ] + } + ], + "attribution": null + } + ], + "passed_case_count": 3, + "failed_case_count": 0, + "not_evaluated_case_count": 0, + "average_score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + } + }, + "candidate_validation": { + "phase": "candidate", + "split": "validation", + "eval_set_id": "eval_optimize_loop_validation", + "cases": [ + { + "eval_id": "val_knowledge_recall", + "status": "passed", + "average_score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "status": "passed", + "score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "reason": null + } + ], + "runs": [ + { + "run_id": 1, + "status": "passed", + "error_message": null, + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "status": "passed", + "score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "reason": null + } + ], + "invocations": [ + { + "invocation_id": "val-2", + "user_text": "How long does standard shipping usually take?", + "expected_response": "{\"route\":\"shipping_policy\",\"message\":\"Standard shipping normally takes 3-5 business days.\"}", + "actual_response": "{\"route\":\"shipping_policy\",\"message\":\"Standard shipping normally takes 3-5 business days.\"}", + "expected_tools": [], + "actual_tools": [], + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "status": "passed", + "score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "reason": null + } + ] + } + ] + } + ], + "attribution": null + }, + { + "eval_id": "val_paraphrase", + "status": "passed", + "average_score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "status": "passed", + "score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "reason": null + } + ], + "runs": [ + { + "run_id": 1, + "status": "passed", + "error_message": null, + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "status": "passed", + "score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "reason": null + } + ], + "invocations": [ + { + "invocation_id": "val-1", + "user_text": "Where do I change the address tied to my account?", + "expected_response": "{\"route\":\"account\",\"message\":\"Open profile settings to update your address.\"}", + "actual_response": "{\"route\":\"account\",\"message\":\"Open profile settings to update your address.\"}", + "expected_tools": [], + "actual_tools": [], + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "status": "passed", + "score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "reason": null + } + ] + } + ] + } + ], + "attribution": null + }, + { + "eval_id": "val_refund_route", + "status": "passed", + "average_score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "status": "passed", + "score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "reason": null + } + ], + "runs": [ + { + "run_id": 1, + "status": "passed", + "error_message": null, + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "status": "passed", + "score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "reason": null + } + ], + "invocations": [ + { + "invocation_id": "val-3", + "user_text": "I was charged twice and need the duplicate payment refunded.", + "expected_response": "{\"route\":\"billing_refund\",\"message\":\"I will route this duplicate charge for refund review.\"}", + "actual_response": "{\"route\":\"billing_refund\",\"message\":\"I will route this duplicate charge for refund review.\"}", + "expected_tools": [], + "actual_tools": [], + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "status": "passed", + "score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "reason": null + } + ] + } + ] + } + ], + "attribution": null + } + ], + "passed_case_count": 3, + "failed_case_count": 0, + "not_evaluated_case_count": 0, + "average_score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + } + }, + "train_diff": { + "split": "train", + "eval_set_id": "eval_optimize_loop_train", + "cases": [ + { + "eval_id": "train_output_format", + "split": "train", + "baseline_status": "passed", + "candidate_status": "passed", + "baseline_score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "candidate_score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "score_delta": { + "status": "available", + "value": 0.0, + "unit": null, + "reason": null + }, + "change": "unchanged", + "metrics": [ + { + "metric_name": "final_response_avg_score", + "baseline_status": "passed", + "candidate_status": "passed", + "baseline_score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "candidate_score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "score_delta": { + "status": "available", + "value": 0.0, + "unit": null, + "reason": null + }, + "change": "unchanged" + } + ], + "baseline_attribution": null, + "candidate_attribution": null, + "is_hard": false, + "is_critical": false, + "severe_regression": false + }, + { + "eval_id": "train_tool_arguments", + "split": "train", + "baseline_status": "failed", + "candidate_status": "passed", + "baseline_score": { + "status": "available", + "value": 0.0, + "unit": null, + "reason": null + }, + "candidate_score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "score_delta": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "change": "newly_passed", + "metrics": [ + { + "metric_name": "final_response_avg_score", + "baseline_status": "failed", + "candidate_status": "passed", + "baseline_score": { + "status": "available", + "value": 0.0, + "unit": null, + "reason": null + }, + "candidate_score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "score_delta": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "change": "newly_passed" + } + ], + "baseline_attribution": { + "primary_category": "routing_error", + "secondary_categories": [ + "final_response_mismatch" + ], + "summary": "Expected route 'order_lookup', got 'general_support'.", + "evidence": [ + { + "evidence_type": "response", + "message": "Expected route 'order_lookup', got 'general_support'.", + "run_id": 1, + "invocation_id": "train-3", + "metric_name": null, + "expected": "order_lookup", + "actual": "general_support" + }, + { + "evidence_type": "response", + "message": "Final response did not satisfy metric 'final_response_avg_score'.", + "run_id": 1, + "invocation_id": "train-3", + "metric_name": "final_response_avg_score", + "expected": "{\"route\":\"order_lookup\",\"message\":\"Checking order B-204 for customer 17.\"}", + "actual": "{\"route\":\"general_support\",\"message\":\"Please provide more details so I can route your request.\"}" + } + ] + }, + "candidate_attribution": null, + "is_hard": false, + "is_critical": false, + "severe_regression": false + }, + { + "eval_id": "train_tool_choice", + "split": "train", + "baseline_status": "failed", + "candidate_status": "passed", + "baseline_score": { + "status": "available", + "value": 0.0, + "unit": null, + "reason": null + }, + "candidate_score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "score_delta": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "change": "newly_passed", + "metrics": [ + { + "metric_name": "final_response_avg_score", + "baseline_status": "failed", + "candidate_status": "passed", + "baseline_score": { + "status": "available", + "value": 0.0, + "unit": null, + "reason": null + }, + "candidate_score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "score_delta": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "change": "newly_passed" + } + ], + "baseline_attribution": { + "primary_category": "routing_error", + "secondary_categories": [ + "final_response_mismatch" + ], + "summary": "Expected route 'order_lookup', got 'general_support'.", + "evidence": [ + { + "evidence_type": "response", + "message": "Expected route 'order_lookup', got 'general_support'.", + "run_id": 1, + "invocation_id": "train-2", + "metric_name": null, + "expected": "order_lookup", + "actual": "general_support" + }, + { + "evidence_type": "response", + "message": "Final response did not satisfy metric 'final_response_avg_score'.", + "run_id": 1, + "invocation_id": "train-2", + "metric_name": "final_response_avg_score", + "expected": "{\"route\":\"order_lookup\",\"message\":\"Checking order A100.\"}", + "actual": "{\"route\":\"general_support\",\"message\":\"Please provide more details so I can route your request.\"}" + } + ] + }, + "candidate_attribution": null, + "is_hard": false, + "is_critical": false, + "severe_regression": false + } + ], + "baseline_average_score": { + "status": "available", + "value": 0.3333333333333333, + "unit": null, + "reason": null + }, + "candidate_average_score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "score_delta": { + "status": "available", + "value": 0.6666666666666667, + "unit": null, + "reason": null + }, + "newly_passed_count": 2, + "newly_failed_count": 0, + "improved_count": 0, + "regressed_count": 0, + "unchanged_count": 1, + "incomparable_count": 0 + }, + "validation_diff": { + "split": "validation", + "eval_set_id": "eval_optimize_loop_validation", + "cases": [ + { + "eval_id": "val_knowledge_recall", + "split": "validation", + "baseline_status": "failed", + "candidate_status": "passed", + "baseline_score": { + "status": "available", + "value": 0.0, + "unit": null, + "reason": null + }, + "candidate_score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "score_delta": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "change": "newly_passed", + "metrics": [ + { + "metric_name": "final_response_avg_score", + "baseline_status": "failed", + "candidate_status": "passed", + "baseline_score": { + "status": "available", + "value": 0.0, + "unit": null, + "reason": null + }, + "candidate_score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "score_delta": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "change": "newly_passed" + } + ], + "baseline_attribution": { + "primary_category": "routing_error", + "secondary_categories": [ + "final_response_mismatch" + ], + "summary": "Expected route 'shipping_policy', got 'general_support'.", + "evidence": [ + { + "evidence_type": "response", + "message": "Expected route 'shipping_policy', got 'general_support'.", + "run_id": 1, + "invocation_id": "val-2", + "metric_name": null, + "expected": "shipping_policy", + "actual": "general_support" + }, + { + "evidence_type": "response", + "message": "Final response did not satisfy metric 'final_response_avg_score'.", + "run_id": 1, + "invocation_id": "val-2", + "metric_name": "final_response_avg_score", + "expected": "{\"route\":\"shipping_policy\",\"message\":\"Standard shipping normally takes 3-5 business days.\"}", + "actual": "{\"route\":\"general_support\",\"message\":\"Please provide more details so I can route your request.\"}" + } + ] + }, + "candidate_attribution": null, + "is_hard": true, + "is_critical": false, + "severe_regression": false + }, + { + "eval_id": "val_paraphrase", + "split": "validation", + "baseline_status": "failed", + "candidate_status": "passed", + "baseline_score": { + "status": "available", + "value": 0.0, + "unit": null, + "reason": null + }, + "candidate_score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "score_delta": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "change": "newly_passed", + "metrics": [ + { + "metric_name": "final_response_avg_score", + "baseline_status": "failed", + "candidate_status": "passed", + "baseline_score": { + "status": "available", + "value": 0.0, + "unit": null, + "reason": null + }, + "candidate_score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "score_delta": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "change": "newly_passed" + } + ], + "baseline_attribution": { + "primary_category": "routing_error", + "secondary_categories": [ + "final_response_mismatch" + ], + "summary": "Expected route 'account', got 'general_support'.", + "evidence": [ + { + "evidence_type": "response", + "message": "Expected route 'account', got 'general_support'.", + "run_id": 1, + "invocation_id": "val-1", + "metric_name": null, + "expected": "account", + "actual": "general_support" + }, + { + "evidence_type": "response", + "message": "Final response did not satisfy metric 'final_response_avg_score'.", + "run_id": 1, + "invocation_id": "val-1", + "metric_name": "final_response_avg_score", + "expected": "{\"route\":\"account\",\"message\":\"Open profile settings to update your address.\"}", + "actual": "{\"route\":\"general_support\",\"message\":\"Please provide more details so I can route your request.\"}" + } + ] + }, + "candidate_attribution": null, + "is_hard": false, + "is_critical": false, + "severe_regression": false + }, + { + "eval_id": "val_refund_route", + "split": "validation", + "baseline_status": "passed", + "candidate_status": "passed", + "baseline_score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "candidate_score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "score_delta": { + "status": "available", + "value": 0.0, + "unit": null, + "reason": null + }, + "change": "unchanged", + "metrics": [ + { + "metric_name": "final_response_avg_score", + "baseline_status": "passed", + "candidate_status": "passed", + "baseline_score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "candidate_score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "score_delta": { + "status": "available", + "value": 0.0, + "unit": null, + "reason": null + }, + "change": "unchanged" + } + ], + "baseline_attribution": null, + "candidate_attribution": null, + "is_hard": false, + "is_critical": true, + "severe_regression": false + } + ], + "baseline_average_score": { + "status": "available", + "value": 0.3333333333333333, + "unit": null, + "reason": null + }, + "candidate_average_score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "score_delta": { + "status": "available", + "value": 0.6666666666666667, + "unit": null, + "reason": null + }, + "newly_passed_count": 2, + "newly_failed_count": 0, + "improved_count": 0, + "regressed_count": 0, + "unchanged_count": 1, + "incomparable_count": 0 + }, + "overfit_status": "not_detected", + "overfit_reason": "Train score delta is 0.666667; validation score delta is 0.666667." + }, + "pipeline_resources": { + "cost_usd": { + "status": "unavailable", + "value": null, + "unit": "USD", + "reason": "Offline deterministic model does not report monetary cost." + }, + "total_tokens": { + "status": "unavailable", + "value": null, + "unit": "tokens", + "reason": "Offline deterministic model does not report token usage." + }, + "duration_seconds": { + "status": "available", + "value": 0.03433158100233413, + "unit": "seconds", + "reason": null + } + }, + "optimizer_resources": { + "scope_note": "Offline mode uses a deterministic candidate provider.", + "total_rounds": { + "status": "not_applicable", + "value": null, + "unit": "rounds", + "reason": "Offline mode uses a deterministic candidate provider." + }, + "reflection_lm_calls": { + "status": "not_applicable", + "value": null, + "unit": "calls", + "reason": "Offline mode uses a deterministic candidate provider." + }, + "cost_usd": { + "status": "not_applicable", + "value": null, + "unit": "USD", + "reason": "Offline mode uses a deterministic candidate provider." + }, + "token_usage": { + "status": "not_applicable", + "value": null, + "unit": "tokens", + "reason": "Offline mode uses a deterministic candidate provider." + }, + "duration_seconds": { + "status": "not_applicable", + "value": null, + "unit": "seconds", + "reason": "Offline mode uses a deterministic candidate provider." + } + }, + "gate_decision": { + "decision": "accept", + "rule_results": [ + { + "rule_id": "evaluation_completeness", + "outcome": "pass", + "message": "All four evaluations contain complete case and metric results.", + "case_ids": [], + "metric_names": [], + "observed": {}, + "threshold": null + }, + { + "rule_id": "minimum_validation_score_delta", + "outcome": "pass", + "message": "Validation score improvement meets the configured minimum.", + "case_ids": [], + "metric_names": [], + "observed": { + "validation_score_delta": { + "status": "available", + "value": 0.6666666666666667, + "unit": null, + "reason": null + } + }, + "threshold": 0.05 + }, + { + "rule_id": "validation_pass_rate_non_decrease", + "outcome": "pass", + "message": "Validation pass rate did not decrease.", + "case_ids": [], + "metric_names": [], + "observed": { + "baseline_validation_pass_rate": { + "status": "available", + "value": 0.3333333333333333, + "unit": "ratio", + "reason": null + }, + "candidate_validation_pass_rate": { + "status": "available", + "value": 1.0, + "unit": "ratio", + "reason": null + } + }, + "threshold": null + }, + { + "rule_id": "no_new_hard_fail", + "outcome": "pass", + "message": "No new hard failures were found.", + "case_ids": [], + "metric_names": [], + "observed": {}, + "threshold": null + }, + { + "rule_id": "no_critical_regression", + "outcome": "pass", + "message": "No critical-case regressions were found.", + "case_ids": [], + "metric_names": [], + "observed": {}, + "threshold": null + }, + { + "rule_id": "no_severe_regression", + "outcome": "pass", + "message": "No severe case regressions were found.", + "case_ids": [], + "metric_names": [], + "observed": {}, + "threshold": null + }, + { + "rule_id": "required_metrics", + "outcome": "pass", + "message": "All required candidate metrics are available and passed.", + "case_ids": [], + "metric_names": [], + "observed": {}, + "threshold": null + }, + { + "rule_id": "no_overfitting", + "outcome": "pass", + "message": "No train-improvement/validation-regression pattern was detected.", + "case_ids": [], + "metric_names": [], + "observed": {}, + "threshold": null + }, + { + "rule_id": "cost_budget", + "outcome": "skipped", + "message": "cost_usd budget is not configured.", + "case_ids": [], + "metric_names": [], + "observed": {}, + "threshold": null + }, + { + "rule_id": "token_budget", + "outcome": "skipped", + "message": "total_tokens budget is not configured.", + "case_ids": [], + "metric_names": [], + "observed": {}, + "threshold": null + }, + { + "rule_id": "duration_budget", + "outcome": "pass", + "message": "duration_seconds is within the configured budget.", + "case_ids": [], + "metric_names": [], + "observed": { + "duration_seconds": { + "status": "available", + "value": 0.03433158100233413, + "unit": "seconds", + "reason": null + } + }, + "threshold": 180.0 + } + ], + "rejection_reasons": [], + "warnings": [] + }, + "writeback": { + "status": "skipped", + "reason": "disabled", + "attempted": false, + "changed_fields": [], + "source_hashes_before": {}, + "source_hashes_after": {}, + "error_message": null + } +} diff --git a/examples/optimization/eval_optimize_loop/sample_output/optimization_report.md b/examples/optimization/eval_optimize_loop/sample_output/optimization_report.md new file mode 100644 index 00000000..f5100362 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/sample_output/optimization_report.md @@ -0,0 +1,44 @@ +# Optimization Report + +- Run: `stage6_sample` +- Mode: `offline` +- Gate decision: ACCEPT +- Candidate: `fake-improve-60cc05b773e8` + +## Full Evaluations + +- Baseline train: 1/3 passed; average score=0.3333333333333333 +- Baseline validation: 1/3 passed; average score=0.3333333333333333 +- Candidate train: 3/3 passed; average score=1.0 +- Candidate validation: 3/3 passed; average score=1.0 + +## Gate + +- No rejection reasons or warnings. + +## Candidate Changes + +- system_prompt + +## Overfit +- Status: not_detected +- Reason: Train score delta is 0.666667; validation score delta is 0.666667. + +## Writeback +- Status: skipped +- Reason: disabled + +## Pipeline Observations +- Cost: unavailable +- Tokens: unavailable +- Duration: available + +## Optimizer Resources +- Rounds: not_applicable; unit=rounds; reason=Offline mode uses a deterministic candidate provider. +- Reflection calls: not_applicable; unit=calls; reason=Offline mode uses a deterministic candidate provider. +- Cost: not_applicable; unit=USD; reason=Offline mode uses a deterministic candidate provider. +- Token usage: not_applicable; unit=tokens; reason=Offline mode uses a deterministic candidate provider. +- Duration: not_applicable; unit=seconds; reason=Offline mode uses a deterministic candidate provider. + +## Optimizer Scope +- Offline mode uses a deterministic candidate provider. diff --git a/examples/optimization/eval_optimize_loop/sample_output/prompts/baseline/000-system_prompt.md b/examples/optimization/eval_optimize_loop/sample_output/prompts/baseline/000-system_prompt.md new file mode 100644 index 00000000..345c342d --- /dev/null +++ b/examples/optimization/eval_optimize_loop/sample_output/prompts/baseline/000-system_prompt.md @@ -0,0 +1,5 @@ +You are a customer-support routing assistant. + +Answer with a compact JSON object containing `route` and `message`. Use the +available account tool for account-specific requests. Never invent account +facts. diff --git a/examples/optimization/eval_optimize_loop/sample_output/prompts/candidate/000-system_prompt.md b/examples/optimization/eval_optimize_loop/sample_output/prompts/candidate/000-system_prompt.md new file mode 100644 index 00000000..42ad8d8f --- /dev/null +++ b/examples/optimization/eval_optimize_loop/sample_output/prompts/candidate/000-system_prompt.md @@ -0,0 +1,13 @@ +You are a customer-support routing assistant. + +Answer with a compact JSON object containing `route` and `message`. Use the +available account tool for account-specific requests. Never invent account +facts. + + +Apply general customer-support routing rules across equivalent user phrasings. + + + + + diff --git a/examples/optimization/eval_optimize_loop/schemas.py b/examples/optimization/eval_optimize_loop/schemas.py new file mode 100644 index 00000000..5698fe5b --- /dev/null +++ b/examples/optimization/eval_optimize_loop/schemas.py @@ -0,0 +1,582 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under the Apache License, Version 2.0. +"""Serializable data schemas owned by the pipeline example. + +The SDK evaluation result types remain the source of truth for raw evaluation +data. These schemas capture run inputs, prompt provenance, fake candidates, +and the full stage-two evaluation outputs consumed by later pipeline phases. +""" + +from __future__ import annotations + +from datetime import datetime +from typing import Any +from typing import Generic +from typing import Literal +from typing import Optional +from typing import TypeVar +from typing import Union + +from pydantic import Field +from pydantic import field_validator +from pydantic import model_validator + +from trpc_agent_sdk.evaluation import EvalBaseModel +from trpc_agent_sdk.evaluation import EvalCaseResult +from trpc_agent_sdk.evaluation import OptimizeResult + + +CandidateScenario = Literal["improve", "no_improvement", "overfit"] +EvaluationStatus = Literal["passed", "failed", "not_evaluated"] +FailureCategory = Literal[ + "evaluation_error", + "tool_name_error", + "tool_argument_error", + "knowledge_recall", + "format_error", + "rubric_failure", + "routing_error", + "final_response_mismatch", + "unknown", +] +ChangeKind = Literal[ + "newly_passed", + "newly_failed", + "improved", + "regressed", + "unchanged", + "incomparable", +] +OverfitStatus = Literal["detected", "not_detected", "unavailable"] +GateRuleId = Literal[ + "evaluation_completeness", + "minimum_validation_score_delta", + "validation_pass_rate_non_decrease", + "no_new_hard_fail", + "no_critical_regression", + "no_severe_regression", + "required_metrics", + "no_overfitting", + "cost_budget", + "token_budget", + "duration_budget", +] +GateRuleOutcome = Literal["pass", "reject", "warning", "skipped"] +GateDecisionValue = Literal["accept", "reject"] +WritebackStatus = Literal["skipped", "written", "blocked", "failed"] +WritebackReason = Literal[ + "gate_rejected", + "disabled", + "source_drift", + "write_error", + "readback_mismatch", + "written", + "trace_replay", +] + + +class OptimizerRuntimeParameters(EvalBaseModel): + """命令行显式传入的反思优化模型参数，不包含任何凭据。""" + + provider_name: str = "openai" + model_name: str + variant: str = "" + temperature: float = Field(default=0.8, ge=0.0, allow_inf_nan=False) + max_tokens: int = Field(default=4096, gt=0) + think: Optional[bool] = None + max_candidate_proposals: int = Field(default=1, gt=0) + + @field_validator("provider_name", "model_name") + @classmethod + def _require_non_empty_model_identity(cls, value: str) -> str: + if not value.strip(): + raise ValueError("model identity must not be empty") + return value.strip() + + +class ObservableValue(EvalBaseModel): + """A measurement whose absence is explicit rather than silently zero.""" + + status: Literal["available", "unavailable"] + value: Optional[float] = None + unit: Optional[str] = None + reason: Optional[str] = None + + @model_validator(mode="after") + def _validate_status(self) -> "ObservableValue": + if self.status == "available" and self.value is None: + raise ValueError("available observable values require value") + if self.status == "unavailable" and self.value is not None: + raise ValueError("unavailable observable values must not carry a value") + return self + + +class PromptSnapshot(EvalBaseModel): + """Content and provenance of one source prompt field at preparation time.""" + + field_name: str + source_path: str + working_path: str + content: str + sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + + +class TracePromptSnapshot(EvalBaseModel): + """Trace 候选随附的只读 Prompt 快照。""" + + field_name: str + path: str + content: str + sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + + +class TraceScenarioInputSnapshot(EvalBaseModel): + train_evalset_path: str + train_evalset_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + validation_evalset_path: str + validation_evalset_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + prompt_snapshots: list[TracePromptSnapshot] + + +class TraceInputSnapshot(EvalBaseModel): + scenarios: dict[CandidateScenario, TraceScenarioInputSnapshot] + + +class InputSnapshot(EvalBaseModel): + """Immutable file identities captured before a pipeline run starts.""" + + pipeline_config_path: str + pipeline_config_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + optimizer_config_path: str + optimizer_config_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + train_evalset_path: str + train_evalset_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + validation_evalset_path: str + validation_evalset_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + prompt_snapshots: list[PromptSnapshot] + seed: int + trace_inputs: Optional[TraceInputSnapshot] = None + + +class WorkspaceSnapshot(EvalBaseModel): + """Directory layout created for one isolated pipeline run.""" + + run_id: str + run_dir: str + workspace_dir: str + prompts_dir: str + + +class CandidateProposal(EvalBaseModel): + """Common, serializable identity and prompt payload for any provider.""" + + provider: Literal["fake", "agent_optimizer", "trace"] + prompts: dict[str, str] + changed_fields: list[str] + rationale: str + parent_prompt_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + candidate_prompt_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + candidate_id: str + + +class FakeCandidateProposal(CandidateProposal): + """One deterministic prompt proposal produced without a real optimizer.""" + + provider: Literal["fake"] = "fake" + scenario: CandidateScenario + seed: int + candidate_id: str = Field(pattern=r"^fake-(improve|no_improvement|overfit)-[0-9a-f]{12}$") + + +class OptimizerCandidateProposal(CandidateProposal): + """Best candidate returned by a successful real AgentOptimizer run.""" + + provider: Literal["agent_optimizer"] = "agent_optimizer" + optimizer_status: Literal["SUCCEEDED"] = "SUCCEEDED" + finish_reason: str + stop_reason: Optional[str] = None + baseline_pass_rate: float = Field(ge=0.0, le=1.0) + best_pass_rate: float = Field(ge=0.0, le=1.0) + optimizer_output_dir: Optional[str] = None + candidate_id: str = Field(pattern=r"^optimizer-[0-9a-f]{12}$") + + +class TraceCandidateProposal(CandidateProposal): + """由预录制轨迹和 Prompt 快照标识的候选版本。""" + + provider: Literal["trace"] = "trace" + scenario: CandidateScenario + source_trace_sha256: dict[Literal["train", "validation"], str] + candidate_id: str = Field(pattern=r"^trace-(improve|no_improvement|overfit)-[0-9a-f]{12}$") + + +class EvaluationSnapshot(EvalBaseModel): + """Complete SDK outputs from one evaluation split.""" + + phase: Literal["baseline", "candidate"] + split: Literal["train", "validation"] + eval_set_id: str + failed_summary: Optional[dict[str, Any]] = None + details_lines: list[str] = Field( + description=( + "SDK detailed-output lines; intentionally empty in stage two because " + "print_detailed_results is disabled." + ) + ) + result_lines: list[str] + eval_results_by_eval_id: dict[str, list[EvalCaseResult]] + passed_case_count: int = Field(ge=0) + total_case_count: int = Field(ge=0) + average_score: Optional[float] = Field( + default=None, + ge=0.0, + le=1.0, + description=( + "Arithmetic mean of every available overall metric score across " + "all cases, configured runs, and metrics." + ), + ) + + +class ToolCallEvidence(EvalBaseModel): + """A compact tool call retained for attribution and reporting.""" + + name: str + arguments: dict[str, Any] = Field(default_factory=dict) + + +class MetricOutcome(EvalBaseModel): + """One normalized metric outcome for a run or an aggregate.""" + + metric_name: str + threshold: float + status: EvaluationStatus + score: ObservableValue + reason: Optional[str] = None + + +class InvocationEvidence(EvalBaseModel): + """Expected and actual evidence from one evaluated invocation.""" + + invocation_id: str + user_text: str + expected_response: Optional[str] = None + actual_response: Optional[str] = None + expected_tools: list[ToolCallEvidence] = Field(default_factory=list) + actual_tools: list[ToolCallEvidence] = Field(default_factory=list) + metrics: list[MetricOutcome] = Field(default_factory=list) + + +class CaseRunOutcome(EvalBaseModel): + """Normalized evidence from one configured run of an eval case.""" + + run_id: int + status: EvaluationStatus + error_message: Optional[str] = None + metrics: list[MetricOutcome] + invocations: list[InvocationEvidence] + + +class AttributionEvidence(EvalBaseModel): + """One concrete observation supporting a failure attribution.""" + + evidence_type: Literal["execution_error", "metric", "response", "tool"] + message: str + run_id: Optional[int] = None + invocation_id: Optional[str] = None + metric_name: Optional[str] = None + expected: Optional[Any] = None + actual: Optional[Any] = None + + +class FailureAttribution(EvalBaseModel): + """Deterministic primary and secondary reasons for a failed case.""" + + primary_category: FailureCategory + secondary_categories: list[FailureCategory] = Field(default_factory=list) + summary: str + evidence: list[AttributionEvidence] = Field(default_factory=list) + + +class CaseEvaluation(EvalBaseModel): + """One eval case aggregated across all configured runs.""" + + eval_id: str + status: EvaluationStatus + average_score: ObservableValue + metrics: list[MetricOutcome] + runs: list[CaseRunOutcome] + attribution: Optional[FailureAttribution] = None + + +class StandardizedEvaluation(EvalBaseModel): + """Stable case-oriented representation of one SDK evaluation snapshot.""" + + phase: Literal["baseline", "candidate"] + split: Literal["train", "validation"] + eval_set_id: str + cases: list[CaseEvaluation] + passed_case_count: int = Field(ge=0) + failed_case_count: int = Field(ge=0) + not_evaluated_case_count: int = Field(ge=0) + average_score: ObservableValue + + +class MetricDelta(EvalBaseModel): + """Before/after comparison for one metric.""" + + metric_name: str + baseline_status: EvaluationStatus + candidate_status: EvaluationStatus + baseline_score: ObservableValue + candidate_score: ObservableValue + score_delta: ObservableValue + change: ChangeKind + + +class CaseDiff(EvalBaseModel): + """Before/after comparison and policy labels for one eval case.""" + + eval_id: str + split: Literal["train", "validation"] + baseline_status: EvaluationStatus + candidate_status: EvaluationStatus + baseline_score: ObservableValue + candidate_score: ObservableValue + score_delta: ObservableValue + change: ChangeKind + metrics: list[MetricDelta] + baseline_attribution: Optional[FailureAttribution] = None + candidate_attribution: Optional[FailureAttribution] = None + is_hard: bool = False + is_critical: bool = False + severe_regression: bool = False + + +class DatasetDiff(EvalBaseModel): + """Case-level changes and aggregate deltas for one dataset split.""" + + split: Literal["train", "validation"] + eval_set_id: str + cases: list[CaseDiff] + baseline_average_score: ObservableValue + candidate_average_score: ObservableValue + score_delta: ObservableValue + newly_passed_count: int = Field(ge=0) + newly_failed_count: int = Field(ge=0) + improved_count: int = Field(ge=0) + regressed_count: int = Field(ge=0) + unchanged_count: int = Field(ge=0) + incomparable_count: int = Field(ge=0) + + +class EvaluationAnalysis(EvalBaseModel): + """All normalized evidence and comparisons produced by stage 3a.""" + + baseline_train: StandardizedEvaluation + baseline_validation: StandardizedEvaluation + candidate_train: StandardizedEvaluation + candidate_validation: StandardizedEvaluation + train_diff: DatasetDiff + validation_diff: DatasetDiff + overfit_status: OverfitStatus + overfit_reason: str + + +class ResourceMeasurements(EvalBaseModel): + """Resource observations available when Gate evaluates a candidate.""" + + cost_usd: ObservableValue + total_tokens: ObservableValue + duration_seconds: ObservableValue + + +class GateRuleResult(EvalBaseModel): + """One deterministic policy result with evidence for later reporting.""" + + rule_id: GateRuleId + outcome: GateRuleOutcome + message: str + case_ids: list[str] = Field(default_factory=list) + metric_names: list[str] = Field(default_factory=list) + observed: dict[str, ObservableValue] = Field(default_factory=dict) + threshold: Optional[float] = None + + +class GateDecision(EvalBaseModel): + """The complete, auditable acceptance decision produced by Gate.""" + + decision: GateDecisionValue + rule_results: list[GateRuleResult] + rejection_reasons: list[str] = Field(default_factory=list) + warnings: list[str] = Field(default_factory=list) + + +class WritebackResult(EvalBaseModel): + """Auditable outcome of the post-Gate source prompt operation.""" + + status: WritebackStatus + reason: WritebackReason + attempted: bool = False + changed_fields: list[str] = Field(default_factory=list) + source_hashes_before: dict[str, str] = Field(default_factory=dict) + source_hashes_after: dict[str, str] = Field(default_factory=dict) + error_message: Optional[str] = None + + +class PipelineStageResult(EvalBaseModel): + """Evaluation, analysis, Gate, and writeback fields shared by all modes.""" + + baseline_train: EvaluationSnapshot + baseline_validation: EvaluationSnapshot + candidate_train: EvaluationSnapshot + candidate_validation: EvaluationSnapshot + analysis: EvaluationAnalysis + measurements: ResourceMeasurements + gate_decision: GateDecision + writeback: WritebackResult + + +class OfflineStageResult(PipelineStageResult): + """Full deterministic offline-mode pipeline result.""" + + scenario: CandidateScenario + candidate: FakeCandidateProposal + + +class RealStageResult(PipelineStageResult): + """Full regression and Gate result for an AgentOptimizer proposal.""" + + candidate: OptimizerCandidateProposal + optimize_result: OptimizeResult + + +class TraceStageResult(PipelineStageResult): + """完整的 Trace 回放、分析与 Gate 结果。""" + + scenario: CandidateScenario + candidate: TraceCandidateProposal + +ReportPhase = Literal[ + "baseline_train", "baseline_validation", "candidate_generation", "candidate_train", + "candidate_validation", "analysis", "gate", "writeback", "reporting", +] + +class ReportProgress(EvalBaseModel): + started_at: datetime + current_phase: ReportPhase + completed_phases: list[ReportPhase] = Field(default_factory=list) + + @model_validator(mode="after") + def _validate_phases(self) -> "ReportProgress": + if len(self.completed_phases) != len(set(self.completed_phases)): + raise ValueError("completed phases must not contain duplicates") + if self.current_phase in self.completed_phases: + raise ValueError("completed phases must not include current phase") + return self + + +OptimizerResourceValueT = TypeVar("OptimizerResourceValueT") + + +class OptimizerResourceValue(EvalBaseModel, Generic[OptimizerResourceValueT]): + status: Literal["available", "unavailable", "not_applicable"] + value: Optional[OptimizerResourceValueT] = None + unit: str = Field(min_length=1) + reason: Optional[str] = None + + @model_validator(mode="after") + def _validate_status(self) -> "OptimizerResourceValue[OptimizerResourceValueT]": + if self.status == "available": + if self.value is None: + raise ValueError("available optimizer resource values require a value") + if isinstance(self.value, (int, float)) and not self.value >= 0: + raise ValueError("optimizer numeric resource values must be non-negative") + else: + if self.value is not None: + raise ValueError("non-available optimizer resource values must not carry a value") + if self.reason is None or not self.reason.strip(): + raise ValueError("non-available optimizer resource values require a reason") + return self + + +class OptimizerResourceObservation(EvalBaseModel): + scope_note: str + total_rounds: OptimizerResourceValue[int] + reflection_lm_calls: OptimizerResourceValue[int] + cost_usd: OptimizerResourceValue[float] + token_usage: OptimizerResourceValue[dict[str, int]] + duration_seconds: OptimizerResourceValue[float] + +class ArtifactReference(EvalBaseModel): + artifact_id: str + artifact_type: Literal["input", "prompt", "evaluation", "candidate", "optimizer_native", "report"] + relative_path: Optional[str] = None + required: bool + produced_by: ReportPhase + status: Literal["available", "unavailable"] + size_bytes: Optional[int] = Field(default=None, ge=0) + sha256: Optional[str] = Field(default=None, pattern=r"^[0-9a-f]{64}$") + unavailable_reason: Optional[str] = None + + @model_validator(mode="after") + def _validate_status(self) -> "ArtifactReference": + if self.status == "available": + if self.relative_path is None or self.size_bytes is None or self.sha256 is None or self.unavailable_reason is not None: + raise ValueError("available artifacts require path, size, hash, and no unavailable reason") + elif self.unavailable_reason is None or self.size_bytes is not None or self.sha256 is not None: + raise ValueError("unavailable artifacts require a reason and no size or hash") + return self + +class ArtifactIndex(EvalBaseModel): + schema_version: Literal[1] = 1 + run_id: str + generated_at: datetime + artifacts: list[ArtifactReference] + + @model_validator(mode="after") + def _validate_artifacts(self) -> "ArtifactIndex": + artifact_ids = [artifact.artifact_id for artifact in self.artifacts] + paths = [artifact.relative_path for artifact in self.artifacts if artifact.relative_path is not None] + if len(artifact_ids) != len(set(artifact_ids)): + raise ValueError("artifact IDs must be unique") + if len(paths) != len(set(paths)): + raise ValueError("artifact relative paths must be unique") + return self + +class OptimizationReport(EvalBaseModel): + schema_version: Literal[1] = 1 + status: Literal["completed"] = "completed" + run_id: str + execution_mode: Literal["offline", "real", "trace"] + seed: int + started_at: datetime + finished_at: datetime + input_snapshot: InputSnapshot + candidate: Union[FakeCandidateProposal, OptimizerCandidateProposal, TraceCandidateProposal] + baseline_train: EvaluationSnapshot + baseline_validation: EvaluationSnapshot + candidate_train: EvaluationSnapshot + candidate_validation: EvaluationSnapshot + analysis: EvaluationAnalysis + pipeline_resources: ResourceMeasurements + optimizer_resources: OptimizerResourceObservation + gate_decision: GateDecision + writeback: WritebackResult + +class FailureReport(EvalBaseModel): + schema_version: Literal[1] = 1 + status: Literal["failed"] = "failed" + run_id: str + execution_mode: Literal["offline", "real", "trace"] + failed_phase: ReportPhase + exception_type: str + error_message: str + generated_at: datetime + input_snapshot: InputSnapshot + source_prompt_hashes: dict[str, str] + completed_phases: list[ReportPhase] + existing_artifacts: list[str] diff --git a/examples/optimization/eval_optimize_loop/sensitive_config.py b/examples/optimization/eval_optimize_loop/sensitive_config.py new file mode 100644 index 00000000..d909f23f --- /dev/null +++ b/examples/optimization/eval_optimize_loop/sensitive_config.py @@ -0,0 +1,124 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under the Apache License, Version 2.0. +"""共享的敏感配置识别、占位符替换与持久化校验规则。""" + +from __future__ import annotations + + +API_KEY_PLACEHOLDER = "${TRPC_AGENT_API_KEY}" +BASE_URL_PLACEHOLDER = "${TRPC_AGENT_BASE_URL}" + +_SENSITIVE_CONFIG_KEYS = { + "accesstoken", + "apikey", + "auth", + "authorization", + "authtoken", + "baseurl", + "bearertoken", + "clientsecret", + "credential", + "credentials", + "password", + "passwd", + "privatekey", + "secret", + "secretkey", + "token", + "xapikey", +} +_SENSITIVE_CONFIG_KEY_SUFFIXES = { + "accesstoken", + "apikey", + "authtoken", + "baseurl", + "bearertoken", + "clientsecret", + "credential", + "credentials", + "endpointurl", + "password", + "passwd", + "privatekey", + "secretkey", +} +_URL_CONFIG_KEY_SUFFIXES = {"baseurl", "endpointurl"} +_APPROVED_SENSITIVE_VALUES = { + "", + API_KEY_PLACEHOLDER, + BASE_URL_PLACEHOLDER, + "fake-not-used-in-offline-mode", +} + + +class SensitiveConfigError(ValueError): + """配置中存在不允许持久化的连接信息或凭据。""" + + +def _normalized_key(key: str) -> str: + return key.replace("_", "").replace("-", "").casefold() + + +def _is_sensitive_key(key: str) -> bool: + normalized = _normalized_key(key) + return normalized in _SENSITIVE_CONFIG_KEYS or any( + normalized.endswith(suffix) + for suffix in _SENSITIVE_CONFIG_KEY_SUFFIXES + ) + + +def _placeholder_for_key(key: str) -> str: + normalized = _normalized_key(key) + if any(normalized.endswith(suffix) for suffix in _URL_CONFIG_KEY_SUFFIXES): + return BASE_URL_PLACEHOLDER + return API_KEY_PLACEHOLDER + + +def replace_persisted_sensitive_values(value: object) -> object: + """递归替换任何可能进入运行产物的连接地址和凭据。""" + if isinstance(value, str): + if value.strip().casefold().startswith(("http://", "https://")): + return BASE_URL_PLACEHOLDER + return value + if isinstance(value, list): + return [replace_persisted_sensitive_values(item) for item in value] + if not isinstance(value, dict): + return value + return { + key: ( + _placeholder_for_key(key) + if _is_sensitive_key(key) + else replace_persisted_sensitive_values(item) + ) + for key, item in value.items() + } + + +def validate_persisted_sensitive_values(value: object, *, path: str = "$") -> None: + """拒绝不符合共享占位符策略的持久化配置。""" + if isinstance(value, str): + if value.strip().casefold().startswith(("http://", "https://")): + raise SensitiveConfigError( + "sensitive optimizer config value is not an approved " + f"placeholder: {path}" + ) + return + if isinstance(value, list): + for index, item in enumerate(value): + validate_persisted_sensitive_values(item, path=f"{path}[{index}]") + return + if not isinstance(value, dict): + return + for key, item in value.items(): + item_path = f"{path}.{key}" + if _is_sensitive_key(key): + if not isinstance(item, str) or item not in _APPROVED_SENSITIVE_VALUES: + raise SensitiveConfigError( + "sensitive optimizer config value is not an approved " + f"placeholder: {item_path}" + ) + else: + validate_persisted_sensitive_values(item, path=item_path) diff --git a/examples/optimization/eval_optimize_loop/writeback.py b/examples/optimization/eval_optimize_loop/writeback.py new file mode 100644 index 00000000..302d1f90 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/writeback.py @@ -0,0 +1,166 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under the Apache License, Version 2.0. +"""Post-Gate source prompt writeback with drift detection and verification.""" + +from __future__ import annotations + +from hashlib import sha256 + +from trpc_agent_sdk.evaluation import TargetPrompt + +from .candidate_provider import prompt_mapping_sha256 +from .config import WritebackConfig +from .prompt_workspace import SourcePromptDriftError +from .prompt_workspace import verify_source_hashes +from .schemas import CandidateProposal +from .schemas import GateDecision +from .schemas import PromptSnapshot +from .schemas import WritebackResult + + +class WritebackIntegrityError(RuntimeError): + """The pipeline cannot prove that source prompts remain in a safe state.""" + + +def _field_hashes(prompts: dict[str, str]) -> dict[str, str]: + return { + name: sha256(content.encode("utf-8")).hexdigest() + for name, content in prompts.items() + } + + +async def _blocked_for_drift( + source_target: TargetPrompt, + message: str, +) -> WritebackResult: + try: + observed = await source_target.read_all() + except Exception: + observed = {} + return WritebackResult( + status="blocked", + reason="source_drift", + source_hashes_before=_field_hashes(observed), + error_message=message, + ) + + +async def _restore_and_verify( + source_target: TargetPrompt, + baseline: dict[str, str], +) -> dict[str, str]: + """Restore only when needed, then prove the exact baseline is present.""" + try: + current = await source_target.read_all() + except Exception: + current = None + if current != baseline: + try: + await source_target.write_all(baseline) + except Exception as exc: + raise WritebackIntegrityError(f"source prompt rollback failed: {exc}") from exc + try: + restored = await source_target.read_all() + except Exception as exc: + raise WritebackIntegrityError(f"failed to verify source prompt rollback: {exc}") from exc + if restored != baseline: + raise WritebackIntegrityError("source prompts do not match the pre-write snapshot after rollback") + return restored + + +async def perform_writeback( + *, + decision: GateDecision, + config: WritebackConfig, + snapshots: list[PromptSnapshot], + source_target: TargetPrompt, + candidate: CandidateProposal, +) -> WritebackResult: + """Apply a candidate only after ACCEPT and return a structured outcome.""" + if decision.decision == "reject": + return WritebackResult(status="skipped", reason="gate_rejected") + if not config.enabled: + return WritebackResult(status="skipped", reason="disabled") + if not config.require_source_hash_match: + raise WritebackIntegrityError("enabled writeback requires source hash verification") + if prompt_mapping_sha256(candidate.prompts) != candidate.candidate_prompt_sha256: + raise WritebackIntegrityError("candidate prompt hash does not match its prompt payload") + + try: + verify_source_hashes(snapshots) + except SourcePromptDriftError as exc: + return await _blocked_for_drift(source_target, str(exc)) + + try: + baseline = await source_target.read_all() + except Exception as exc: + return WritebackResult( + status="failed", + reason="write_error", + error_message=f"failed to read source prompts before writeback: {exc}", + ) + expected_baseline = {snapshot.field_name: snapshot.content for snapshot in snapshots} + if baseline != expected_baseline: + return await _blocked_for_drift( + source_target, + "source prompts changed after the initial hash check", + ) + hashes_before = _field_hashes(baseline) + + # This synchronous check is intentionally adjacent to the path-backed + # write. It narrows the compare/write window after the awaited read above. + try: + verify_source_hashes(snapshots) + except SourcePromptDriftError as exc: + return await _blocked_for_drift(source_target, str(exc)) + + try: + await source_target.write_all(candidate.prompts) + except Exception as exc: + restored = await _restore_and_verify(source_target, baseline) + return WritebackResult( + status="failed", + reason="write_error", + attempted=True, + changed_fields=list(candidate.changed_fields), + source_hashes_before=hashes_before, + source_hashes_after=_field_hashes(restored), + error_message=str(exc), + ) + + try: + written = await source_target.read_all() + except Exception as exc: + restored = await _restore_and_verify(source_target, baseline) + return WritebackResult( + status="failed", + reason="readback_mismatch", + attempted=True, + changed_fields=list(candidate.changed_fields), + source_hashes_before=hashes_before, + source_hashes_after=_field_hashes(restored), + error_message=f"failed to read source prompts after writeback: {exc}", + ) + if written != candidate.prompts: + restored = await _restore_and_verify(source_target, baseline) + return WritebackResult( + status="failed", + reason="readback_mismatch", + attempted=True, + changed_fields=list(candidate.changed_fields), + source_hashes_before=hashes_before, + source_hashes_after=_field_hashes(restored), + error_message="source prompt readback did not match the accepted candidate", + ) + + return WritebackResult( + status="written", + reason="written", + attempted=True, + changed_fields=list(candidate.changed_fields), + source_hashes_before=hashes_before, + source_hashes_after=_field_hashes(written), + ) diff --git a/tests/evaluation/test_eval_optimize_loop_real_integration.py b/tests/evaluation/test_eval_optimize_loop_real_integration.py new file mode 100644 index 00000000..7c626066 --- /dev/null +++ b/tests/evaluation/test_eval_optimize_loop_real_integration.py @@ -0,0 +1,447 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under the Apache License, Version 2.0. +"""真实业务 Agent 与真实反思模型集成入口的离线契约测试。""" + +from __future__ import annotations + +import json +import shutil +import subprocess +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from examples.optimization.eval_optimize_loop.candidate_provider import ( + AgentOptimizerCandidateProvider, +) +from examples.optimization.eval_optimize_loop.candidate_provider import CandidateRequest +from examples.optimization.eval_optimize_loop.pipeline import prepare_run +from examples.optimization.eval_optimize_loop.real_agent import BusinessModelConfig +from examples.optimization.eval_optimize_loop.real_agent import RealBusinessAgent +from examples.optimization.eval_optimize_loop.real_agent import load_business_model_config +from examples.optimization.eval_optimize_loop import run_real_integration +from examples.optimization.eval_optimize_loop.schemas import OptimizerRuntimeParameters +from trpc_agent_sdk.evaluation import OptimizeResult + + +_REPO_ROOT = Path(__file__).resolve().parents[2] +_EXAMPLE = _REPO_ROOT / "examples" / "optimization" / "eval_optimize_loop" + + +def _copy_example(tmp_path: Path) -> Path: + target = tmp_path / "eval_optimize_loop" + shutil.copytree(_EXAMPLE, target, ignore=shutil.ignore_patterns("runs", "__pycache__")) + return target + + +def _result(baseline: dict[str, str]) -> OptimizeResult: + best = {name: f"{value}\noptimized" for name, value in baseline.items()} + return OptimizeResult( + algorithm="gepa_reflective", + status="SUCCEEDED", + finish_reason="completed", + baseline_pass_rate=0.3, + best_pass_rate=0.7, + pass_rate_improvement=0.4, + baseline_prompts=baseline, + best_prompts=best, + total_rounds=1, + rounds=[], + total_reflection_lm_calls=1, + total_llm_cost=0.0, + total_token_usage={"prompt": 0, "completion": 0, "total": 0}, + duration_seconds=0.1, + started_at="2026-07-18T00:00:00Z", + finished_at="2026-07-18T00:00:01Z", + ) + + +def test_optimizer_runtime_parameters_are_serializable_and_validated(): + parameters = OptimizerRuntimeParameters( + model_name="mimo-v2.5", + temperature=0.2, + max_tokens=2048, + think=False, + max_candidate_proposals=1, + ) + + assert OptimizerRuntimeParameters.model_validate_json(parameters.model_dump_json()) == parameters + with pytest.raises(ValueError): + OptimizerRuntimeParameters(model_name="", max_candidate_proposals=1) + with pytest.raises(ValueError): + OptimizerRuntimeParameters(model_name="model", max_candidate_proposals=0) + with pytest.raises(ValueError): + OptimizerRuntimeParameters(model_name="model", temperature=float("inf")) + + +def test_business_model_config_requires_all_three_environment_values(): + with pytest.raises(ValueError) as exc_info: + load_business_model_config({}) + + message = str(exc_info.value) + assert "TRPC_AGENT_API_KEY" in message + assert "TRPC_AGENT_BASE_URL" in message + assert "TRPC_AGENT_MODEL_NAME" in message + + normalized = load_business_model_config( + { + "TRPC_AGENT_API_KEY": " key ", + "TRPC_AGENT_BASE_URL": " https://example.test ", + "TRPC_AGENT_MODEL_NAME": " model ", + } + ) + assert normalized == BusinessModelConfig( + api_key="key", + base_url="https://example.test", + model_name="model", + ) + + +@pytest.mark.asyncio +async def test_real_business_agent_rereads_prompts_and_returns_only_final_non_thought_text( + monkeypatch: pytest.MonkeyPatch, +): + prompt_versions = [ + {"system_prompt": "baseline instruction"}, + {"system_prompt": "candidate instruction"}, + ] + + class FakeTarget: + async def read_all(self): + return prompt_versions.pop(0) + + class FakePart: + def __init__(self, text: str, *, thought: bool = False): + self.text = text + self.thought = thought + + class FakeEvent: + def __init__(self, final: bool, parts: list[FakePart]): + self._final = final + self.content = type("Content", (), {"parts": parts})() + + def is_final_response(self): + return self._final + + captured_instructions: list[str] = [] + + class FakeLlmAgent: + def __init__(self, **kwargs): + captured_instructions.append(kwargs["instruction"]) + + class FakeSessionService: + async def create_session(self, **kwargs): + return None + + class FakeRunner: + def __init__(self, **kwargs): + pass + + async def run_async(self, **kwargs): + yield FakeEvent(False, [FakePart("intermediate")]) + yield FakeEvent(True, [FakePart("thinking", thought=True), FakePart("answer")]) + + monkeypatch.setattr("examples.optimization.eval_optimize_loop.real_agent.OpenAIModel", lambda **kwargs: object()) + monkeypatch.setattr( + "examples.optimization.eval_optimize_loop.business_agent.LlmAgent", + FakeLlmAgent, + ) + monkeypatch.setattr( + "examples.optimization.eval_optimize_loop.business_agent.InMemorySessionService", + FakeSessionService, + ) + monkeypatch.setattr( + "examples.optimization.eval_optimize_loop.business_agent.Runner", + FakeRunner, + ) + agent = RealBusinessAgent( + FakeTarget(), + BusinessModelConfig(api_key="secret", base_url="https://example.test", model_name="model"), + ) + + assert await agent.call_agent("first") == "answer" + assert await agent.call_agent("second") == "answer" + assert captured_instructions == ["baseline instruction", "candidate instruction"] + + +@pytest.mark.asyncio +async def test_provider_writes_runtime_config_without_persisting_secrets( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + root = _copy_example(tmp_path) + prepared = prepare_run(root / "pipeline.json", run_id="runtime_config") + baseline = await prepared.working_target.read_all() + original_config = Path(prepared.input_snapshot.optimizer_config_path).read_text(encoding="utf-8") + captured: dict[str, object] = {} + + async def fake_optimize(**kwargs): + captured.update(kwargs) + output_dir = Path(str(kwargs["output_dir"])) + output_dir.mkdir(parents=True, exist_ok=True) + shutil.copyfile(str(kwargs["config_path"]), output_dir / "config.snapshot.json") + return _result(baseline) + + monkeypatch.setattr( + "examples.optimization.eval_optimize_loop.candidate_provider.AgentOptimizer.optimize", + fake_optimize, + ) + request = CandidateRequest( + current_prompts=baseline, + target_prompt=prepared.working_target, + optimizer_config_path=Path(prepared.input_snapshot.optimizer_config_path), + train_evalset_path=Path(prepared.input_snapshot.train_evalset_path), + validation_evalset_path=Path(prepared.input_snapshot.validation_evalset_path), + output_dir=Path(prepared.workspace.run_dir) / "optimizer", + seed=42, + runtime_parameters=OptimizerRuntimeParameters( + provider_name="openai", + model_name="mimo-v2.5", + variant="chat_completions", + temperature=0.25, + max_tokens=1024, + think=True, + max_candidate_proposals=1, + ), + ) + + template = json.loads(request.optimizer_config_path.read_text(encoding="utf-8")) + template["evaluate"]["credential_probe"] = { + "api_key": "judge-secret", + "base_url": "https://judge.example.test", + "authorization": "Bearer judge-authorization-secret", + "access_token": "judge-access-token-secret", + "client_secret": "judge-client-secret", + "password": "judge-password-secret", + "provider_endpoint_url": "https://judge-alt.example.test/v1", + } + request.optimizer_config_path.write_text(json.dumps(template), encoding="utf-8") + + await AgentOptimizerCandidateProvider(lambda query: query).propose(request) + + runtime_path = Path(str(captured["config_path"])) + runtime = json.loads(runtime_path.read_text(encoding="utf-8")) + reflection = runtime["optimize"]["algorithm"]["reflection_lm"] + assert runtime_path == Path(prepared.workspace.run_dir) / "optimizer.runtime.json" + assert reflection == { + "provider_name": "openai", + "model_name": "mimo-v2.5", + "variant": "chat_completions", + "base_url": "${TRPC_AGENT_BASE_URL}", + "api_key": "${TRPC_AGENT_API_KEY}", + "generation_config": {"temperature": 0.25, "max_tokens": 1024}, + "think": True, + } + assert runtime["optimize"]["algorithm"]["max_candidate_proposals"] == 1 + assert runtime["evaluate"]["credential_probe"] == { + "api_key": "${TRPC_AGENT_API_KEY}", + "base_url": "${TRPC_AGENT_BASE_URL}", + "authorization": "${TRPC_AGENT_API_KEY}", + "access_token": "${TRPC_AGENT_API_KEY}", + "client_secret": "${TRPC_AGENT_API_KEY}", + "password": "${TRPC_AGENT_API_KEY}", + "provider_endpoint_url": "${TRPC_AGENT_BASE_URL}", + } + persisted_texts = [ + runtime_path.read_text(encoding="utf-8"), + (request.output_dir / "config.snapshot.json").read_text(encoding="utf-8"), + ] + for secret in ( + "judge-secret", + "judge-authorization-secret", + "judge-access-token-secret", + "judge-client-secret", + "judge-password-secret", + "https://judge-alt.example.test/v1", + ): + assert all(secret not in text for text in persisted_texts) + assert original_config != request.optimizer_config_path.read_text(encoding="utf-8") + + +@pytest.mark.asyncio +async def test_run_checks_source_prompt_even_when_pipeline_fails(monkeypatch: pytest.MonkeyPatch): + class SourceTarget: + reads = 0 + + async def read_all(self): + self.reads += 1 + return {"system_prompt": "before" if self.reads == 1 else "drifted"} + + prepared = SimpleNamespace( + source_target=SourceTarget(), + working_target=object(), + ) + + monkeypatch.setattr(run_real_integration, "prepare_run", lambda *args, **kwargs: prepared) + monkeypatch.setattr(run_real_integration, "RealBusinessAgent", lambda *args, **kwargs: object()) + + async def fail_stage(*args, **kwargs): + raise RuntimeError("pipeline failed") + + monkeypatch.setattr(run_real_integration, "run_real_stage", fail_stage) + args = SimpleNamespace(config=Path("pipeline.real.json"), run_id="failure") + + with pytest.raises(RuntimeError, match="source Prompt changed"): + await run_real_integration._run( + args, + BusinessModelConfig(api_key="key", base_url="url", model_name="model"), + OptimizerRuntimeParameters(model_name="model"), + ) + + +def test_real_cli_complete_gate_reject_still_exits_zero( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +): + snapshot = SimpleNamespace(passed_case_count=1, total_case_count=3, average_score=1 / 3) + result = SimpleNamespace( + baseline_train=snapshot, + baseline_validation=snapshot, + candidate_train=snapshot, + candidate_validation=snapshot, + optimize_result=SimpleNamespace(status="SUCCEEDED", total_rounds=1), + candidate=SimpleNamespace(candidate_id="optimizer-123456789abc"), + gate_decision=SimpleNamespace(decision="reject", rejection_reasons=["no improvement"]), + writeback=SimpleNamespace(status="skipped", reason="gate_rejected"), + ) + prepared = SimpleNamespace(workspace=SimpleNamespace(run_dir="runs/offline")) + + async def complete(*args, **kwargs): + return prepared, result + + monkeypatch.setenv("TRPC_AGENT_API_KEY", "key") + monkeypatch.setenv("TRPC_AGENT_BASE_URL", "https://example.test") + monkeypatch.setenv("TRPC_AGENT_MODEL_NAME", "business-model") + monkeypatch.setattr(run_real_integration, "_run", complete) + monkeypatch.setattr( + sys, + "argv", + ["run_real_integration.py", "--run-real", "--optimizer-model-name", "optimizer-model"], + ) + + assert run_real_integration.main() == 0 + output = capsys.readouterr().out + assert "Gate decision: REJECT" in output + assert "Baseline train" in output + assert "Baseline validation" in output + assert "Candidate train" in output + assert "Candidate validation" in output + + +def test_real_cli_pipeline_error_exits_one( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +): + async def fail(*args, **kwargs): + raise RuntimeError("offline failure") + + monkeypatch.setenv("TRPC_AGENT_API_KEY", "key") + monkeypatch.setenv("TRPC_AGENT_BASE_URL", "https://example.test") + monkeypatch.setenv("TRPC_AGENT_MODEL_NAME", "business-model") + monkeypatch.setattr(run_real_integration, "_run", fail) + monkeypatch.setattr( + sys, + "argv", + ["run_real_integration.py", "--run-real", "--optimizer-model-name", "optimizer-model"], + ) + + assert run_real_integration.main() == 1 + assert "offline failure" in capsys.readouterr().err + + +def test_real_cli_redacts_environment_secrets_from_pipeline_error( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +): + api_key = "integration-secret-key" + base_url = "https://private-gateway.example.test/v1" + + async def fail(*args, **kwargs): + raise RuntimeError( + f"request failed api_key={api_key} base_url={base_url}" + ) + + monkeypatch.setenv("TRPC_AGENT_API_KEY", api_key) + monkeypatch.setenv("TRPC_AGENT_BASE_URL", base_url) + monkeypatch.setenv("TRPC_AGENT_MODEL_NAME", "business-model") + monkeypatch.setattr(run_real_integration, "_run", fail) + monkeypatch.setattr( + sys, + "argv", + [ + "run_real_integration.py", + "--run-real", + "--optimizer-model-name", + "optimizer-model", + ], + ) + + assert run_real_integration.main() == 1 + error = capsys.readouterr().err + assert api_key not in error + assert base_url not in error + assert "[REDACTED]" in error + + +def test_real_cli_requires_explicit_confirmation_before_creating_workspace(tmp_path: Path): + root = _copy_example(tmp_path) + completed = subprocess.run( + [ + sys.executable, + str(root / "run_real_integration.py"), + "--config", + str(root / "pipeline.real.json"), + "--optimizer-model-name", + "mimo-v2.5", + "--run-id", + "must_not_exist", + ], + cwd=_REPO_ROOT, + capture_output=True, + text=True, + check=False, + ) + + assert completed.returncode == 2 + assert "--run-real" in completed.stderr + assert not (root / "runs" / "must_not_exist").exists() + + +def test_real_cli_requires_business_model_environment_before_creating_workspace( + tmp_path: Path, +): + root = _copy_example(tmp_path) + env = { + key: value + for key, value in __import__("os").environ.items() + if key not in {"TRPC_AGENT_API_KEY", "TRPC_AGENT_BASE_URL", "TRPC_AGENT_MODEL_NAME"} + } + completed = subprocess.run( + [ + sys.executable, + str(root / "run_real_integration.py"), + "--run-real", + "--config", + str(root / "pipeline.real.json"), + "--optimizer-model-name", + "mimo-v2.5", + "--run-id", + "missing_env", + ], + cwd=_REPO_ROOT, + env=env, + capture_output=True, + text=True, + check=False, + ) + + assert completed.returncode == 2 + assert "TRPC_AGENT_API_KEY" in completed.stderr + assert "TRPC_AGENT_BASE_URL" in completed.stderr + assert "TRPC_AGENT_MODEL_NAME" in completed.stderr + assert not (root / "runs" / "missing_env").exists() diff --git a/tests/evaluation/test_eval_optimize_loop_stage1.py b/tests/evaluation/test_eval_optimize_loop_stage1.py new file mode 100644 index 00000000..bd6f8df0 --- /dev/null +++ b/tests/evaluation/test_eval_optimize_loop_stage1.py @@ -0,0 +1,261 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under the Apache License, Version 2.0. +"""Offline tests for stage-one eval/optimization pipeline preparation.""" + +from __future__ import annotations + +import json +import shutil +from pathlib import Path + +import pytest +from pydantic import ValidationError + +from examples.optimization.eval_optimize_loop.config import PipelineConfig +from examples.optimization.eval_optimize_loop.config import load_pipeline_config +from examples.optimization.eval_optimize_loop.pipeline import PipelinePreparationError +from examples.optimization.eval_optimize_loop.pipeline import prepare_run +from examples.optimization.eval_optimize_loop.prompt_workspace import SourcePromptDriftError +from examples.optimization.eval_optimize_loop.prompt_workspace import verify_source_hashes +from examples.optimization.eval_optimize_loop.schemas import ObservableValue + + +_REPO_ROOT = Path(__file__).resolve().parents[2] +_EXAMPLE = _REPO_ROOT / "examples" / "optimization" / "eval_optimize_loop" + + +def _copy_example(tmp_path: Path) -> Path: + target = tmp_path / "eval_optimize_loop" + shutil.copytree(_EXAMPLE, target, ignore=shutil.ignore_patterns("runs", "__pycache__")) + return target + + +def _read_config(root: Path) -> dict: + return json.loads((root / "pipeline.json").read_text(encoding="utf-8")) + + +def _write_config(root: Path, payload: dict) -> Path: + path = root / "pipeline.json" + path.write_text(json.dumps(payload), encoding="utf-8") + return path + + +def _write_evalset(root: Path, relative: str, eval_set_id: str, case_ids: list[str]) -> None: + payload = { + "eval_set_id": eval_set_id, + "eval_cases": [ + { + "eval_id": case_id, + "conversation": [ + { + "invocation_id": case_id, + "user_content": {"parts": [{"text": "input"}], "role": "user"}, + "final_response": {"parts": [{"text": "output"}], "role": "model"}, + } + ], + } + for case_id in case_ids + ], + } + (root / relative).write_text(json.dumps(payload), encoding="utf-8") + + +def test_pipeline_config_loads_complete_example_and_camel_case(tmp_path: Path): + root = _copy_example(tmp_path) + config = load_pipeline_config(root / "pipeline.json") + assert config.config_version == 1 + assert config.execution.mode == "offline" + assert config.prompts[0].name == "system_prompt" + assert config.gate.required_metrics == ["final_response_avg_score"] + + payload = _read_config(root) + payload["configVersion"] = payload.pop("config_version") + payload["caseLabels"] = payload.pop("case_labels") + payload["caseLabels"]["hardCaseIds"] = payload["caseLabels"].pop("hard_case_ids") + camel_path = root / "camel.json" + camel_path.write_text(json.dumps(payload), encoding="utf-8") + camel_config = load_pipeline_config(camel_path) + assert camel_config.case_labels.hard_case_ids == ["val_knowledge_recall"] + + +def test_pipeline_config_rejects_legacy_fake_mode_with_migration_message( + tmp_path: Path, +): + root = _copy_example(tmp_path) + payload = _read_config(root) + payload["execution"]["mode"] = "fake" + + with pytest.raises(ValidationError, match="renamed to 'offline'"): + PipelineConfig.model_validate(payload) + + +def test_pipeline_config_rejects_removed_fake_judge_option(tmp_path: Path): + root = _copy_example(tmp_path) + payload = _read_config(root) + payload["execution"]["use_fake_judge"] = True + + with pytest.raises(ValidationError, match="evaluation metrics or rubric"): + PipelineConfig.model_validate(payload) + + +@pytest.mark.parametrize( + "mutate", + [ + lambda payload: payload.update({"unexpected": True}), + lambda payload: payload["execution"].update({"mode": "maybe"}), + lambda payload: payload["gate"].update({"min_validation_score_delta": -0.01}), + lambda payload: payload["gate"].update({"severe_case_score_drop": -0.1}), + lambda payload: payload["gate"].update({"severe_case_score_drop": 1.1}), + lambda payload: payload["budget"].update({"max_tokens": -1}), + lambda payload: payload["reporting"].update({"write_json": False, "write_markdown": False}), + lambda payload: payload.update({"prompts": [{"name": "system_prompt", "callback": "x"}]}), + ], +) +def test_pipeline_config_rejects_invalid_values(tmp_path: Path, mutate): + root = _copy_example(tmp_path) + payload = _read_config(root) + mutate(payload) + with pytest.raises(ValidationError): + PipelineConfig.model_validate(payload) + + +def test_pipeline_config_rejects_duplicate_prompt_names(tmp_path: Path): + root = _copy_example(tmp_path) + payload = _read_config(root) + payload["prompts"].append({"name": "system_prompt", "path": "prompts/system.md"}) + with pytest.raises(ValidationError, match="duplicate field names"): + PipelineConfig.model_validate(payload) + + +def test_observable_value_never_treats_unavailable_as_zero(): + unavailable = ObservableValue(status="unavailable", reason="SDK does not expose cost") + assert unavailable.value is None + with pytest.raises(ValidationError): + ObservableValue(status="unavailable", value=0) + with pytest.raises(ValidationError): + ObservableValue(status="available") + + +@pytest.mark.asyncio +async def test_prepare_run_creates_isolated_prompt_workspace(tmp_path: Path): + root = _copy_example(tmp_path) + source = root / "prompts" / "system.md" + baseline = source.read_text(encoding="utf-8") + + prepared = prepare_run(root / "pipeline.json", run_id="stage1_test") + + assert Path(prepared.workspace.run_dir).is_dir() + assert prepared.source_target.names() == ["system_prompt"] + assert prepared.working_target.names() == ["system_prompt"] + source_values = await prepared.source_target.read_all() + working_values = await prepared.working_target.read_all() + assert source_values == working_values == {"system_prompt": baseline} + snapshot = prepared.input_snapshot.prompt_snapshots[0] + assert snapshot.source_path == str(source.resolve()) + assert snapshot.working_path != snapshot.source_path + assert len(snapshot.sha256) == 64 + assert len(prepared.input_snapshot.pipeline_config_sha256) == 64 + assert len(prepared.input_snapshot.optimizer_config_sha256) == 64 + assert len(prepared.input_snapshot.train_evalset_sha256) == 64 + assert len(prepared.input_snapshot.validation_evalset_sha256) == 64 + verify_source_hashes(prepared.input_snapshot.prompt_snapshots) + + await prepared.working_target.write_all({"system_prompt": "candidate"}) + assert source.read_text(encoding="utf-8") == baseline + assert await prepared.working_target.read_all() == {"system_prompt": "candidate"} + + source.write_text("concurrent source edit", encoding="utf-8") + with pytest.raises(SourcePromptDriftError, match="system_prompt"): + verify_source_hashes(prepared.input_snapshot.prompt_snapshots) + + +def test_prepare_run_does_not_overwrite_existing_run(tmp_path: Path): + root = _copy_example(tmp_path) + prepare_run(root / "pipeline.json", run_id="same_run") + with pytest.raises(FileExistsError, match="already exists"): + prepare_run(root / "pipeline.json", run_id="same_run") + + +@pytest.mark.parametrize("label_key", ["hard_case_ids", "critical_case_ids"]) +def test_prepare_run_rejects_unknown_case_labels(tmp_path: Path, label_key: str): + root = _copy_example(tmp_path) + payload = _read_config(root) + payload["case_labels"][label_key] = ["missing_case"] + with pytest.raises(PipelinePreparationError, match="unknown eval_id"): + prepare_run(_write_config(root, payload), run_id="bad_labels") + assert not (root / "runs").exists() + + +def test_prepare_run_rejects_shared_dataset_path(tmp_path: Path): + root = _copy_example(tmp_path) + payload = _read_config(root) + payload["inputs"]["validation_evalset"] = payload["inputs"]["train_evalset"] + with pytest.raises(PipelinePreparationError, match="different files"): + prepare_run(_write_config(root, payload), run_id="shared_data") + + +def test_prepare_run_rejects_duplicate_or_overlapping_eval_ids(tmp_path: Path): + root = _copy_example(tmp_path) + _write_evalset(root, "data/train.evalset.json", "train", ["same", "same"]) + with pytest.raises(PipelinePreparationError, match="duplicate eval_id"): + prepare_run(root / "pipeline.json", run_id="duplicate_case") + + _write_evalset(root, "data/train.evalset.json", "train", ["same"]) + _write_evalset(root, "data/val.evalset.json", "val", ["same"]) + payload = _read_config(root) + payload["case_labels"] = {} + with pytest.raises(PipelinePreparationError, match="must not share eval_id"): + prepare_run(_write_config(root, payload), run_id="overlap_case") + + +def test_prepare_run_rejects_unknown_required_metric(tmp_path: Path): + root = _copy_example(tmp_path) + payload = _read_config(root) + payload["gate"]["required_metrics"] = ["does_not_exist"] + with pytest.raises(PipelinePreparationError, match="unknown metrics"): + prepare_run(_write_config(root, payload), run_id="bad_metric") + + +def test_prepare_run_rejects_path_escape_and_non_utf8_prompt(tmp_path: Path): + root = _copy_example(tmp_path) + outside = tmp_path / "outside.md" + outside.write_text("secret", encoding="utf-8") + payload = _read_config(root) + payload["prompts"][0]["path"] = "../outside.md" + with pytest.raises(PipelinePreparationError, match="escapes the example root"): + prepare_run(_write_config(root, payload), run_id="escape") + + payload["prompts"][0]["path"] = "prompts/binary.md" + (root / "prompts" / "binary.md").write_bytes(b"\xff\xfe") + with pytest.raises(PipelinePreparationError, match="not UTF-8"): + prepare_run(_write_config(root, payload), run_id="binary") + + +def test_prepare_run_rejects_duplicate_prompt_paths(tmp_path: Path): + root = _copy_example(tmp_path) + payload = _read_config(root) + payload["prompts"].append({"name": "router_prompt", "path": "prompts/system.md"}) + with pytest.raises(PipelinePreparationError, match="multiple prompt fields"): + prepare_run(_write_config(root, payload), run_id="duplicate_path") + + +def test_prepare_run_removes_staging_workspace_on_failure(tmp_path: Path, monkeypatch): + root = _copy_example(tmp_path) + + def fail_staging(**kwargs): + (kwargs["staging_run_dir"] / "workspace").mkdir() + raise RuntimeError("injected staging failure") + + monkeypatch.setattr( + "examples.optimization.eval_optimize_loop.pipeline.stage_prompt_workspace", + fail_staging, + ) + with pytest.raises(RuntimeError, match="injected"): + prepare_run(root / "pipeline.json", run_id="failed_run") + + runs_dir = root / "runs" + assert not (runs_dir / "failed_run").exists() + assert list(runs_dir.iterdir()) == [] diff --git a/tests/evaluation/test_eval_optimize_loop_stage2.py b/tests/evaluation/test_eval_optimize_loop_stage2.py new file mode 100644 index 00000000..383f7a87 --- /dev/null +++ b/tests/evaluation/test_eval_optimize_loop_stage2.py @@ -0,0 +1,331 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under the Apache License, Version 2.0. +"""Offline tests for the deterministic stage-two eval/optimization loop.""" + +from __future__ import annotations + +import asyncio +import json +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest + +from examples.optimization.eval_optimize_loop.business_agent import BusinessAgent +from examples.optimization.eval_optimize_loop.fake import DeterministicFakeModel +from examples.optimization.eval_optimize_loop.fake import DeterministicFakeCandidateProvider +from examples.optimization.eval_optimize_loop.pipeline import PipelineStageExecutionError +from examples.optimization.eval_optimize_loop.pipeline import prepare_run +from examples.optimization.eval_optimize_loop.pipeline import run_offline_stage +from examples.optimization.eval_optimize_loop.schemas import OfflineStageResult +from trpc_agent_sdk.evaluation import EvalStatus +from trpc_agent_sdk.evaluation import TargetPrompt + + +_REPO_ROOT = Path(__file__).resolve().parents[2] +_EXAMPLE = _REPO_ROOT / "examples" / "optimization" / "eval_optimize_loop" + + +def _offline_agent(target: TargetPrompt) -> BusinessAgent: + return BusinessAgent( + target, + DeterministicFakeModel, + agent_name="stage2_offline_agent", + app_name="stage2_offline_test", + user_id="stage2-test", + ) + + +def _copy_example(tmp_path: Path, name: str = "eval_optimize_loop") -> Path: + target = tmp_path / name + shutil.copytree(_EXAMPLE, target, ignore=shutil.ignore_patterns("runs", "__pycache__")) + return target + + +def _read_json(path: Path) -> dict: + return json.loads(path.read_text(encoding="utf-8")) + + +def _write_json(path: Path, payload: dict) -> None: + path.write_text(json.dumps(payload), encoding="utf-8") + + +def _target_for_prompt(path: Path, content: str) -> TargetPrompt: + path.write_text(content, encoding="utf-8") + return TargetPrompt().add_path("system_prompt", str(path)) + + +def test_fake_candidate_provider_is_pure_deterministic_and_preserves_fields(): + provider = DeterministicFakeCandidateProvider() + current = {"system_prompt": "baseline", "skill": "unchanged"} + original = dict(current) + + first = provider.propose(current, scenario="improve", seed=42) + second = provider.propose(current, scenario="improve", seed=42) + + assert first == second + assert current == original + assert first.changed_fields == ["system_prompt"] + assert first.prompts["skill"] == "unchanged" + assert first.parent_prompt_sha256 != first.candidate_prompt_sha256 + assert first.candidate_id == f"fake-improve-{first.candidate_prompt_sha256[:12]}" + + +def test_fake_candidate_provider_rejects_missing_field_and_unknown_scenario(): + provider = DeterministicFakeCandidateProvider() + with pytest.raises(ValueError, match="target field is missing"): + provider.propose({"skill": "x"}, scenario="improve", seed=42) + with pytest.raises(ValueError, match="unknown fake candidate scenario"): + provider.propose( + {"system_prompt": "x"}, + scenario="unknown", # type: ignore[arg-type] + seed=42, + ) + + +def test_overfit_candidate_contains_rules_not_eval_ids_or_full_examples(): + proposal = DeterministicFakeCandidateProvider().propose( + {"system_prompt": "baseline"}, + scenario="overfit", + seed=42, + ) + prompt = proposal.prompts["system_prompt"] + for forbidden in ( + "train_output_format", + "train_tool_choice", + "train_tool_arguments", + "How can I update my email address?", + "A100", + "B-204", + ): + assert forbidden not in prompt + assert "account_terms=email" in prompt + assert "order_lookup=true" in prompt + assert "refund_route=false" in prompt + + +@pytest.mark.asyncio +async def test_fake_agent_rereads_prompt_and_is_deterministic(tmp_path: Path): + target = _target_for_prompt(tmp_path / "system.md", "baseline") + agent = _offline_agent(target) + email_query = "How can I update my email address?" + order_query = "Check the status of order A100." + + assert await agent.call_agent(email_query) == ( + '{"route":"account","message":"Open profile settings to update your email."}' + ) + baseline_order = await agent.call_agent(order_query) + assert '"route":"general_support"' in baseline_order + + proposal = DeterministicFakeCandidateProvider().propose( + await target.read_all(), + scenario="improve", + seed=42, + ) + await target.write_all(proposal.prompts) + expected_order = '{"route":"order_lookup","message":"Checking order A100."}' + assert await agent.call_agent(order_query) == expected_order + assert await asyncio.gather(*(agent.call_agent(order_query) for _ in range(8))) == [expected_order] * 8 + + +@pytest.mark.asyncio +async def test_fake_agent_unknown_query_has_stable_fallback(tmp_path: Path): + target = _target_for_prompt(tmp_path / "system.md", "baseline") + agent = _offline_agent(target) + expected = ( + '{"route":"general_support","message":' + '"Please provide more details so I can route your request."}' + ) + assert await agent.call_agent("Tell me a joke") == expected + assert await agent.call_agent(" Tell me a JOKE ") == expected + with pytest.raises(TypeError, match="query must be a string"): + await agent.call_agent(None) # type: ignore[arg-type] + + +@pytest.mark.parametrize( + ("scenario", "candidate_train_passed", "candidate_validation_passed"), + [ + ("improve", 3, 3), + ("no_improvement", 1, 1), + ("overfit", 3, 0), + ], +) +@pytest.mark.asyncio +async def test_fake_stage_scenario_matrix( + tmp_path: Path, + scenario: str, + candidate_train_passed: int, + candidate_validation_passed: int, +): + root = _copy_example(tmp_path, scenario) + source = root / "prompts" / "system.md" + baseline_source = source.read_text(encoding="utf-8") + prepared = prepare_run(root / "pipeline.json", run_id=f"stage2_{scenario}") + + result = await run_offline_stage(prepared, scenario=scenario) # type: ignore[arg-type] + + assert result.baseline_train.passed_case_count == 1 + assert result.baseline_validation.passed_case_count == 1 + assert result.candidate_train.passed_case_count == candidate_train_passed + assert result.candidate_validation.passed_case_count == candidate_validation_passed + assert result.baseline_train.average_score == pytest.approx(1 / 3) + assert result.baseline_validation.average_score == pytest.approx(1 / 3) + assert len(result.baseline_train.eval_results_by_eval_id) == 3 + assert len(result.baseline_validation.eval_results_by_eval_id) == 3 + assert result.baseline_train.failed_summary is not None + assert result.baseline_validation.failed_summary is not None + assert source.read_text(encoding="utf-8") == baseline_source + assert await prepared.working_target.read_all() == result.candidate.prompts + + if scenario == "improve": + assert result.candidate_train.failed_summary is None + assert result.candidate_validation.failed_summary is None + restored = OfflineStageResult.model_validate_json(result.model_dump_json()) + assert restored.candidate.candidate_id == result.candidate.candidate_id + elif scenario == "overfit": + baseline_refund = result.baseline_validation.eval_results_by_eval_id["val_refund_route"][0] + candidate_refund = result.candidate_validation.eval_results_by_eval_id["val_refund_route"][0] + assert baseline_refund.final_eval_status == EvalStatus.PASSED + assert candidate_refund.final_eval_status == EvalStatus.FAILED + + +@pytest.mark.asyncio +async def test_fake_stage_keeps_all_configured_runs(tmp_path: Path): + root = _copy_example(tmp_path) + optimizer_path = root / "optimizer.json" + optimizer = _read_json(optimizer_path) + optimizer["evaluate"]["num_runs"] = 2 + _write_json(optimizer_path, optimizer) + prepared = prepare_run(root / "pipeline.json", run_id="two_runs") + + result = await run_offline_stage(prepared, scenario="improve") + + for snapshot in ( + result.baseline_train, + result.baseline_validation, + result.candidate_train, + result.candidate_validation, + ): + assert all(len(runs) == 2 for runs in snapshot.eval_results_by_eval_id.values()) + + +@pytest.mark.asyncio +async def test_offline_stage_rejects_real_execution_mode(tmp_path: Path): + mode = "real" + root = _copy_example(tmp_path, mode) + config_path = root / "pipeline.json" + config = _read_json(config_path) + config["execution"]["mode"] = mode + _write_json(config_path, config) + prepared = prepare_run(config_path, run_id=f"mode_{mode}") + + with pytest.raises( + PipelineStageExecutionError, + match=rf"requires execution.mode='offline', got '{mode}'", + ): + await run_offline_stage(prepared) + + +@pytest.mark.parametrize( + ("relative_path", "expected_label"), + [ + ("data/train.evalset.json", "train_evalset"), + ("data/val.evalset.json", "validation_evalset"), + ], +) +@pytest.mark.asyncio +async def test_fake_stage_rejects_evalset_changes_after_prepare_run( + tmp_path: Path, + relative_path: str, + expected_label: str, +): + root = _copy_example(tmp_path, expected_label) + prepared = prepare_run(root / "pipeline.json", run_id=f"changed_{expected_label}") + evalset_path = root / relative_path + evalset = _read_json(evalset_path) + evalset["description"] = "changed after prepare_run" + _write_json(evalset_path, evalset) + + with pytest.raises( + PipelineStageExecutionError, + match=rf"{expected_label} changed after prepare_run", + ): + await run_offline_stage(prepared) + + +@pytest.mark.asyncio +async def test_fake_stage_wraps_baseline_evaluator_errors(tmp_path: Path, monkeypatch): + root = _copy_example(tmp_path) + prepared = prepare_run(root / "pipeline.json", run_id="evaluator_error") + + async def fail_evaluation(*_args, **_kwargs): + raise RuntimeError("injected evaluator failure") + + monkeypatch.setattr( + "examples.optimization.eval_optimize_loop.pipeline.AgentEvaluator.evaluate_eval_set", + fail_evaluation, + ) + with pytest.raises(PipelineStageExecutionError, match="baseline train evaluation failed"): + await run_offline_stage(prepared) + assert await prepared.working_target.read_all() == { + snapshot.field_name: snapshot.content for snapshot in prepared.input_snapshot.prompt_snapshots + } + + +@pytest.mark.asyncio +async def test_candidate_evaluation_error_retains_working_candidate(tmp_path: Path, monkeypatch): + root = _copy_example(tmp_path) + source = root / "prompts" / "system.md" + baseline_source = source.read_text(encoding="utf-8") + prepared = prepare_run(root / "pipeline.json", run_id="candidate_error") + + from examples.optimization.eval_optimize_loop import pipeline as pipeline_module + + original = pipeline_module.AgentEvaluator.evaluate_eval_set + calls = 0 + + async def fail_candidate_train(*args, **kwargs): + nonlocal calls + calls += 1 + if calls == 3: + raise RuntimeError("injected candidate failure") + return await original(*args, **kwargs) + + monkeypatch.setattr(pipeline_module.AgentEvaluator, "evaluate_eval_set", fail_candidate_train) + with pytest.raises(PipelineStageExecutionError, match="candidate train evaluation failed"): + await run_offline_stage(prepared, scenario="improve") + + working = await prepared.working_target.read_all() + assert "deterministic-fake-candidate:start" in working["system_prompt"] + assert source.read_text(encoding="utf-8") == baseline_source + + +def test_stage_two_cli_smoke(tmp_path: Path): + root = _copy_example(tmp_path) + completed = subprocess.run( + [ + sys.executable, + str(_EXAMPLE / "run_pipeline.py"), + "--config", + str(root / "pipeline.json"), + "--run-id", + "cli_smoke", + "--scenario", + "overfit", + ], + cwd=_REPO_ROOT, + check=True, + capture_output=True, + text=True, + timeout=180, + ) + assert "Baseline train: 1/3 passed" in completed.stdout + assert "Baseline validation: 1/3 passed" in completed.stdout + assert "Candidate train: 3/3 passed" in completed.stdout + assert "Candidate validation: 0/3 passed" in completed.stdout + assert "Gate decision: REJECT" in completed.stdout diff --git a/tests/evaluation/test_eval_optimize_loop_stage3a.py b/tests/evaluation/test_eval_optimize_loop_stage3a.py new file mode 100644 index 00000000..a9680c75 --- /dev/null +++ b/tests/evaluation/test_eval_optimize_loop_stage3a.py @@ -0,0 +1,593 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under the Apache License, Version 2.0. +"""Tests for stage 3a evaluation normalization, attribution, and case diff.""" + +from __future__ import annotations + +import shutil +from pathlib import Path + +import pytest + +from examples.optimization.eval_optimize_loop import pipeline as pipeline_module +from examples.optimization.eval_optimize_loop.attribution import attribute_evaluation +from examples.optimization.eval_optimize_loop.case_diff import compare_evaluations +from examples.optimization.eval_optimize_loop.evaluation_adapter import EvaluationAnalysisError +from examples.optimization.eval_optimize_loop.evaluation_adapter import standardize_snapshot +from examples.optimization.eval_optimize_loop.pipeline import prepare_run +from examples.optimization.eval_optimize_loop.pipeline import run_offline_stage +from examples.optimization.eval_optimize_loop.schemas import EvaluationSnapshot +from examples.optimization.eval_optimize_loop.schemas import OfflineStageResult +from trpc_agent_sdk.evaluation import EvalCaseResult +from trpc_agent_sdk.evaluation import EvalMetricResult +from trpc_agent_sdk.evaluation import EvalMetricResultDetails +from trpc_agent_sdk.evaluation import EvalMetricResultPerInvocation +from trpc_agent_sdk.evaluation import EvalStatus +from trpc_agent_sdk.evaluation import IntermediateData +from trpc_agent_sdk.evaluation import Invocation +from trpc_agent_sdk.types import Content +from trpc_agent_sdk.types import FunctionCall +from trpc_agent_sdk.types import Part + + +_REPO_ROOT = Path(__file__).resolve().parents[2] +_EXAMPLE = _REPO_ROOT / "examples" / "optimization" / "eval_optimize_loop" + + +def _copy_example(tmp_path: Path, name: str) -> Path: + target = tmp_path / name + shutil.copytree(_EXAMPLE, target, ignore=shutil.ignore_patterns("runs", "__pycache__")) + return target + + +def _content(text: str) -> Content: + return Content(parts=[Part(text=text)]) + + +def _metric( + *, + score: float | None, + status: EvalStatus, + name: str = "final_response_avg_score", + threshold: float = 1.0, + reason: str | None = None, +) -> EvalMetricResult: + return EvalMetricResult( + metric_name=name, + threshold=threshold, + score=score, + eval_status=status, + details=EvalMetricResultDetails(reason=reason, score=score) if reason else None, + ) + + +def _case_run( + *, + run_id: int, + status: EvalStatus, + score: float | None, + actual_text: str = '{"route":"general_support"}', + expected_text: str = '{"route":"order_lookup"}', + actual_tools: list[FunctionCall] | None = None, + expected_tools: list[FunctionCall] | None = None, + error_message: str | None = None, + metrics: list[EvalMetricResult] | None = None, +) -> EvalCaseResult: + actual = Invocation( + invocation_id="inv-1", + user_content=_content("Check order A100"), + final_response=_content(actual_text), + intermediate_data=IntermediateData(tool_uses=actual_tools or []), + ) + expected = Invocation( + invocation_id="inv-1", + user_content=_content("Check order A100"), + final_response=_content(expected_text), + intermediate_data=IntermediateData(tool_uses=expected_tools or []), + ) + metric_results = metrics or [_metric(score=score, status=status)] + return EvalCaseResult( + eval_set_id="train-set", + eval_id="case-order", + run_id=run_id, + final_eval_status=status, + error_message=error_message, + overall_eval_metric_results=metric_results, + eval_metric_result_per_invocation=[ + EvalMetricResultPerInvocation( + actual_invocation=actual, + expected_invocation=expected, + eval_metric_results=metric_results, + ) + ], + session_id=f"session-{run_id}", + ) + + +def _snapshot(*runs: EvalCaseResult) -> EvaluationSnapshot: + return EvaluationSnapshot( + phase="baseline", + split="train", + eval_set_id="train-set", + details_lines=[], + result_lines=[], + eval_results_by_eval_id={"case-order": list(runs)}, + passed_case_count=0, + total_case_count=1, + average_score=0.5, + ) + + +def test_standardize_snapshot_preserves_multirun_metric_and_invocation_evidence(): + expected_tool = FunctionCall(name="lookup_order", args={"order_id": "A100"}) + actual_tool = FunctionCall(name="general_support", args={}) + snapshot = _snapshot( + _case_run( + run_id=2, + status=EvalStatus.FAILED, + score=0.0, + actual_tools=[actual_tool], + expected_tools=[expected_tool], + ), + _case_run( + run_id=1, + status=EvalStatus.PASSED, + score=1.0, + actual_text='{"route":"order_lookup"}', + actual_tools=[expected_tool], + expected_tools=[expected_tool], + ), + ) + + standardized = standardize_snapshot(snapshot) + + case = standardized.cases[0] + assert case.eval_id == "case-order" + assert case.status == "failed" + assert case.average_score.status == "available" + assert case.average_score.value == pytest.approx(0.5) + assert case.metrics[0].metric_name == "final_response_avg_score" + assert case.metrics[0].score.value == pytest.approx(0.5) + assert [run.run_id for run in case.runs] == [1, 2] + failed_evidence = case.runs[1].invocations[0] + assert failed_evidence.user_text == "Check order A100" + assert failed_evidence.actual_response == '{"route":"general_support"}' + assert failed_evidence.expected_response == '{"route":"order_lookup"}' + assert failed_evidence.actual_tools[0].name == "general_support" + assert failed_evidence.expected_tools[0].arguments == {"order_id": "A100"} + assert standardized.average_score.value == pytest.approx(0.5) + + +def test_standardize_snapshot_keeps_missing_scores_unavailable_instead_of_zero(): + standardized = standardize_snapshot( + _snapshot( + _case_run( + run_id=1, + status=EvalStatus.NOT_EVALUATED, + score=None, + error_message="metric failed", + ) + ) + ) + + case = standardized.cases[0] + assert case.status == "not_evaluated" + assert case.average_score.status == "unavailable" + assert case.average_score.value is None + assert case.metrics[0].score.status == "unavailable" + assert standardized.average_score.status == "unavailable" + assert standardized.not_evaluated_case_count == 1 + + +def test_standardize_snapshot_treats_failed_metric_without_score_as_not_evaluated(): + standardized = standardize_snapshot( + _snapshot( + _case_run( + run_id=1, + status=EvalStatus.FAILED, + score=None, + ) + ) + ) + + case = standardized.cases[0] + assert case.status == "not_evaluated" + assert case.runs[0].status == "not_evaluated" + assert case.runs[0].metrics[0].status == "not_evaluated" + + +@pytest.mark.asyncio +async def test_evaluation_snapshot_uses_standardized_pass_count( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + root = _copy_example(tmp_path, "standardized_snapshot_summary") + prepared = prepare_run(root / "pipeline.json", run_id="standardized_snapshot_summary") + eval_set = pipeline_module._reload_prepared_evalset( + Path(prepared.input_snapshot.train_evalset_path), + label="train_evalset", + expected_sha256=prepared.input_snapshot.train_evalset_sha256, + ) + results: dict[str, list[EvalCaseResult]] = {} + for index, case in enumerate(eval_set.eval_cases): + score = None if index == 0 else 1.0 + run = _case_run( + run_id=1, + status=EvalStatus.PASSED, + score=score, + ).model_copy( + update={"eval_set_id": eval_set.eval_set_id, "eval_id": case.eval_id} + ) + results[case.eval_id] = [run] + + async def fake_evaluate_eval_set(*args, **kwargs): + return None, [], [], results + + monkeypatch.setattr( + pipeline_module.AgentEvaluator, + "evaluate_eval_set", + fake_evaluate_eval_set, + ) + + snapshot = await pipeline_module._evaluate_split( + prepared=prepared, + eval_set=eval_set, + call_agent=None, + phase="baseline", + split="train", + ) + standardized = standardize_snapshot(snapshot) + + assert snapshot.passed_case_count == len(eval_set.eval_cases) - 1 + assert snapshot.passed_case_count == standardized.passed_case_count + + +def test_standardize_snapshot_rejects_duplicate_run_ids_and_metric_threshold_drift(): + duplicate_runs = _snapshot( + _case_run(run_id=1, status=EvalStatus.PASSED, score=1.0), + _case_run(run_id=1, status=EvalStatus.PASSED, score=1.0), + ) + with pytest.raises(EvaluationAnalysisError, match="duplicate run ids"): + standardize_snapshot(duplicate_runs) + + threshold_drift = _snapshot( + _case_run(run_id=1, status=EvalStatus.PASSED, score=1.0), + _case_run( + run_id=2, + status=EvalStatus.PASSED, + score=1.0, + metrics=[_metric(score=1.0, status=EvalStatus.PASSED, threshold=0.5)], + ), + ) + with pytest.raises(EvaluationAnalysisError, match="inconsistent thresholds"): + standardize_snapshot(threshold_drift) + + +def test_standardize_snapshot_rejects_result_from_a_different_evalset(): + wrong_evalset_run = _case_run( + run_id=1, + status=EvalStatus.PASSED, + score=1.0, + ).model_copy(update={"eval_set_id": "other-set"}) + + with pytest.raises(EvaluationAnalysisError, match="eval_set_id"): + standardize_snapshot(_snapshot(wrong_evalset_run)) + + +def test_attribution_prefers_tool_name_error_and_retains_secondary_response_evidence(): + standardized = standardize_snapshot( + _snapshot( + _case_run( + run_id=1, + status=EvalStatus.FAILED, + score=0.0, + actual_tools=[FunctionCall(name="general_support", args={})], + expected_tools=[FunctionCall(name="lookup_order", args={"order_id": "A100"})], + ) + ) + ) + + attributed = attribute_evaluation(standardized) + + attribution = attributed.cases[0].attribution + assert attribution is not None + assert attribution.primary_category == "tool_name_error" + assert attribution.secondary_categories == ["routing_error", "final_response_mismatch"] + assert attribution.summary == "Expected tool names ['lookup_order'], got ['general_support']." + assert attribution.evidence[0].expected == ["lookup_order"] + assert attribution.evidence[0].actual == ["general_support"] + + +@pytest.mark.parametrize( + ("case_run", "expected_category"), + [ + ( + _case_run( + run_id=1, + status=EvalStatus.NOT_EVALUATED, + score=None, + error_message="judge unavailable", + ), + "evaluation_error", + ), + ( + _case_run( + run_id=1, + status=EvalStatus.FAILED, + score=0.0, + actual_tools=[FunctionCall(name="lookup_order", args={"order_id": "B200"})], + expected_tools=[FunctionCall(name="lookup_order", args={"order_id": "A100"})], + ), + "tool_argument_error", + ), + ( + _case_run( + run_id=1, + status=EvalStatus.FAILED, + score=0.0, + metrics=[ + _metric( + name="llm_rubric_knowledge_recall", + score=0.0, + status=EvalStatus.FAILED, + reason="Shipping policy was not recalled.", + ) + ], + ), + "knowledge_recall", + ), + ( + _case_run( + run_id=1, + status=EvalStatus.FAILED, + score=0.0, + actual_text="not-json", + ), + "format_error", + ), + ( + _case_run( + run_id=1, + status=EvalStatus.FAILED, + score=0.0, + actual_text="same", + expected_text="same", + metrics=[ + _metric( + name="llm_rubric_response", + score=0.0, + status=EvalStatus.FAILED, + reason="Rubric requirement was not met.", + ) + ], + ), + "rubric_failure", + ), + ( + _case_run( + run_id=1, + status=EvalStatus.FAILED, + score=0.0, + actual_text="same", + expected_text="same", + metrics=[_metric(name="custom_metric", score=0.0, status=EvalStatus.FAILED)], + ), + "unknown", + ), + ], +) +def test_attribution_uses_generic_evidence_categories( + case_run: EvalCaseResult, + expected_category: str, +): + attributed = attribute_evaluation(standardize_snapshot(_snapshot(case_run))) + + attribution = attributed.cases[0].attribution + assert attribution is not None + assert attribution.primary_category == expected_category + + +def test_attribution_is_absent_for_passed_case(): + attributed = attribute_evaluation( + standardize_snapshot( + _snapshot( + _case_run( + run_id=1, + status=EvalStatus.PASSED, + score=1.0, + actual_text='{"route":"order_lookup"}', + ) + ) + ) + ) + + assert attributed.cases[0].attribution is None + + +def test_compare_evaluations_marks_newly_passed_score_delta_and_hard_label(): + baseline = attribute_evaluation( + standardize_snapshot( + _snapshot(_case_run(run_id=1, status=EvalStatus.FAILED, score=0.0)) + ) + ) + candidate_snapshot = _snapshot( + _case_run( + run_id=1, + status=EvalStatus.PASSED, + score=1.0, + actual_text='{"route":"order_lookup"}', + ) + ).model_copy(update={"phase": "candidate", "average_score": 1.0}) + candidate = attribute_evaluation(standardize_snapshot(candidate_snapshot)) + + diff = compare_evaluations( + baseline, + candidate, + hard_case_ids={"case-order"}, + critical_case_ids=set(), + severe_case_score_drop=0.2, + ) + + case_diff = diff.cases[0] + assert case_diff.change == "newly_passed" + assert case_diff.score_delta.status == "available" + assert case_diff.score_delta.value == pytest.approx(1.0) + assert case_diff.is_hard is True + assert case_diff.is_critical is False + assert case_diff.severe_regression is False + assert diff.newly_passed_count == 1 + + +def test_compare_evaluations_marks_critical_severe_regression_and_unavailable_as_incomparable(): + passing_snapshot = _snapshot( + _case_run( + run_id=1, + status=EvalStatus.PASSED, + score=1.0, + actual_text='{"route":"order_lookup"}', + ) + ) + baseline = attribute_evaluation(standardize_snapshot(passing_snapshot)) + failing_snapshot = _snapshot(_case_run(run_id=1, status=EvalStatus.FAILED, score=0.0)).model_copy( + update={"phase": "candidate", "average_score": 0.0} + ) + candidate = attribute_evaluation(standardize_snapshot(failing_snapshot)) + + regression = compare_evaluations( + baseline, + candidate, + hard_case_ids=set(), + critical_case_ids={"case-order"}, + severe_case_score_drop=0.2, + ).cases[0] + assert regression.change == "newly_failed" + assert regression.is_critical is True + assert regression.severe_regression is True + + unavailable_snapshot = _snapshot( + _case_run( + run_id=1, + status=EvalStatus.NOT_EVALUATED, + score=None, + error_message="candidate metric unavailable", + ) + ).model_copy(update={"phase": "candidate", "average_score": None}) + unavailable = attribute_evaluation(standardize_snapshot(unavailable_snapshot)) + incomparable = compare_evaluations( + baseline, + unavailable, + hard_case_ids=set(), + critical_case_ids=set(), + severe_case_score_drop=0.2, + ).cases[0] + assert incomparable.change == "incomparable" + assert incomparable.score_delta.status == "unavailable" + assert incomparable.severe_regression is False + + +def test_compare_evaluations_rejects_case_and_metric_threshold_mismatch(): + baseline = attribute_evaluation( + standardize_snapshot( + _snapshot(_case_run(run_id=1, status=EvalStatus.FAILED, score=0.0)) + ) + ) + candidate = attribute_evaluation( + standardize_snapshot( + _snapshot( + _case_run( + run_id=1, + status=EvalStatus.PASSED, + score=1.0, + metrics=[_metric(score=1.0, status=EvalStatus.PASSED, threshold=0.5)], + ) + ).model_copy(update={"phase": "candidate"}) + ) + ) + with pytest.raises(EvaluationAnalysisError, match="threshold changed"): + compare_evaluations( + baseline, + candidate, + hard_case_ids=set(), + critical_case_ids=set(), + severe_case_score_drop=0.2, + ) + + different_case = candidate.model_copy( + update={"cases": [candidate.cases[0].model_copy(update={"eval_id": "other-case"})]} + ) + with pytest.raises(EvaluationAnalysisError, match="case ids do not match"): + compare_evaluations( + baseline, + different_case, + hard_case_ids=set(), + critical_case_ids=set(), + severe_case_score_drop=0.2, + ) + + +@pytest.mark.parametrize( + ( + "scenario", + "train_newly_passed", + "validation_newly_passed", + "validation_newly_failed", + "validation_unchanged", + "overfit_status", + ), + [ + ("improve", 2, 2, 0, 1, "not_detected"), + ("no_improvement", 0, 0, 0, 3, "not_detected"), + ("overfit", 2, 0, 1, 2, "detected"), + ], +) +@pytest.mark.asyncio +async def test_stage3a_scenario_analysis_matrix( + tmp_path: Path, + scenario: str, + train_newly_passed: int, + validation_newly_passed: int, + validation_newly_failed: int, + validation_unchanged: int, + overfit_status: str, +): + root = _copy_example(tmp_path, scenario) + prepared = prepare_run(root / "pipeline.json", run_id=f"stage3a_{scenario}") + + result = await run_offline_stage(prepared, scenario=scenario) # type: ignore[arg-type] + + analysis = result.analysis + assert analysis.train_diff.newly_passed_count == train_newly_passed + assert analysis.validation_diff.newly_passed_count == validation_newly_passed + assert analysis.validation_diff.newly_failed_count == validation_newly_failed + assert analysis.validation_diff.unchanged_count == validation_unchanged + assert analysis.overfit_status == overfit_status + + if scenario == "overfit": + refund = next( + case for case in analysis.validation_diff.cases if case.eval_id == "val_refund_route" + ) + assert refund.change == "newly_failed" + assert refund.is_critical is True + assert refund.severe_regression is True + assert refund.candidate_attribution is not None + assert refund.candidate_attribution.primary_category == "routing_error" + + restored = OfflineStageResult.model_validate_json(result.model_dump_json()) + assert restored.analysis == analysis + + +@pytest.mark.asyncio +async def test_stage3a_does_not_hide_unexpected_value_errors(tmp_path: Path, monkeypatch): + root = _copy_example(tmp_path, "unexpected_analysis_error") + prepared = prepare_run(root / "pipeline.json", run_id="unexpected_analysis_error") + + def fail_analysis(**_kwargs): + raise ValueError("injected programming error") + + monkeypatch.setattr(pipeline_module, "build_evaluation_analysis", fail_analysis) + + with pytest.raises(ValueError, match="injected programming error"): + await run_offline_stage(prepared, scenario="improve") diff --git a/tests/evaluation/test_eval_optimize_loop_stage3b.py b/tests/evaluation/test_eval_optimize_loop_stage3b.py new file mode 100644 index 00000000..4a9a7223 --- /dev/null +++ b/tests/evaluation/test_eval_optimize_loop_stage3b.py @@ -0,0 +1,627 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under the Apache License, Version 2.0. +"""Tests for stage 3b deterministic Gate decisions.""" + +from __future__ import annotations + +import shutil +import subprocess +import sys +from pathlib import Path +from statistics import mean + +import pytest + +from examples.optimization.eval_optimize_loop import pipeline as pipeline_module +from examples.optimization.eval_optimize_loop.case_diff import compare_evaluations +from examples.optimization.eval_optimize_loop.config import BudgetConfig +from examples.optimization.eval_optimize_loop.config import GateConfig +from examples.optimization.eval_optimize_loop.pipeline import prepare_run +from examples.optimization.eval_optimize_loop.pipeline import PipelineStageExecutionError +from examples.optimization.eval_optimize_loop.pipeline import run_offline_stage +from examples.optimization.eval_optimize_loop.schemas import CaseEvaluation +from examples.optimization.eval_optimize_loop.schemas import EvaluationAnalysis +from examples.optimization.eval_optimize_loop.schemas import OfflineStageResult +from examples.optimization.eval_optimize_loop.schemas import MetricOutcome +from examples.optimization.eval_optimize_loop.schemas import ObservableValue +from examples.optimization.eval_optimize_loop.schemas import ResourceMeasurements +from examples.optimization.eval_optimize_loop.schemas import StandardizedEvaluation + + +_REPO_ROOT = Path(__file__).resolve().parents[2] +_EXAMPLE = _REPO_ROOT / "examples" / "optimization" / "eval_optimize_loop" + + +def _copy_example(tmp_path: Path, name: str) -> Path: + target = tmp_path / name + shutil.copytree(_EXAMPLE, target, ignore=shutil.ignore_patterns("runs", "__pycache__")) + return target + + +def _available(value: float, unit: str | None = None) -> ObservableValue: + return ObservableValue(status="available", value=value, unit=unit) + + +def _case( + eval_id: str, + *, + status: str, + score: float, + metric_name: str = "final_response_avg_score", +) -> CaseEvaluation: + return CaseEvaluation( + eval_id=eval_id, + status=status, + average_score=_available(score), + metrics=[ + MetricOutcome( + metric_name=metric_name, + threshold=0.5, + status=status, + score=_available(score), + ) + ], + runs=[], + ) + + +def _evaluation( + phase: str, + split: str, + cases: list[CaseEvaluation], +) -> StandardizedEvaluation: + return StandardizedEvaluation( + phase=phase, + split=split, + eval_set_id=f"{split}-set", + cases=cases, + passed_case_count=sum(case.status == "passed" for case in cases), + failed_case_count=sum(case.status == "failed" for case in cases), + not_evaluated_case_count=sum(case.status == "not_evaluated" for case in cases), + average_score=_available(mean(float(case.average_score.value) for case in cases)), + ) + + +def _analysis( + *, + baseline_train: list[CaseEvaluation] | None = None, + candidate_train: list[CaseEvaluation] | None = None, + baseline_validation: list[CaseEvaluation] | None = None, + candidate_validation: list[CaseEvaluation] | None = None, + hard_case_ids: set[str] | None = None, + critical_case_ids: set[str] | None = None, + overfit_status: str = "not_detected", +) -> EvaluationAnalysis: + normalized_baseline_train = _evaluation( + "baseline", + "train", + baseline_train or [_case("train_case", status="failed", score=0.0)], + ) + normalized_candidate_train = _evaluation( + "candidate", + "train", + candidate_train or [_case("train_case", status="passed", score=1.0)], + ) + normalized_baseline_validation = _evaluation( + "baseline", + "validation", + baseline_validation or [_case("validation_case", status="failed", score=0.0)], + ) + normalized_candidate_validation = _evaluation( + "candidate", + "validation", + candidate_validation or [_case("validation_case", status="passed", score=1.0)], + ) + train_diff = compare_evaluations( + normalized_baseline_train, + normalized_candidate_train, + hard_case_ids=hard_case_ids or set(), + critical_case_ids=critical_case_ids or set(), + severe_case_score_drop=0.2, + ) + validation_diff = compare_evaluations( + normalized_baseline_validation, + normalized_candidate_validation, + hard_case_ids=hard_case_ids or set(), + critical_case_ids=critical_case_ids or set(), + severe_case_score_drop=0.2, + ) + return EvaluationAnalysis( + baseline_train=normalized_baseline_train, + baseline_validation=normalized_baseline_validation, + candidate_train=normalized_candidate_train, + candidate_validation=normalized_candidate_validation, + train_diff=train_diff, + validation_diff=validation_diff, + overfit_status=overfit_status, + overfit_reason="test overfit status", + ) + + +def _measurements() -> ResourceMeasurements: + return ResourceMeasurements( + cost_usd=ObservableValue(status="unavailable", reason="not observed"), + total_tokens=ObservableValue(status="unavailable", reason="not observed"), + duration_seconds=_available(1.0, "seconds"), + ) + + +def _rule(decision, rule_id: str): + return next(result for result in decision.rule_results if result.rule_id == rule_id) + + +def test_gate_models_are_serializable_and_auditable(): + from examples.optimization.eval_optimize_loop.schemas import GateDecision + from examples.optimization.eval_optimize_loop.schemas import GateRuleResult + from examples.optimization.eval_optimize_loop.schemas import ObservableValue + from examples.optimization.eval_optimize_loop.schemas import ResourceMeasurements + + measurements = ResourceMeasurements( + cost_usd=ObservableValue(status="unavailable", reason="fake mode"), + total_tokens=ObservableValue(status="available", value=12, unit="tokens"), + duration_seconds=ObservableValue(status="available", value=0.5, unit="seconds"), + ) + decision = GateDecision( + decision="reject", + rule_results=[ + GateRuleResult( + rule_id="minimum_validation_score_delta", + outcome="reject", + message="Validation score improvement is below the configured minimum.", + observed={"validation_score_delta": ObservableValue(status="available", value=0)}, + threshold=0.05, + ) + ], + rejection_reasons=["Validation score improvement is below the configured minimum."], + warnings=[], + ) + + restored_measurements = ResourceMeasurements.model_validate_json( + measurements.model_dump_json() + ) + restored_decision = GateDecision.model_validate_json(decision.model_dump_json()) + + assert restored_measurements == measurements + assert restored_decision == decision + assert restored_decision.rule_results[0].case_ids == [] + assert restored_decision.rule_results[0].metric_names == [] + + +def test_gate_accepts_candidate_and_returns_every_rule_in_stable_order(): + from examples.optimization.eval_optimize_loop.gate import evaluate_gate + + decision = evaluate_gate( + _analysis(), + GateConfig(), + BudgetConfig(max_duration_seconds=10.0, on_unavailable="warning"), + _measurements(), + ) + + assert decision.decision == "accept" + assert decision.rejection_reasons == [] + assert decision.warnings == [] + assert [result.rule_id for result in decision.rule_results] == [ + "evaluation_completeness", + "minimum_validation_score_delta", + "validation_pass_rate_non_decrease", + "no_new_hard_fail", + "no_critical_regression", + "no_severe_regression", + "required_metrics", + "no_overfitting", + "cost_budget", + "token_budget", + "duration_budget", + ] + assert [result.outcome for result in decision.rule_results[-3:]] == [ + "skipped", + "skipped", + "pass", + ] + + +def test_gate_rejects_incomplete_evaluation_and_unavailable_validation_delta(): + from examples.optimization.eval_optimize_loop.gate import evaluate_gate + + analysis = _analysis() + unavailable = ObservableValue(status="unavailable", reason="metric unavailable") + candidate_case = analysis.candidate_validation.cases[0] + incomplete_case = candidate_case.model_copy( + update={ + "status": "not_evaluated", + "average_score": unavailable, + "metrics": [ + candidate_case.metrics[0].model_copy( + update={"status": "not_evaluated", "score": unavailable} + ) + ], + } + ) + incomplete_candidate = analysis.candidate_validation.model_copy( + update={ + "cases": [incomplete_case], + "passed_case_count": 0, + "not_evaluated_case_count": 1, + "average_score": unavailable, + } + ) + incomplete_diff = analysis.validation_diff.model_copy( + update={"candidate_average_score": unavailable, "score_delta": unavailable} + ) + incomplete_analysis = analysis.model_copy( + update={ + "candidate_validation": incomplete_candidate, + "validation_diff": incomplete_diff, + } + ) + + decision = evaluate_gate( + incomplete_analysis, + GateConfig(), + BudgetConfig(), + _measurements(), + ) + + assert decision.decision == "reject" + assert _rule(decision, "evaluation_completeness").outcome == "reject" + assert _rule(decision, "minimum_validation_score_delta").outcome == "reject" + assert len(decision.rejection_reasons) >= 2 + + +def test_minimum_validation_delta_accepts_exact_threshold(): + from examples.optimization.eval_optimize_loop.gate import evaluate_gate + + analysis = _analysis( + baseline_validation=[_case("validation_case", status="failed", score=0.0)], + candidate_validation=[_case("validation_case", status="failed", score=0.05)], + ) + + decision = evaluate_gate( + analysis, + GateConfig(min_validation_score_delta=0.05), + BudgetConfig(), + _measurements(), + ) + + assert _rule(decision, "minimum_validation_score_delta").outcome == "pass" + + +def test_validation_pass_rate_drop_rejects_unless_rule_is_disabled(): + from examples.optimization.eval_optimize_loop.gate import evaluate_gate + + analysis = _analysis( + baseline_validation=[ + _case("validation_a", status="passed", score=1.0), + _case("validation_b", status="failed", score=0.0), + ], + candidate_validation=[ + _case("validation_a", status="failed", score=0.0), + _case("validation_b", status="failed", score=0.0), + ], + ) + + rejected = evaluate_gate(analysis, GateConfig(), BudgetConfig(), _measurements()) + disabled = evaluate_gate( + analysis, + GateConfig(reject_on_validation_pass_rate_drop=False), + BudgetConfig(), + _measurements(), + ) + + result = _rule(rejected, "validation_pass_rate_non_decrease") + assert result.outcome == "reject" + assert result.observed["baseline_validation_pass_rate"].value == 0.5 + assert result.observed["candidate_validation_pass_rate"].value == 0.0 + assert _rule(disabled, "validation_pass_rate_non_decrease").outcome == "skipped" + + +def test_gate_collects_hard_critical_and_severe_train_regressions(): + from examples.optimization.eval_optimize_loop.gate import evaluate_gate + + analysis = _analysis( + baseline_train=[ + _case("hard_case", status="passed", score=1.0), + _case("critical_case", status="passed", score=1.0), + _case("severe_case", status="passed", score=1.0), + ], + candidate_train=[ + _case("hard_case", status="failed", score=0.0), + _case("critical_case", status="passed", score=0.9), + _case("severe_case", status="passed", score=0.7), + ], + hard_case_ids={"hard_case"}, + critical_case_ids={"critical_case"}, + ) + + decision = evaluate_gate(analysis, GateConfig(), BudgetConfig(), _measurements()) + + assert _rule(decision, "no_new_hard_fail").outcome == "reject" + assert _rule(decision, "no_new_hard_fail").case_ids == ["hard_case"] + assert _rule(decision, "no_critical_regression").outcome == "reject" + assert _rule(decision, "no_critical_regression").case_ids == ["critical_case"] + assert _rule(decision, "no_severe_regression").outcome == "reject" + assert _rule(decision, "no_severe_regression").case_ids == ["hard_case", "severe_case"] + + disabled = evaluate_gate( + analysis, + GateConfig(reject_new_hard_fail=False, reject_critical_regression=False), + BudgetConfig(), + _measurements(), + ) + assert _rule(disabled, "no_new_hard_fail").outcome == "skipped" + assert _rule(disabled, "no_critical_regression").outcome == "skipped" + assert _rule(disabled, "no_severe_regression").outcome == "reject" + + +@pytest.mark.parametrize("required_metrics", [["required_metric"], "all"]) +def test_required_metrics_reject_missing_or_empty_candidate_metrics(required_metrics): + from examples.optimization.eval_optimize_loop.gate import evaluate_gate + + analysis = _analysis() + if required_metrics == "all": + empty_case = analysis.candidate_train.cases[0].model_copy(update={"metrics": []}) + analysis = analysis.model_copy( + update={ + "candidate_train": analysis.candidate_train.model_copy( + update={"cases": [empty_case]} + ) + } + ) + + decision = evaluate_gate( + analysis, + GateConfig(required_metrics=required_metrics), + BudgetConfig(), + _measurements(), + ) + + result = _rule(decision, "required_metrics") + assert result.outcome == "reject" + if required_metrics == ["required_metric"]: + assert result.metric_names == ["required_metric"] + assert result.case_ids == ["train_case", "validation_case"] + else: + assert result.case_ids == ["train_case"] + + +def test_required_metrics_reject_failed_and_unavailable_candidate_metrics(): + from examples.optimization.eval_optimize_loop.gate import evaluate_gate + + failed_case = _case("train_case", status="failed", score=0.0) + unavailable = ObservableValue(status="unavailable", reason="judge failed") + unavailable_case = _case("validation_case", status="passed", score=1.0) + unavailable_case = unavailable_case.model_copy( + update={ + "metrics": [ + unavailable_case.metrics[0].model_copy( + update={"status": "not_evaluated", "score": unavailable} + ) + ] + } + ) + analysis = _analysis( + candidate_train=[failed_case], + candidate_validation=[unavailable_case], + ) + + decision = evaluate_gate(analysis, GateConfig(), BudgetConfig(), _measurements()) + + result = _rule(decision, "required_metrics") + assert result.outcome == "reject" + assert result.case_ids == ["train_case", "validation_case"] + assert result.metric_names == ["final_response_avg_score"] + + +@pytest.mark.parametrize("overfit_status", ["detected", "unavailable"]) +def test_overfit_detected_or_unavailable_rejects(overfit_status: str): + from examples.optimization.eval_optimize_loop.gate import evaluate_gate + + decision = evaluate_gate( + _analysis(overfit_status=overfit_status), + GateConfig(), + BudgetConfig(), + _measurements(), + ) + + assert _rule(decision, "no_overfitting").outcome == "reject" + + +def test_budget_rules_handle_limits_unavailable_policy_and_disabled_limits(): + from examples.optimization.eval_optimize_loop.gate import evaluate_gate + + measurements = ResourceMeasurements( + cost_usd=_available(2.0, "USD"), + total_tokens=ObservableValue(status="unavailable", reason="usage missing"), + duration_seconds=_available(1.0, "seconds"), + ) + warning_decision = evaluate_gate( + _analysis(), + GateConfig(), + BudgetConfig(max_cost_usd=1.0, max_tokens=100, on_unavailable="warning"), + measurements, + ) + + assert _rule(warning_decision, "cost_budget").outcome == "reject" + assert _rule(warning_decision, "token_budget").outcome == "warning" + assert _rule(warning_decision, "duration_budget").outcome == "skipped" + assert len(warning_decision.warnings) == 1 + + reject_decision = evaluate_gate( + _analysis(), + GateConfig(), + BudgetConfig(max_tokens=100, max_duration_seconds=1.0, on_unavailable="reject"), + measurements, + ) + assert _rule(reject_decision, "cost_budget").outcome == "skipped" + assert _rule(reject_decision, "token_budget").outcome == "reject" + assert _rule(reject_decision, "duration_budget").outcome == "pass" + + +def test_gate_rejects_structurally_inconsistent_analysis(): + from examples.optimization.eval_optimize_loop.gate import GateEvaluationError + from examples.optimization.eval_optimize_loop.gate import evaluate_gate + + analysis = _analysis() + wrong_split = analysis.model_copy( + update={"train_diff": analysis.train_diff.model_copy(update={"split": "validation"})} + ) + with pytest.raises(GateEvaluationError, match="train_diff.split"): + evaluate_gate(wrong_split, GateConfig(), BudgetConfig(), _measurements()) + + missing_diff_case = analysis.model_copy( + update={ + "train_diff": analysis.train_diff.model_copy(update={"cases": []}), + } + ) + with pytest.raises(GateEvaluationError, match="case ids"): + evaluate_gate(missing_diff_case, GateConfig(), BudgetConfig(), _measurements()) + + candidate_case = analysis.candidate_train.cases[0] + duplicate_metric_case = candidate_case.model_copy( + update={"metrics": [candidate_case.metrics[0], candidate_case.metrics[0]]} + ) + duplicate_metric_analysis = analysis.model_copy( + update={ + "candidate_train": analysis.candidate_train.model_copy( + update={"cases": [duplicate_metric_case]} + ) + } + ) + with pytest.raises(GateEvaluationError, match="duplicate metric"): + evaluate_gate( + duplicate_metric_analysis, + GateConfig(), + BudgetConfig(), + _measurements(), + ) + + +@pytest.mark.asyncio +async def test_fake_stage_returns_measurements_and_accepts_improving_candidate(tmp_path: Path): + root = _copy_example(tmp_path, "improve") + source = root / "prompts" / "system.md" + baseline_source = source.read_text(encoding="utf-8") + prepared = prepare_run(root / "pipeline.json", run_id="stage3b_improve") + + result = await run_offline_stage(prepared, scenario="improve") + + assert result.gate_decision.decision == "accept" + assert result.gate_decision.rejection_reasons == [] + assert result.measurements.duration_seconds.status == "available" + assert result.measurements.duration_seconds.value >= 0 + assert result.measurements.cost_usd.status == "unavailable" + assert result.measurements.total_tokens.status == "unavailable" + assert source.read_text(encoding="utf-8") == baseline_source + assert "deterministic-fake-candidate:start" in ( + await prepared.working_target.read_all() + )["system_prompt"] + + +@pytest.mark.parametrize( + ("scenario", "expected_decision", "expected_reject_rules"), + [ + ("improve", "accept", []), + ( + "no_improvement", + "reject", + ["minimum_validation_score_delta", "required_metrics"], + ), + ( + "overfit", + "reject", + [ + "minimum_validation_score_delta", + "validation_pass_rate_non_decrease", + "no_critical_regression", + "no_severe_regression", + "required_metrics", + "no_overfitting", + ], + ), + ], +) +@pytest.mark.asyncio +async def test_stage3b_scenario_gate_matrix( + tmp_path: Path, + scenario: str, + expected_decision: str, + expected_reject_rules: list[str], +): + root = _copy_example(tmp_path, f"matrix_{scenario}") + source = root / "prompts" / "system.md" + baseline_source = source.read_text(encoding="utf-8") + prepared = prepare_run(root / "pipeline.json", run_id=f"stage3b_{scenario}") + + result = await run_offline_stage(prepared, scenario=scenario) # type: ignore[arg-type] + + assert result.gate_decision.decision == expected_decision + assert [ + rule.rule_id + for rule in result.gate_decision.rule_results + if rule.outcome == "reject" + ] == expected_reject_rules + assert source.read_text(encoding="utf-8") == baseline_source + assert OfflineStageResult.model_validate_json(result.model_dump_json()) == result + + +@pytest.mark.asyncio +async def test_fake_stage_wraps_gate_structure_errors_only(tmp_path: Path, monkeypatch): + from examples.optimization.eval_optimize_loop.gate import GateEvaluationError + + root = _copy_example(tmp_path, "gate_structure_error") + prepared = prepare_run(root / "pipeline.json", run_id="gate_structure_error") + + def fail_gate(*_args, **_kwargs): + raise GateEvaluationError("injected unsafe analysis") + + monkeypatch.setattr(pipeline_module, "evaluate_gate", fail_gate) + + with pytest.raises(PipelineStageExecutionError, match="stage 3b gate failed"): + await run_offline_stage(prepared, scenario="improve") + + +@pytest.mark.asyncio +async def test_fake_stage_does_not_hide_unexpected_gate_value_errors(tmp_path: Path, monkeypatch): + root = _copy_example(tmp_path, "gate_programming_error") + prepared = prepare_run(root / "pipeline.json", run_id="gate_programming_error") + + def fail_gate(*_args, **_kwargs): + raise ValueError("injected programming error") + + monkeypatch.setattr(pipeline_module, "evaluate_gate", fail_gate) + + with pytest.raises(ValueError, match="injected programming error"): + await run_offline_stage(prepared, scenario="improve") + + +def test_stage3b_cli_prints_gate_decision_and_rejection_reasons(tmp_path: Path): + root = _copy_example(tmp_path, "cli_overfit") + + completed = subprocess.run( + [ + sys.executable, + str(_EXAMPLE / "run_pipeline.py"), + "--config", + str(root / "pipeline.json"), + "--run-id", + "stage3b_cli", + "--scenario", + "overfit", + ], + cwd=_REPO_ROOT, + check=True, + capture_output=True, + text=True, + timeout=180, + ) + + assert "Gate decision: REJECT" in completed.stdout + assert "Rejection reasons:" in completed.stdout + assert "[no_overfitting]" in completed.stdout + assert "Writeback: SKIPPED (gate_rejected)" in completed.stdout + assert "optimization_report.json" in completed.stdout + assert "optimization_report.md" in completed.stdout + assert "artifact_index.json" in completed.stdout diff --git a/tests/evaluation/test_eval_optimize_loop_stage4.py b/tests/evaluation/test_eval_optimize_loop_stage4.py new file mode 100644 index 00000000..8b2c84ab --- /dev/null +++ b/tests/evaluation/test_eval_optimize_loop_stage4.py @@ -0,0 +1,634 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under the Apache License, Version 2.0. +"""Tests for Stage 4 real optimization and guarded prompt writeback.""" + +from __future__ import annotations + +import json +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest + +from examples.optimization.eval_optimize_loop import candidate_provider as candidate_provider_module +from examples.optimization.eval_optimize_loop import pipeline as pipeline_module +from examples.optimization.eval_optimize_loop import writeback as writeback_module +from examples.optimization.eval_optimize_loop.candidate_provider import AgentOptimizerCandidateProvider +from examples.optimization.eval_optimize_loop.candidate_provider import CandidateProviderError +from examples.optimization.eval_optimize_loop.candidate_provider import CandidateRequest +from examples.optimization.eval_optimize_loop.config import WritebackConfig +from examples.optimization.eval_optimize_loop.business_agent import BusinessAgent +from examples.optimization.eval_optimize_loop.fake import DeterministicFakeModel +from examples.optimization.eval_optimize_loop.fake import DeterministicFakeCandidateProvider +from examples.optimization.eval_optimize_loop.pipeline import prepare_run +from examples.optimization.eval_optimize_loop.pipeline import PipelineStageExecutionError +from examples.optimization.eval_optimize_loop.pipeline import run_offline_stage +from examples.optimization.eval_optimize_loop.pipeline import run_real_stage +from examples.optimization.eval_optimize_loop.schemas import GateDecision +from examples.optimization.eval_optimize_loop.schemas import RealStageResult +from examples.optimization.eval_optimize_loop.writeback import perform_writeback +from trpc_agent_sdk.evaluation import OptimizeResult +from trpc_agent_sdk.evaluation import TargetPrompt + + +_REPO_ROOT = Path(__file__).resolve().parents[2] +_EXAMPLE = _REPO_ROOT / "examples" / "optimization" / "eval_optimize_loop" + + +def _offline_agent(target: TargetPrompt) -> BusinessAgent: + return BusinessAgent( + target, + DeterministicFakeModel, + agent_name="stage4_offline_agent", + app_name="stage4_offline_test", + user_id="stage4-test", + ) + + +def _copy_example(tmp_path: Path, name: str = "eval_optimize_loop") -> Path: + target = tmp_path / name + shutil.copytree(_EXAMPLE, target, ignore=shutil.ignore_patterns("runs", "__pycache__")) + return target + + +def _set_real_mode(root: Path, *, writeback: bool = False) -> None: + path = root / "pipeline.json" + payload = json.loads(path.read_text(encoding="utf-8")) + payload["execution"]["mode"] = "real" + payload["writeback"]["enabled"] = writeback + path.write_text(json.dumps(payload), encoding="utf-8") + + +def _optimize_result( + baseline_prompts: dict[str, str], + best_prompts: dict[str, str], + *, + status: str = "SUCCEEDED", +) -> OptimizeResult: + return OptimizeResult( + algorithm="gepa_reflective", + status=status, + finish_reason="completed" if status == "SUCCEEDED" else "error", + error_message="" if status == "SUCCEEDED" else "optimizer failed", + baseline_pass_rate=1 / 3, + best_pass_rate=1.0, + pass_rate_improvement=2 / 3, + baseline_prompts=baseline_prompts, + best_prompts=best_prompts, + total_rounds=0, + rounds=[], + total_reflection_lm_calls=1, + total_llm_cost=0.25, + total_token_usage={"prompt": 10, "completion": 5, "total": 15}, + duration_seconds=0.5, + started_at="2026-07-17T00:00:00Z", + finished_at="2026-07-17T00:00:01Z", + ) + + +def _install_optimizer_result( + monkeypatch: pytest.MonkeyPatch, + baseline: dict[str, str], + best: dict[str, str], +) -> None: + async def fake_optimize(**kwargs): + output_dir = Path(str(kwargs["output_dir"])) + output_dir.mkdir(parents=True) + result = _optimize_result(baseline, best) + result.dump_to(str(output_dir / "result.json")) + return result + + monkeypatch.setattr( + "examples.optimization.eval_optimize_loop.candidate_provider.AgentOptimizer.optimize", + fake_optimize, + ) + + +@pytest.mark.asyncio +async def test_real_candidate_provider_uses_isolated_target_and_never_updates_source( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + root = _copy_example(tmp_path) + prepared = prepare_run(root / "pipeline.json", run_id="stage4_provider") + baseline = await prepared.working_target.read_all() + best = {name: f"{content}\noptimized" for name, content in baseline.items()} + captured: dict[str, object] = {} + + async def call_agent(query: str) -> str: + return query + + async def fake_optimize(**kwargs): + captured.update(kwargs) + output_dir = Path(str(kwargs["output_dir"])) + output_dir.mkdir(parents=True) + result = _optimize_result(baseline, best) + result.dump_to(str(output_dir / "result.json")) + return result + + monkeypatch.setattr( + "examples.optimization.eval_optimize_loop.candidate_provider.AgentOptimizer.optimize", + fake_optimize, + ) + request = CandidateRequest( + current_prompts=baseline, + target_prompt=prepared.working_target, + optimizer_config_path=Path(prepared.input_snapshot.optimizer_config_path), + train_evalset_path=Path(prepared.input_snapshot.train_evalset_path), + validation_evalset_path=Path(prepared.input_snapshot.validation_evalset_path), + output_dir=Path(prepared.workspace.run_dir) / "optimizer", + seed=prepared.input_snapshot.seed, + ) + + generated = await AgentOptimizerCandidateProvider(call_agent).propose(request) + + assert captured["target_prompt"] is prepared.working_target + assert captured["update_source"] is False + assert captured["verbose"] == 0 + assert generated.proposal.provider == "agent_optimizer" + assert generated.proposal.prompts == best + assert generated.proposal.changed_fields == ["system_prompt"] + assert generated.proposal.candidate_id.startswith("optimizer-") + assert generated.optimize_result == _optimize_result(baseline, best) + assert (request.output_dir / "result.json").is_file() + assert await prepared.working_target.read_all() == baseline + + +@pytest.mark.asyncio +async def test_real_stage_runs_full_regression_and_skips_disabled_writeback( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + root = _copy_example(tmp_path, "real_stage") + _set_real_mode(root) + source_path = root / "prompts" / "system.md" + source_before = source_path.read_text(encoding="utf-8") + prepared = prepare_run(root / "pipeline.json", run_id="stage4_real") + baseline = await prepared.working_target.read_all() + best = DeterministicFakeCandidateProvider().propose( + baseline, + scenario="improve", + seed=42, + ).prompts + + async def fake_optimize(**kwargs): + output_dir = Path(str(kwargs["output_dir"])) + output_dir.mkdir(parents=True) + result = _optimize_result(baseline, best) + result.dump_to(str(output_dir / "result.json")) + return result + + monkeypatch.setattr( + "examples.optimization.eval_optimize_loop.candidate_provider.AgentOptimizer.optimize", + fake_optimize, + ) + agent = _offline_agent(prepared.working_target) + + result = await run_real_stage(prepared, call_agent=agent.call_agent) + + assert isinstance(result, RealStageResult) + assert result.baseline_train.passed_case_count == 1 + assert result.baseline_validation.passed_case_count == 1 + assert result.candidate_train.passed_case_count == 3 + assert result.candidate_validation.passed_case_count == 3 + assert result.gate_decision.decision == "accept" + assert result.writeback.status == "skipped" + assert result.writeback.reason == "disabled" + assert result.optimize_result.best_prompts == best + assert source_path.read_text(encoding="utf-8") == source_before + assert await prepared.working_target.read_all() == best + assert RealStageResult.model_validate_json(result.model_dump_json()) == result + + +@pytest.mark.asyncio +async def test_real_stage_writes_accepted_candidate_after_hash_check( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + root = _copy_example(tmp_path, "write_accepted") + _set_real_mode(root, writeback=True) + prepared = prepare_run(root / "pipeline.json", run_id="stage4_write") + baseline = await prepared.working_target.read_all() + best = DeterministicFakeCandidateProvider().propose( + baseline, + scenario="improve", + seed=42, + ).prompts + _install_optimizer_result(monkeypatch, baseline, best) + + result = await run_real_stage( + prepared, + call_agent=_offline_agent(prepared.working_target).call_agent, + ) + + assert result.gate_decision.decision == "accept" + assert result.writeback.status == "written" + assert result.writeback.reason == "written" + assert result.writeback.attempted is True + assert result.writeback.changed_fields == ["system_prompt"] + assert await prepared.source_target.read_all() == best + assert result.writeback.source_hashes_before != result.writeback.source_hashes_after + + +@pytest.mark.asyncio +async def test_real_stage_blocks_writeback_when_source_drifted( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + root = _copy_example(tmp_path, "source_drift") + _set_real_mode(root, writeback=True) + prepared = prepare_run(root / "pipeline.json", run_id="stage4_drift") + baseline = await prepared.working_target.read_all() + best = DeterministicFakeCandidateProvider().propose( + baseline, + scenario="improve", + seed=42, + ).prompts + _install_optimizer_result(monkeypatch, baseline, best) + source_path = root / "prompts" / "system.md" + source_path.write_text("concurrent user edit", encoding="utf-8") + + result = await run_real_stage( + prepared, + call_agent=_offline_agent(prepared.working_target).call_agent, + ) + + assert result.gate_decision.decision == "accept" + assert result.writeback.status == "blocked" + assert result.writeback.reason == "source_drift" + assert result.writeback.attempted is False + assert source_path.read_text(encoding="utf-8") == "concurrent user edit" + + +@pytest.mark.asyncio +async def test_real_stage_returns_failed_writeback_after_verified_rollback( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + root = _copy_example(tmp_path, "write_failure") + _set_real_mode(root, writeback=True) + prepared = prepare_run(root / "pipeline.json", run_id="stage4_write_failure") + baseline = await prepared.working_target.read_all() + best = DeterministicFakeCandidateProvider().propose( + baseline, + scenario="improve", + seed=42, + ).prompts + _install_optimizer_result(monkeypatch, baseline, best) + + async def fail_write(prompts: dict[str, str]) -> None: + raise OSError("simulated source write failure") + + monkeypatch.setattr(prepared.source_target, "write_all", fail_write) + + result = await run_real_stage( + prepared, + call_agent=_offline_agent(prepared.working_target).call_agent, + ) + + assert result.writeback.status == "failed" + assert result.writeback.reason == "write_error" + assert result.writeback.attempted is True + assert "simulated source write failure" in (result.writeback.error_message or "") + assert await prepared.source_target.read_all() == baseline + + +@pytest.mark.asyncio +async def test_fake_stage_uses_the_same_guarded_writeback_path(tmp_path: Path): + root = _copy_example(tmp_path, "fake_writeback") + path = root / "pipeline.json" + payload = json.loads(path.read_text(encoding="utf-8")) + payload["writeback"]["enabled"] = True + path.write_text(json.dumps(payload), encoding="utf-8") + prepared = prepare_run(root / "pipeline.json", run_id="stage4_fake_write") + + result = await run_offline_stage(prepared, scenario="improve") + + assert result.gate_decision.decision == "accept" + assert result.writeback.status == "written" + assert await prepared.source_target.read_all() == result.candidate.prompts + + +@pytest.mark.asyncio +async def test_fake_stage_never_writes_a_gate_rejected_candidate(tmp_path: Path): + root = _copy_example(tmp_path, "fake_reject") + path = root / "pipeline.json" + payload = json.loads(path.read_text(encoding="utf-8")) + payload["writeback"]["enabled"] = True + path.write_text(json.dumps(payload), encoding="utf-8") + prepared = prepare_run(root / "pipeline.json", run_id="stage4_fake_reject") + source_before = await prepared.source_target.read_all() + + result = await run_offline_stage(prepared, scenario="no_improvement") + + assert result.gate_decision.decision == "reject" + assert result.writeback.status == "skipped" + assert result.writeback.reason == "gate_rejected" + assert await prepared.source_target.read_all() == source_before + + +@pytest.mark.asyncio +async def test_real_candidate_provider_rejects_failed_optimizer_result( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + root = _copy_example(tmp_path, "optimizer_failed") + prepared = prepare_run(root / "pipeline.json", run_id="stage4_optimizer_failed") + baseline = await prepared.working_target.read_all() + + async def failed_optimize(**kwargs): + return _optimize_result(baseline, baseline, status="FAILED") + + monkeypatch.setattr( + "examples.optimization.eval_optimize_loop.candidate_provider.AgentOptimizer.optimize", + failed_optimize, + ) + request = CandidateRequest( + current_prompts=baseline, + target_prompt=prepared.working_target, + optimizer_config_path=Path(prepared.input_snapshot.optimizer_config_path), + train_evalset_path=Path(prepared.input_snapshot.train_evalset_path), + validation_evalset_path=Path(prepared.input_snapshot.validation_evalset_path), + output_dir=Path(prepared.workspace.run_dir) / "optimizer", + seed=42, + ) + + with pytest.raises(CandidateProviderError, match="returned FAILED"): + await AgentOptimizerCandidateProvider(_offline_agent(prepared.working_target).call_agent).propose( + request + ) + + +@pytest.mark.asyncio +async def test_real_stage_rejects_optimizer_config_drift_before_optimization( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + root = _copy_example(tmp_path, "optimizer_drift") + _set_real_mode(root) + prepared = prepare_run(root / "pipeline.json", run_id="stage4_optimizer_drift") + optimizer_path = root / "optimizer.json" + optimizer_path.write_text( + optimizer_path.read_text(encoding="utf-8") + "\n", + encoding="utf-8", + ) + called = False + + async def should_not_optimize(**kwargs): + nonlocal called + called = True + raise AssertionError("optimizer must not run after config drift") + + monkeypatch.setattr( + "examples.optimization.eval_optimize_loop.candidate_provider.AgentOptimizer.optimize", + should_not_optimize, + ) + + with pytest.raises(PipelineStageExecutionError, match="optimizer_config changed"): + await run_real_stage( + prepared, + call_agent=_offline_agent(prepared.working_target).call_agent, + ) + assert called is False + + +@pytest.mark.asyncio +async def test_writeback_readback_mismatch_restores_source( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + root = _copy_example(tmp_path, "readback_mismatch") + prepared = prepare_run(root / "pipeline.json", run_id="stage4_readback") + baseline = await prepared.source_target.read_all() + candidate = DeterministicFakeCandidateProvider().propose( + baseline, + scenario="improve", + seed=42, + ) + original_read_all = prepared.source_target.read_all + read_count = 0 + + async def mismatched_readback(): + nonlocal read_count + read_count += 1 + actual = await original_read_all() + if read_count == 2: + return {"system_prompt": "unexpected readback"} + return actual + + monkeypatch.setattr(prepared.source_target, "read_all", mismatched_readback) + + result = await perform_writeback( + decision=GateDecision(decision="accept", rule_results=[]), + config=WritebackConfig(enabled=True), + snapshots=prepared.input_snapshot.prompt_snapshots, + source_target=prepared.source_target, + candidate=candidate, + ) + + assert result.status == "failed" + assert result.reason == "readback_mismatch" + assert await original_read_all() == baseline + + +def test_enabled_writeback_cannot_disable_source_hash_guard(): + with pytest.raises(ValueError, match="requires require_source_hash_match=true"): + WritebackConfig(enabled=True, require_source_hash_match=False) + + +@pytest.mark.asyncio +async def test_real_stage_restores_working_prompt_after_optimizer_exception( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + root = _copy_example(tmp_path, "optimizer_exception") + _set_real_mode(root) + prepared = prepare_run(root / "pipeline.json", run_id="stage4_optimizer_exception") + baseline = await prepared.working_target.read_all() + source_before = await prepared.source_target.read_all() + changed = {name: f"{value}\nleft by failed optimizer" for name, value in baseline.items()} + + async def mutating_failure(**kwargs): + await kwargs["target_prompt"].write_all(changed) + raise RuntimeError("simulated optimizer crash") + + monkeypatch.setattr( + "examples.optimization.eval_optimize_loop.candidate_provider.AgentOptimizer.optimize", + mutating_failure, + ) + + with pytest.raises(PipelineStageExecutionError, match="real candidate generation failed"): + await run_real_stage( + prepared, + call_agent=_offline_agent(prepared.working_target).call_agent, + ) + + assert await prepared.working_target.read_all() == baseline + assert await prepared.source_target.read_all() == source_before + + +@pytest.mark.asyncio +async def test_restore_working_baseline_falls_back_when_initial_read_fails( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + root = _copy_example(tmp_path, "restore_after_read_failure") + prepared = prepare_run(root / "pipeline.json", run_id="stage4_restore_read_failure") + baseline = await prepared.working_target.read_all() + changed = { + name: f"{content}\nleft by optimizer" + for name, content in baseline.items() + } + await prepared.working_target.write_all(changed) + original_read_all = prepared.working_target.read_all + read_count = 0 + + async def fail_initial_read(): + nonlocal read_count + read_count += 1 + if read_count == 1: + raise OSError("simulated transient read failure") + return await original_read_all() + + monkeypatch.setattr(prepared.working_target, "read_all", fail_initial_read) + + was_modified = await pipeline_module._restore_working_baseline( + prepared, + baseline, + ) + + assert was_modified is True + assert read_count == 2 + assert await original_read_all() == baseline + + +@pytest.mark.asyncio +async def test_real_candidate_provider_can_discard_native_artifacts_after_extraction( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + root = _copy_example(tmp_path, "discard_artifacts") + prepared = prepare_run(root / "pipeline.json", run_id="stage4_discard_artifacts") + baseline = await prepared.working_target.read_all() + best = {name: f"{content}\noptimized" for name, content in baseline.items()} + _install_optimizer_result(monkeypatch, baseline, best) + output_dir = Path(prepared.workspace.run_dir) / "optimizer" + request = CandidateRequest( + current_prompts=baseline, + target_prompt=prepared.working_target, + optimizer_config_path=Path(prepared.input_snapshot.optimizer_config_path), + train_evalset_path=Path(prepared.input_snapshot.train_evalset_path), + validation_evalset_path=Path(prepared.input_snapshot.validation_evalset_path), + output_dir=output_dir, + seed=42, + retain_native_artifacts=False, + ) + + generated = await AgentOptimizerCandidateProvider( + _offline_agent(prepared.working_target).call_agent + ).propose(request) + + assert generated.optimize_result is not None + assert generated.proposal.optimizer_output_dir is None + assert not output_dir.exists() + + +def test_cli_explains_that_real_mode_requires_injected_call_agent(tmp_path: Path): + root = _copy_example(tmp_path, "real_cli") + _set_real_mode(root) + + completed = subprocess.run( + [ + sys.executable, + str(root / "run_pipeline.py"), + "--config", + str(root / "pipeline.json"), + "--run-id", + "stage4_real_cli", + ], + cwd=_REPO_ROOT, + capture_output=True, + text=True, + check=False, + ) + + assert completed.returncode == 2 + assert "run_real_stage(prepared, call_agent=...)" in completed.stderr + assert not (root / "runs" / "stage4_real_cli").exists() + + +@pytest.mark.asyncio +async def test_writeback_blocks_drift_introduced_after_initial_hash_check( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + root = _copy_example(tmp_path, "late_source_drift") + prepared = prepare_run(root / "pipeline.json", run_id="stage4_late_drift") + baseline = await prepared.source_target.read_all() + candidate = DeterministicFakeCandidateProvider().propose( + baseline, + scenario="improve", + seed=42, + ) + source_path = root / "prompts" / "system.md" + original_verify = writeback_module.verify_source_hashes + checks = 0 + + def verify_then_drift(snapshots): + nonlocal checks + checks += 1 + original_verify(snapshots) + if checks == 1: + source_path.write_text("late concurrent edit", encoding="utf-8") + + monkeypatch.setattr(writeback_module, "verify_source_hashes", verify_then_drift) + + result = await perform_writeback( + decision=GateDecision(decision="accept", rule_results=[]), + config=WritebackConfig(enabled=True), + snapshots=prepared.input_snapshot.prompt_snapshots, + source_target=prepared.source_target, + candidate=candidate, + ) + + assert result.status == "blocked" + assert result.reason == "source_drift" + assert result.attempted is False + assert source_path.read_text(encoding="utf-8") == "late concurrent edit" + + +@pytest.mark.asyncio +async def test_real_candidate_provider_wraps_artifact_cleanup_failure( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + root = _copy_example(tmp_path, "artifact_cleanup_failure") + prepared = prepare_run(root / "pipeline.json", run_id="stage4_cleanup_failure") + baseline = await prepared.working_target.read_all() + best = {name: f"{content}\noptimized" for name, content in baseline.items()} + _install_optimizer_result(monkeypatch, baseline, best) + request = CandidateRequest( + current_prompts=baseline, + target_prompt=prepared.working_target, + optimizer_config_path=Path(prepared.input_snapshot.optimizer_config_path), + train_evalset_path=Path(prepared.input_snapshot.train_evalset_path), + validation_evalset_path=Path(prepared.input_snapshot.validation_evalset_path), + output_dir=Path(prepared.workspace.run_dir) / "optimizer", + seed=42, + retain_native_artifacts=False, + ) + + def fail_cleanup(path: Path, **kwargs) -> None: + raise OSError("simulated cleanup failure") + + monkeypatch.setattr(candidate_provider_module.shutil, "rmtree", fail_cleanup) + + with pytest.raises(CandidateProviderError, match="discard optimizer artifacts"): + await AgentOptimizerCandidateProvider( + _offline_agent(prepared.working_target).call_agent + ).propose(request) diff --git a/tests/evaluation/test_eval_optimize_loop_stage5_artifacts.py b/tests/evaluation/test_eval_optimize_loop_stage5_artifacts.py new file mode 100644 index 00000000..d6bf7137 --- /dev/null +++ b/tests/evaluation/test_eval_optimize_loop_stage5_artifacts.py @@ -0,0 +1,509 @@ +"""Tests for safe and atomic Stage 5 report artifact publication.""" + +from datetime import datetime, timezone +from hashlib import sha256 +import json +from pathlib import Path +import shutil + +import pytest + +from examples.optimization.eval_optimize_loop import artifact_writer +from examples.optimization.eval_optimize_loop import pipeline as pipeline_module +from examples.optimization.eval_optimize_loop.artifact_writer import ( + ArtifactWriteError, + discover_run_artifacts, + publish_report_bundle, + write_failure_report, +) +from examples.optimization.eval_optimize_loop.pipeline import prepare_run +from examples.optimization.eval_optimize_loop.report_builder import ( + build_failure_report, + build_optimization_report, +) +from examples.optimization.eval_optimize_loop.schemas import FailureReport, ReportProgress + + +_ROOT = Path(__file__).resolve().parents[2] +_EXAMPLE = _ROOT / "examples" / "optimization" / "eval_optimize_loop" + + +def _copy_example(tmp_path: Path) -> Path: + target = tmp_path / "eval_optimize_loop" + shutil.copytree(_EXAMPLE, target, ignore=shutil.ignore_patterns("runs", "__pycache__")) + return target + + +async def _build_fake_report_for_root(root: Path, run_id: str): + prepared = prepare_run(root / "pipeline.json", run_id=run_id) + execution_progress = pipeline_module._MutableReportProgress( + started_at=datetime(2026, 7, 18, tzinfo=timezone.utc) + ) + result = await pipeline_module._execute_offline_stage( + prepared, + scenario="improve", + progress=execution_progress, + ) + progress = ReportProgress( + started_at=datetime(2026, 7, 18, tzinfo=timezone.utc), + current_phase="reporting", + completed_phases=[ + "baseline_train", + "baseline_validation", + "candidate_generation", + "candidate_train", + "candidate_validation", + "analysis", + "gate", + "writeback", + ], + ) + report = build_optimization_report( + prepared, + result, + progress=progress, + finished_at=datetime(2026, 7, 18, 0, 1, tzinfo=timezone.utc), + ) + return prepared, report + + +async def _build_fake_report(tmp_path: Path, run_id: str): + return await _build_fake_report_for_root(_copy_example(tmp_path), run_id) + + +@pytest.mark.asyncio +async def test_publish_report_bundle_materializes_and_indexes_required_files(tmp_path): + prepared, report = await _build_fake_report(tmp_path, "artifact_complete") + run_dir = Path(prepared.workspace.run_dir) + + index = publish_report_bundle(report, run_dir=run_dir, copy_input_files=True) + + report_dir = run_dir / "report" + assert (report_dir / "optimization_report.json").is_file() + assert (report_dir / "optimization_report.md").is_file() + assert (report_dir / "artifact_index.json").is_file() + assert all(ref.relative_path != "report/artifact_index.json" for ref in index.artifacts) + + input_paths = { + ref.artifact_id: ref.relative_path + for ref in index.artifacts + if ref.artifact_type == "input" + } + assert input_paths == { + "input.pipeline_config": "report/inputs/pipeline_config.json", + "input.optimizer_config": "report/inputs/optimizer_config.json", + "input.train_evalset": "report/inputs/train_evalset.json", + "input.validation_evalset": "report/inputs/validation_evalset.json", + } + assert { + ref.artifact_id: ref.relative_path + for ref in index.artifacts + if ref.artifact_type == "prompt" + } == { + "prompt.baseline.system_prompt": ( + "report/prompts/baseline/000-system_prompt.md" + ), + "prompt.candidate.system_prompt": ( + "report/prompts/candidate/000-system_prompt.md" + ), + } + baseline_prompt = report_dir / "prompts" / "baseline" / "000-system_prompt.md" + candidate_prompt = report_dir / "prompts" / "candidate" / "000-system_prompt.md" + assert baseline_prompt.read_text(encoding="utf-8") == ( + report.input_snapshot.prompt_snapshots[0].content + ) + assert candidate_prompt.read_text(encoding="utf-8") == ( + report.candidate.prompts["system_prompt"] + ) + evaluation_paths = { + ref.artifact_id: ref.relative_path + for ref in index.artifacts + if ref.artifact_type == "evaluation" + } + assert evaluation_paths == { + f"evaluation.{name}": f"report/evaluations/{name}.json" + for name in ( + "baseline_train", + "baseline_validation", + "candidate_train", + "candidate_validation", + ) + } + assert { + path.name for path in (report_dir / "evaluations").iterdir() + } == { + "baseline_train.json", + "baseline_validation.json", + "candidate_train.json", + "candidate_validation.json", + } + + available = [ref for ref in index.artifacts if ref.status == "available"] + assert available + for ref in available: + assert ref.relative_path is not None + relative_path = Path(ref.relative_path) + assert not relative_path.is_absolute() + assert ".." not in relative_path.parts + path = run_dir / relative_path + assert path.is_file() + assert ref.sha256 == sha256(path.read_bytes()).hexdigest() + assert ref.size_bytes == path.stat().st_size + + +@pytest.mark.asyncio +async def test_publish_rejects_input_hash_drift(tmp_path): + prepared, report = await _build_fake_report(tmp_path, "artifact_drift") + train_path = Path(prepared.input_snapshot.train_evalset_path) + train_path.write_text( + train_path.read_text(encoding="utf-8") + "\n", + encoding="utf-8", + ) + run_dir = Path(prepared.workspace.run_dir) + + with pytest.raises(ArtifactWriteError, match="input hash"): + publish_report_bundle(report, run_dir=run_dir, copy_input_files=True) + + assert not (run_dir / "report").exists() + assert list(run_dir.glob(".report.tmp-*")) == [] + + +@pytest.mark.parametrize( + "sensitive_key", + [ + "api_key", + "access_token", + "client_secret", + "password", + "x-api-key", + "openai_api_key", + "provider_api_key", + "endpoint_url", + ], +) +@pytest.mark.asyncio +async def test_publish_rejects_plaintext_secret_in_optimizer_config_without_leaking_it( + tmp_path, sensitive_key +): + root = _copy_example(tmp_path) + optimizer_path = root / "optimizer.json" + payload = json.loads(optimizer_path.read_text(encoding="utf-8")) + secret = "sk-real-sentinel-secret-must-not-be-copied" + payload["optimize"]["algorithm"]["reflection_lm"]["extra_fields"] = { + sensitive_key: secret + } + optimizer_path.write_text( + json.dumps(payload, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + prepared, report = await _build_fake_report_for_root(root, "artifact_secret") + run_dir = Path(prepared.workspace.run_dir) + + with pytest.raises(ArtifactWriteError, match="sensitive optimizer config"): + publish_report_bundle(report, run_dir=run_dir, copy_input_files=True) + + assert secret in optimizer_path.read_text(encoding="utf-8") + assert not (run_dir / "report").exists() + assert list(run_dir.glob(".report.tmp-*")) == [] + secret_bytes = secret.encode("utf-8") + assert all( + secret_bytes not in path.read_bytes() + for path in run_dir.rglob("*") + if path.is_file() + ) + + +@pytest.mark.asyncio +async def test_publish_rejects_literal_url_hidden_under_provider_specific_key( + tmp_path, +): + root = _copy_example(tmp_path) + optimizer_path = root / "optimizer.json" + payload = json.loads(optimizer_path.read_text(encoding="utf-8")) + endpoint = "https://private-provider.example.test/v1" + payload["optimize"]["algorithm"]["reflection_lm"]["extra_fields"] = { + "service_location": endpoint + } + optimizer_path.write_text( + json.dumps(payload, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + prepared, report = await _build_fake_report_for_root(root, "artifact_url") + run_dir = Path(prepared.workspace.run_dir) + + with pytest.raises(ArtifactWriteError, match="sensitive optimizer config"): + publish_report_bundle(report, run_dir=run_dir, copy_input_files=True) + + assert not (run_dir / "report").exists() + endpoint_bytes = endpoint.encode("utf-8") + assert all( + endpoint_bytes not in path.read_bytes() + for path in run_dir.rglob("*") + if path.is_file() + ) + + +@pytest.mark.asyncio +async def test_publish_rejects_plaintext_secret_in_optimizer_runtime_artifact(tmp_path): + prepared, report = await _build_fake_report(tmp_path, "artifact_runtime_secret") + run_dir = Path(prepared.workspace.run_dir) + runtime_path = run_dir / "optimizer.runtime.json" + secret = "runtime-authorization-secret" + runtime_path.write_text( + json.dumps({"reflection_lm": {"authorization": secret}}), + encoding="utf-8", + ) + + with pytest.raises(ArtifactWriteError, match="sensitive optimizer config"): + publish_report_bundle(report, run_dir=run_dir, copy_input_files=True) + + assert not (run_dir / "report").exists() + assert list(run_dir.glob(".report.tmp-*")) == [] + + +@pytest.mark.asyncio +async def test_publish_rejects_existing_report_directory(tmp_path): + prepared, report = await _build_fake_report(tmp_path, "artifact_existing") + report_dir = Path(prepared.workspace.run_dir) / "report" + report_dir.mkdir() + + with pytest.raises(ArtifactWriteError, match="already exists"): + publish_report_bundle( + report, + run_dir=Path(prepared.workspace.run_dir), + copy_input_files=True, + ) + + +@pytest.mark.asyncio +async def test_publish_does_not_replace_report_directory_created_during_staging( + tmp_path, monkeypatch +): + prepared, report = await _build_fake_report(tmp_path, "artifact_report_race") + run_dir = Path(prepared.workspace.run_dir) + report_dir = run_dir / "report" + original_write = artifact_writer._write_text + + def create_competing_report_before_publish(path, content): + original_write(path, content) + if path.name == "artifact_index.json": + report_dir.mkdir() + + monkeypatch.setattr( + artifact_writer, + "_write_text", + create_competing_report_before_publish, + ) + + with pytest.raises(ArtifactWriteError, match="already exists"): + publish_report_bundle(report, run_dir=run_dir, copy_input_files=True) + + assert report_dir.is_dir() + assert list(report_dir.iterdir()) == [] + assert list(run_dir.glob(".report.tmp-*")) == [] + + +def test_publish_fails_closed_when_atomic_no_replace_is_unsupported( + tmp_path, monkeypatch +): + source = tmp_path / "staging" + target = tmp_path / "report" + source.mkdir() + + def unexpected_rename(self, destination): + raise AssertionError("普通 Path.rename 不得作为 no-replace 回退") + + monkeypatch.setattr(artifact_writer.sys, "platform", "freebsd") + monkeypatch.setattr(Path, "rename", unexpected_rename) + + with pytest.raises(ArtifactWriteError, match="atomic no-replace unavailable"): + artifact_writer._rename_directory_no_replace(source, target) + + assert source.is_dir() + assert not target.exists() + + +@pytest.mark.parametrize("target_inside_run", [False, True]) +def test_discovery_rejects_any_file_symlink(tmp_path, target_inside_run): + run_dir = tmp_path / "run" + run_dir.mkdir() + target_root = run_dir if target_inside_run else tmp_path + target = target_root / "target.txt" + target.write_text("target", encoding="utf-8") + (run_dir / "link.txt").symlink_to(target) + + with pytest.raises(ArtifactWriteError, match="symbolic link"): + discover_run_artifacts(run_dir) + + +def test_discovery_rejects_directory_symlink_without_following_it(tmp_path): + run_dir = tmp_path / "run" + run_dir.mkdir() + outside = tmp_path / "outside" + outside.mkdir() + (outside / "artifact.txt").write_text("outside", encoding="utf-8") + (run_dir / "linked-directory").symlink_to(outside, target_is_directory=True) + + with pytest.raises(ArtifactWriteError, match="symbolic link"): + discover_run_artifacts(run_dir) + + +@pytest.mark.asyncio +async def test_publish_rejects_symlink_in_optimizer_artifacts(tmp_path): + prepared, report = await _build_fake_report(tmp_path, "artifact_optimizer_link") + run_dir = Path(prepared.workspace.run_dir) + optimizer_dir = run_dir / "optimizer" + optimizer_dir.mkdir() + outside = tmp_path / "optimizer-result.json" + outside.write_text("{}\n", encoding="utf-8") + (optimizer_dir / "result.json").symlink_to(outside) + + with pytest.raises(ArtifactWriteError, match="symbolic link"): + publish_report_bundle(report, run_dir=run_dir, copy_input_files=True) + + assert not (run_dir / "report").exists() + assert list(run_dir.glob(".report.tmp-*")) == [] + + +@pytest.mark.asyncio +async def test_publish_failure_removes_staging_directory(tmp_path, monkeypatch): + prepared, report = await _build_fake_report(tmp_path, "artifact_partial") + calls = 0 + original = artifact_writer._write_text + + def fail_second_write(path, content): + nonlocal calls + calls += 1 + if calls == 2: + raise OSError("simulated write failure") + original(path, content) + + monkeypatch.setattr(artifact_writer, "_write_text", fail_second_write) + run_dir = Path(prepared.workspace.run_dir) + + with pytest.raises(ArtifactWriteError, match="simulated write failure"): + publish_report_bundle(report, run_dir=run_dir, copy_input_files=True) + + assert not (run_dir / "report").exists() + assert list(run_dir.glob(".report.tmp-*")) == [] + + +@pytest.mark.asyncio +async def test_failure_report_is_atomic_and_round_trips(tmp_path): + prepared, _ = await _build_fake_report(tmp_path, "artifact_failure") + progress = ReportProgress( + started_at=datetime(2026, 7, 18, tzinfo=timezone.utc), + current_phase="analysis", + completed_phases=[ + "baseline_train", + "baseline_validation", + "candidate_generation", + "candidate_train", + "candidate_validation", + ], + ) + failure = build_failure_report( + prepared, + progress=progress, + error=RuntimeError("simulated analysis failure"), + source_prompt_hashes={ + snapshot.field_name: snapshot.sha256 + for snapshot in prepared.input_snapshot.prompt_snapshots + }, + existing_artifacts=[], + generated_at=datetime(2026, 7, 18, 0, 1, tzinfo=timezone.utc), + ) + + path = write_failure_report(failure, run_dir=Path(prepared.workspace.run_dir)) + + assert FailureReport.model_validate_json(path.read_text(encoding="utf-8")) == failure + assert list(path.parent.glob(".failure_report.tmp-*")) == [] + + +@pytest.mark.asyncio +async def test_failure_report_rejects_existing_first_failure_evidence(tmp_path): + prepared, _ = await _build_fake_report(tmp_path, "artifact_failure_existing") + run_dir = Path(prepared.workspace.run_dir) + target = run_dir / "failure_report.json" + target.write_text('{"first": true}\n', encoding="utf-8") + original = target.read_bytes() + progress = ReportProgress( + started_at=datetime(2026, 7, 18, tzinfo=timezone.utc), + current_phase="reporting", + completed_phases=[], + ) + failure = build_failure_report( + prepared, + progress=progress, + error=RuntimeError("second failure"), + source_prompt_hashes={}, + existing_artifacts=[], + generated_at=datetime(2026, 7, 18, 0, 1, tzinfo=timezone.utc), + ) + + with pytest.raises(ArtifactWriteError, match="already exists"): + write_failure_report(failure, run_dir=run_dir) + + assert target.read_bytes() == original + assert list(run_dir.glob(".failure_report.tmp-*")) == [] + + +@pytest.mark.asyncio +async def test_failure_report_does_not_overwrite_target_created_during_write( + tmp_path, monkeypatch +): + prepared, _ = await _build_fake_report(tmp_path, "artifact_failure_race") + run_dir = Path(prepared.workspace.run_dir) + target = run_dir / "failure_report.json" + first_failure = b'{"first": true}\n' + original_write = artifact_writer._write_text + + def create_first_failure_after_temporary_write(path, content): + original_write(path, content) + target.write_bytes(first_failure) + + monkeypatch.setattr( + artifact_writer, + "_write_text", + create_first_failure_after_temporary_write, + ) + progress = ReportProgress( + started_at=datetime(2026, 7, 18, tzinfo=timezone.utc), + current_phase="reporting", + completed_phases=[], + ) + failure = build_failure_report( + prepared, + progress=progress, + error=RuntimeError("racing failure"), + source_prompt_hashes={}, + existing_artifacts=[], + generated_at=datetime(2026, 7, 18, 0, 1, tzinfo=timezone.utc), + ) + + with pytest.raises(ArtifactWriteError, match="already exists"): + write_failure_report(failure, run_dir=run_dir) + + assert target.read_bytes() == first_failure + assert list(run_dir.glob(".failure_report.tmp-*")) == [] + + +@pytest.mark.asyncio +async def test_copy_input_files_false_records_unavailable_inputs(tmp_path): + prepared, report = await _build_fake_report(tmp_path, "artifact_no_inputs") + + index = publish_report_bundle( + report, + run_dir=Path(prepared.workspace.run_dir), + copy_input_files=False, + ) + + input_refs = [ + ref for ref in index.artifacts if ref.artifact_id.startswith("input.") + ] + assert input_refs + assert {ref.status for ref in input_refs} == {"unavailable"} + assert all( + ref.unavailable_reason == "artifacts.copy_input_files=false" + for ref in input_refs + ) diff --git a/tests/evaluation/test_eval_optimize_loop_stage5_pipeline.py b/tests/evaluation/test_eval_optimize_loop_stage5_pipeline.py new file mode 100644 index 00000000..808efbbd --- /dev/null +++ b/tests/evaluation/test_eval_optimize_loop_stage5_pipeline.py @@ -0,0 +1,475 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under the Apache License, Version 2.0. +"""Stage 5 integration tests for success and failure report orchestration.""" + +from __future__ import annotations + +import json +from pathlib import Path +import shutil +import subprocess +import sys +import time +from types import SimpleNamespace + +import pytest + +from examples.optimization.eval_optimize_loop import pipeline as pipeline_module +from examples.optimization.eval_optimize_loop import run_real_integration +from examples.optimization.eval_optimize_loop.business_agent import BusinessAgent +from examples.optimization.eval_optimize_loop.fake import DeterministicFakeModel +from examples.optimization.eval_optimize_loop.fake import DeterministicFakeCandidateProvider +from examples.optimization.eval_optimize_loop.pipeline import PipelineExecutionError +from examples.optimization.eval_optimize_loop.pipeline import prepare_run +from examples.optimization.eval_optimize_loop.pipeline import run_offline_stage +from examples.optimization.eval_optimize_loop.pipeline import run_real_stage +from examples.optimization.eval_optimize_loop.schemas import ArtifactIndex +from examples.optimization.eval_optimize_loop.schemas import FailureReport +from examples.optimization.eval_optimize_loop.schemas import OptimizationReport +from examples.optimization.eval_optimize_loop.schemas import OptimizerRuntimeParameters +from trpc_agent_sdk.evaluation import OptimizeResult +from trpc_agent_sdk.evaluation import TargetPrompt + + +_REPO_ROOT = Path(__file__).resolve().parents[2] +_EXAMPLE = _REPO_ROOT / "examples" / "optimization" / "eval_optimize_loop" + + +def _offline_agent(target: TargetPrompt) -> BusinessAgent: + return BusinessAgent( + target, + DeterministicFakeModel, + agent_name="stage5_offline_agent", + app_name="stage5_offline_test", + user_id="stage5-test", + ) + + +def _copy_example(tmp_path: Path, name: str = "eval_optimize_loop") -> Path: + target = tmp_path / name + shutil.copytree( + _EXAMPLE, + target, + ignore=shutil.ignore_patterns("runs", "__pycache__"), + ) + return target + + +def _set_real_mode(root: Path) -> None: + path = root / "pipeline.json" + payload = json.loads(path.read_text(encoding="utf-8")) + payload["execution"]["mode"] = "real" + path.write_text(json.dumps(payload), encoding="utf-8") + + +def _optimize_result( + baseline_prompts: dict[str, str], + best_prompts: dict[str, str], +) -> OptimizeResult: + return OptimizeResult( + algorithm="gepa_reflective", + status="SUCCEEDED", + finish_reason="completed", + error_message="", + baseline_pass_rate=1 / 3, + best_pass_rate=1.0, + pass_rate_improvement=2 / 3, + baseline_prompts=baseline_prompts, + best_prompts=best_prompts, + total_rounds=1, + rounds=[], + total_reflection_lm_calls=1, + total_llm_cost=0.25, + total_token_usage={"prompt": 10, "completion": 5, "total": 15}, + duration_seconds=0.5, + started_at="2026-07-18T00:00:00Z", + finished_at="2026-07-18T00:00:01Z", + ) + + +@pytest.mark.parametrize("scenario", ["improve", "no_improvement", "overfit"]) +@pytest.mark.asyncio +async def test_fake_stage_publishes_complete_report_bundle( + tmp_path: Path, + scenario: str, +) -> None: + root = _copy_example(tmp_path) + prepared = prepare_run(root / "pipeline.json", run_id=f"stage5_{scenario}") + + result = await run_offline_stage(prepared, scenario=scenario) + + run_dir = Path(prepared.workspace.run_dir) + report_dir = run_dir / "report" + report = OptimizationReport.model_validate_json( + (report_dir / "optimization_report.json").read_text(encoding="utf-8") + ) + assert report.gate_decision == result.gate_decision + assert report.candidate == result.candidate + assert (report_dir / "artifact_index.json").is_file() + assert not (run_dir / "failure_report.json").exists() + + +@pytest.mark.asyncio +async def test_real_stage_reports_optimizer_native_artifacts( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + root = _copy_example(tmp_path) + _set_real_mode(root) + pipeline_path = root / "pipeline.json" + pipeline_payload = json.loads(pipeline_path.read_text(encoding="utf-8")) + pipeline_payload["artifacts"]["copy_input_files"] = False + pipeline_path.write_text(json.dumps(pipeline_payload), encoding="utf-8") + optimizer_path = root / "optimizer.json" + optimizer_payload = json.loads(optimizer_path.read_text(encoding="utf-8")) + optimizer_payload["optimize"]["algorithm"]["reflection_lm"]["extra_fields"] = { + "authorization": "Bearer stage5-authorization-secret", + "token": "stage5-provider-token", + "client_secret": "stage5-client-secret", + } + optimizer_path.write_text(json.dumps(optimizer_payload), encoding="utf-8") + prepared = prepare_run(root / "pipeline.json", run_id="stage5_real") + baseline = await prepared.working_target.read_all() + best = DeterministicFakeCandidateProvider().propose( + baseline, + scenario="improve", + seed=prepared.input_snapshot.seed, + ).prompts + + async def fake_optimize(**kwargs: object) -> OptimizeResult: + output_dir = Path(str(kwargs["output_dir"])) + rounds_dir = output_dir / "rounds" + rounds_dir.mkdir(parents=True) + result = _optimize_result(baseline, best) + result.dump_to(str(output_dir / "result.json")) + (rounds_dir / "round_001.json").write_text("{}\n", encoding="utf-8") + return result + + monkeypatch.setattr( + "examples.optimization.eval_optimize_loop.candidate_provider.AgentOptimizer.optimize", + fake_optimize, + ) + + result = await run_real_stage( + prepared, + call_agent=_offline_agent(prepared.working_target).call_agent, + optimizer_parameters=OptimizerRuntimeParameters( + model_name="stage5-reflection-model", + max_candidate_proposals=1, + ), + ) + + report_dir = Path(prepared.workspace.run_dir) / "report" + report = OptimizationReport.model_validate_json( + (report_dir / "optimization_report.json").read_text(encoding="utf-8") + ) + index = ArtifactIndex.model_validate_json( + (report_dir / "artifact_index.json").read_text(encoding="utf-8") + ) + native_paths = { + reference.relative_path + for reference in index.artifacts + if reference.artifact_type == "optimizer_native" + } + assert native_paths == { + "optimizer.runtime.json", + "optimizer/result.json", + "optimizer/rounds/round_001.json", + } + runtime_text = ( + Path(prepared.workspace.run_dir) / "optimizer.runtime.json" + ).read_text(encoding="utf-8") + for secret in ( + "stage5-authorization-secret", + "stage5-provider-token", + "stage5-client-secret", + ): + assert secret not in runtime_text + assert report.candidate == result.candidate + assert report.optimizer_resources.total_rounds.status == "available" + assert report.optimizer_resources.reflection_lm_calls.status == "available" + assert report.optimizer_resources.duration_seconds.status == "available" + + +_FAILURE_CASES = [ + ("_evaluate_split", "baseline_train"), + ("FakeCandidateProviderAdapter.propose", "candidate_generation"), + ("build_evaluation_analysis", "analysis"), + ("evaluate_gate", "gate"), + ("perform_writeback", "writeback"), + ("publish_report_bundle", "reporting"), +] + + +def _install_stage_failure( + monkeypatch: pytest.MonkeyPatch, + target: str, +) -> None: + def fail_sync(*args: object, **kwargs: object) -> None: + raise RuntimeError("simulated stage failure") + + async def fail_async(*args: object, **kwargs: object) -> None: + raise RuntimeError("simulated stage failure") + + if target == "FakeCandidateProviderAdapter.propose": + monkeypatch.setattr( + pipeline_module.FakeCandidateProviderAdapter, + "propose", + fail_async, + ) + elif target in {"_evaluate_split", "perform_writeback"}: + monkeypatch.setattr(pipeline_module, target, fail_async) + else: + monkeypatch.setattr(pipeline_module, target, fail_sync, raising=False) + + +@pytest.mark.parametrize(("target", "expected_phase"), _FAILURE_CASES) +@pytest.mark.asyncio +async def test_fake_stage_writes_phase_specific_failure_report( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + target: str, + expected_phase: str, +) -> None: + root = _copy_example(tmp_path) + prepared = prepare_run( + root / "pipeline.json", + run_id=f"stage5_failure_{expected_phase}", + ) + _install_stage_failure(monkeypatch, target) + + with pytest.raises(Exception, match="simulated stage failure"): + await run_offline_stage(prepared, scenario="improve") + + run_dir = Path(prepared.workspace.run_dir) + failure = FailureReport.model_validate_json( + (run_dir / "failure_report.json").read_text(encoding="utf-8") + ) + assert failure.failed_phase == expected_phase + assert failure.exception_type in { + "RuntimeError", + "PipelineExecutionError", + "ArtifactWriteError", + } + assert "simulated stage failure" in failure.error_message + assert not (run_dir / "report").exists() + + +@pytest.mark.asyncio +async def test_failure_report_error_preserves_original_exception_as_cause( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + root = _copy_example(tmp_path) + prepared = prepare_run(root / "pipeline.json", run_id="stage5_double_failure") + + async def fail_evaluation(*args: object, **kwargs: object) -> None: + raise RuntimeError("simulated stage failure") + + def fail_failure_report(*args: object, **kwargs: object) -> None: + raise RuntimeError("failed to write failure report") + + monkeypatch.setattr(pipeline_module, "_evaluate_split", fail_evaluation) + monkeypatch.setattr( + pipeline_module, + "write_failure_report", + fail_failure_report, + raising=False, + ) + + with pytest.raises(PipelineExecutionError) as raised: + await run_offline_stage(prepared, scenario="improve") + + assert "simulated stage failure" in str(raised.value) + assert "failed to write failure report" in str(raised.value) + assert isinstance(raised.value.__cause__, RuntimeError) + assert str(raised.value.__cause__) == "simulated stage failure" + + +@pytest.mark.asyncio +async def test_reporting_failure_rolls_back_successful_source_writeback( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + root = _copy_example(tmp_path) + config_path = root / "pipeline.json" + payload = json.loads(config_path.read_text(encoding="utf-8")) + payload["writeback"]["enabled"] = True + config_path.write_text(json.dumps(payload), encoding="utf-8") + prepared = prepare_run(config_path, run_id="stage5_reporting_rollback") + source_before = await prepared.source_target.read_all() + + def fail_report(*args: object, **kwargs: object) -> None: + raise RuntimeError("simulated reporting failure") + + monkeypatch.setattr(pipeline_module, "publish_report_bundle", fail_report) + + with pytest.raises(RuntimeError, match="simulated reporting failure"): + await run_offline_stage(prepared, scenario="improve") + + assert await prepared.source_target.read_all() == source_before + failure = FailureReport.model_validate_json( + (Path(prepared.workspace.run_dir) / "failure_report.json").read_text( + encoding="utf-8" + ) + ) + assert failure.failed_phase == "reporting" + + +@pytest.mark.asyncio +async def test_reporting_and_writeback_rollback_failure_are_both_exposed( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + root = _copy_example(tmp_path) + config_path = root / "pipeline.json" + payload = json.loads(config_path.read_text(encoding="utf-8")) + payload["writeback"]["enabled"] = True + config_path.write_text(json.dumps(payload), encoding="utf-8") + prepared = prepare_run(config_path, run_id="stage5_reporting_rollback_failure") + + def fail_report(*args: object, **kwargs: object) -> None: + raise RuntimeError("simulated reporting failure") + + async def fail_rollback(*args: object, **kwargs: object) -> None: + raise RuntimeError("simulated source rollback failure") + + monkeypatch.setattr(pipeline_module, "publish_report_bundle", fail_report) + monkeypatch.setattr(pipeline_module, "_rollback_written_source", fail_rollback) + + with pytest.raises(PipelineExecutionError) as raised: + await run_offline_stage(prepared, scenario="improve") + + assert "simulated reporting failure" in str(raised.value) + assert "simulated source rollback failure" in str(raised.value) + assert isinstance(raised.value.__cause__, RuntimeError) + assert str(raised.value.__cause__) == "simulated reporting failure" + + +@pytest.mark.asyncio +async def test_failure_report_remains_writable_when_source_prompt_is_unreadable( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + root = _copy_example(tmp_path) + prepared = prepare_run(root / "pipeline.json", run_id="stage5_source_unreadable") + progress = pipeline_module._MutableReportProgress( + started_at=pipeline_module.datetime.now(pipeline_module.timezone.utc) + ) + + async def fail_source_read() -> dict[str, str]: + raise OSError("simulated unreadable source Prompt") + + monkeypatch.setattr(prepared.source_target, "read_all", fail_source_read) + + await pipeline_module._record_failure( + prepared, + progress, + RuntimeError("simulated pipeline failure"), + ) + + failure = FailureReport.model_validate_json( + (Path(prepared.workspace.run_dir) / "failure_report.json").read_text( + encoding="utf-8" + ) + ) + assert failure.source_prompt_hashes == {} + + +def test_fake_cli_prints_report_paths(tmp_path: Path) -> None: + root = _copy_example(tmp_path) + + completed = subprocess.run( + [ + sys.executable, + str(root / "run_pipeline.py"), + "--config", + str(root / "pipeline.json"), + "--run-id", + "stage5_cli", + ], + cwd=_REPO_ROOT, + capture_output=True, + text=True, + check=False, + ) + + assert completed.returncode == 0, completed.stderr + assert "optimization_report.json" in completed.stdout + assert "optimization_report.md" in completed.stdout + assert "artifact_index.json" in completed.stdout + + +@pytest.mark.asyncio +async def test_all_fake_scenarios_duration_is_under_three_minutes_with_reports( + tmp_path: Path, +) -> None: + started_at = time.perf_counter() + + for scenario in ("improve", "no_improvement", "overfit"): + root = _copy_example(tmp_path, name=f"eval_optimize_loop_{scenario}") + prepared = prepare_run(root / "pipeline.json", run_id=f"stage5_{scenario}") + + await run_offline_stage(prepared, scenario=scenario) + + report_dir = Path(prepared.workspace.run_dir) / "report" + assert (report_dir / "optimization_report.json").is_file() + assert (report_dir / "optimization_report.md").is_file() + assert (report_dir / "artifact_index.json").is_file() + + assert time.perf_counter() - started_at < 180.0 + + +def test_real_cli_gate_reject_prints_report_paths_and_exits_zero( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + snapshot = SimpleNamespace( + passed_case_count=1, + total_case_count=3, + average_score=1 / 3, + ) + result = SimpleNamespace( + baseline_train=snapshot, + baseline_validation=snapshot, + candidate_train=snapshot, + candidate_validation=snapshot, + optimize_result=SimpleNamespace(status="SUCCEEDED", total_rounds=1), + candidate=SimpleNamespace(candidate_id="optimizer-stage5"), + gate_decision=SimpleNamespace( + decision="reject", + rejection_reasons=["no improvement"], + ), + writeback=SimpleNamespace(status="skipped", reason="gate_rejected"), + ) + prepared = SimpleNamespace( + workspace=SimpleNamespace(run_dir="runs/stage5_real_cli") + ) + + async def complete(*args: object, **kwargs: object) -> tuple[object, object]: + return prepared, result + + monkeypatch.setenv("TRPC_AGENT_API_KEY", "key") + monkeypatch.setenv("TRPC_AGENT_BASE_URL", "https://example.test") + monkeypatch.setenv("TRPC_AGENT_MODEL_NAME", "business-model") + monkeypatch.setattr(run_real_integration, "_run", complete) + monkeypatch.setattr( + sys, + "argv", + [ + "run_real_integration.py", + "--run-real", + "--optimizer-model-name", + "optimizer-model", + ], + ) + + assert run_real_integration.main() == 0 + output = capsys.readouterr().out + assert "Gate decision: REJECT" in output + assert "optimization_report.json" in output + assert "optimization_report.md" in output + assert "artifact_index.json" in output diff --git a/tests/evaluation/test_eval_optimize_loop_stage5_report_builder.py b/tests/evaluation/test_eval_optimize_loop_stage5_report_builder.py new file mode 100644 index 00000000..4bc4badd --- /dev/null +++ b/tests/evaluation/test_eval_optimize_loop_stage5_report_builder.py @@ -0,0 +1,352 @@ +"""Tests for Stage 5 serializable optimization reports.""" + +from datetime import datetime, timezone +import shutil +from pathlib import Path + +import pytest + +from examples.optimization.eval_optimize_loop.pipeline import prepare_run, run_offline_stage +from examples.optimization.eval_optimize_loop.report_builder import ( + build_failure_report, build_optimization_report, render_optimization_markdown, +) +from examples.optimization.eval_optimize_loop.schemas import ( + ArtifactIndex, ArtifactReference, OptimizationReport, OptimizerResourceValue, ReportProgress, +) +from examples.optimization.eval_optimize_loop.schemas import OptimizerCandidateProposal, RealStageResult +from trpc_agent_sdk.evaluation import OptimizeResult + +_ROOT = Path(__file__).resolve().parents[2] +_EXAMPLE = _ROOT / "examples" / "optimization" / "eval_optimize_loop" + +def _copy_example(tmp_path: Path, name: str = "eval_optimize_loop") -> Path: + target = tmp_path / name + shutil.copytree(_EXAMPLE, target, ignore=shutil.ignore_patterns("runs", "__pycache__")) + return target + +def _progress() -> ReportProgress: + return ReportProgress( + started_at=datetime(2026, 7, 18, tzinfo=timezone.utc), current_phase="reporting", + completed_phases=[ + "baseline_train", "baseline_validation", "candidate_generation", + "candidate_train", "candidate_validation", "analysis", "gate", "writeback", + ], + ) + + +def _optimize_result(baseline_prompts: dict[str, str], best_prompts: dict[str, str]) -> OptimizeResult: + return OptimizeResult( + algorithm="gepa_reflective", status="SUCCEEDED", finish_reason="completed", error_message="", + baseline_pass_rate=1 / 3, best_pass_rate=1.0, pass_rate_improvement=2 / 3, + baseline_prompts=baseline_prompts, best_prompts=best_prompts, total_rounds=1, rounds=[], + total_reflection_lm_calls=1, total_llm_cost=0.25, + total_token_usage={"prompt": 10, "completion": 5, "total": 15}, duration_seconds=0.5, + started_at="2026-07-17T00:00:00Z", finished_at="2026-07-17T00:00:01Z", + ) + +@pytest.mark.asyncio +async def test_fake_report_is_serializable_and_marks_optimizer_resources_not_applicable(tmp_path): + root = _copy_example(tmp_path) + prepared = prepare_run(root / "pipeline.json", run_id="report_fake") + report = build_optimization_report( + prepared, await run_offline_stage(prepared, scenario="improve"), progress=_progress(), + finished_at=datetime(2026, 7, 18, 0, 1, tzinfo=timezone.utc), + ) + assert report.status == "completed" + assert report.execution_mode == "offline" + optimizer_resources = report.optimizer_resources + for observation in ( + optimizer_resources.total_rounds, + optimizer_resources.reflection_lm_calls, + optimizer_resources.cost_usd, + optimizer_resources.token_usage, + optimizer_resources.duration_seconds, + ): + assert observation.status == "not_applicable" + assert observation.value is None + assert observation.reason == ( + "Offline mode uses a deterministic candidate provider." + ) + assert report.pipeline_resources.total_tokens.status == "unavailable" + assert OptimizationReport.model_validate_json(report.model_dump_json()) == report + + +@pytest.mark.asyncio +async def test_real_report_includes_optimizer_resource_observations(tmp_path): + root = _copy_example(tmp_path) + prepared = prepare_run(root / "pipeline.json", run_id="report_real") + fake_result = await run_offline_stage(prepared, scenario="improve") + candidate = OptimizerCandidateProposal.model_validate({ + **fake_result.candidate.model_dump(exclude={"scenario", "seed"}), "provider": "agent_optimizer", + "optimizer_status": "SUCCEEDED", "finish_reason": "completed", + "baseline_pass_rate": 1 / 3, "best_pass_rate": 1.0, + "candidate_id": f"optimizer-{fake_result.candidate.candidate_id[-12:]}", + }) + result = RealStageResult( + **fake_result.model_dump(exclude={"scenario", "candidate"}), candidate=candidate, + optimize_result=_optimize_result(await prepared.source_target.read_all(), candidate.prompts), + ) + report = build_optimization_report( + prepared, result, progress=_progress(), + finished_at=datetime(2026, 7, 18, 0, 1, tzinfo=timezone.utc), + ) + assert report.optimizer_resources.total_rounds.status == "available" + assert report.optimizer_resources.total_rounds.value == 1 + assert report.optimizer_resources.total_rounds.unit == "rounds" + assert report.optimizer_resources.reflection_lm_calls.status == "available" + assert report.optimizer_resources.reflection_lm_calls.value == 1 + assert report.optimizer_resources.cost_usd.status == "available" + assert report.optimizer_resources.token_usage.status == "available" + assert report.optimizer_resources.duration_seconds.status == "available" + assert report.optimizer_resources.scope_note == ( + "Optimizer-only observation; excludes complete business Agent evaluation usage." + ) + + +@pytest.mark.asyncio +async def test_real_report_marks_incomplete_cost_and_token_telemetry_unavailable(tmp_path): + root = _copy_example(tmp_path) + prepared = prepare_run(root / "pipeline.json", run_id="report_real_incomplete") + fake_result = await run_offline_stage(prepared, scenario="improve") + candidate = OptimizerCandidateProposal.model_validate({ + **fake_result.candidate.model_dump(exclude={"scenario", "seed"}), "provider": "agent_optimizer", + "optimizer_status": "SUCCEEDED", "finish_reason": "completed", + "baseline_pass_rate": 1 / 3, "best_pass_rate": 1.0, + "candidate_id": f"optimizer-{fake_result.candidate.candidate_id[-12:]}", + }) + native = _optimize_result( + await prepared.source_target.read_all(), candidate.prompts, + ).model_copy(update={ + "total_reflection_lm_calls": 2, + "total_llm_cost": 0.0, + "total_token_usage": {"prompt": 0, "completion": 0, "total": 0}, + }) + result = RealStageResult( + **fake_result.model_dump(exclude={"scenario", "candidate"}), candidate=candidate, + optimize_result=native, + ) + + report = build_optimization_report( + prepared, result, progress=_progress(), + finished_at=datetime(2026, 7, 18, 0, 1, tzinfo=timezone.utc), + ) + + assert report.optimizer_resources.total_rounds.status == "available" + assert report.optimizer_resources.total_rounds.value == 1 + assert report.optimizer_resources.reflection_lm_calls.status == "available" + assert report.optimizer_resources.reflection_lm_calls.value == 2 + assert report.optimizer_resources.duration_seconds.status == "available" + assert report.optimizer_resources.duration_seconds.value == 0.5 + assert report.optimizer_resources.cost_usd.status == "unavailable" + assert report.optimizer_resources.cost_usd.value is None + assert report.optimizer_resources.cost_usd.reason + assert report.optimizer_resources.token_usage.status == "unavailable" + assert report.optimizer_resources.token_usage.value is None + assert report.optimizer_resources.token_usage.reason + + markdown = render_optimization_markdown(report) + assert "Rounds: available" in markdown + assert "Reflection calls: available" in markdown + assert "Cost: unavailable" in markdown + assert "Token usage: unavailable" in markdown + assert "Duration: available" in markdown + +@pytest.mark.asyncio +async def test_markdown_includes_accept_and_reject_report_evidence(tmp_path): + root = _copy_example(tmp_path) + prepared = prepare_run(root / "pipeline.json", run_id="report_markdown") + accepted = build_optimization_report( + prepared, await run_offline_stage(prepared, scenario="improve"), progress=_progress(), + finished_at=datetime(2026, 7, 18, 0, 1, tzinfo=timezone.utc), + ) + rejected_root = _copy_example(tmp_path, "eval_optimize_loop_reject") + rejected_prepared = prepare_run(rejected_root / "pipeline.json", run_id="report_markdown_reject") + rejected = build_optimization_report( + rejected_prepared, await run_offline_stage(rejected_prepared, scenario="no_improvement"), progress=_progress(), + finished_at=datetime(2026, 7, 18, 0, 1, tzinfo=timezone.utc), + ) + markdown = render_optimization_markdown(accepted) + assert "# Optimization Report" in markdown + assert "Gate decision: ACCEPT" in markdown + assert "Baseline train" in markdown + assert "Candidate validation" in markdown + assert "Writeback" in markdown + assert "unavailable" in markdown + assert "Gate decision: REJECT" in render_optimization_markdown(rejected) + +@pytest.mark.asyncio +async def test_failure_report_is_deterministic_and_records_only_error_identity(tmp_path): + root = _copy_example(tmp_path) + prepared = prepare_run(root / "pipeline.json", run_id="report_failure") + report = build_failure_report( + prepared, progress=_progress(), error=RuntimeError("pipeline failed"), + source_prompt_hashes={"z": "last", "a": "first"}, existing_artifacts=["z.json", "a.json"], + generated_at=datetime(2026, 7, 18, 0, 1, tzinfo=timezone.utc), + ) + assert report.status == "failed" + assert report.failed_phase == "reporting" + assert report.exception_type == "RuntimeError" + assert report.error_message == "pipeline failed" + assert list(report.source_prompt_hashes) == ["a", "z"] + assert report.existing_artifacts == ["a.json", "z.json"] + + +@pytest.mark.asyncio +async def test_failure_report_redacts_environment_values_and_common_secret_forms(tmp_path, monkeypatch): + root = _copy_example(tmp_path) + prepared = prepare_run(root / "pipeline.json", run_id="report_failure_redacted") + api_key = "env-api-key-secret" + base_url = "https://env-base-url-secret.example/v1" + monkeypatch.setenv("TRPC_AGENT_API_KEY", api_key) + monkeypatch.setenv("TRPC_AGENT_BASE_URL", base_url) + error = RuntimeError( + f"timeout contacting {base_url} with {api_key}; " + "api_key=snake-secret apiKey:'camel-secret' " + "Authorization: Bearer authorization-secret Bearer loose-bearer-secret " + "base_url=https://snake-url-secret.example/v1 " + 'baseUrl="https://camel-url-secret.example/v1"; retryable=true' + ) + + report = build_failure_report( + prepared, progress=_progress(), error=error, + source_prompt_hashes={}, existing_artifacts=[], + generated_at=datetime(2026, 7, 18, 0, 1, tzinfo=timezone.utc), + ) + + for secret in ( + api_key, + base_url, + "snake-secret", + "camel-secret", + "authorization-secret", + "loose-bearer-secret", + "https://snake-url-secret.example/v1", + "https://camel-url-secret.example/v1", + ): + assert secret not in report.error_message + assert report.error_message.count("[REDACTED]") >= 8 + assert "timeout contacting" in report.error_message + assert "retryable=true" in report.error_message + + +def test_failure_report_redacts_hyphenated_keys_and_bare_urls(tmp_path): + root = _copy_example(tmp_path) + prepared = prepare_run(root / "pipeline.json", run_id="report_failure_hyphenated") + error = RuntimeError( + "X-API-Key: hyphen-api-secret x-api-key=lower-hyphen-secret; " + "endpoint https://private.example/v1?token=secret and http://10.0.0.8:8080/run" + ) + + report = build_failure_report( + prepared, progress=_progress(), error=error, + source_prompt_hashes={}, existing_artifacts=[], + generated_at=datetime(2026, 7, 18, 0, 1, tzinfo=timezone.utc), + ) + + assert "hyphen-api-secret" not in report.error_message + assert "lower-hyphen-secret" not in report.error_message + assert "https://private.example/v1?token=secret" not in report.error_message + assert "http://10.0.0.8:8080/run" not in report.error_message + assert "endpoint [REDACTED]" in report.error_message + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "token_usage", + [ + {}, + {"prompt": 1, "completion": 2}, + {"prompt": 1, "completion": "2", "total": 3}, + {"prompt": True, "completion": 2, "total": 3}, + {"prompt": -1, "completion": 2, "total": 1}, + {"prompt": 1, "completion": 2, "total": 4}, + ["not", "a", "dict"], + ], +) +async def test_real_report_marks_malformed_token_usage_unavailable(tmp_path, token_usage): + root = _copy_example(tmp_path) + prepared = prepare_run(root / "pipeline.json", run_id="report_real_bad_tokens") + fake_result = await run_offline_stage(prepared, scenario="improve") + candidate = OptimizerCandidateProposal.model_validate({ + **fake_result.candidate.model_dump(exclude={"scenario", "seed"}), "provider": "agent_optimizer", + "optimizer_status": "SUCCEEDED", "finish_reason": "completed", + "baseline_pass_rate": 1 / 3, "best_pass_rate": 1.0, + "candidate_id": f"optimizer-{fake_result.candidate.candidate_id[-12:]}", + }) + native = _optimize_result( + await prepared.source_target.read_all(), candidate.prompts, + ).model_copy(update={"total_token_usage": token_usage}) + result = RealStageResult( + **fake_result.model_dump(exclude={"scenario", "candidate"}), candidate=candidate, + optimize_result=native, + ) + + report = build_optimization_report( + prepared, result, progress=_progress(), + finished_at=datetime(2026, 7, 18, 0, 1, tzinfo=timezone.utc), + ) + + assert report.optimizer_resources.token_usage.status == "unavailable" + assert report.optimizer_resources.token_usage.value is None + + +@pytest.mark.asyncio +async def test_real_report_allows_consistent_zero_token_usage_without_reflection_calls(tmp_path): + root = _copy_example(tmp_path) + prepared = prepare_run(root / "pipeline.json", run_id="report_real_zero_tokens") + fake_result = await run_offline_stage(prepared, scenario="improve") + candidate = OptimizerCandidateProposal.model_validate({ + **fake_result.candidate.model_dump(exclude={"scenario", "seed"}), "provider": "agent_optimizer", + "optimizer_status": "SUCCEEDED", "finish_reason": "completed", + "baseline_pass_rate": 1 / 3, "best_pass_rate": 1.0, + "candidate_id": f"optimizer-{fake_result.candidate.candidate_id[-12:]}", + }) + native = _optimize_result( + await prepared.source_target.read_all(), candidate.prompts, + ).model_copy(update={ + "total_reflection_lm_calls": 0, + "total_token_usage": {"prompt": 0, "completion": 0, "total": 0, "cached": 7}, + }) + result = RealStageResult( + **fake_result.model_dump(exclude={"scenario", "candidate"}), candidate=candidate, + optimize_result=native, + ) + + report = build_optimization_report( + prepared, result, progress=_progress(), + finished_at=datetime(2026, 7, 18, 0, 1, tzinfo=timezone.utc), + ) + + assert report.optimizer_resources.token_usage.status == "available" + assert report.optimizer_resources.token_usage.value == { + "prompt": 0, "completion": 0, "total": 0, "cached": 7, + } + + +def test_report_dto_invariants_reject_inconsistent_artifact_states(): + with pytest.raises(ValueError, match="available artifacts require"): + ArtifactReference( + artifact_id="input", artifact_type="input", required=True, + produced_by="baseline_train", status="available", + ) + with pytest.raises(ValueError, match="artifact IDs must be unique"): + ArtifactIndex( + run_id="report", generated_at=datetime(2026, 7, 18, tzinfo=timezone.utc), + artifacts=[ + ArtifactReference( + artifact_id="input", artifact_type="input", required=True, + produced_by="baseline_train", status="unavailable", unavailable_reason="not found", + ), + ArtifactReference( + artifact_id="input", artifact_type="prompt", required=True, + produced_by="candidate_generation", status="unavailable", unavailable_reason="not found", + ), + ], + ) + + +def test_optimizer_resource_value_rejects_negative_numeric_value(): + with pytest.raises(ValueError, match="non-negative"): + OptimizerResourceValue[int]( + status="available", value=-1, unit="calls", + ) diff --git a/tests/evaluation/test_eval_optimize_loop_stage6_acceptance.py b/tests/evaluation/test_eval_optimize_loop_stage6_acceptance.py new file mode 100644 index 00000000..c7e4f272 --- /dev/null +++ b/tests/evaluation/test_eval_optimize_loop_stage6_acceptance.py @@ -0,0 +1,60 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under the Apache License, Version 2.0. +"""Repository-level Stage 6 sample-output acceptance tests.""" + +from __future__ import annotations + +from hashlib import sha256 +from pathlib import Path + +from examples.optimization.eval_optimize_loop.schemas import ArtifactIndex +from examples.optimization.eval_optimize_loop.schemas import OptimizationReport + + +_REPO_ROOT = Path(__file__).resolve().parents[2] +_SAMPLE = ( + _REPO_ROOT + / "examples" + / "optimization" + / "eval_optimize_loop" + / "sample_output" +) + + +def test_sample_output_is_schema_valid_and_self_consistent(): + report_path = _SAMPLE / "optimization_report.json" + markdown_path = _SAMPLE / "optimization_report.md" + index_path = _SAMPLE / "artifact_index.json" + report = OptimizationReport.model_validate_json( + report_path.read_text(encoding="utf-8") + ) + index = ArtifactIndex.model_validate_json(index_path.read_text(encoding="utf-8")) + + assert report.execution_mode == "offline" + assert report.gate_decision.decision == "accept" + assert index.run_id == report.run_id + assert report.run_id in markdown_path.read_text(encoding="utf-8") + for reference in index.artifacts: + if reference.status != "available": + continue + path = _SAMPLE / reference.relative_path # type: ignore[arg-type] + payload = path.read_bytes() + assert len(payload) == reference.size_bytes + assert sha256(payload).hexdigest() == reference.sha256 + + +def test_sample_output_does_not_contain_credentials_or_machine_paths(): + combined = "\n".join( + path.read_text(encoding="utf-8") + for path in ( + _SAMPLE / "optimization_report.json", + _SAMPLE / "optimization_report.md", + _SAMPLE / "artifact_index.json", + ) + ) + assert "/home/" not in combined + assert "TRPC_AGENT_API_KEY" not in combined + assert "TRPC_AGENT_BASE_URL" not in combined diff --git a/tests/evaluation/test_eval_optimize_loop_stage6_offline.py b/tests/evaluation/test_eval_optimize_loop_stage6_offline.py new file mode 100644 index 00000000..2a4f4c30 --- /dev/null +++ b/tests/evaluation/test_eval_optimize_loop_stage6_offline.py @@ -0,0 +1,143 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under the Apache License, Version 2.0. +"""Stage 6 offline-mode tests for the SDK-backed deterministic model.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from examples.optimization.eval_optimize_loop.business_agent import BusinessAgent +from examples.optimization.eval_optimize_loop.fake.model import ( + DeterministicFakeModel, +) +from trpc_agent_sdk.evaluation import TargetPrompt +from trpc_agent_sdk.models import LlmRequest +from trpc_agent_sdk.types import Content +from trpc_agent_sdk.types import GenerateContentConfig +from trpc_agent_sdk.types import Part + + +def _request(instruction: str, user_text: str) -> LlmRequest: + return LlmRequest( + contents=[ + Content(role="user", parts=[Part.from_text(text=user_text)]), + ], + config=GenerateContentConfig(system_instruction=instruction), + ) + + +async def _text(model: DeterministicFakeModel, request: LlmRequest) -> str: + responses = [response async for response in model.generate_async(request)] + assert len(responses) == 1 + assert responses[0].content is not None + return "".join(part.text or "" for part in responses[0].content.parts or []) + + +def test_deterministic_fake_model_registers_a_stable_model_pattern(): + assert DeterministicFakeModel.supported_models() == [ + "deterministic-fake-model" + ] + + +@pytest.mark.asyncio +async def test_deterministic_fake_model_uses_system_instruction(): + model = DeterministicFakeModel() + query = "Check the status of order A100." + + baseline = await _text(model, _request("baseline", query)) + improved = await _text( + model, + _request( + "", + query, + ), + ) + + assert '"route":"general_support"' in baseline + assert improved == ( + '{"route":"order_lookup","message":"Checking order A100."}' + ) + assert await _text(model, _request("baseline", query)) == baseline + + +@pytest.mark.asyncio +async def test_deterministic_fake_model_uses_last_user_message(): + request = _request( + "", + "ignored", + ) + request.contents.extend( + [ + Content(role="model", parts=[Part.from_text(text="intermediate")]), + Content( + role="user", + parts=[Part.from_text(text="How long is standard shipping?")], + ), + ] + ) + + assert await _text(DeterministicFakeModel(), request) == ( + '{"route":"shipping_policy","message":' + '"Standard shipping normally takes 3-5 business days."}' + ) + + +@pytest.mark.asyncio +async def test_deterministic_fake_model_rejects_missing_user_text(): + request = LlmRequest( + contents=[Content(role="model", parts=[Part.from_text(text="only model")])], + config=GenerateContentConfig(system_instruction="baseline"), + ) + + responses = [ + response + async for response in DeterministicFakeModel().generate_async(request) + ] + + assert len(responses) == 1 + assert responses[0].content is None + assert responses[0].error_code == "API_ERROR" + assert "user text" in (responses[0].error_message or "") + + +@pytest.mark.asyncio +async def test_business_agent_rereads_prompt_through_llm_agent(tmp_path: Path): + prompt_path = tmp_path / "system.md" + prompt_path.write_text("baseline", encoding="utf-8") + target = TargetPrompt().add_path("system_prompt", str(prompt_path)) + created_models: list[DeterministicFakeModel] = [] + + def model_factory() -> DeterministicFakeModel: + model = DeterministicFakeModel() + created_models.append(model) + return model + + agent = BusinessAgent( + target, + model_factory, + agent_name="stage6_offline_agent", + app_name="stage6_offline_test", + user_id="stage6-test", + ) + query = "Check the status of order A100." + + baseline = await agent.call_agent(query) + await target.write_all( + { + "system_prompt": ( + "" + ) + } + ) + candidate = await agent.call_agent(query) + + assert '"route":"general_support"' in baseline + assert candidate == ( + '{"route":"order_lookup","message":"Checking order A100."}' + ) + assert len(created_models) == 2 diff --git a/tests/evaluation/test_eval_optimize_loop_stage6_trace.py b/tests/evaluation/test_eval_optimize_loop_stage6_trace.py new file mode 100644 index 00000000..1fb682de --- /dev/null +++ b/tests/evaluation/test_eval_optimize_loop_stage6_trace.py @@ -0,0 +1,138 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under the Apache License, Version 2.0. +"""Stage 6 tests for deterministic SDK Trace replay.""" + +from __future__ import annotations + +import shutil +from pathlib import Path + +import pytest + +from examples.optimization.eval_optimize_loop import pipeline as pipeline_module +from examples.optimization.eval_optimize_loop.pipeline import prepare_run +from examples.optimization.eval_optimize_loop.pipeline import PipelineStageExecutionError +from examples.optimization.eval_optimize_loop.pipeline import run_trace_stage +from examples.optimization.eval_optimize_loop.schemas import ArtifactIndex +from examples.optimization.eval_optimize_loop.schemas import OptimizationReport + + +_REPO_ROOT = Path(__file__).resolve().parents[2] +_EXAMPLE = _REPO_ROOT / "examples" / "optimization" / "eval_optimize_loop" + + +def _copy_example(tmp_path: Path, name: str) -> Path: + target = tmp_path / name + shutil.copytree( + _EXAMPLE, + target, + ignore=shutil.ignore_patterns("runs", "__pycache__"), + ) + return target + + +@pytest.mark.parametrize( + ("scenario", "train_passed", "validation_passed", "decision"), + [ + ("improve", 3, 3, "accept"), + ("no_improvement", 1, 1, "reject"), + ("overfit", 3, 0, "reject"), + ], +) +@pytest.mark.asyncio +async def test_trace_stage_replays_four_evalsets_without_writeback( + tmp_path: Path, + scenario: str, + train_passed: int, + validation_passed: int, + decision: str, + monkeypatch: pytest.MonkeyPatch, +): + class _ForbiddenComponent: + def __init__(self, *_args, **_kwargs): + raise AssertionError("trace mode must not construct runtime components") + + monkeypatch.setattr(pipeline_module, "BusinessAgent", _ForbiddenComponent) + monkeypatch.setattr( + pipeline_module, + "FakeCandidateProviderAdapter", + _ForbiddenComponent, + ) + root = _copy_example(tmp_path, f"trace_{scenario}") + source = root / "prompts" / "system.md" + baseline_source = source.read_text(encoding="utf-8") + prepared = prepare_run( + root / "pipeline.trace.json", + run_id=f"trace_{scenario}", + ) + baseline_working = await prepared.working_target.read_all() + + result = await run_trace_stage( + prepared, + scenario=scenario, # type: ignore[arg-type] + ) + + assert result.baseline_train.passed_case_count == 1 + assert result.baseline_validation.passed_case_count == 1 + assert result.candidate_train.passed_case_count == train_passed + assert result.candidate_validation.passed_case_count == validation_passed + assert result.gate_decision.decision == decision + assert result.writeback.status == "skipped" + assert result.writeback.reason == "trace_replay" + assert result.writeback.attempted is False + assert source.read_text(encoding="utf-8") == baseline_source + assert await prepared.working_target.read_all() == baseline_working + assert len(result.analysis.train_diff.cases) == 3 + assert len(result.analysis.validation_diff.cases) == 3 + report_dir = Path(prepared.workspace.run_dir) / "report" + report = OptimizationReport.model_validate_json( + (report_dir / "optimization_report.json").read_text(encoding="utf-8") + ) + index = ArtifactIndex.model_validate_json( + (report_dir / "artifact_index.json").read_text(encoding="utf-8") + ) + assert report.execution_mode == "trace" + artifact_ids = {artifact.artifact_id for artifact in index.artifacts} + assert "input.trace.candidate_train" in artifact_ids + assert "input.trace.candidate_validation" in artifact_ids + + +@pytest.mark.parametrize( + ("target", "error_type", "expected"), + [ + ( + "build_evaluation_analysis", + pipeline_module.EvaluationAnalysisError, + "stage 3a analysis failed", + ), + ( + "evaluate_gate", + pipeline_module.GateEvaluationError, + "stage 3b gate failed", + ), + ], +) +@pytest.mark.asyncio +async def test_trace_stage_wraps_analysis_and_gate_errors_with_phase_context( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + target: str, + error_type: type[Exception], + expected: str, +): + root = _copy_example(tmp_path, f"trace_{target}") + prepared = prepare_run( + root / "pipeline.trace.json", + run_id=f"trace_{target}", + ) + + def fail(*_args, **_kwargs): + raise error_type("injected failure") + + monkeypatch.setattr(pipeline_module, target, fail) + + with pytest.raises(PipelineStageExecutionError, match=expected): + await run_trace_stage(prepared, scenario="improve")