From 6c47ddd28b36c2180a0cc2ee69d0c99a84dcdef6 Mon Sep 17 00:00:00 2001 From: guocfu Date: Wed, 15 Jul 2026 15:07:51 +0800 Subject: [PATCH 01/18] =?UTF-8?q?feat:=20=E6=96=B0=E5=A2=9E=E8=AF=84?= =?UTF-8?q?=E6=B5=8B=E4=BC=98=E5=8C=96=E9=97=AD=E7=8E=AF=E5=87=86=E5=A4=87?= =?UTF-8?q?=E9=98=B6=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 仅实现 Pipeline 配置校验和 Prompt 隔离工作区创建，不执行实际评测、提示词优化、Gate 或报告生成。 --- .gitignore | 3 + .../optimization/eval_optimize_loop/README.md | 38 +++ .../eval_optimize_loop/__init__.py | 6 + .../optimization/eval_optimize_loop/config.py | 200 +++++++++++++++ .../data/train.evalset.json | 40 +++ .../eval_optimize_loop/data/val.evalset.json | 40 +++ .../eval_optimize_loop/optimizer.json | 36 +++ .../eval_optimize_loop/pipeline.json | 53 ++++ .../eval_optimize_loop/pipeline.py | 201 +++++++++++++++ .../eval_optimize_loop/prompt_workspace.py | 123 +++++++++ .../eval_optimize_loop/prompts/system.md | 5 + .../eval_optimize_loop/run_pipeline.py | 41 +++ .../eval_optimize_loop/schemas.py | 72 ++++++ .../test_eval_optimize_loop_stage1.py | 241 ++++++++++++++++++ 14 files changed, 1099 insertions(+) create mode 100644 examples/optimization/eval_optimize_loop/README.md create mode 100644 examples/optimization/eval_optimize_loop/__init__.py create mode 100644 examples/optimization/eval_optimize_loop/config.py create mode 100644 examples/optimization/eval_optimize_loop/data/train.evalset.json create mode 100644 examples/optimization/eval_optimize_loop/data/val.evalset.json create mode 100644 examples/optimization/eval_optimize_loop/optimizer.json create mode 100644 examples/optimization/eval_optimize_loop/pipeline.json create mode 100644 examples/optimization/eval_optimize_loop/pipeline.py create mode 100644 examples/optimization/eval_optimize_loop/prompt_workspace.py create mode 100644 examples/optimization/eval_optimize_loop/prompts/system.md create mode 100644 examples/optimization/eval_optimize_loop/run_pipeline.py create mode 100644 examples/optimization/eval_optimize_loop/schemas.py create mode 100644 tests/evaluation/test_eval_optimize_loop_stage1.py diff --git a/.gitignore b/.gitignore index 233248dd..8c703b63 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,6 @@ test-ngtest-ut-trpc-agent-py.xml node_modules package-lock.json pyrightconfig.json + +# Generated artifacts for the evaluation/optimization loop example. +examples/optimization/eval_optimize_loop/runs/ diff --git a/examples/optimization/eval_optimize_loop/README.md b/examples/optimization/eval_optimize_loop/README.md new file mode 100644 index 00000000..fb66f3d6 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/README.md @@ -0,0 +1,38 @@ +# Evaluation + Optimization Loop — Stage 1 + +This example is the foundation for an auditable evaluation and prompt- +optimization loop. Stage 1 validates the complete pipeline configuration and +inputs, snapshots source prompts, and creates an isolated path-backed +`TargetPrompt` under a unique run directory. + +It deliberately does not run evaluation, candidate generation, gating, +reporting, or source writeback yet. Later stages consume the stable +`PreparedRun` returned by `pipeline.prepare_run`. + +Run the preparation step from the repository root: + +```bash +python examples/optimization/eval_optimize_loop/run_pipeline.py \ + --run-id local_stage1 +``` + +The command creates: + +```text +runs/local_stage1/ +└── workspace/ + └── prompts/ + └── 01_system_prompt.md +``` + +All paths in `pipeline.json` are relative to the example directory. They may +not escape that directory. Prompt sources must be unique, regular UTF-8 files; +the preparation step never writes them. `optimizer.json` remains the native +SDK optimizer configuration, while gate, budget, artifact, and writeback +settings stay in `pipeline.json`. + +Run the stage-one test suite with: + +```bash +pytest -q tests/evaluation/test_eval_optimize_loop_stage1.py +``` diff --git a/examples/optimization/eval_optimize_loop/__init__.py b/examples/optimization/eval_optimize_loop/__init__.py new file mode 100644 index 00000000..0f6ebd16 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/__init__.py @@ -0,0 +1,6 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under the Apache License, Version 2.0. +"""A safe, auditable evaluation and prompt-optimization pipeline example.""" diff --git a/examples/optimization/eval_optimize_loop/config.py b/examples/optimization/eval_optimize_loop/config.py new file mode 100644 index 00000000..0bb62e26 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/config.py @@ -0,0 +1,200 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under the Apache License, Version 2.0. +"""Pipeline-specific configuration. + +``optimizer.json`` deliberately remains an SDK ``OptimizeConfigFile``. The +configuration in this module contains only orchestration concerns that do not +belong in the SDK optimizer schema: isolated prompt sources, gate policy, +budgets, reporting, and artifact retention. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Literal +from typing import Optional +from typing import Union + +from pydantic import Field +from pydantic import field_validator +from pydantic import model_validator + +from trpc_agent_sdk.evaluation import EvalBaseModel + + +_PROMPT_NAME_PATTERN = r"^[A-Za-z][A-Za-z0-9_]*$" +_RUN_ID_PATTERN = r"^[A-Za-z0-9][A-Za-z0-9_-]*$" + + +class ExecutionConfig(EvalBaseModel): + """How a later phase obtains an optimization candidate.""" + + mode: Literal["fake", "real", "trace"] = "fake" + fake_candidate_scenario: Literal["improve", "no_improvement", "overfit"] = "improve" + use_fake_judge: bool = True + + +class InputPathsConfig(EvalBaseModel): + """Files shared by the baseline, candidate, and optimizer runs.""" + + train_evalset: str + validation_evalset: str + optimizer_config: str + + @field_validator("train_evalset", "validation_evalset", "optimizer_config") + @classmethod + def _require_non_empty_relative_path(cls, value: str) -> str: + path = Path(value) + if not value.strip(): + raise ValueError("path must not be empty") + if path.is_absolute(): + raise ValueError("path must be relative to pipeline.json") + return value + + +class PromptFieldConfig(EvalBaseModel): + """One file-backed field that forms the pipeline TargetPrompt.""" + + name: str = Field(pattern=_PROMPT_NAME_PATTERN) + path: str + + @field_validator("path") + @classmethod + def _require_non_empty_relative_path(cls, value: str) -> str: + path = Path(value) + if not value.strip(): + raise ValueError("prompt path must not be empty") + if path.is_absolute(): + raise ValueError("prompt path must be relative to pipeline.json") + return value + + +class RunConfig(EvalBaseModel): + """Reproducibility and workspace location settings.""" + + runs_dir: str = "runs" + run_id: Optional[str] = Field(default=None, pattern=_RUN_ID_PATTERN) + seed: int = 42 + + @field_validator("runs_dir") + @classmethod + def _require_non_empty_relative_path(cls, value: str) -> str: + path = Path(value) + if not value.strip(): + raise ValueError("runs_dir must not be empty") + if path.is_absolute(): + raise ValueError("runs_dir must be relative to pipeline.json") + return value + + +class CaseLabelsConfig(EvalBaseModel): + """Case identifiers with stronger gate guarantees.""" + + hard_case_ids: list[str] = Field(default_factory=list) + critical_case_ids: list[str] = Field(default_factory=list) + + @field_validator("hard_case_ids", "critical_case_ids") + @classmethod + def _require_unique_non_empty_ids(cls, values: list[str]) -> list[str]: + if any(not value.strip() for value in values): + raise ValueError("case labels must not contain empty IDs") + if len(values) != len(set(values)): + raise ValueError("case labels must not contain duplicate IDs") + return values + + +class GateConfig(EvalBaseModel): + """Acceptance policy consumed by the gate phase.""" + + min_validation_score_delta: float = Field(default=0.01, ge=0.0) + reject_on_validation_pass_rate_drop: bool = True + reject_new_hard_fail: bool = True + reject_critical_regression: bool = True + severe_case_score_drop: float = Field(default=0.20, ge=0.0, le=1.0) + required_metrics: Union[Literal["all"], list[str]] = "all" + + @field_validator("required_metrics") + @classmethod + def _require_unique_metric_names(cls, value: Union[str, list[str]]) -> Union[str, list[str]]: + if not isinstance(value, list): + return value + if any(not item.strip() for item in value): + raise ValueError("required_metrics must not contain empty metric names") + if len(value) != len(set(value)): + raise ValueError("required_metrics must not contain duplicates") + return value + + +class BudgetConfig(EvalBaseModel): + """Resource limits and the policy for measurements unavailable from the SDK.""" + + max_cost_usd: Optional[float] = Field(default=None, ge=0.0) + max_tokens: Optional[int] = Field(default=None, ge=0) + max_duration_seconds: Optional[float] = Field(default=None, gt=0.0) + on_unavailable: Literal["reject", "warning"] = "reject" + + +class ReportingConfig(EvalBaseModel): + """Report formats selected for a successful future pipeline run.""" + + write_json: bool = True + write_markdown: bool = True + include_case_evidence: bool = True + + @model_validator(mode="after") + def _require_report_format(self) -> "ReportingConfig": + if not self.write_json and not self.write_markdown: + raise ValueError("at least one of write_json or write_markdown must be enabled") + return self + + +class ArtifactConfig(EvalBaseModel): + """Which reproducibility artifacts future phases must retain.""" + + copy_input_files: bool = True + retain_optimizer_native_artifacts: bool = True + audit_all_candidates: bool = False + + +class WritebackConfig(EvalBaseModel): + """Safety settings used only after a future ACCEPT decision.""" + + enabled: bool = False + require_source_hash_match: bool = True + + +class PipelineConfig(EvalBaseModel): + """The complete, example-local ``pipeline.json`` schema (version 1).""" + + config_version: Literal[1] = 1 + execution: ExecutionConfig = Field(default_factory=ExecutionConfig) + inputs: InputPathsConfig + prompts: list[PromptFieldConfig] = Field(min_length=1) + run: RunConfig = Field(default_factory=RunConfig) + case_labels: CaseLabelsConfig = Field(default_factory=CaseLabelsConfig) + gate: GateConfig = Field(default_factory=GateConfig) + budget: BudgetConfig = Field(default_factory=BudgetConfig) + reporting: ReportingConfig = Field(default_factory=ReportingConfig) + artifacts: ArtifactConfig = Field(default_factory=ArtifactConfig) + writeback: WritebackConfig = Field(default_factory=WritebackConfig) + + @model_validator(mode="after") + def _require_unique_prompt_names(self) -> "PipelineConfig": + names = [prompt.name for prompt in self.prompts] + if len(names) != len(set(names)): + raise ValueError("prompts must not contain duplicate field names") + return self + + +def load_pipeline_config(path: str | Path) -> PipelineConfig: + """Load a pipeline config while retaining path resolution at the caller. + + Paths intentionally remain relative strings in the model so a copied example + directory remains relocatable. ``prepare_run`` resolves and validates them + relative to this file. + """ + config_path = Path(path) + return PipelineConfig.model_validate_json(config_path.read_text(encoding="utf-8")) diff --git a/examples/optimization/eval_optimize_loop/data/train.evalset.json b/examples/optimization/eval_optimize_loop/data/train.evalset.json new file mode 100644 index 00000000..6b41ca3a --- /dev/null +++ b/examples/optimization/eval_optimize_loop/data/train.evalset.json @@ -0,0 +1,40 @@ +{ + "eval_set_id": "eval_optimize_loop_train", + "name": "Evaluation optimization loop - train", + "description": "Three deterministic training cases for format, tool choice, and tool arguments.", + "eval_cases": [ + { + "eval_id": "train_output_format", + "conversation": [ + { + "invocation_id": "train-1", + "user_content": {"parts": [{"text": "How can I update my email address?"}], "role": "user"}, + "final_response": {"parts": [{"text": "{\"route\":\"account\",\"message\":\"Open profile settings to update your email.\"}"}], "role": "model"} + } + ], + "session_input": {"app_name": "eval_optimize_loop", "user_id": "demo", "state": {}} + }, + { + "eval_id": "train_tool_choice", + "conversation": [ + { + "invocation_id": "train-2", + "user_content": {"parts": [{"text": "Check the status of order A100."}], "role": "user"}, + "final_response": {"parts": [{"text": "{\"route\":\"order_lookup\",\"message\":\"Checking order A100.\"}"}], "role": "model"} + } + ], + "session_input": {"app_name": "eval_optimize_loop", "user_id": "demo", "state": {}} + }, + { + "eval_id": "train_tool_arguments", + "conversation": [ + { + "invocation_id": "train-3", + "user_content": {"parts": [{"text": "Look up order B-204 for customer 17."}], "role": "user"}, + "final_response": {"parts": [{"text": "{\"route\":\"order_lookup\",\"message\":\"Checking order B-204 for customer 17.\"}"}], "role": "model"} + } + ], + "session_input": {"app_name": "eval_optimize_loop", "user_id": "demo", "state": {}} + } + ] +} diff --git a/examples/optimization/eval_optimize_loop/data/val.evalset.json b/examples/optimization/eval_optimize_loop/data/val.evalset.json new file mode 100644 index 00000000..387e6de7 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/data/val.evalset.json @@ -0,0 +1,40 @@ +{ + "eval_set_id": "eval_optimize_loop_validation", + "name": "Evaluation optimization loop - validation", + "description": "Three deterministic validation cases for generalization, recall, and critical routing.", + "eval_cases": [ + { + "eval_id": "val_paraphrase", + "conversation": [ + { + "invocation_id": "val-1", + "user_content": {"parts": [{"text": "Where do I change the address tied to my account?"}], "role": "user"}, + "final_response": {"parts": [{"text": "{\"route\":\"account\",\"message\":\"Open profile settings to update your address.\"}"}], "role": "model"} + } + ], + "session_input": {"app_name": "eval_optimize_loop", "user_id": "demo", "state": {}} + }, + { + "eval_id": "val_knowledge_recall", + "conversation": [ + { + "invocation_id": "val-2", + "user_content": {"parts": [{"text": "How long does standard shipping usually take?"}], "role": "user"}, + "final_response": {"parts": [{"text": "{\"route\":\"shipping_policy\",\"message\":\"Standard shipping normally takes 3-5 business days.\"}"}], "role": "model"} + } + ], + "session_input": {"app_name": "eval_optimize_loop", "user_id": "demo", "state": {}} + }, + { + "eval_id": "val_refund_route", + "conversation": [ + { + "invocation_id": "val-3", + "user_content": {"parts": [{"text": "I was charged twice and need the duplicate payment refunded."}], "role": "user"}, + "final_response": {"parts": [{"text": "{\"route\":\"billing_refund\",\"message\":\"I will route this duplicate charge for refund review.\"}"}], "role": "model"} + } + ], + "session_input": {"app_name": "eval_optimize_loop", "user_id": "demo", "state": {}} + } + ] +} diff --git a/examples/optimization/eval_optimize_loop/optimizer.json b/examples/optimization/eval_optimize_loop/optimizer.json new file mode 100644 index 00000000..2f03e377 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/optimizer.json @@ -0,0 +1,36 @@ +{ + "evaluate": { + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + } + } + ], + "num_runs": 1 + }, + "optimize": { + "eval_case_parallelism": 1, + "stop": { + "required_metrics": "all" + }, + "algorithm": { + "name": "gepa_reflective", + "seed": 42, + "reflection_lm": { + "model_name": "fake-not-used-in-stage-one", + "api_key": "fake-not-used-in-stage-one" + }, + "reflection_minibatch_size": 3, + "skip_perfect_score": false, + "max_candidate_proposals": 3 + } + } +} diff --git a/examples/optimization/eval_optimize_loop/pipeline.json b/examples/optimization/eval_optimize_loop/pipeline.json new file mode 100644 index 00000000..93c5c995 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline.json @@ -0,0 +1,53 @@ +{ + "config_version": 1, + "execution": { + "mode": "fake", + "fake_candidate_scenario": "improve", + "use_fake_judge": true + }, + "inputs": { + "train_evalset": "data/train.evalset.json", + "validation_evalset": "data/val.evalset.json", + "optimizer_config": "optimizer.json" + }, + "prompts": [ + { + "name": "system_prompt", + "path": "prompts/system.md" + } + ], + "run": { + "runs_dir": "runs", + "seed": 42 + }, + "case_labels": { + "hard_case_ids": ["val_knowledge_recall"], + "critical_case_ids": ["val_refund_route"] + }, + "gate": { + "min_validation_score_delta": 0.05, + "reject_on_validation_pass_rate_drop": true, + "reject_new_hard_fail": true, + "reject_critical_regression": true, + "severe_case_score_drop": 0.2, + "required_metrics": ["final_response_avg_score"] + }, + "budget": { + "max_duration_seconds": 180, + "on_unavailable": "warning" + }, + "reporting": { + "write_json": true, + "write_markdown": true, + "include_case_evidence": true + }, + "artifacts": { + "copy_input_files": true, + "retain_optimizer_native_artifacts": true, + "audit_all_candidates": false + }, + "writeback": { + "enabled": false, + "require_source_hash_match": true + } +} diff --git a/examples/optimization/eval_optimize_loop/pipeline.py b/examples/optimization/eval_optimize_loop/pipeline.py new file mode 100644 index 00000000..8c8598dc --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline.py @@ -0,0 +1,201 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under the Apache License, Version 2.0. +"""First-stage pipeline preparation: validate inputs and isolate prompts.""" + +from __future__ import annotations + +import re +import shutil +from dataclasses import dataclass +from datetime import datetime +from datetime import timezone +from hashlib import sha256 +from pathlib import Path +from uuid import uuid4 + +from trpc_agent_sdk.evaluation import EvalSet +from trpc_agent_sdk.evaluation import OptimizeConfigFile +from trpc_agent_sdk.evaluation import TargetPrompt +from trpc_agent_sdk.evaluation import load_optimize_config + +from .config import PipelineConfig +from .config import load_pipeline_config +from .prompt_workspace import PromptWorkspaceError +from .prompt_workspace import resolve_inside_example_root +from .prompt_workspace import stage_prompt_workspace +from .prompt_workspace import validate_prompt_sources +from .schemas import InputSnapshot +from .schemas import WorkspaceSnapshot + + +class PipelinePreparationError(ValueError): + """The example cannot safely prepare an evaluation/optimization run.""" + + +@dataclass(frozen=True) +class PreparedRun: + """Validated inputs and isolated prompts handed to the next pipeline phase.""" + + config: PipelineConfig + optimizer_config: OptimizeConfigFile + input_snapshot: InputSnapshot + workspace: WorkspaceSnapshot + source_target: TargetPrompt + working_target: TargetPrompt + example_root: Path + + +_RUN_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_-]*$") + + +def _load_evalset(path: Path, label: str) -> EvalSet: + if not path.is_file(): + raise PipelinePreparationError(f"{label} must be a file: {path}") + try: + return EvalSet.model_validate_json(path.read_text(encoding="utf-8")) + except UnicodeDecodeError as exc: + raise PipelinePreparationError(f"{label} is not UTF-8: {path}") from exc + except Exception as exc: + raise PipelinePreparationError(f"{label} is not a valid EvalSet: {path}: {exc}") from exc + + +def _validate_eval_case_ids(train: EvalSet, validation: EvalSet, config: PipelineConfig) -> None: + train_ids = [case.eval_id for case in train.eval_cases] + validation_ids = [case.eval_id for case in validation.eval_cases] + for label, ids in (("train", train_ids), ("validation", validation_ids)): + if len(ids) != len(set(ids)): + raise PipelinePreparationError(f"{label} evalset contains duplicate eval_id values") + if set(train_ids) & set(validation_ids): + raise PipelinePreparationError("train and validation evalsets must not share eval_id values") + + known_ids = set(train_ids) | set(validation_ids) + labels = set(config.case_labels.hard_case_ids) | set(config.case_labels.critical_case_ids) + unknown = sorted(labels - known_ids) + if unknown: + raise PipelinePreparationError(f"case_labels reference unknown eval_id values: {unknown}") + + +def _validate_gate_metrics(config: PipelineConfig, optimizer_config: object) -> None: + required = config.gate.required_metrics + if not isinstance(required, list): + return + available = {metric.metric_name for metric in optimizer_config.evaluate.get_eval_metrics()} + unknown = sorted(set(required) - available) + if unknown: + raise PipelinePreparationError( + f"gate.required_metrics references unknown metrics {unknown}; available metrics: {sorted(available)}") + + +def _resolve_inputs(example_root: Path, config: PipelineConfig) -> tuple[Path, Path, Path]: + train_path = resolve_inside_example_root(example_root, config.inputs.train_evalset, "train_evalset") + validation_path = resolve_inside_example_root(example_root, config.inputs.validation_evalset, "validation_evalset") + optimizer_path = resolve_inside_example_root(example_root, config.inputs.optimizer_config, "optimizer_config") + if train_path == validation_path: + raise PipelinePreparationError("train_evalset and validation_evalset must be different files") + if not optimizer_path.is_file(): + raise PipelinePreparationError(f"optimizer_config must be a file: {optimizer_path}") + return train_path, validation_path, optimizer_path + + +def _validate_run_id(run_id: str) -> str: + if not _RUN_ID_RE.fullmatch(run_id): + raise PipelinePreparationError("run_id may contain only letters, numbers, underscores, and hyphens") + return run_id + + +def _new_run_id() -> str: + return datetime.now(timezone.utc).strftime("run_%Y%m%dT%H%M%S_%fZ") + + +def _file_sha256(path: Path) -> str: + return sha256(path.read_bytes()).hexdigest() + + +def prepare_run(pipeline_config_path: str | Path, *, run_id: str | None = None) -> PreparedRun: + """Prepare a run without evaluating, optimizing, reporting, or writing a source prompt. + + All configuration and input validation completes before a staging directory + is created. The final run directory appears only through an atomic rename, + and an exception removes the staging directory. This keeps failed setup + from looking like a runnable or audited pipeline result. + """ + config_path = Path(pipeline_config_path).resolve() + config = load_pipeline_config(config_path) + example_root = config_path.parent + + train_path, validation_path, optimizer_path = _resolve_inputs(example_root, config) + train_evalset = _load_evalset(train_path, "train_evalset") + validation_evalset = _load_evalset(validation_path, "validation_evalset") + _validate_eval_case_ids(train_evalset, validation_evalset, config) + + try: + optimizer_config = load_optimize_config(str(optimizer_path)) + except Exception as exc: + raise PipelinePreparationError(f"optimizer_config is invalid: {optimizer_path}: {exc}") from exc + if not optimizer_config.evaluate.get_eval_metrics(): + raise PipelinePreparationError("optimizer_config must define at least one evaluation metric") + if optimizer_config.evaluate.num_runs < 1: + raise PipelinePreparationError("optimizer_config evaluate.num_runs must be at least 1") + if optimizer_config.optimize.eval_case_parallelism < 1: + raise PipelinePreparationError("optimizer_config optimize.eval_case_parallelism must be at least 1") + _validate_gate_metrics(config, optimizer_config) + + try: + prompt_sources = validate_prompt_sources(example_root, config.prompts) + runs_dir = resolve_inside_example_root(example_root, config.run.runs_dir, "runs_dir") + except PromptWorkspaceError as exc: + raise PipelinePreparationError(str(exc)) from exc + + configured_run_id = run_id if run_id is not None else config.run.run_id + selected_run_id = _validate_run_id(configured_run_id or _new_run_id()) + runs_dir.mkdir(parents=True, exist_ok=True) + final_run_dir = runs_dir / selected_run_id + if final_run_dir.exists(): + raise FileExistsError(f"run directory already exists: {final_run_dir}") + + staging_run_dir = runs_dir / f".{selected_run_id}.tmp-{uuid4().hex}" + try: + staging_run_dir.mkdir() + prompt_snapshots, source_target, working_target = stage_prompt_workspace( + example_root=example_root, + staging_run_dir=staging_run_dir, + final_run_dir=final_run_dir, + prompts=config.prompts, + sources=prompt_sources, + ) + workspace_dir = final_run_dir / "workspace" + workspace = WorkspaceSnapshot( + run_id=selected_run_id, + run_dir=str(final_run_dir), + workspace_dir=str(workspace_dir), + prompts_dir=str(workspace_dir / "prompts"), + ) + input_snapshot = InputSnapshot( + pipeline_config_path=str(config_path), + pipeline_config_sha256=_file_sha256(config_path), + optimizer_config_path=str(optimizer_path), + optimizer_config_sha256=_file_sha256(optimizer_path), + train_evalset_path=str(train_path), + train_evalset_sha256=_file_sha256(train_path), + validation_evalset_path=str(validation_path), + validation_evalset_sha256=_file_sha256(validation_path), + prompt_snapshots=prompt_snapshots, + seed=config.run.seed, + ) + prepared = PreparedRun( + config=config, + optimizer_config=optimizer_config, + input_snapshot=input_snapshot, + workspace=workspace, + source_target=source_target, + working_target=working_target, + example_root=example_root, + ) + staging_run_dir.replace(final_run_dir) + return prepared + except BaseException: + shutil.rmtree(staging_run_dir, ignore_errors=True) + raise diff --git a/examples/optimization/eval_optimize_loop/prompt_workspace.py b/examples/optimization/eval_optimize_loop/prompt_workspace.py new file mode 100644 index 00000000..1e30a6e4 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/prompt_workspace.py @@ -0,0 +1,123 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under the Apache License, Version 2.0. +"""Prompt snapshots and isolated path-backed ``TargetPrompt`` instances.""" + +from __future__ import annotations + +from hashlib import sha256 +from pathlib import Path + +from trpc_agent_sdk.evaluation import TargetPrompt + +from .config import PromptFieldConfig +from .schemas import PromptSnapshot + + +class PromptWorkspaceError(ValueError): + """A prompt source cannot safely participate in an isolated run.""" + + +class SourcePromptDriftError(RuntimeError): + """One or more source prompts changed after the baseline snapshot.""" + + +def resolve_inside_example_root(example_root: Path, relative_path: str, label: str) -> Path: + """Resolve a configured path and reject traversal or symlink escape.""" + root = example_root.resolve() + candidate = (root / relative_path).resolve() + try: + candidate.relative_to(root) + except ValueError as exc: + raise PromptWorkspaceError(f"{label} escapes the example root: {relative_path}") from exc + return candidate + + +def validate_prompt_sources(example_root: Path, prompts: list[PromptFieldConfig]) -> list[Path]: + """Validate path-backed, UTF-8 prompt files and return resolved sources.""" + sources: list[Path] = [] + seen_paths: set[Path] = set() + for prompt in prompts: + source = resolve_inside_example_root(example_root, prompt.path, f"prompt {prompt.name!r}") + raw_source = example_root.resolve() / prompt.path + if raw_source.is_symlink(): + raise PromptWorkspaceError(f"prompt {prompt.name!r} must not be a symlink") + if not source.is_file(): + raise PromptWorkspaceError(f"prompt {prompt.name!r} is not a regular file: {prompt.path}") + try: + source.read_text(encoding="utf-8") + except UnicodeDecodeError as exc: + raise PromptWorkspaceError(f"prompt {prompt.name!r} is not UTF-8: {prompt.path}") from exc + if source in seen_paths: + raise PromptWorkspaceError(f"multiple prompt fields reference {prompt.path}") + seen_paths.add(source) + sources.append(source) + return sources + + +def verify_source_hashes(snapshots: list[PromptSnapshot]) -> None: + """Fail if a source prompt no longer matches its preparation snapshot. + + Later writeback code must call this immediately before an ACCEPT write. It + is useful in stage one as a read-only concurrency guard; this module does + not expose a source-writing operation. + """ + drifted: list[str] = [] + for snapshot in snapshots: + source = Path(snapshot.source_path) + try: + content = source.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError): + drifted.append(snapshot.field_name) + continue + digest = sha256(content.encode("utf-8")).hexdigest() + if digest != snapshot.sha256: + drifted.append(snapshot.field_name) + if drifted: + raise SourcePromptDriftError(f"source prompt hash changed for fields: {sorted(drifted)}") + + +def stage_prompt_workspace( + *, + example_root: Path, + staging_run_dir: Path, + final_run_dir: Path, + prompts: list[PromptFieldConfig], + sources: list[Path], +) -> tuple[list[PromptSnapshot], TargetPrompt, TargetPrompt]: + """Copy prompt sources into a staging run and build source/working targets. + + The returned working target intentionally points at *final* paths. The + caller atomically renames ``staging_run_dir`` into ``final_run_dir`` only + once every source has been copied, so no later phase can observe a partial + prompt workspace. + """ + prompts_dir = staging_run_dir / "workspace" / "prompts" + prompts_dir.mkdir(parents=True) + + source_target = TargetPrompt() + working_target = TargetPrompt() + snapshots: list[PromptSnapshot] = [] + + for index, (prompt, source) in enumerate(zip(prompts, sources, strict=True), start=1): + content = source.read_text(encoding="utf-8") + suffix = source.suffix or ".txt" + working_name = f"{index:02d}_{prompt.name}{suffix}" + staged_path = prompts_dir / working_name + final_path = final_run_dir / "workspace" / "prompts" / working_name + staged_path.write_text(content, encoding="utf-8") + + source_target.add_path(prompt.name, str(source)) + working_target.add_path(prompt.name, str(final_path)) + snapshots.append( + PromptSnapshot( + field_name=prompt.name, + source_path=str(source), + working_path=str(final_path), + content=content, + sha256=sha256(content.encode("utf-8")).hexdigest(), + )) + + return snapshots, source_target, working_target diff --git a/examples/optimization/eval_optimize_loop/prompts/system.md b/examples/optimization/eval_optimize_loop/prompts/system.md new file mode 100644 index 00000000..345c342d --- /dev/null +++ b/examples/optimization/eval_optimize_loop/prompts/system.md @@ -0,0 +1,5 @@ +You are a customer-support routing assistant. + +Answer with a compact JSON object containing `route` and `message`. Use the +available account tool for account-specific requests. Never invent account +facts. diff --git a/examples/optimization/eval_optimize_loop/run_pipeline.py b/examples/optimization/eval_optimize_loop/run_pipeline.py new file mode 100644 index 00000000..7081c550 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/run_pipeline.py @@ -0,0 +1,41 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under the Apache License, Version 2.0. +"""Prepare an isolated evaluation/optimization run. + +This is intentionally a stage-one-only entry point. It validates the complete +pipeline contract and makes a prompt workspace, but does not evaluate, optimize, +gate, report, or write source prompts. +""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +_HERE = Path(__file__).resolve().parent +if __package__ in (None, ""): + _REPO_ROOT = _HERE.parents[2] + if str(_REPO_ROOT) not in sys.path: + sys.path.insert(0, str(_REPO_ROOT)) + from examples.optimization.eval_optimize_loop.pipeline import prepare_run +else: + from .pipeline import prepare_run + + +def main() -> None: + parser = argparse.ArgumentParser(description="Prepare an isolated eval/optimization pipeline run.") + parser.add_argument("--config", type=Path, default=_HERE / "pipeline.json") + parser.add_argument("--run-id", help="Optional reproducible run identifier.") + args = parser.parse_args() + + prepared = prepare_run(args.config, run_id=args.run_id) + print(f"Prepared pipeline run: {prepared.workspace.run_dir}") + print("Stage 1 complete: evaluation, optimization, gate, report, and source writeback are not implemented yet.") + + +if __name__ == "__main__": + main() diff --git a/examples/optimization/eval_optimize_loop/schemas.py b/examples/optimization/eval_optimize_loop/schemas.py new file mode 100644 index 00000000..5aae4b2e --- /dev/null +++ b/examples/optimization/eval_optimize_loop/schemas.py @@ -0,0 +1,72 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under the Apache License, Version 2.0. +"""Serializable data schemas owned by the pipeline example. + +The SDK evaluation result types remain the source of truth for raw evaluation +data. These schemas only capture run inputs and prompt provenance needed by +the pipeline report and artifact index. +""" + +from __future__ import annotations + +from typing import Literal +from typing import Optional + +from pydantic import Field +from pydantic import model_validator + +from trpc_agent_sdk.evaluation import EvalBaseModel + + +class ObservableValue(EvalBaseModel): + """A measurement whose absence is explicit rather than silently zero.""" + + status: Literal["available", "unavailable"] + value: Optional[float] = None + unit: Optional[str] = None + reason: Optional[str] = None + + @model_validator(mode="after") + def _validate_status(self) -> "ObservableValue": + if self.status == "available" and self.value is None: + raise ValueError("available observable values require value") + if self.status == "unavailable" and self.value is not None: + raise ValueError("unavailable observable values must not carry a value") + return self + + +class PromptSnapshot(EvalBaseModel): + """Content and provenance of one source prompt field at preparation time.""" + + field_name: str + source_path: str + working_path: str + content: str + sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + + +class InputSnapshot(EvalBaseModel): + """Immutable file identities captured before a pipeline run starts.""" + + pipeline_config_path: str + pipeline_config_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + optimizer_config_path: str + optimizer_config_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + train_evalset_path: str + train_evalset_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + validation_evalset_path: str + validation_evalset_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + prompt_snapshots: list[PromptSnapshot] + seed: int + + +class WorkspaceSnapshot(EvalBaseModel): + """Directory layout created for one isolated pipeline run.""" + + run_id: str + run_dir: str + workspace_dir: str + prompts_dir: str diff --git a/tests/evaluation/test_eval_optimize_loop_stage1.py b/tests/evaluation/test_eval_optimize_loop_stage1.py new file mode 100644 index 00000000..58adc76f --- /dev/null +++ b/tests/evaluation/test_eval_optimize_loop_stage1.py @@ -0,0 +1,241 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under the Apache License, Version 2.0. +"""Offline tests for stage-one eval/optimization pipeline preparation.""" + +from __future__ import annotations + +import json +import shutil +from pathlib import Path + +import pytest +from pydantic import ValidationError + +from examples.optimization.eval_optimize_loop.config import PipelineConfig +from examples.optimization.eval_optimize_loop.config import load_pipeline_config +from examples.optimization.eval_optimize_loop.pipeline import PipelinePreparationError +from examples.optimization.eval_optimize_loop.pipeline import prepare_run +from examples.optimization.eval_optimize_loop.prompt_workspace import SourcePromptDriftError +from examples.optimization.eval_optimize_loop.prompt_workspace import verify_source_hashes +from examples.optimization.eval_optimize_loop.schemas import ObservableValue + + +_REPO_ROOT = Path(__file__).resolve().parents[2] +_EXAMPLE = _REPO_ROOT / "examples" / "optimization" / "eval_optimize_loop" + + +def _copy_example(tmp_path: Path) -> Path: + target = tmp_path / "eval_optimize_loop" + shutil.copytree(_EXAMPLE, target, ignore=shutil.ignore_patterns("runs", "__pycache__")) + return target + + +def _read_config(root: Path) -> dict: + return json.loads((root / "pipeline.json").read_text(encoding="utf-8")) + + +def _write_config(root: Path, payload: dict) -> Path: + path = root / "pipeline.json" + path.write_text(json.dumps(payload), encoding="utf-8") + return path + + +def _write_evalset(root: Path, relative: str, eval_set_id: str, case_ids: list[str]) -> None: + payload = { + "eval_set_id": eval_set_id, + "eval_cases": [ + { + "eval_id": case_id, + "conversation": [ + { + "invocation_id": case_id, + "user_content": {"parts": [{"text": "input"}], "role": "user"}, + "final_response": {"parts": [{"text": "output"}], "role": "model"}, + } + ], + } + for case_id in case_ids + ], + } + (root / relative).write_text(json.dumps(payload), encoding="utf-8") + + +def test_pipeline_config_loads_complete_example_and_camel_case(tmp_path: Path): + root = _copy_example(tmp_path) + config = load_pipeline_config(root / "pipeline.json") + assert config.config_version == 1 + assert config.execution.mode == "fake" + assert config.prompts[0].name == "system_prompt" + assert config.gate.required_metrics == ["final_response_avg_score"] + + payload = _read_config(root) + payload["configVersion"] = payload.pop("config_version") + payload["caseLabels"] = payload.pop("case_labels") + payload["caseLabels"]["hardCaseIds"] = payload["caseLabels"].pop("hard_case_ids") + camel_path = root / "camel.json" + camel_path.write_text(json.dumps(payload), encoding="utf-8") + camel_config = load_pipeline_config(camel_path) + assert camel_config.case_labels.hard_case_ids == ["val_knowledge_recall"] + + +@pytest.mark.parametrize( + "mutate", + [ + lambda payload: payload.update({"unexpected": True}), + lambda payload: payload["execution"].update({"mode": "maybe"}), + lambda payload: payload["gate"].update({"min_validation_score_delta": -0.01}), + lambda payload: payload["gate"].update({"severe_case_score_drop": -0.1}), + lambda payload: payload["gate"].update({"severe_case_score_drop": 1.1}), + lambda payload: payload["budget"].update({"max_tokens": -1}), + lambda payload: payload["reporting"].update({"write_json": False, "write_markdown": False}), + lambda payload: payload.update({"prompts": [{"name": "system_prompt", "callback": "x"}]}), + ], +) +def test_pipeline_config_rejects_invalid_values(tmp_path: Path, mutate): + root = _copy_example(tmp_path) + payload = _read_config(root) + mutate(payload) + with pytest.raises(ValidationError): + PipelineConfig.model_validate(payload) + + +def test_pipeline_config_rejects_duplicate_prompt_names(tmp_path: Path): + root = _copy_example(tmp_path) + payload = _read_config(root) + payload["prompts"].append({"name": "system_prompt", "path": "prompts/system.md"}) + with pytest.raises(ValidationError, match="duplicate field names"): + PipelineConfig.model_validate(payload) + + +def test_observable_value_never_treats_unavailable_as_zero(): + unavailable = ObservableValue(status="unavailable", reason="SDK does not expose cost") + assert unavailable.value is None + with pytest.raises(ValidationError): + ObservableValue(status="unavailable", value=0) + with pytest.raises(ValidationError): + ObservableValue(status="available") + + +@pytest.mark.asyncio +async def test_prepare_run_creates_isolated_prompt_workspace(tmp_path: Path): + root = _copy_example(tmp_path) + source = root / "prompts" / "system.md" + baseline = source.read_text(encoding="utf-8") + + prepared = prepare_run(root / "pipeline.json", run_id="stage1_test") + + assert Path(prepared.workspace.run_dir).is_dir() + assert prepared.source_target.names() == ["system_prompt"] + assert prepared.working_target.names() == ["system_prompt"] + source_values = await prepared.source_target.read_all() + working_values = await prepared.working_target.read_all() + assert source_values == working_values == {"system_prompt": baseline} + snapshot = prepared.input_snapshot.prompt_snapshots[0] + assert snapshot.source_path == str(source.resolve()) + assert snapshot.working_path != snapshot.source_path + assert len(snapshot.sha256) == 64 + assert len(prepared.input_snapshot.pipeline_config_sha256) == 64 + assert len(prepared.input_snapshot.optimizer_config_sha256) == 64 + assert len(prepared.input_snapshot.train_evalset_sha256) == 64 + assert len(prepared.input_snapshot.validation_evalset_sha256) == 64 + verify_source_hashes(prepared.input_snapshot.prompt_snapshots) + + await prepared.working_target.write_all({"system_prompt": "candidate"}) + assert source.read_text(encoding="utf-8") == baseline + assert await prepared.working_target.read_all() == {"system_prompt": "candidate"} + + source.write_text("concurrent source edit", encoding="utf-8") + with pytest.raises(SourcePromptDriftError, match="system_prompt"): + verify_source_hashes(prepared.input_snapshot.prompt_snapshots) + + +def test_prepare_run_does_not_overwrite_existing_run(tmp_path: Path): + root = _copy_example(tmp_path) + prepare_run(root / "pipeline.json", run_id="same_run") + with pytest.raises(FileExistsError, match="already exists"): + prepare_run(root / "pipeline.json", run_id="same_run") + + +@pytest.mark.parametrize("label_key", ["hard_case_ids", "critical_case_ids"]) +def test_prepare_run_rejects_unknown_case_labels(tmp_path: Path, label_key: str): + root = _copy_example(tmp_path) + payload = _read_config(root) + payload["case_labels"][label_key] = ["missing_case"] + with pytest.raises(PipelinePreparationError, match="unknown eval_id"): + prepare_run(_write_config(root, payload), run_id="bad_labels") + assert not (root / "runs").exists() + + +def test_prepare_run_rejects_shared_dataset_path(tmp_path: Path): + root = _copy_example(tmp_path) + payload = _read_config(root) + payload["inputs"]["validation_evalset"] = payload["inputs"]["train_evalset"] + with pytest.raises(PipelinePreparationError, match="different files"): + prepare_run(_write_config(root, payload), run_id="shared_data") + + +def test_prepare_run_rejects_duplicate_or_overlapping_eval_ids(tmp_path: Path): + root = _copy_example(tmp_path) + _write_evalset(root, "data/train.evalset.json", "train", ["same", "same"]) + with pytest.raises(PipelinePreparationError, match="duplicate eval_id"): + prepare_run(root / "pipeline.json", run_id="duplicate_case") + + _write_evalset(root, "data/train.evalset.json", "train", ["same"]) + _write_evalset(root, "data/val.evalset.json", "val", ["same"]) + payload = _read_config(root) + payload["case_labels"] = {} + with pytest.raises(PipelinePreparationError, match="must not share eval_id"): + prepare_run(_write_config(root, payload), run_id="overlap_case") + + +def test_prepare_run_rejects_unknown_required_metric(tmp_path: Path): + root = _copy_example(tmp_path) + payload = _read_config(root) + payload["gate"]["required_metrics"] = ["does_not_exist"] + with pytest.raises(PipelinePreparationError, match="unknown metrics"): + prepare_run(_write_config(root, payload), run_id="bad_metric") + + +def test_prepare_run_rejects_path_escape_and_non_utf8_prompt(tmp_path: Path): + root = _copy_example(tmp_path) + outside = tmp_path / "outside.md" + outside.write_text("secret", encoding="utf-8") + payload = _read_config(root) + payload["prompts"][0]["path"] = "../outside.md" + with pytest.raises(PipelinePreparationError, match="escapes the example root"): + prepare_run(_write_config(root, payload), run_id="escape") + + payload["prompts"][0]["path"] = "prompts/binary.md" + (root / "prompts" / "binary.md").write_bytes(b"\xff\xfe") + with pytest.raises(PipelinePreparationError, match="not UTF-8"): + prepare_run(_write_config(root, payload), run_id="binary") + + +def test_prepare_run_rejects_duplicate_prompt_paths(tmp_path: Path): + root = _copy_example(tmp_path) + payload = _read_config(root) + payload["prompts"].append({"name": "router_prompt", "path": "prompts/system.md"}) + with pytest.raises(PipelinePreparationError, match="multiple prompt fields"): + prepare_run(_write_config(root, payload), run_id="duplicate_path") + + +def test_prepare_run_removes_staging_workspace_on_failure(tmp_path: Path, monkeypatch): + root = _copy_example(tmp_path) + + def fail_staging(**kwargs): + (kwargs["staging_run_dir"] / "workspace").mkdir() + raise RuntimeError("injected staging failure") + + monkeypatch.setattr( + "examples.optimization.eval_optimize_loop.pipeline.stage_prompt_workspace", + fail_staging, + ) + with pytest.raises(RuntimeError, match="injected"): + prepare_run(root / "pipeline.json", run_id="failed_run") + + runs_dir = root / "runs" + assert not (runs_dir / "failed_run").exists() + assert list(runs_dir.iterdir()) == [] From 115c914f50cec335aac912206ea01662fdab17f1 Mon Sep 17 00:00:00 2001 From: guocfu Date: Wed, 15 Jul 2026 19:59:56 +0800 Subject: [PATCH 02/18] =?UTF-8?q?feat(evaluation):=20=E5=AE=9E=E7=8E=B0?= =?UTF-8?q?=E7=A1=AE=E5=AE=9A=E6=80=A7=E8=AF=84=E6=B5=8B=E4=BC=98=E5=8C=96?= =?UTF-8?q?=E9=97=AD=E7=8E=AF=E7=AC=AC=E4=BA=8C=E9=98=B6=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 添加 Prompt 感知的 fake agent 与确定性候选提供器。 完成 baseline/candidate 的 train/validation 四次评测，并补充输入漂移保护和离线回归测试。 --- .../optimization/eval_optimize_loop/README.md | 41 ++- .../optimization/eval_optimize_loop/config.py | 2 +- .../eval_optimize_loop/fake/__init__.py | 11 + .../eval_optimize_loop/fake/agent.py | 127 +++++++ .../fake/candidate_provider.py | 110 ++++++ .../eval_optimize_loop/optimizer.json | 4 +- .../eval_optimize_loop/pipeline.json | 2 +- .../eval_optimize_loop/pipeline.py | 229 +++++++++++- .../eval_optimize_loop/run_pipeline.py | 36 +- .../eval_optimize_loop/schemas.py | 61 +++- .../test_eval_optimize_loop_stage2.py | 333 ++++++++++++++++++ 11 files changed, 922 insertions(+), 34 deletions(-) create mode 100644 examples/optimization/eval_optimize_loop/fake/__init__.py create mode 100644 examples/optimization/eval_optimize_loop/fake/agent.py create mode 100644 examples/optimization/eval_optimize_loop/fake/candidate_provider.py create mode 100644 tests/evaluation/test_eval_optimize_loop_stage2.py diff --git a/examples/optimization/eval_optimize_loop/README.md b/examples/optimization/eval_optimize_loop/README.md index fb66f3d6..98f9f63f 100644 --- a/examples/optimization/eval_optimize_loop/README.md +++ b/examples/optimization/eval_optimize_loop/README.md @@ -1,38 +1,43 @@ -# Evaluation + Optimization Loop — Stage 1 +# Evaluation + Optimization Loop — Stage 2 -This example is the foundation for an auditable evaluation and prompt- -optimization loop. Stage 1 validates the complete pipeline configuration and -inputs, snapshots source prompts, and creates an isolated path-backed -`TargetPrompt` under a unique run directory. +This example now provides the deterministic offline foundation for an auditable +evaluation and prompt-optimization loop. Stage 1 validates inputs and creates +an isolated prompt workspace. Stage 2 evaluates that baseline on train and +validation data, generates a fake prompt candidate, and evaluates the candidate +on both datasets again. -It deliberately does not run evaluation, candidate generation, gating, -reporting, or source writeback yet. Later stages consume the stable -`PreparedRun` returned by `pipeline.prepare_run`. +No model, API key, judge, or optimizer is used. The fake agent reads explicit +capability rules from the current working prompt, while the fake candidate +provider can generate an improving, behaviorally equivalent, or overfit +candidate. Gate, reporting, and source writeback remain later-stage work. -Run the preparation step from the repository root: +Run the deterministic stage from the repository root: ```bash python examples/optimization/eval_optimize_loop/run_pipeline.py \ - --run-id local_stage1 + --run-id local_stage2 \ + --scenario improve ``` The command creates: ```text -runs/local_stage1/ +runs/local_stage2/ └── workspace/ └── prompts/ └── 01_system_prompt.md ``` -All paths in `pipeline.json` are relative to the example directory. They may -not escape that directory. Prompt sources must be unique, regular UTF-8 files; -the preparation step never writes them. `optimizer.json` remains the native -SDK optimizer configuration, while gate, budget, artifact, and writeback -settings stay in `pipeline.json`. +All paths in `pipeline.json` are relative to the example directory. Prompt +sources are never written. The candidate remains in the isolated run workspace +after evaluation so it can be inspected. `optimizer.json` remains the native +SDK optimizer configuration, while orchestration settings stay in +`pipeline.json`. -Run the stage-one test suite with: +Run the stage-one and stage-two tests with: ```bash -pytest -q tests/evaluation/test_eval_optimize_loop_stage1.py +pytest -q \ + tests/evaluation/test_eval_optimize_loop_stage1.py \ + tests/evaluation/test_eval_optimize_loop_stage2.py ``` diff --git a/examples/optimization/eval_optimize_loop/config.py b/examples/optimization/eval_optimize_loop/config.py index 0bb62e26..1c02e63d 100644 --- a/examples/optimization/eval_optimize_loop/config.py +++ b/examples/optimization/eval_optimize_loop/config.py @@ -34,7 +34,7 @@ class ExecutionConfig(EvalBaseModel): mode: Literal["fake", "real", "trace"] = "fake" fake_candidate_scenario: Literal["improve", "no_improvement", "overfit"] = "improve" - use_fake_judge: bool = True + use_fake_judge: bool = False class InputPathsConfig(EvalBaseModel): diff --git a/examples/optimization/eval_optimize_loop/fake/__init__.py b/examples/optimization/eval_optimize_loop/fake/__init__.py new file mode 100644 index 00000000..c6895fa9 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/fake/__init__.py @@ -0,0 +1,11 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under the Apache License, Version 2.0. +"""Deterministic offline components for the eval/optimization loop.""" + +from .agent import DeterministicFakeAgent +from .candidate_provider import DeterministicFakeCandidateProvider + +__all__ = ["DeterministicFakeAgent", "DeterministicFakeCandidateProvider"] diff --git a/examples/optimization/eval_optimize_loop/fake/agent.py b/examples/optimization/eval_optimize_loop/fake/agent.py new file mode 100644 index 00000000..2912ab04 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/fake/agent.py @@ -0,0 +1,127 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under the Apache License, Version 2.0. +"""Prompt-sensitive deterministic agent used by the offline pipeline mode.""" + +from __future__ import annotations + +import json +import re +from dataclasses import dataclass + +from trpc_agent_sdk.evaluation import TargetPrompt + + +RULE_PREFIX = "deterministic-fake-rule" +_RULE_RE = re.compile( + rf"", + re.IGNORECASE, +) +_ORDER_ID_RE = re.compile(r"\border\s+([A-Za-z0-9][A-Za-z0-9-]*)", re.IGNORECASE) +_CUSTOMER_ID_RE = re.compile(r"\bcustomer\s+([A-Za-z0-9][A-Za-z0-9-]*)", re.IGNORECASE) + + +@dataclass(frozen=True) +class _RoutingPolicy: + account_terms: frozenset[str] = frozenset({"email"}) + order_lookup: bool = False + shipping_policy: bool = False + refund_route: bool = True + + +def _parse_bool(value: str, *, default: bool) -> bool: + normalized = value.strip().lower() + if normalized in {"true", "yes", "1", "enabled"}: + return True + if normalized in {"false", "no", "0", "disabled"}: + return False + return default + + +def _parse_policy(prompt_text: str) -> _RoutingPolicy: + values: dict[str, str] = {} + for key, value in _RULE_RE.findall(prompt_text): + values[key.lower()] = value.strip() + + account_terms = frozenset( + term.strip().lower() + for term in values.get("account_terms", "email").split(",") + if term.strip() + ) + return _RoutingPolicy( + account_terms=account_terms, + order_lookup=_parse_bool(values.get("order_lookup", "false"), default=False), + shipping_policy=_parse_bool(values.get("shipping_policy", "false"), default=False), + refund_route=_parse_bool(values.get("refund_route", "true"), default=True), + ) + + +def _compact_response(route: str, message: str) -> str: + # Keep route before message: the example intentionally uses exact text matching. + return json.dumps({"route": route, "message": message}, ensure_ascii=False, separators=(",", ":")) + + +class DeterministicFakeAgent: + """A black-box ``call_agent`` implementation whose behavior follows prompt rules. + + The agent deliberately receives only the user query. It cannot inspect an + eval id, expected response, scenario name, call count, or external state. + Every call rereads ``target_prompt`` so a candidate written between the + baseline and candidate evaluations takes effect immediately. + """ + + def __init__(self, target_prompt: TargetPrompt) -> None: + self._target_prompt = target_prompt + + async def call_agent(self, query: str) -> str: + if not isinstance(query, str): + raise TypeError("query must be a string") + + prompt_values = await self._target_prompt.read_all() + policy = _parse_policy("\n".join(prompt_values.values())) + normalized = " ".join(query.casefold().split()) + + if policy.refund_route and ( + "charged twice" in normalized + or ("duplicate" in normalized and ("payment" in normalized or "charge" in normalized)) + ): + return _compact_response( + "billing_refund", + "I will route this duplicate charge for refund review.", + ) + + if policy.shipping_policy and "shipping" in normalized and ( + "standard" in normalized or "how long" in normalized + ): + return _compact_response( + "shipping_policy", + "Standard shipping normally takes 3-5 business days.", + ) + + if ( + policy.order_lookup + and "order" in normalized + and (order_match := _ORDER_ID_RE.search(query)) is not None + ): + order_id = order_match.group(1) + customer_match = _CUSTOMER_ID_RE.search(query) + if customer_match is not None: + message = f"Checking order {order_id} for customer {customer_match.group(1)}." + else: + message = f"Checking order {order_id}." + return _compact_response("order_lookup", message) + + account_term = next((term for term in sorted(policy.account_terms) if term in normalized), None) + if account_term and ("update" in normalized or "change" in normalized): + attribute = "email" if "email" in normalized else "address" + return _compact_response( + "account", + f"Open profile settings to update your {attribute}.", + ) + + return _compact_response( + "general_support", + "Please provide more details so I can route your request.", + ) diff --git a/examples/optimization/eval_optimize_loop/fake/candidate_provider.py b/examples/optimization/eval_optimize_loop/fake/candidate_provider.py new file mode 100644 index 00000000..1724e29b --- /dev/null +++ b/examples/optimization/eval_optimize_loop/fake/candidate_provider.py @@ -0,0 +1,110 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under the Apache License, Version 2.0. +"""Pure deterministic candidate generation for the offline pipeline mode.""" + +from __future__ import annotations + +import json +from hashlib import sha256 +from typing import Mapping + +from ..schemas import FakeCandidateProposal +from ..schemas import FakeCandidateScenario +from .agent import RULE_PREFIX + + +_SCENARIO_BLOCKS: dict[FakeCandidateScenario, tuple[str, str]] = { + "improve": ( + "Generalize routing across account synonyms, order lookup, shipping policy, and refunds.", + "\n".join( + [ + "", + "Apply general customer-support routing rules across equivalent user phrasings.", + f"", + f"", + f"", + f"", + "", + ] + ), + ), + "no_improvement": ( + "Add an auditable wording-only change that leaves routing behavior unchanged.", + "\n".join( + [ + "", + "Keep responses concise, direct, and suitable for customer support.", + "", + ] + ), + ), + "overfit": ( + "Narrow routing to email changes and order lookups while disabling unseen intents.", + "\n".join( + [ + "", + "Handle only email profile changes and order lookups; use general support otherwise.", + f"", + f"", + f"", + f"", + "", + ] + ), + ), +} + + +def _prompt_mapping_sha256(prompts: Mapping[str, str]) -> str: + canonical = json.dumps( + dict(prompts), + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ) + return sha256(canonical.encode("utf-8")).hexdigest() + + +class DeterministicFakeCandidateProvider: + """Generate one structured candidate without performing I/O or mutation.""" + + def __init__(self, target_field: str = "system_prompt") -> None: + if not target_field: + raise ValueError("target_field must not be empty") + self._target_field = target_field + + def propose( + self, + current_prompts: Mapping[str, str], + *, + scenario: FakeCandidateScenario, + seed: int, + ) -> FakeCandidateProposal: + if self._target_field not in current_prompts: + raise ValueError(f"fake candidate target field is missing: {self._target_field}") + if scenario not in _SCENARIO_BLOCKS: + raise ValueError(f"unknown fake candidate scenario: {scenario}") + if any(not isinstance(name, str) or not isinstance(value, str) for name, value in current_prompts.items()): + raise TypeError("current_prompts must map string field names to string values") + + rationale, rule_block = _SCENARIO_BLOCKS[scenario] + prompts = dict(current_prompts) + baseline = prompts[self._target_field].rstrip() + prompts[self._target_field] = f"{baseline}\n\n{rule_block}\n" + + parent_hash = _prompt_mapping_sha256(current_prompts) + candidate_hash = _prompt_mapping_sha256(prompts) + changed_fields = [name for name in current_prompts if current_prompts[name] != prompts[name]] + return FakeCandidateProposal( + scenario=scenario, + prompts=prompts, + changed_fields=changed_fields, + rationale=rationale, + seed=seed, + parent_prompt_sha256=parent_hash, + candidate_prompt_sha256=candidate_hash, + candidate_id=f"fake-{scenario}-{candidate_hash[:12]}", + ) diff --git a/examples/optimization/eval_optimize_loop/optimizer.json b/examples/optimization/eval_optimize_loop/optimizer.json index 2f03e377..9efff8be 100644 --- a/examples/optimization/eval_optimize_loop/optimizer.json +++ b/examples/optimization/eval_optimize_loop/optimizer.json @@ -25,8 +25,8 @@ "name": "gepa_reflective", "seed": 42, "reflection_lm": { - "model_name": "fake-not-used-in-stage-one", - "api_key": "fake-not-used-in-stage-one" + "model_name": "fake-not-used-in-offline-mode", + "api_key": "fake-not-used-in-offline-mode" }, "reflection_minibatch_size": 3, "skip_perfect_score": false, diff --git a/examples/optimization/eval_optimize_loop/pipeline.json b/examples/optimization/eval_optimize_loop/pipeline.json index 93c5c995..f3b862e4 100644 --- a/examples/optimization/eval_optimize_loop/pipeline.json +++ b/examples/optimization/eval_optimize_loop/pipeline.json @@ -3,7 +3,7 @@ "execution": { "mode": "fake", "fake_candidate_scenario": "improve", - "use_fake_judge": true + "use_fake_judge": false }, "inputs": { "train_evalset": "data/train.evalset.json", diff --git a/examples/optimization/eval_optimize_loop/pipeline.py b/examples/optimization/eval_optimize_loop/pipeline.py index 8c8598dc..3eb48ebd 100644 --- a/examples/optimization/eval_optimize_loop/pipeline.py +++ b/examples/optimization/eval_optimize_loop/pipeline.py @@ -3,7 +3,7 @@ # Copyright (C) 2026 Tencent. All rights reserved. # # tRPC-Agent-Python is licensed under the Apache License, Version 2.0. -"""First-stage pipeline preparation: validate inputs and isolate prompts.""" +"""Pipeline preparation and the deterministic offline stage-two loop.""" from __future__ import annotations @@ -14,20 +14,29 @@ from datetime import timezone from hashlib import sha256 from pathlib import Path +from typing import Literal from uuid import uuid4 +from trpc_agent_sdk.evaluation import AgentEvaluator +from trpc_agent_sdk.evaluation import EvalCaseResult from trpc_agent_sdk.evaluation import EvalSet +from trpc_agent_sdk.evaluation import EvalStatus from trpc_agent_sdk.evaluation import OptimizeConfigFile from trpc_agent_sdk.evaluation import TargetPrompt from trpc_agent_sdk.evaluation import load_optimize_config from .config import PipelineConfig from .config import load_pipeline_config +from .fake import DeterministicFakeAgent +from .fake import DeterministicFakeCandidateProvider from .prompt_workspace import PromptWorkspaceError from .prompt_workspace import resolve_inside_example_root from .prompt_workspace import stage_prompt_workspace from .prompt_workspace import validate_prompt_sources from .schemas import InputSnapshot +from .schemas import FakeCandidateScenario +from .schemas import FakeEvaluationSnapshot +from .schemas import FakeStageResult from .schemas import WorkspaceSnapshot @@ -35,6 +44,10 @@ class PipelinePreparationError(ValueError): """The example cannot safely prepare an evaluation/optimization run.""" +class FakeStageExecutionError(RuntimeError): + """The deterministic stage-two loop could not complete safely.""" + + @dataclass(frozen=True) class PreparedRun: """Validated inputs and isolated prompts handed to the next pipeline phase.""" @@ -114,6 +127,31 @@ def _file_sha256(path: Path) -> str: return sha256(path.read_bytes()).hexdigest() +def _reload_prepared_evalset( + path: Path, + *, + label: str, + expected_sha256: str, +) -> EvalSet: + """Reload exactly the evalset bytes whose identity was prepared.""" + try: + payload = path.read_bytes() + except OSError as exc: + raise FakeStageExecutionError(f"failed to reload prepared {label}: {path}: {exc}") from exc + + actual_sha256 = sha256(payload).hexdigest() + if actual_sha256 != expected_sha256: + raise FakeStageExecutionError( + f"{label} changed after prepare_run: {path}; " + f"expected sha256 {expected_sha256}, got {actual_sha256}" + ) + + try: + return EvalSet.model_validate_json(payload) + except Exception as exc: + raise FakeStageExecutionError(f"prepared {label} is no longer a valid EvalSet: {path}: {exc}") from exc + + def prepare_run(pipeline_config_path: str | Path, *, run_id: str | None = None) -> PreparedRun: """Prepare a run without evaluating, optimizing, reporting, or writing a source prompt. @@ -199,3 +237,192 @@ def prepare_run(pipeline_config_path: str | Path, *, run_id: str | None = None) except BaseException: shutil.rmtree(staging_run_dir, ignore_errors=True) raise + + +def _summarize_fake_results( + eval_results_by_eval_id: dict[str, list[EvalCaseResult]], +) -> tuple[int, int, float | None]: + passed_cases = 0 + scores: list[float] = [] + for runs in eval_results_by_eval_id.values(): + if runs and all(getattr(run, "final_eval_status", None) == EvalStatus.PASSED for run in runs): + passed_cases += 1 + for run in runs: + for metric in getattr(run, "overall_eval_metric_results", []): + if metric.score is not None: + scores.append(float(metric.score)) + average_score = sum(scores) / len(scores) if scores else None + return passed_cases, len(eval_results_by_eval_id), average_score + + +def _validate_fake_results( + *, + eval_set: EvalSet, + eval_results_by_eval_id: dict[str, list[EvalCaseResult]], + num_runs: int, + phase: Literal["baseline", "candidate"], + split: Literal["train", "validation"], +) -> None: + expected_ids = {case.eval_id for case in eval_set.eval_cases} + actual_ids = set(eval_results_by_eval_id) + if actual_ids != expected_ids: + raise FakeStageExecutionError( + f"{phase} {split} evaluation returned case ids {sorted(actual_ids)}; " + f"expected {sorted(expected_ids)}" + ) + wrong_run_counts = { + eval_id: len(results) + for eval_id, results in eval_results_by_eval_id.items() + if len(results) != num_runs + } + if wrong_run_counts: + raise FakeStageExecutionError( + f"{phase} {split} evaluation returned unexpected run counts: {wrong_run_counts}; " + f"expected {num_runs}" + ) + + +async def _evaluate_fake_split( + *, + prepared: PreparedRun, + eval_set: EvalSet, + agent: DeterministicFakeAgent, + phase: Literal["baseline", "candidate"], + split: Literal["train", "validation"], +) -> FakeEvaluationSnapshot: + num_runs = prepared.optimizer_config.evaluate.num_runs + try: + failed_summary, details_lines, result_lines, eval_results_by_eval_id = ( + await AgentEvaluator.evaluate_eval_set( + eval_set, + call_agent=agent.call_agent, + eval_config=prepared.optimizer_config.evaluate, + num_runs=num_runs, + print_detailed_results=False, + case_parallelism=prepared.optimizer_config.optimize.eval_case_parallelism, + case_eval_parallelism=prepared.optimizer_config.optimize.eval_case_parallelism, + ) + ) + except Exception as exc: + raise FakeStageExecutionError(f"{phase} {split} evaluation failed: {exc}") from exc + + _validate_fake_results( + eval_set=eval_set, + eval_results_by_eval_id=eval_results_by_eval_id, + num_runs=num_runs, + phase=phase, + split=split, + ) + passed_cases, total_cases, average_score = _summarize_fake_results(eval_results_by_eval_id) + return FakeEvaluationSnapshot( + phase=phase, + split=split, + eval_set_id=eval_set.eval_set_id, + failed_summary=failed_summary, + details_lines=details_lines, + result_lines=result_lines, + eval_results_by_eval_id=eval_results_by_eval_id, + passed_case_count=passed_cases, + total_case_count=total_cases, + average_score=average_score, + ) + + +async def run_fake_stage( + prepared: PreparedRun, + *, + scenario: FakeCandidateScenario | None = None, +) -> FakeStageResult: + """Run four deterministic evaluations without a model, judge, or optimizer. + + Source prompts are never written. Once generated, the candidate remains in + the isolated working target on success or candidate-evaluation failure so + the run can be inspected later. + """ + if prepared.config.execution.mode != "fake": + raise FakeStageExecutionError( + f"run_fake_stage requires execution.mode='fake', got {prepared.config.execution.mode!r}" + ) + if prepared.config.execution.use_fake_judge: + raise FakeStageExecutionError( + "execution.use_fake_judge=true is not supported in stage two; " + "the example uses deterministic final-response matching" + ) + + selected_scenario = scenario or prepared.config.execution.fake_candidate_scenario + train_evalset = _reload_prepared_evalset( + Path(prepared.input_snapshot.train_evalset_path), + label="train_evalset", + expected_sha256=prepared.input_snapshot.train_evalset_sha256, + ) + validation_evalset = _reload_prepared_evalset( + Path(prepared.input_snapshot.validation_evalset_path), + label="validation_evalset", + expected_sha256=prepared.input_snapshot.validation_evalset_sha256, + ) + + try: + baseline_prompts = await prepared.working_target.read_all() + except Exception as exc: + raise FakeStageExecutionError(f"failed to read prepared working prompts: {exc}") from exc + expected_baseline = { + snapshot.field_name: snapshot.content for snapshot in prepared.input_snapshot.prompt_snapshots + } + if baseline_prompts != expected_baseline: + raise FakeStageExecutionError("working prompts no longer match the prepared baseline snapshot") + + agent = DeterministicFakeAgent(prepared.working_target) + baseline_train = await _evaluate_fake_split( + prepared=prepared, + eval_set=train_evalset, + agent=agent, + phase="baseline", + split="train", + ) + baseline_validation = await _evaluate_fake_split( + prepared=prepared, + eval_set=validation_evalset, + agent=agent, + phase="baseline", + split="validation", + ) + + try: + candidate = DeterministicFakeCandidateProvider().propose( + baseline_prompts, + scenario=selected_scenario, + seed=prepared.input_snapshot.seed, + ) + except Exception as exc: + raise FakeStageExecutionError(f"fake candidate generation failed: {exc}") from exc + + try: + await prepared.working_target.write_all(candidate.prompts) + written_prompts = await prepared.working_target.read_all() + except Exception as exc: + raise FakeStageExecutionError(f"candidate prompt write failed: {exc}") from exc + if written_prompts != candidate.prompts: + raise FakeStageExecutionError("candidate prompt readback did not match the generated proposal") + + candidate_train = await _evaluate_fake_split( + prepared=prepared, + eval_set=train_evalset, + agent=agent, + phase="candidate", + split="train", + ) + candidate_validation = await _evaluate_fake_split( + prepared=prepared, + eval_set=validation_evalset, + agent=agent, + phase="candidate", + split="validation", + ) + return FakeStageResult( + scenario=selected_scenario, + candidate=candidate, + baseline_train=baseline_train, + baseline_validation=baseline_validation, + candidate_train=candidate_train, + candidate_validation=candidate_validation, + ) diff --git a/examples/optimization/eval_optimize_loop/run_pipeline.py b/examples/optimization/eval_optimize_loop/run_pipeline.py index 7081c550..473c2e7b 100644 --- a/examples/optimization/eval_optimize_loop/run_pipeline.py +++ b/examples/optimization/eval_optimize_loop/run_pipeline.py @@ -3,16 +3,12 @@ # Copyright (C) 2026 Tencent. All rights reserved. # # tRPC-Agent-Python is licensed under the Apache License, Version 2.0. -"""Prepare an isolated evaluation/optimization run. - -This is intentionally a stage-one-only entry point. It validates the complete -pipeline contract and makes a prompt workspace, but does not evaluate, optimize, -gate, report, or write source prompts. -""" +"""Run the deterministic offline evaluation/candidate stage.""" from __future__ import annotations import argparse +import asyncio import sys from pathlib import Path @@ -22,19 +18,41 @@ if str(_REPO_ROOT) not in sys.path: sys.path.insert(0, str(_REPO_ROOT)) from examples.optimization.eval_optimize_loop.pipeline import prepare_run + from examples.optimization.eval_optimize_loop.pipeline import run_fake_stage else: from .pipeline import prepare_run + from .pipeline import run_fake_stage + + +def _format_snapshot(label: str, snapshot: object) -> str: + score = getattr(snapshot, "average_score", None) + score_text = "unavailable" if score is None else f"{score:.3f}" + return ( + f"{label}: {snapshot.passed_case_count}/{snapshot.total_case_count} passed, " + f"average score={score_text}" + ) def main() -> None: - parser = argparse.ArgumentParser(description="Prepare an isolated eval/optimization pipeline run.") + parser = argparse.ArgumentParser(description="Run the deterministic offline eval/optimization stage.") parser.add_argument("--config", type=Path, default=_HERE / "pipeline.json") parser.add_argument("--run-id", help="Optional reproducible run identifier.") + parser.add_argument( + "--scenario", + choices=("improve", "no_improvement", "overfit"), + help="Override execution.fake_candidate_scenario for this run.", + ) args = parser.parse_args() prepared = prepare_run(args.config, run_id=args.run_id) - print(f"Prepared pipeline run: {prepared.workspace.run_dir}") - print("Stage 1 complete: evaluation, optimization, gate, report, and source writeback are not implemented yet.") + result = asyncio.run(run_fake_stage(prepared, scenario=args.scenario)) + print(f"Completed deterministic pipeline stage: {prepared.workspace.run_dir}") + print(f"Candidate: {result.candidate.candidate_id} ({result.scenario})") + print(_format_snapshot("Baseline train", result.baseline_train)) + print(_format_snapshot("Baseline validation", result.baseline_validation)) + print(_format_snapshot("Candidate train", result.candidate_train)) + print(_format_snapshot("Candidate validation", result.candidate_validation)) + print("Stage 2 does not run Gate, write reports, or update source prompts.") if __name__ == "__main__": diff --git a/examples/optimization/eval_optimize_loop/schemas.py b/examples/optimization/eval_optimize_loop/schemas.py index 5aae4b2e..04c12f48 100644 --- a/examples/optimization/eval_optimize_loop/schemas.py +++ b/examples/optimization/eval_optimize_loop/schemas.py @@ -6,12 +6,13 @@ """Serializable data schemas owned by the pipeline example. The SDK evaluation result types remain the source of truth for raw evaluation -data. These schemas only capture run inputs and prompt provenance needed by -the pipeline report and artifact index. +data. These schemas capture run inputs, prompt provenance, fake candidates, +and the full stage-two evaluation outputs consumed by later pipeline phases. """ from __future__ import annotations +from typing import Any from typing import Literal from typing import Optional @@ -19,6 +20,10 @@ from pydantic import model_validator from trpc_agent_sdk.evaluation import EvalBaseModel +from trpc_agent_sdk.evaluation import EvalCaseResult + + +FakeCandidateScenario = Literal["improve", "no_improvement", "overfit"] class ObservableValue(EvalBaseModel): @@ -70,3 +75,55 @@ class WorkspaceSnapshot(EvalBaseModel): run_dir: str workspace_dir: str prompts_dir: str + + +class FakeCandidateProposal(EvalBaseModel): + """One deterministic prompt proposal produced without a real optimizer.""" + + scenario: FakeCandidateScenario + prompts: dict[str, str] + changed_fields: list[str] + rationale: str + seed: int + parent_prompt_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + candidate_prompt_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + candidate_id: str = Field(pattern=r"^fake-(improve|no_improvement|overfit)-[0-9a-f]{12}$") + + +class FakeEvaluationSnapshot(EvalBaseModel): + """Complete SDK outputs from one fake-agent dataset evaluation.""" + + phase: Literal["baseline", "candidate"] + split: Literal["train", "validation"] + eval_set_id: str + failed_summary: Optional[dict[str, Any]] = None + details_lines: list[str] = Field( + description=( + "SDK detailed-output lines; intentionally empty in stage two because " + "print_detailed_results is disabled." + ) + ) + result_lines: list[str] + eval_results_by_eval_id: dict[str, list[EvalCaseResult]] + passed_case_count: int = Field(ge=0) + total_case_count: int = Field(ge=0) + average_score: Optional[float] = Field( + default=None, + ge=0.0, + le=1.0, + description=( + "Arithmetic mean of every available overall metric score across " + "all cases, configured runs, and metrics." + ), + ) + + +class FakeStageResult(EvalBaseModel): + """The four full evaluations and candidate metadata produced in stage two.""" + + scenario: FakeCandidateScenario + candidate: FakeCandidateProposal + baseline_train: FakeEvaluationSnapshot + baseline_validation: FakeEvaluationSnapshot + candidate_train: FakeEvaluationSnapshot + candidate_validation: FakeEvaluationSnapshot diff --git a/tests/evaluation/test_eval_optimize_loop_stage2.py b/tests/evaluation/test_eval_optimize_loop_stage2.py new file mode 100644 index 00000000..a34b92da --- /dev/null +++ b/tests/evaluation/test_eval_optimize_loop_stage2.py @@ -0,0 +1,333 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under the Apache License, Version 2.0. +"""Offline tests for the deterministic stage-two eval/optimization loop.""" + +from __future__ import annotations + +import asyncio +import json +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest + +from examples.optimization.eval_optimize_loop.fake import DeterministicFakeAgent +from examples.optimization.eval_optimize_loop.fake import DeterministicFakeCandidateProvider +from examples.optimization.eval_optimize_loop.pipeline import FakeStageExecutionError +from examples.optimization.eval_optimize_loop.pipeline import prepare_run +from examples.optimization.eval_optimize_loop.pipeline import run_fake_stage +from examples.optimization.eval_optimize_loop.schemas import FakeStageResult +from trpc_agent_sdk.evaluation import EvalStatus +from trpc_agent_sdk.evaluation import TargetPrompt + + +_REPO_ROOT = Path(__file__).resolve().parents[2] +_EXAMPLE = _REPO_ROOT / "examples" / "optimization" / "eval_optimize_loop" + + +def _copy_example(tmp_path: Path, name: str = "eval_optimize_loop") -> Path: + target = tmp_path / name + shutil.copytree(_EXAMPLE, target, ignore=shutil.ignore_patterns("runs", "__pycache__")) + return target + + +def _read_json(path: Path) -> dict: + return json.loads(path.read_text(encoding="utf-8")) + + +def _write_json(path: Path, payload: dict) -> None: + path.write_text(json.dumps(payload), encoding="utf-8") + + +def _target_for_prompt(path: Path, content: str) -> TargetPrompt: + path.write_text(content, encoding="utf-8") + return TargetPrompt().add_path("system_prompt", str(path)) + + +def test_fake_candidate_provider_is_pure_deterministic_and_preserves_fields(): + provider = DeterministicFakeCandidateProvider() + current = {"system_prompt": "baseline", "skill": "unchanged"} + original = dict(current) + + first = provider.propose(current, scenario="improve", seed=42) + second = provider.propose(current, scenario="improve", seed=42) + + assert first == second + assert current == original + assert first.changed_fields == ["system_prompt"] + assert first.prompts["skill"] == "unchanged" + assert first.parent_prompt_sha256 != first.candidate_prompt_sha256 + assert first.candidate_id == f"fake-improve-{first.candidate_prompt_sha256[:12]}" + + +def test_fake_candidate_provider_rejects_missing_field_and_unknown_scenario(): + provider = DeterministicFakeCandidateProvider() + with pytest.raises(ValueError, match="target field is missing"): + provider.propose({"skill": "x"}, scenario="improve", seed=42) + with pytest.raises(ValueError, match="unknown fake candidate scenario"): + provider.propose( + {"system_prompt": "x"}, + scenario="unknown", # type: ignore[arg-type] + seed=42, + ) + + +def test_overfit_candidate_contains_rules_not_eval_ids_or_full_examples(): + proposal = DeterministicFakeCandidateProvider().propose( + {"system_prompt": "baseline"}, + scenario="overfit", + seed=42, + ) + prompt = proposal.prompts["system_prompt"] + for forbidden in ( + "train_output_format", + "train_tool_choice", + "train_tool_arguments", + "How can I update my email address?", + "A100", + "B-204", + ): + assert forbidden not in prompt + assert "account_terms=email" in prompt + assert "order_lookup=true" in prompt + assert "refund_route=false" in prompt + + +@pytest.mark.asyncio +async def test_fake_agent_rereads_prompt_and_is_deterministic(tmp_path: Path): + target = _target_for_prompt(tmp_path / "system.md", "baseline") + agent = DeterministicFakeAgent(target) + email_query = "How can I update my email address?" + order_query = "Check the status of order A100." + + assert await agent.call_agent(email_query) == ( + '{"route":"account","message":"Open profile settings to update your email."}' + ) + baseline_order = await agent.call_agent(order_query) + assert '"route":"general_support"' in baseline_order + + proposal = DeterministicFakeCandidateProvider().propose( + await target.read_all(), + scenario="improve", + seed=42, + ) + await target.write_all(proposal.prompts) + expected_order = '{"route":"order_lookup","message":"Checking order A100."}' + assert await agent.call_agent(order_query) == expected_order + assert await asyncio.gather(*(agent.call_agent(order_query) for _ in range(8))) == [expected_order] * 8 + + +@pytest.mark.asyncio +async def test_fake_agent_unknown_query_has_stable_fallback(tmp_path: Path): + target = _target_for_prompt(tmp_path / "system.md", "baseline") + agent = DeterministicFakeAgent(target) + expected = ( + '{"route":"general_support","message":' + '"Please provide more details so I can route your request."}' + ) + assert await agent.call_agent("Tell me a joke") == expected + assert await agent.call_agent(" Tell me a JOKE ") == expected + with pytest.raises(TypeError, match="query must be a string"): + await agent.call_agent(None) # type: ignore[arg-type] + + +@pytest.mark.parametrize( + ("scenario", "candidate_train_passed", "candidate_validation_passed"), + [ + ("improve", 3, 3), + ("no_improvement", 1, 1), + ("overfit", 3, 0), + ], +) +@pytest.mark.asyncio +async def test_fake_stage_scenario_matrix( + tmp_path: Path, + scenario: str, + candidate_train_passed: int, + candidate_validation_passed: int, +): + root = _copy_example(tmp_path, scenario) + source = root / "prompts" / "system.md" + baseline_source = source.read_text(encoding="utf-8") + prepared = prepare_run(root / "pipeline.json", run_id=f"stage2_{scenario}") + + result = await run_fake_stage(prepared, scenario=scenario) # type: ignore[arg-type] + + assert result.baseline_train.passed_case_count == 1 + assert result.baseline_validation.passed_case_count == 1 + assert result.candidate_train.passed_case_count == candidate_train_passed + assert result.candidate_validation.passed_case_count == candidate_validation_passed + assert result.baseline_train.average_score == pytest.approx(1 / 3) + assert result.baseline_validation.average_score == pytest.approx(1 / 3) + assert len(result.baseline_train.eval_results_by_eval_id) == 3 + assert len(result.baseline_validation.eval_results_by_eval_id) == 3 + assert result.baseline_train.failed_summary is not None + assert result.baseline_validation.failed_summary is not None + assert source.read_text(encoding="utf-8") == baseline_source + assert await prepared.working_target.read_all() == result.candidate.prompts + + if scenario == "improve": + assert result.candidate_train.failed_summary is None + assert result.candidate_validation.failed_summary is None + restored = FakeStageResult.model_validate_json(result.model_dump_json()) + assert restored.candidate.candidate_id == result.candidate.candidate_id + elif scenario == "overfit": + baseline_refund = result.baseline_validation.eval_results_by_eval_id["val_refund_route"][0] + candidate_refund = result.candidate_validation.eval_results_by_eval_id["val_refund_route"][0] + assert baseline_refund.final_eval_status == EvalStatus.PASSED + assert candidate_refund.final_eval_status == EvalStatus.FAILED + + +@pytest.mark.asyncio +async def test_fake_stage_keeps_all_configured_runs(tmp_path: Path): + root = _copy_example(tmp_path) + optimizer_path = root / "optimizer.json" + optimizer = _read_json(optimizer_path) + optimizer["evaluate"]["num_runs"] = 2 + _write_json(optimizer_path, optimizer) + prepared = prepare_run(root / "pipeline.json", run_id="two_runs") + + result = await run_fake_stage(prepared, scenario="improve") + + for snapshot in ( + result.baseline_train, + result.baseline_validation, + result.candidate_train, + result.candidate_validation, + ): + assert all(len(runs) == 2 for runs in snapshot.eval_results_by_eval_id.values()) + + +@pytest.mark.asyncio +async def test_fake_stage_rejects_unimplemented_fake_judge(tmp_path: Path): + root = _copy_example(tmp_path) + config_path = root / "pipeline.json" + config = _read_json(config_path) + config["execution"]["use_fake_judge"] = True + _write_json(config_path, config) + prepared = prepare_run(config_path, run_id="fake_judge") + + with pytest.raises(FakeStageExecutionError, match="use_fake_judge=true"): + await run_fake_stage(prepared) + + +@pytest.mark.parametrize("mode", ["real", "trace"]) +@pytest.mark.asyncio +async def test_fake_stage_rejects_non_fake_execution_modes(tmp_path: Path, mode: str): + root = _copy_example(tmp_path, mode) + config_path = root / "pipeline.json" + config = _read_json(config_path) + config["execution"]["mode"] = mode + _write_json(config_path, config) + prepared = prepare_run(config_path, run_id=f"mode_{mode}") + + with pytest.raises( + FakeStageExecutionError, + match=rf"requires execution.mode='fake', got '{mode}'", + ): + await run_fake_stage(prepared) + + +@pytest.mark.parametrize( + ("relative_path", "expected_label"), + [ + ("data/train.evalset.json", "train_evalset"), + ("data/val.evalset.json", "validation_evalset"), + ], +) +@pytest.mark.asyncio +async def test_fake_stage_rejects_evalset_changes_after_prepare_run( + tmp_path: Path, + relative_path: str, + expected_label: str, +): + root = _copy_example(tmp_path, expected_label) + prepared = prepare_run(root / "pipeline.json", run_id=f"changed_{expected_label}") + evalset_path = root / relative_path + evalset = _read_json(evalset_path) + evalset["description"] = "changed after prepare_run" + _write_json(evalset_path, evalset) + + with pytest.raises( + FakeStageExecutionError, + match=rf"{expected_label} changed after prepare_run", + ): + await run_fake_stage(prepared) + + +@pytest.mark.asyncio +async def test_fake_stage_wraps_baseline_evaluator_errors(tmp_path: Path, monkeypatch): + root = _copy_example(tmp_path) + prepared = prepare_run(root / "pipeline.json", run_id="evaluator_error") + + async def fail_evaluation(*_args, **_kwargs): + raise RuntimeError("injected evaluator failure") + + monkeypatch.setattr( + "examples.optimization.eval_optimize_loop.pipeline.AgentEvaluator.evaluate_eval_set", + fail_evaluation, + ) + with pytest.raises(FakeStageExecutionError, match="baseline train evaluation failed"): + await run_fake_stage(prepared) + assert await prepared.working_target.read_all() == { + snapshot.field_name: snapshot.content for snapshot in prepared.input_snapshot.prompt_snapshots + } + + +@pytest.mark.asyncio +async def test_candidate_evaluation_error_retains_working_candidate(tmp_path: Path, monkeypatch): + root = _copy_example(tmp_path) + source = root / "prompts" / "system.md" + baseline_source = source.read_text(encoding="utf-8") + prepared = prepare_run(root / "pipeline.json", run_id="candidate_error") + + from examples.optimization.eval_optimize_loop import pipeline as pipeline_module + + original = pipeline_module.AgentEvaluator.evaluate_eval_set + calls = 0 + + async def fail_candidate_train(*args, **kwargs): + nonlocal calls + calls += 1 + if calls == 3: + raise RuntimeError("injected candidate failure") + return await original(*args, **kwargs) + + monkeypatch.setattr(pipeline_module.AgentEvaluator, "evaluate_eval_set", fail_candidate_train) + with pytest.raises(FakeStageExecutionError, match="candidate train evaluation failed"): + await run_fake_stage(prepared, scenario="improve") + + working = await prepared.working_target.read_all() + assert "deterministic-fake-candidate:start" in working["system_prompt"] + assert source.read_text(encoding="utf-8") == baseline_source + + +def test_stage_two_cli_smoke(tmp_path: Path): + root = _copy_example(tmp_path) + completed = subprocess.run( + [ + sys.executable, + str(_EXAMPLE / "run_pipeline.py"), + "--config", + str(root / "pipeline.json"), + "--run-id", + "cli_smoke", + "--scenario", + "overfit", + ], + cwd=_REPO_ROOT, + check=True, + capture_output=True, + text=True, + timeout=180, + ) + assert "Baseline train: 1/3 passed" in completed.stdout + assert "Baseline validation: 1/3 passed" in completed.stdout + assert "Candidate train: 3/3 passed" in completed.stdout + assert "Candidate validation: 0/3 passed" in completed.stdout + assert "does not run Gate" in completed.stdout From 9023b034509cafddc7cd2046f428207eefd31170 Mon Sep 17 00:00:00 2001 From: guocfu Date: Wed, 15 Jul 2026 21:20:14 +0800 Subject: [PATCH 03/18] =?UTF-8?q?feat(evaluation):=20=E5=AE=8C=E6=88=90?= =?UTF-8?q?=E8=AF=84=E6=B5=8B=E6=A0=87=E5=87=86=E5=8C=96=E3=80=81=E5=A4=B1?= =?UTF-8?q?=E8=B4=A5=E5=BD=92=E5=9B=A0=E4=B8=8E=E6=A1=88=E4=BE=8B=E5=B7=AE?= =?UTF-8?q?=E5=BC=82=E5=88=86=E6=9E=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...65\350\267\257\347\272\277\345\233\276.md" | 125 ++++ .../optimization/eval_optimize_loop/README.md | 27 +- .../eval_optimize_loop/analysis.py | 83 +++ .../eval_optimize_loop/attribution.py | 261 +++++++++ .../eval_optimize_loop/case_diff.py | 181 ++++++ .../eval_optimize_loop/evaluation_adapter.py | 227 ++++++++ .../eval_optimize_loop/pipeline.py | 15 + .../eval_optimize_loop/schemas.py | 167 ++++++ .../test_eval_optimize_loop_stage3a.py | 547 ++++++++++++++++++ 9 files changed, 1626 insertions(+), 7 deletions(-) create mode 100644 "analysis/evaluation optimization pipeline/\345\256\236\346\226\275\351\230\266\346\256\265\350\267\257\347\272\277\345\233\276.md" create mode 100644 examples/optimization/eval_optimize_loop/analysis.py create mode 100644 examples/optimization/eval_optimize_loop/attribution.py create mode 100644 examples/optimization/eval_optimize_loop/case_diff.py create mode 100644 examples/optimization/eval_optimize_loop/evaluation_adapter.py create mode 100644 tests/evaluation/test_eval_optimize_loop_stage3a.py diff --git "a/analysis/evaluation optimization pipeline/\345\256\236\346\226\275\351\230\266\346\256\265\350\267\257\347\272\277\345\233\276.md" "b/analysis/evaluation optimization pipeline/\345\256\236\346\226\275\351\230\266\346\256\265\350\267\257\347\272\277\345\233\276.md" new file mode 100644 index 00000000..bf7b8787 --- /dev/null +++ "b/analysis/evaluation optimization pipeline/\345\256\236\346\226\275\351\230\266\346\256\265\350\267\257\347\272\277\345\233\276.md" @@ -0,0 +1,125 @@ +# Evaluation + Optimization Pipeline 实施阶段路线图 + +本文记录完成 Evaluation + Optimization 自动回归与提示词优化闭环所需的实施阶段。每个阶段应形成一组可以独立测试、评审和提交的能力，规模大致与当前前两次实现提交相当。整体沿用 1–6 的主阶段编号，其中范围较大的阶段 3 拆成两个独立提交单元。 + +状态说明： + +- `[x]` 已完成 +- `[ ]` 待完成 + +## 阶段总览 + +| 阶段 | 状态 | 核心目标 | +|---|---|---| +| 1. 输入准备与 Prompt 工作区 | 已完成 | 校验配置和输入，创建可复现、隔离的 Prompt 工作副本 | +| 2. 确定性离线评测闭环 | 已完成 | 使用 fake agent/provider 完成 baseline 和 candidate 的四次完整评测 | +| 3a. 评测标准化、失败归因与 Case Diff | 已完成 | 统一评测数据，解释失败并比较候选变化 | +| 3b. 独立 Gate | 待完成 | 根据 diff、关键 case 和预算规则给出接受或拒绝决策 | +| 4. 真实优化器与安全写回 | 待完成 | 接入 AgentOptimizer，审计候选，并在 Gate 接受后安全更新源 Prompt | +| 5. 报告、产物与资源观测 | 待完成 | 输出结构化报告、可读报告和完整审计产物 | +| 6. 离线模式与端到端验收 | 待完成 | 补齐 fake judge/trace 场景、示例输出、文档和完整验收测试 | + +## 1. 输入准备与 Prompt 工作区 + +**状态：`[x]` 已完成** + +- 建立示例目录、`pipeline.json`、`optimizer.json`、train/validation evalset 和 baseline Prompt。 +- 校验配置、路径、评测集、metric、case 标签和运行参数。 +- 保存输入文件及 Prompt 的内容和哈希快照。 +- 将源 Prompt 复制到独立 run 工作区，区分 `source_target` 和 `working_target`。 +- 保证准备失败时不留下伪完整运行目录，且准备阶段不修改源 Prompt。 +- 提供阶段一单元测试和命令行入口。 + +对应提交：`6c47ddd feat: 新增评测优化闭环准备阶段` + +## 2. 确定性离线评测闭环 + +**状态：`[x]` 已完成** + +- 实现每次调用都重新读取工作 Prompt 的确定性 fake agent。 +- 实现 `improve`、`no_improvement`、`overfit` 三种 fake candidate。 +- 对 baseline 和 candidate 分别执行 train、validation 完整评测。 +- 保存 SDK 原始评测结果、通过数量、平均分和候选 Prompt 元数据。 +- 校验 evalset 和工作 Prompt 在准备后没有漂移，并保留 candidate 工作副本供审计。 +- 提供三种场景矩阵、异常路径和 CLI smoke test。 + +对应提交：`115c914 feat(evaluation): 实现确定性评测优化闭环第二阶段` + +## 3a. 评测标准化、失败归因与 Case Diff + +**状态：`[x]` 已完成** + +- 将 SDK 评测结果转换为稳定的逐 case、逐 metric 数据模型。 +- 根据 metric、预期/实际回复和调用轨迹执行确定性失败归因。 +- 比较 baseline 与 candidate，识别新增通过、新增失败、提升、退化和不变 case。 +- 标记 hard、critical 和 severe regression，并识别训练提升但验证退化的过拟合。 +- 保留归因依据和前后变化证据，供 Gate 与报告阶段直接消费。 + +阶段完成标准：可以仅根据四次评测结果生成可序列化、可解释的逐 case 差异，且不修改源 Prompt。 + +## 3b. 独立 Gate + +**状态：`[ ]` 待完成** + +- 实现与优化器解耦的 Gate，消费阶段 3a 生成的评测与 diff 数据。 +- 执行验证集最小提升、通过率不得下降、hard/critical case、severe regression 和必需 metric 规则。 +- 纳入成本、token、耗时及不可观测数据策略，但不将未知数据误记为零。 +- 收集全部拒绝理由，不采用只返回第一个错误的方式。 +- 固定三种离线场景的决策：improve ACCEPT、no improvement REJECT、overfit REJECT。 +- 覆盖多规则同时失败、关键 case 退化、训练提升但验证下降等集成场景。 + +阶段完成标准：Gate 能稳定给出完整决策证据，且 ACCEPT/REJECT 本身不触发源 Prompt 写回。 + +## 4. 真实优化器与安全写回 + +**状态：`[ ]` 待完成** + +- 抽象统一 Candidate Provider 接口，接入 `AgentOptimizer` 真实候选生成。 +- 始终使用 `update_source=False`，让优化器只操作隔离工作区。 +- 保留优化器原生候选、轮次记录、分数和配置快照。 +- 对最终候选重新执行完整 train/validation 回归，并交给统一 diff 和 Gate。 +- 只有 Gate ACCEPT 且源 Prompt 哈希未变化时才允许写回；写回后必须回读校验。 +- REJECT、异常、源文件漂移或校验失败时保持源 Prompt 不变。 + +阶段完成标准：真实模式和 fake 模式共用同一条候选验证及 Gate 链路，写回行为具备明确的安全边界。 + +## 5. 报告、产物与资源观测 + +**状态：`[ ]` 待完成** + +- 生成 `optimization_report.json`，包含 baseline、candidate、归因、case diff、Gate 和写回状态。 +- 生成面向使用者的 `optimization_report.md`，解释候选是否值得接受及具体原因。 +- 建立 artifact index，索引输入快照、四次评测、候选 Prompt、优化器原生产物和报告。 +- 记录随机种子、配置、耗时以及可观测的 token、成本和调用信息。 +- 对无法可靠观测的资源数据明确记录为 `unavailable`，并按预算策略产生 reject 或 warning。 +- 使用原子写入，避免失败运行留下看似完整的报告和索引。 + +阶段完成标准：一次运行的输入、Prompt 变化、评测证据、决策和写回结果都可以从产物中复现和审计。 + +## 6. 离线模式与端到端验收 + +**状态：`[ ]` 待完成** + +必须完成： + +- 保持无 API Key 时可以运行 improve、no improvement 和 overfit 三个完整场景。 +- 如果保留 `use_fake_judge` 配置，则实现确定性 fake judge；如果保留 `trace` 模式，则让它能够驱动归因、diff、Gate 和报告链路。 +- 提供示例输出、完整 README、运行命令和各模式适用边界。 +- 覆盖 evaluator/optimizer 异常、Gate 拒绝、写回失败、输入漂移和产物不完整等端到端路径。 +- 验证离线完整 pipeline 在三分钟内完成，并核对 issue 要求的报告字段和交付物。 + +可选且默认跳过： + +- 通过显式环境变量启用真实 API 集成测试。 +- 真实 API 测试不作为普通 CI 或无 API Key 核心流程的必要条件。 + +阶段完成标准：公开样例能够稳定生成完整报告和正确决策，项目具备提交 issue 验收所需的文档、测试和审计产物。 + +## 全局实施约束 + +- Baseline 和最终 Candidate 都必须分别执行完整 train、validation 评测。 +- 真实优化器的内部 minibatch 或轮次分数不能替代 pipeline 的完整回归。 +- Gate 决策前不得修改源 Prompt，任何写回都必须经过源哈希校验和回读验证。 +- fake agent/provider/judge 必须保持确定性，不得读取 `eval_id`、期望答案或调用次数作弊。 +- 所有阶段优先提供无 API Key 的自动化测试，并保留后续真实模式的清晰接口。 +- 新增数据模型和产物需要保持可序列化、可解释和可审计。 diff --git a/examples/optimization/eval_optimize_loop/README.md b/examples/optimization/eval_optimize_loop/README.md index 98f9f63f..61980376 100644 --- a/examples/optimization/eval_optimize_loop/README.md +++ b/examples/optimization/eval_optimize_loop/README.md @@ -1,28 +1,32 @@ -# Evaluation + Optimization Loop — Stage 2 +# Evaluation + Optimization Loop — Stage 3a This example now provides the deterministic offline foundation for an auditable evaluation and prompt-optimization loop. Stage 1 validates inputs and creates an isolated prompt workspace. Stage 2 evaluates that baseline on train and validation data, generates a fake prompt candidate, and evaluates the candidate -on both datasets again. +on both datasets again. Stage 3a normalizes the four SDK result sets, attributes +failures from available evidence, and builds train/validation case diffs. No model, API key, judge, or optimizer is used. The fake agent reads explicit capability rules from the current working prompt, while the fake candidate provider can generate an improving, behaviorally equivalent, or overfit -candidate. Gate, reporting, and source writeback remain later-stage work. +candidate. The returned result retains both the raw SDK outputs and a +serializable analysis with metric deltas, hard/critical labels, severe +regressions, and overfit status. Gate, reporting, and source writeback remain +later-stage work. Run the deterministic stage from the repository root: ```bash python examples/optimization/eval_optimize_loop/run_pipeline.py \ - --run-id local_stage2 \ + --run-id local_stage3a \ --scenario improve ``` The command creates: ```text -runs/local_stage2/ +runs/local_stage3a/ └── workspace/ └── prompts/ └── 01_system_prompt.md @@ -34,10 +38,19 @@ after evaluation so it can be inspected. `optimizer.json` remains the native SDK optimizer configuration, while orchestration settings stay in `pipeline.json`. -Run the stage-one and stage-two tests with: +For cases with multiple runs or invocations, normalization keeps every +invocation's expected/actual response, tool calls, and metric results. Case +status and score are aggregated from the SDK's overall metrics; invocation +metrics are evidence and do not receive extra weight in that aggregate. +Attribution scans evidence from every run and invocation, retains all matching +evidence, and chooses one primary category using the fixed priority in +`attribution.py`; the other distinct matches become secondary categories. + +Run the stage-one through stage-3a tests with: ```bash pytest -q \ tests/evaluation/test_eval_optimize_loop_stage1.py \ - tests/evaluation/test_eval_optimize_loop_stage2.py + tests/evaluation/test_eval_optimize_loop_stage2.py \ + tests/evaluation/test_eval_optimize_loop_stage3a.py ``` diff --git a/examples/optimization/eval_optimize_loop/analysis.py b/examples/optimization/eval_optimize_loop/analysis.py new file mode 100644 index 00000000..2e4ab48b --- /dev/null +++ b/examples/optimization/eval_optimize_loop/analysis.py @@ -0,0 +1,83 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under the Apache License, Version 2.0. +"""Stage 3a orchestration for normalization, attribution, and case diff.""" + +from __future__ import annotations + +from .attribution import attribute_evaluation +from .case_diff import compare_evaluations +from .evaluation_adapter import standardize_snapshot +from .schemas import EvaluationAnalysis +from .schemas import FakeEvaluationSnapshot +from .schemas import ObservableValue +from .schemas import OverfitStatus + + +def _overfit_status( + train_delta: ObservableValue, + validation_delta: ObservableValue, +) -> tuple[OverfitStatus, str]: + if train_delta.status != "available" or validation_delta.status != "available": + return "unavailable", "Train or validation score delta is unavailable." + train_value = float(train_delta.value) + validation_value = float(validation_delta.value) + if train_value > 0.0 and validation_value < 0.0: + return ( + "detected", + f"Train score improved by {train_value:.6f} while validation regressed by " + f"{validation_value:.6f}.", + ) + return ( + "not_detected", + f"Train score delta is {train_value:.6f}; validation score delta is " + f"{validation_value:.6f}.", + ) + + +def build_evaluation_analysis( + *, + baseline_train: FakeEvaluationSnapshot, + baseline_validation: FakeEvaluationSnapshot, + candidate_train: FakeEvaluationSnapshot, + candidate_validation: FakeEvaluationSnapshot, + hard_case_ids: set[str], + critical_case_ids: set[str], + severe_case_score_drop: float, +) -> EvaluationAnalysis: + """Build stage 3a analysis from the four complete evaluation snapshots.""" + normalized_baseline_train = attribute_evaluation(standardize_snapshot(baseline_train)) + normalized_baseline_validation = attribute_evaluation(standardize_snapshot(baseline_validation)) + normalized_candidate_train = attribute_evaluation(standardize_snapshot(candidate_train)) + normalized_candidate_validation = attribute_evaluation(standardize_snapshot(candidate_validation)) + + train_diff = compare_evaluations( + normalized_baseline_train, + normalized_candidate_train, + hard_case_ids=hard_case_ids, + critical_case_ids=critical_case_ids, + severe_case_score_drop=severe_case_score_drop, + ) + validation_diff = compare_evaluations( + normalized_baseline_validation, + normalized_candidate_validation, + hard_case_ids=hard_case_ids, + critical_case_ids=critical_case_ids, + severe_case_score_drop=severe_case_score_drop, + ) + overfit_status, overfit_reason = _overfit_status( + train_diff.score_delta, + validation_diff.score_delta, + ) + return EvaluationAnalysis( + baseline_train=normalized_baseline_train, + baseline_validation=normalized_baseline_validation, + candidate_train=normalized_candidate_train, + candidate_validation=normalized_candidate_validation, + train_diff=train_diff, + validation_diff=validation_diff, + overfit_status=overfit_status, + overfit_reason=overfit_reason, + ) diff --git a/examples/optimization/eval_optimize_loop/attribution.py b/examples/optimization/eval_optimize_loop/attribution.py new file mode 100644 index 00000000..351ccd28 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/attribution.py @@ -0,0 +1,261 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under the Apache License, Version 2.0. +"""Deterministic, evidence-driven failure attribution for stage 3a.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass + +from .schemas import AttributionEvidence +from .schemas import CaseEvaluation +from .schemas import FailureAttribution +from .schemas import FailureCategory +from .schemas import StandardizedEvaluation + + +@dataclass(frozen=True) +class _CandidateReason: + priority: int + category: FailureCategory + summary: str + evidence: AttributionEvidence + + +def _json_object(text: str | None) -> dict | None: + if text is None: + return None + try: + value = json.loads(text) + except (TypeError, ValueError): + return None + return value if isinstance(value, dict) else None + + +def _case_attribution(case: CaseEvaluation) -> FailureAttribution | None: + if case.status == "passed": + return None + + reasons: list[_CandidateReason] = [] + for run in case.runs: + if run.status == "not_evaluated" or run.error_message: + summary = run.error_message or "Evaluation did not produce a usable result." + reasons.append( + _CandidateReason( + priority=10, + category="evaluation_error", + summary=summary, + evidence=AttributionEvidence( + evidence_type="execution_error", + message=summary, + run_id=run.run_id, + actual=run.error_message, + ), + ) + ) + for invocation in run.invocations: + expected_names = [tool.name for tool in invocation.expected_tools] + actual_names = [tool.name for tool in invocation.actual_tools] + if expected_names != actual_names and (expected_names or actual_names): + summary = f"Expected tool names {expected_names}, got {actual_names}." + reasons.append( + _CandidateReason( + priority=20, + category="tool_name_error", + summary=summary, + evidence=AttributionEvidence( + evidence_type="tool", + message=summary, + run_id=run.run_id, + invocation_id=invocation.invocation_id, + expected=expected_names, + actual=actual_names, + ), + ) + ) + + expected_arguments = [tool.arguments for tool in invocation.expected_tools] + actual_arguments = [tool.arguments for tool in invocation.actual_tools] + if ( + expected_names == actual_names + and (expected_names or actual_names) + and expected_arguments != actual_arguments + ): + summary = f"Expected tool arguments {expected_arguments}, got {actual_arguments}." + reasons.append( + _CandidateReason( + priority=30, + category="tool_argument_error", + summary=summary, + evidence=AttributionEvidence( + evidence_type="tool", + message=summary, + run_id=run.run_id, + invocation_id=invocation.invocation_id, + expected=expected_arguments, + actual=actual_arguments, + ), + ) + ) + + failed_knowledge_metrics = [ + metric + for metric in invocation.metrics + if metric.status == "failed" and metric.metric_name == "llm_rubric_knowledge_recall" + ] + if failed_knowledge_metrics: + metric = failed_knowledge_metrics[0] + summary = metric.reason or "Knowledge recall rubric was not satisfied." + reasons.append( + _CandidateReason( + priority=40, + category="knowledge_recall", + summary=summary, + evidence=AttributionEvidence( + evidence_type="metric", + message=summary, + run_id=run.run_id, + invocation_id=invocation.invocation_id, + metric_name=metric.metric_name, + actual=metric.reason, + ), + ) + ) + + expected_json = _json_object(invocation.expected_response) + actual_json = _json_object(invocation.actual_response) + if expected_json is not None and ( + actual_json is None or not set(expected_json).issubset(actual_json) + ): + summary = "Actual response is not valid JSON with the expected top-level fields." + reasons.append( + _CandidateReason( + priority=50, + category="format_error", + summary=summary, + evidence=AttributionEvidence( + evidence_type="response", + message=summary, + run_id=run.run_id, + invocation_id=invocation.invocation_id, + expected=invocation.expected_response, + actual=invocation.actual_response, + ), + ) + ) + + failed_rubric_metrics = [ + metric + for metric in invocation.metrics + if metric.status == "failed" + and metric.metric_name.startswith("llm_rubric_") + and metric.metric_name != "llm_rubric_knowledge_recall" + ] + if failed_rubric_metrics: + metric = failed_rubric_metrics[0] + summary = metric.reason or f"Rubric metric {metric.metric_name!r} was not satisfied." + reasons.append( + _CandidateReason( + priority=60, + category="rubric_failure", + summary=summary, + evidence=AttributionEvidence( + evidence_type="metric", + message=summary, + run_id=run.run_id, + invocation_id=invocation.invocation_id, + metric_name=metric.metric_name, + actual=metric.reason, + ), + ) + ) + + if ( + expected_json is not None + and actual_json is not None + and expected_json.get("route") != actual_json.get("route") + ): + summary = ( + f"Expected route {expected_json.get('route')!r}, " + f"got {actual_json.get('route')!r}." + ) + reasons.append( + _CandidateReason( + priority=70, + category="routing_error", + summary=summary, + evidence=AttributionEvidence( + evidence_type="response", + message=summary, + run_id=run.run_id, + invocation_id=invocation.invocation_id, + expected=expected_json.get("route"), + actual=actual_json.get("route"), + ), + ) + ) + + failed_final_metrics = [ + metric + for metric in invocation.metrics + if metric.status == "failed" + and metric.metric_name + in {"final_response_avg_score", "response_match_score", "llm_final_response"} + ] + if failed_final_metrics: + metric = failed_final_metrics[0] + summary = f"Final response did not satisfy metric {metric.metric_name!r}." + reasons.append( + _CandidateReason( + priority=80, + category="final_response_mismatch", + summary=summary, + evidence=AttributionEvidence( + evidence_type="response", + message=summary, + run_id=run.run_id, + invocation_id=invocation.invocation_id, + metric_name=metric.metric_name, + expected=invocation.expected_response, + actual=invocation.actual_response, + ), + ) + ) + + if not reasons: + summary = "Available evaluation evidence does not identify a specific failure category." + reasons.append( + _CandidateReason( + priority=90, + category="unknown", + summary=summary, + evidence=AttributionEvidence(evidence_type="metric", message=summary), + ) + ) + + reasons.sort(key=lambda reason: reason.priority) + categories: list[FailureCategory] = [] + for reason in reasons: + if reason.category not in categories: + categories.append(reason.category) + return FailureAttribution( + primary_category=categories[0], + secondary_categories=categories[1:], + summary=reasons[0].summary, + evidence=[reason.evidence for reason in reasons], + ) + + +def attribute_evaluation(evaluation: StandardizedEvaluation) -> StandardizedEvaluation: + """Return a copy with deterministic attribution attached to failed cases.""" + return evaluation.model_copy( + update={ + "cases": [ + case.model_copy(update={"attribution": _case_attribution(case)}) + for case in evaluation.cases + ] + } + ) diff --git a/examples/optimization/eval_optimize_loop/case_diff.py b/examples/optimization/eval_optimize_loop/case_diff.py new file mode 100644 index 00000000..ad7ba724 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/case_diff.py @@ -0,0 +1,181 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under the Apache License, Version 2.0. +"""Deterministic baseline/candidate case comparison for stage 3a.""" + +from __future__ import annotations + +import math +from typing import Literal + +from .evaluation_adapter import EvaluationAnalysisError +from .schemas import CaseDiff +from .schemas import CaseEvaluation +from .schemas import ChangeKind +from .schemas import DatasetDiff +from .schemas import EvaluationStatus +from .schemas import MetricDelta +from .schemas import MetricOutcome +from .schemas import ObservableValue +from .schemas import StandardizedEvaluation + + +def _unavailable(reason: str) -> ObservableValue: + return ObservableValue(status="unavailable", reason=reason) + + +def _delta(baseline: ObservableValue, candidate: ObservableValue, *, reason: str) -> ObservableValue: + if baseline.status != "available" or candidate.status != "available": + return _unavailable(reason) + return ObservableValue(status="available", value=float(candidate.value) - float(baseline.value)) + + +def _change( + baseline_status: EvaluationStatus, + candidate_status: EvaluationStatus, + score_delta: ObservableValue, +) -> ChangeKind: + if "not_evaluated" in {baseline_status, candidate_status}: + return "incomparable" + if baseline_status == "failed" and candidate_status == "passed": + return "newly_passed" + if baseline_status == "passed" and candidate_status == "failed": + return "newly_failed" + if score_delta.status != "available": + return "incomparable" + if float(score_delta.value) > 0.0 and not math.isclose(float(score_delta.value), 0.0, abs_tol=1e-12): + return "improved" + if float(score_delta.value) < 0.0 and not math.isclose(float(score_delta.value), 0.0, abs_tol=1e-12): + return "regressed" + return "unchanged" + + +def _metric_map(case: CaseEvaluation) -> dict[str, MetricOutcome]: + return {metric.metric_name: metric for metric in case.metrics} + + +def _metric_deltas(baseline: CaseEvaluation, candidate: CaseEvaluation) -> list[MetricDelta]: + baseline_metrics = _metric_map(baseline) + candidate_metrics = _metric_map(candidate) + if set(baseline_metrics) != set(candidate_metrics): + raise EvaluationAnalysisError( + f"case {baseline.eval_id!r} metric sets differ between baseline and candidate" + ) + deltas: list[MetricDelta] = [] + for name in sorted(baseline_metrics): + before = baseline_metrics[name] + after = candidate_metrics[name] + if before.threshold != after.threshold: + raise EvaluationAnalysisError( + f"case {baseline.eval_id!r} metric {name!r} threshold changed " + f"from {before.threshold} to {after.threshold}" + ) + score_delta = _delta( + before.score, + after.score, + reason=f"case {baseline.eval_id!r} metric {name!r} score delta is unavailable", + ) + deltas.append( + MetricDelta( + metric_name=name, + baseline_status=before.status, + candidate_status=after.status, + baseline_score=before.score, + candidate_score=after.score, + score_delta=score_delta, + change=_change(before.status, after.status, score_delta), + ) + ) + return deltas + + +def _case_diff( + baseline: CaseEvaluation, + candidate: CaseEvaluation, + *, + split: Literal["train", "validation"], + hard_case_ids: set[str], + critical_case_ids: set[str], + severe_case_score_drop: float, +) -> CaseDiff: + score_delta = _delta( + baseline.average_score, + candidate.average_score, + reason=f"case {baseline.eval_id!r} aggregate score delta is unavailable", + ) + severe = ( + score_delta.status == "available" + and float(score_delta.value) <= -severe_case_score_drop + and not math.isclose(float(score_delta.value), 0.0, abs_tol=1e-12) + ) + return CaseDiff( + eval_id=baseline.eval_id, + split=split, + baseline_status=baseline.status, + candidate_status=candidate.status, + baseline_score=baseline.average_score, + candidate_score=candidate.average_score, + score_delta=score_delta, + change=_change(baseline.status, candidate.status, score_delta), + metrics=_metric_deltas(baseline, candidate), + baseline_attribution=baseline.attribution, + candidate_attribution=candidate.attribution, + is_hard=baseline.eval_id in hard_case_ids, + is_critical=baseline.eval_id in critical_case_ids, + severe_regression=severe, + ) + + +def compare_evaluations( + baseline: StandardizedEvaluation, + candidate: StandardizedEvaluation, + *, + hard_case_ids: set[str], + critical_case_ids: set[str], + severe_case_score_drop: float, +) -> DatasetDiff: + """Compare matching baseline and candidate evaluations for one split.""" + if baseline.phase != "baseline" or candidate.phase != "candidate": + raise EvaluationAnalysisError("evaluation comparison requires baseline then candidate phases") + if baseline.split != candidate.split: + raise EvaluationAnalysisError("baseline and candidate splits do not match") + if baseline.eval_set_id != candidate.eval_set_id: + raise EvaluationAnalysisError("baseline and candidate eval_set_id values do not match") + + baseline_cases = {case.eval_id: case for case in baseline.cases} + candidate_cases = {case.eval_id: case for case in candidate.cases} + if set(baseline_cases) != set(candidate_cases): + raise EvaluationAnalysisError("baseline and candidate case ids do not match") + + cases = [ + _case_diff( + baseline_cases[eval_id], + candidate_cases[eval_id], + split=baseline.split, + hard_case_ids=hard_case_ids, + critical_case_ids=critical_case_ids, + severe_case_score_drop=severe_case_score_drop, + ) + for eval_id in sorted(baseline_cases) + ] + score_delta = _delta( + baseline.average_score, + candidate.average_score, + reason=f"{baseline.split} dataset score delta is unavailable", + ) + return DatasetDiff( + split=baseline.split, + eval_set_id=baseline.eval_set_id, + cases=cases, + baseline_average_score=baseline.average_score, + candidate_average_score=candidate.average_score, + score_delta=score_delta, + newly_passed_count=sum(case.change == "newly_passed" for case in cases), + newly_failed_count=sum(case.change == "newly_failed" for case in cases), + improved_count=sum(case.change == "improved" for case in cases), + regressed_count=sum(case.change == "regressed" for case in cases), + unchanged_count=sum(case.change == "unchanged" for case in cases), + incomparable_count=sum(case.change == "incomparable" for case in cases), + ) diff --git a/examples/optimization/eval_optimize_loop/evaluation_adapter.py b/examples/optimization/eval_optimize_loop/evaluation_adapter.py new file mode 100644 index 00000000..f0f70391 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/evaluation_adapter.py @@ -0,0 +1,227 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under the Apache License, Version 2.0. +"""Convert SDK evaluation results into stable, lightweight analysis data.""" + +from __future__ import annotations + +from collections.abc import Iterable +from statistics import mean + +from trpc_agent_sdk.evaluation import EvalCaseResult +from trpc_agent_sdk.evaluation import EvalMetricResult +from trpc_agent_sdk.evaluation import EvalMetricResultPerInvocation +from trpc_agent_sdk.evaluation import EvalStatus +from trpc_agent_sdk.evaluation import Invocation +from trpc_agent_sdk.evaluation import get_all_tool_calls + +from .schemas import CaseEvaluation +from .schemas import CaseRunOutcome +from .schemas import EvaluationStatus +from .schemas import FakeEvaluationSnapshot +from .schemas import InvocationEvidence +from .schemas import MetricOutcome +from .schemas import ObservableValue +from .schemas import StandardizedEvaluation +from .schemas import ToolCallEvidence + + +class EvaluationAnalysisError(ValueError): + """Evaluation evidence is structurally inconsistent and unsafe to compare.""" + + +def _status(value: EvalStatus, *, error_message: str | None = None) -> EvaluationStatus: + if error_message or value == EvalStatus.NOT_EVALUATED: + return "not_evaluated" + if value == EvalStatus.PASSED: + return "passed" + return "failed" + + +def _content_text(content: object | None) -> str | None: + if content is None: + return None + parts = getattr(content, "parts", None) or [] + text = "\n".join(part.text for part in parts if getattr(part, "text", None)) + return text or None + + +def _tool_evidence(invocation: Invocation | None) -> list[ToolCallEvidence]: + if invocation is None: + return [] + return [ + ToolCallEvidence(name=call.name or "", arguments=dict(call.args or {})) + for call in get_all_tool_calls(invocation.intermediate_data) + ] + + +def _observable(scores: Iterable[float | None], *, reason: str) -> ObservableValue: + values = list(scores) + if not values or any(score is None for score in values): + return ObservableValue(status="unavailable", reason=reason) + return ObservableValue(status="available", value=mean(float(score) for score in values)) + + +def _metric_map(metrics: list[EvalMetricResult], *, context: str) -> dict[str, EvalMetricResult]: + result: dict[str, EvalMetricResult] = {} + for metric in metrics: + if metric.metric_name in result: + raise EvaluationAnalysisError(f"{context} contains duplicate metric {metric.metric_name!r}") + result[metric.metric_name] = metric + return result + + +def _metric_outcome(metric: EvalMetricResult, *, context: str) -> MetricOutcome: + reason = metric.details.reason if metric.details is not None else None + score = _observable([metric.score], reason=f"{context} metric score is unavailable") + return MetricOutcome( + metric_name=metric.metric_name, + threshold=metric.threshold, + status="not_evaluated" if metric.score is None else _status(metric.eval_status), + score=score, + reason=reason, + ) + + +def _invocation_evidence(result: EvalMetricResultPerInvocation, *, context: str) -> InvocationEvidence: + actual = result.actual_invocation + expected = result.expected_invocation + metrics = _metric_map(result.eval_metric_results, context=context) + return InvocationEvidence( + invocation_id=actual.invocation_id, + user_text=_content_text(actual.user_content) or "", + expected_response=_content_text(expected.final_response) if expected is not None else None, + actual_response=_content_text(actual.final_response), + expected_tools=_tool_evidence(expected), + actual_tools=_tool_evidence(actual), + metrics=[_metric_outcome(metrics[name], context=context) for name in sorted(metrics)], + ) + + +def _case_evaluation( + eval_id: str, + raw_runs: list[EvalCaseResult], + *, + eval_set_id: str, +) -> CaseEvaluation: + if not raw_runs: + raise EvaluationAnalysisError(f"case {eval_id!r} has no run results") + + ordered_runs = sorted(raw_runs, key=lambda run: run.run_id if run.run_id is not None else 0) + run_ids = [run.run_id if run.run_id is not None else index for index, run in enumerate(ordered_runs, 1)] + if len(run_ids) != len(set(run_ids)): + raise EvaluationAnalysisError(f"case {eval_id!r} contains duplicate run ids") + + metric_maps: list[dict[str, EvalMetricResult]] = [] + normalized_runs: list[CaseRunOutcome] = [] + for run_id, run in zip(run_ids, ordered_runs): + if run.eval_id != eval_id: + raise EvaluationAnalysisError( + f"case mapping key {eval_id!r} does not match result eval_id {run.eval_id!r}" + ) + if run.eval_set_id != eval_set_id: + raise EvaluationAnalysisError( + f"case {eval_id!r} run {run_id} has eval_set_id {run.eval_set_id!r}; " + f"expected {eval_set_id!r}" + ) + context = f"case {eval_id!r} run {run_id}" + metric_map = _metric_map(run.overall_eval_metric_results, context=context) + metric_maps.append(metric_map) + normalized_metrics = [ + _metric_outcome(metric_map[name], context=context) for name in sorted(metric_map) + ] + run_status = _status(run.final_eval_status, error_message=run.error_message) + if not normalized_metrics or any(metric.status == "not_evaluated" for metric in normalized_metrics): + run_status = "not_evaluated" + normalized_runs.append( + CaseRunOutcome( + run_id=run_id, + status=run_status, + error_message=run.error_message, + metrics=normalized_metrics, + invocations=[ + _invocation_evidence(invocation, context=f"{context} invocation {index}") + for index, invocation in enumerate(run.eval_metric_result_per_invocation, 1) + ], + ) + ) + + metric_names = sorted(set().union(*(metrics.keys() for metrics in metric_maps))) + aggregate_metrics: list[MetricOutcome] = [] + for name in metric_names: + present = [metrics.get(name) for metrics in metric_maps] + thresholds = {metric.threshold for metric in present if metric is not None} + if len(thresholds) > 1: + raise EvaluationAnalysisError(f"case {eval_id!r} metric {name!r} has inconsistent thresholds") + available_metrics = [metric for metric in present if metric is not None] + metric_status: EvaluationStatus + if len(available_metrics) != len(present) or any( + metric.eval_status == EvalStatus.NOT_EVALUATED or metric.score is None for metric in available_metrics + ): + metric_status = "not_evaluated" + elif all(metric.eval_status == EvalStatus.PASSED for metric in available_metrics): + metric_status = "passed" + else: + metric_status = "failed" + reasons = [ + metric.details.reason + for metric in available_metrics + if metric.details is not None and metric.details.reason + ] + aggregate_metrics.append( + MetricOutcome( + metric_name=name, + threshold=next(iter(thresholds), 0.0), + status=metric_status, + score=_observable( + [metric.score if metric is not None else None for metric in present], + reason=f"case {eval_id!r} metric {name!r} is unavailable in one or more runs", + ), + reason="; ".join(reasons) or None, + ) + ) + + statuses = [run.status for run in normalized_runs] + if "not_evaluated" in statuses or any(metric.status == "not_evaluated" for metric in aggregate_metrics): + case_status: EvaluationStatus = "not_evaluated" + elif all(status == "passed" for status in statuses): + case_status = "passed" + else: + case_status = "failed" + return CaseEvaluation( + eval_id=eval_id, + status=case_status, + average_score=_observable( + [metric.score.value if metric.score.status == "available" else None for metric in aggregate_metrics], + reason=f"case {eval_id!r} has unavailable metric scores", + ), + metrics=aggregate_metrics, + runs=normalized_runs, + ) + + +def standardize_snapshot(snapshot: FakeEvaluationSnapshot) -> StandardizedEvaluation: + """Normalize one complete SDK snapshot without discarding raw evidence.""" + cases = [ + _case_evaluation( + eval_id, + snapshot.eval_results_by_eval_id[eval_id], + eval_set_id=snapshot.eval_set_id, + ) + for eval_id in sorted(snapshot.eval_results_by_eval_id) + ] + return StandardizedEvaluation( + phase=snapshot.phase, + split=snapshot.split, + eval_set_id=snapshot.eval_set_id, + cases=cases, + passed_case_count=sum(case.status == "passed" for case in cases), + failed_case_count=sum(case.status == "failed" for case in cases), + not_evaluated_case_count=sum(case.status == "not_evaluated" for case in cases), + average_score=_observable( + [case.average_score.value if case.average_score.status == "available" else None for case in cases], + reason="one or more case scores are unavailable", + ), + ) diff --git a/examples/optimization/eval_optimize_loop/pipeline.py b/examples/optimization/eval_optimize_loop/pipeline.py index 3eb48ebd..2de7f040 100644 --- a/examples/optimization/eval_optimize_loop/pipeline.py +++ b/examples/optimization/eval_optimize_loop/pipeline.py @@ -25,8 +25,10 @@ from trpc_agent_sdk.evaluation import TargetPrompt from trpc_agent_sdk.evaluation import load_optimize_config +from .analysis import build_evaluation_analysis from .config import PipelineConfig from .config import load_pipeline_config +from .evaluation_adapter import EvaluationAnalysisError from .fake import DeterministicFakeAgent from .fake import DeterministicFakeCandidateProvider from .prompt_workspace import PromptWorkspaceError @@ -418,6 +420,18 @@ async def run_fake_stage( phase="candidate", split="validation", ) + try: + analysis = build_evaluation_analysis( + baseline_train=baseline_train, + baseline_validation=baseline_validation, + candidate_train=candidate_train, + candidate_validation=candidate_validation, + hard_case_ids=set(prepared.config.case_labels.hard_case_ids), + critical_case_ids=set(prepared.config.case_labels.critical_case_ids), + severe_case_score_drop=prepared.config.gate.severe_case_score_drop, + ) + except EvaluationAnalysisError as exc: + raise FakeStageExecutionError(f"stage 3a analysis failed: {exc}") from exc return FakeStageResult( scenario=selected_scenario, candidate=candidate, @@ -425,4 +439,5 @@ async def run_fake_stage( baseline_validation=baseline_validation, candidate_train=candidate_train, candidate_validation=candidate_validation, + analysis=analysis, ) diff --git a/examples/optimization/eval_optimize_loop/schemas.py b/examples/optimization/eval_optimize_loop/schemas.py index 04c12f48..bc559418 100644 --- a/examples/optimization/eval_optimize_loop/schemas.py +++ b/examples/optimization/eval_optimize_loop/schemas.py @@ -24,6 +24,27 @@ FakeCandidateScenario = Literal["improve", "no_improvement", "overfit"] +EvaluationStatus = Literal["passed", "failed", "not_evaluated"] +FailureCategory = Literal[ + "evaluation_error", + "tool_name_error", + "tool_argument_error", + "knowledge_recall", + "format_error", + "rubric_failure", + "routing_error", + "final_response_mismatch", + "unknown", +] +ChangeKind = Literal[ + "newly_passed", + "newly_failed", + "improved", + "regressed", + "unchanged", + "incomparable", +] +OverfitStatus = Literal["detected", "not_detected", "unavailable"] class ObservableValue(EvalBaseModel): @@ -118,6 +139,151 @@ class FakeEvaluationSnapshot(EvalBaseModel): ) +class ToolCallEvidence(EvalBaseModel): + """A compact tool call retained for attribution and reporting.""" + + name: str + arguments: dict[str, Any] = Field(default_factory=dict) + + +class MetricOutcome(EvalBaseModel): + """One normalized metric outcome for a run or an aggregate.""" + + metric_name: str + threshold: float + status: EvaluationStatus + score: ObservableValue + reason: Optional[str] = None + + +class InvocationEvidence(EvalBaseModel): + """Expected and actual evidence from one evaluated invocation.""" + + invocation_id: str + user_text: str + expected_response: Optional[str] = None + actual_response: Optional[str] = None + expected_tools: list[ToolCallEvidence] = Field(default_factory=list) + actual_tools: list[ToolCallEvidence] = Field(default_factory=list) + metrics: list[MetricOutcome] = Field(default_factory=list) + + +class CaseRunOutcome(EvalBaseModel): + """Normalized evidence from one configured run of an eval case.""" + + run_id: int + status: EvaluationStatus + error_message: Optional[str] = None + metrics: list[MetricOutcome] + invocations: list[InvocationEvidence] + + +class AttributionEvidence(EvalBaseModel): + """One concrete observation supporting a failure attribution.""" + + evidence_type: Literal["execution_error", "metric", "response", "tool"] + message: str + run_id: Optional[int] = None + invocation_id: Optional[str] = None + metric_name: Optional[str] = None + expected: Optional[Any] = None + actual: Optional[Any] = None + + +class FailureAttribution(EvalBaseModel): + """Deterministic primary and secondary reasons for a failed case.""" + + primary_category: FailureCategory + secondary_categories: list[FailureCategory] = Field(default_factory=list) + summary: str + evidence: list[AttributionEvidence] = Field(default_factory=list) + + +class CaseEvaluation(EvalBaseModel): + """One eval case aggregated across all configured runs.""" + + eval_id: str + status: EvaluationStatus + average_score: ObservableValue + metrics: list[MetricOutcome] + runs: list[CaseRunOutcome] + attribution: Optional[FailureAttribution] = None + + +class StandardizedEvaluation(EvalBaseModel): + """Stable case-oriented representation of one SDK evaluation snapshot.""" + + phase: Literal["baseline", "candidate"] + split: Literal["train", "validation"] + eval_set_id: str + cases: list[CaseEvaluation] + passed_case_count: int = Field(ge=0) + failed_case_count: int = Field(ge=0) + not_evaluated_case_count: int = Field(ge=0) + average_score: ObservableValue + + +class MetricDelta(EvalBaseModel): + """Before/after comparison for one metric.""" + + metric_name: str + baseline_status: EvaluationStatus + candidate_status: EvaluationStatus + baseline_score: ObservableValue + candidate_score: ObservableValue + score_delta: ObservableValue + change: ChangeKind + + +class CaseDiff(EvalBaseModel): + """Before/after comparison and policy labels for one eval case.""" + + eval_id: str + split: Literal["train", "validation"] + baseline_status: EvaluationStatus + candidate_status: EvaluationStatus + baseline_score: ObservableValue + candidate_score: ObservableValue + score_delta: ObservableValue + change: ChangeKind + metrics: list[MetricDelta] + baseline_attribution: Optional[FailureAttribution] = None + candidate_attribution: Optional[FailureAttribution] = None + is_hard: bool = False + is_critical: bool = False + severe_regression: bool = False + + +class DatasetDiff(EvalBaseModel): + """Case-level changes and aggregate deltas for one dataset split.""" + + split: Literal["train", "validation"] + eval_set_id: str + cases: list[CaseDiff] + baseline_average_score: ObservableValue + candidate_average_score: ObservableValue + score_delta: ObservableValue + newly_passed_count: int = Field(ge=0) + newly_failed_count: int = Field(ge=0) + improved_count: int = Field(ge=0) + regressed_count: int = Field(ge=0) + unchanged_count: int = Field(ge=0) + incomparable_count: int = Field(ge=0) + + +class EvaluationAnalysis(EvalBaseModel): + """All normalized evidence and comparisons produced by stage 3a.""" + + baseline_train: StandardizedEvaluation + baseline_validation: StandardizedEvaluation + candidate_train: StandardizedEvaluation + candidate_validation: StandardizedEvaluation + train_diff: DatasetDiff + validation_diff: DatasetDiff + overfit_status: OverfitStatus + overfit_reason: str + + class FakeStageResult(EvalBaseModel): """The four full evaluations and candidate metadata produced in stage two.""" @@ -127,3 +293,4 @@ class FakeStageResult(EvalBaseModel): baseline_validation: FakeEvaluationSnapshot candidate_train: FakeEvaluationSnapshot candidate_validation: FakeEvaluationSnapshot + analysis: EvaluationAnalysis diff --git a/tests/evaluation/test_eval_optimize_loop_stage3a.py b/tests/evaluation/test_eval_optimize_loop_stage3a.py new file mode 100644 index 00000000..22370bda --- /dev/null +++ b/tests/evaluation/test_eval_optimize_loop_stage3a.py @@ -0,0 +1,547 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under the Apache License, Version 2.0. +"""Tests for stage 3a evaluation normalization, attribution, and case diff.""" + +from __future__ import annotations + +import shutil +from pathlib import Path + +import pytest + +from examples.optimization.eval_optimize_loop import pipeline as pipeline_module +from examples.optimization.eval_optimize_loop.attribution import attribute_evaluation +from examples.optimization.eval_optimize_loop.case_diff import compare_evaluations +from examples.optimization.eval_optimize_loop.evaluation_adapter import EvaluationAnalysisError +from examples.optimization.eval_optimize_loop.evaluation_adapter import standardize_snapshot +from examples.optimization.eval_optimize_loop.pipeline import prepare_run +from examples.optimization.eval_optimize_loop.pipeline import run_fake_stage +from examples.optimization.eval_optimize_loop.schemas import FakeEvaluationSnapshot +from examples.optimization.eval_optimize_loop.schemas import FakeStageResult +from trpc_agent_sdk.evaluation import EvalCaseResult +from trpc_agent_sdk.evaluation import EvalMetricResult +from trpc_agent_sdk.evaluation import EvalMetricResultDetails +from trpc_agent_sdk.evaluation import EvalMetricResultPerInvocation +from trpc_agent_sdk.evaluation import EvalStatus +from trpc_agent_sdk.evaluation import IntermediateData +from trpc_agent_sdk.evaluation import Invocation +from trpc_agent_sdk.types import Content +from trpc_agent_sdk.types import FunctionCall +from trpc_agent_sdk.types import Part + + +_REPO_ROOT = Path(__file__).resolve().parents[2] +_EXAMPLE = _REPO_ROOT / "examples" / "optimization" / "eval_optimize_loop" + + +def _copy_example(tmp_path: Path, name: str) -> Path: + target = tmp_path / name + shutil.copytree(_EXAMPLE, target, ignore=shutil.ignore_patterns("runs", "__pycache__")) + return target + + +def _content(text: str) -> Content: + return Content(parts=[Part(text=text)]) + + +def _metric( + *, + score: float | None, + status: EvalStatus, + name: str = "final_response_avg_score", + threshold: float = 1.0, + reason: str | None = None, +) -> EvalMetricResult: + return EvalMetricResult( + metric_name=name, + threshold=threshold, + score=score, + eval_status=status, + details=EvalMetricResultDetails(reason=reason, score=score) if reason else None, + ) + + +def _case_run( + *, + run_id: int, + status: EvalStatus, + score: float | None, + actual_text: str = '{"route":"general_support"}', + expected_text: str = '{"route":"order_lookup"}', + actual_tools: list[FunctionCall] | None = None, + expected_tools: list[FunctionCall] | None = None, + error_message: str | None = None, + metrics: list[EvalMetricResult] | None = None, +) -> EvalCaseResult: + actual = Invocation( + invocation_id="inv-1", + user_content=_content("Check order A100"), + final_response=_content(actual_text), + intermediate_data=IntermediateData(tool_uses=actual_tools or []), + ) + expected = Invocation( + invocation_id="inv-1", + user_content=_content("Check order A100"), + final_response=_content(expected_text), + intermediate_data=IntermediateData(tool_uses=expected_tools or []), + ) + metric_results = metrics or [_metric(score=score, status=status)] + return EvalCaseResult( + eval_set_id="train-set", + eval_id="case-order", + run_id=run_id, + final_eval_status=status, + error_message=error_message, + overall_eval_metric_results=metric_results, + eval_metric_result_per_invocation=[ + EvalMetricResultPerInvocation( + actual_invocation=actual, + expected_invocation=expected, + eval_metric_results=metric_results, + ) + ], + session_id=f"session-{run_id}", + ) + + +def _snapshot(*runs: EvalCaseResult) -> FakeEvaluationSnapshot: + return FakeEvaluationSnapshot( + phase="baseline", + split="train", + eval_set_id="train-set", + details_lines=[], + result_lines=[], + eval_results_by_eval_id={"case-order": list(runs)}, + passed_case_count=0, + total_case_count=1, + average_score=0.5, + ) + + +def test_standardize_snapshot_preserves_multirun_metric_and_invocation_evidence(): + expected_tool = FunctionCall(name="lookup_order", args={"order_id": "A100"}) + actual_tool = FunctionCall(name="general_support", args={}) + snapshot = _snapshot( + _case_run( + run_id=2, + status=EvalStatus.FAILED, + score=0.0, + actual_tools=[actual_tool], + expected_tools=[expected_tool], + ), + _case_run( + run_id=1, + status=EvalStatus.PASSED, + score=1.0, + actual_text='{"route":"order_lookup"}', + actual_tools=[expected_tool], + expected_tools=[expected_tool], + ), + ) + + standardized = standardize_snapshot(snapshot) + + case = standardized.cases[0] + assert case.eval_id == "case-order" + assert case.status == "failed" + assert case.average_score.status == "available" + assert case.average_score.value == pytest.approx(0.5) + assert case.metrics[0].metric_name == "final_response_avg_score" + assert case.metrics[0].score.value == pytest.approx(0.5) + assert [run.run_id for run in case.runs] == [1, 2] + failed_evidence = case.runs[1].invocations[0] + assert failed_evidence.user_text == "Check order A100" + assert failed_evidence.actual_response == '{"route":"general_support"}' + assert failed_evidence.expected_response == '{"route":"order_lookup"}' + assert failed_evidence.actual_tools[0].name == "general_support" + assert failed_evidence.expected_tools[0].arguments == {"order_id": "A100"} + assert standardized.average_score.value == pytest.approx(0.5) + + +def test_standardize_snapshot_keeps_missing_scores_unavailable_instead_of_zero(): + standardized = standardize_snapshot( + _snapshot( + _case_run( + run_id=1, + status=EvalStatus.NOT_EVALUATED, + score=None, + error_message="metric failed", + ) + ) + ) + + case = standardized.cases[0] + assert case.status == "not_evaluated" + assert case.average_score.status == "unavailable" + assert case.average_score.value is None + assert case.metrics[0].score.status == "unavailable" + assert standardized.average_score.status == "unavailable" + assert standardized.not_evaluated_case_count == 1 + + +def test_standardize_snapshot_treats_failed_metric_without_score_as_not_evaluated(): + standardized = standardize_snapshot( + _snapshot( + _case_run( + run_id=1, + status=EvalStatus.FAILED, + score=None, + ) + ) + ) + + case = standardized.cases[0] + assert case.status == "not_evaluated" + assert case.runs[0].status == "not_evaluated" + assert case.runs[0].metrics[0].status == "not_evaluated" + + +def test_standardize_snapshot_rejects_duplicate_run_ids_and_metric_threshold_drift(): + duplicate_runs = _snapshot( + _case_run(run_id=1, status=EvalStatus.PASSED, score=1.0), + _case_run(run_id=1, status=EvalStatus.PASSED, score=1.0), + ) + with pytest.raises(EvaluationAnalysisError, match="duplicate run ids"): + standardize_snapshot(duplicate_runs) + + threshold_drift = _snapshot( + _case_run(run_id=1, status=EvalStatus.PASSED, score=1.0), + _case_run( + run_id=2, + status=EvalStatus.PASSED, + score=1.0, + metrics=[_metric(score=1.0, status=EvalStatus.PASSED, threshold=0.5)], + ), + ) + with pytest.raises(EvaluationAnalysisError, match="inconsistent thresholds"): + standardize_snapshot(threshold_drift) + + +def test_standardize_snapshot_rejects_result_from_a_different_evalset(): + wrong_evalset_run = _case_run( + run_id=1, + status=EvalStatus.PASSED, + score=1.0, + ).model_copy(update={"eval_set_id": "other-set"}) + + with pytest.raises(EvaluationAnalysisError, match="eval_set_id"): + standardize_snapshot(_snapshot(wrong_evalset_run)) + + +def test_attribution_prefers_tool_name_error_and_retains_secondary_response_evidence(): + standardized = standardize_snapshot( + _snapshot( + _case_run( + run_id=1, + status=EvalStatus.FAILED, + score=0.0, + actual_tools=[FunctionCall(name="general_support", args={})], + expected_tools=[FunctionCall(name="lookup_order", args={"order_id": "A100"})], + ) + ) + ) + + attributed = attribute_evaluation(standardized) + + attribution = attributed.cases[0].attribution + assert attribution is not None + assert attribution.primary_category == "tool_name_error" + assert attribution.secondary_categories == ["routing_error", "final_response_mismatch"] + assert attribution.summary == "Expected tool names ['lookup_order'], got ['general_support']." + assert attribution.evidence[0].expected == ["lookup_order"] + assert attribution.evidence[0].actual == ["general_support"] + + +@pytest.mark.parametrize( + ("case_run", "expected_category"), + [ + ( + _case_run( + run_id=1, + status=EvalStatus.NOT_EVALUATED, + score=None, + error_message="judge unavailable", + ), + "evaluation_error", + ), + ( + _case_run( + run_id=1, + status=EvalStatus.FAILED, + score=0.0, + actual_tools=[FunctionCall(name="lookup_order", args={"order_id": "B200"})], + expected_tools=[FunctionCall(name="lookup_order", args={"order_id": "A100"})], + ), + "tool_argument_error", + ), + ( + _case_run( + run_id=1, + status=EvalStatus.FAILED, + score=0.0, + metrics=[ + _metric( + name="llm_rubric_knowledge_recall", + score=0.0, + status=EvalStatus.FAILED, + reason="Shipping policy was not recalled.", + ) + ], + ), + "knowledge_recall", + ), + ( + _case_run( + run_id=1, + status=EvalStatus.FAILED, + score=0.0, + actual_text="not-json", + ), + "format_error", + ), + ( + _case_run( + run_id=1, + status=EvalStatus.FAILED, + score=0.0, + actual_text="same", + expected_text="same", + metrics=[ + _metric( + name="llm_rubric_response", + score=0.0, + status=EvalStatus.FAILED, + reason="Rubric requirement was not met.", + ) + ], + ), + "rubric_failure", + ), + ( + _case_run( + run_id=1, + status=EvalStatus.FAILED, + score=0.0, + actual_text="same", + expected_text="same", + metrics=[_metric(name="custom_metric", score=0.0, status=EvalStatus.FAILED)], + ), + "unknown", + ), + ], +) +def test_attribution_uses_generic_evidence_categories( + case_run: EvalCaseResult, + expected_category: str, +): + attributed = attribute_evaluation(standardize_snapshot(_snapshot(case_run))) + + attribution = attributed.cases[0].attribution + assert attribution is not None + assert attribution.primary_category == expected_category + + +def test_attribution_is_absent_for_passed_case(): + attributed = attribute_evaluation( + standardize_snapshot( + _snapshot( + _case_run( + run_id=1, + status=EvalStatus.PASSED, + score=1.0, + actual_text='{"route":"order_lookup"}', + ) + ) + ) + ) + + assert attributed.cases[0].attribution is None + + +def test_compare_evaluations_marks_newly_passed_score_delta_and_hard_label(): + baseline = attribute_evaluation( + standardize_snapshot( + _snapshot(_case_run(run_id=1, status=EvalStatus.FAILED, score=0.0)) + ) + ) + candidate_snapshot = _snapshot( + _case_run( + run_id=1, + status=EvalStatus.PASSED, + score=1.0, + actual_text='{"route":"order_lookup"}', + ) + ).model_copy(update={"phase": "candidate", "average_score": 1.0}) + candidate = attribute_evaluation(standardize_snapshot(candidate_snapshot)) + + diff = compare_evaluations( + baseline, + candidate, + hard_case_ids={"case-order"}, + critical_case_ids=set(), + severe_case_score_drop=0.2, + ) + + case_diff = diff.cases[0] + assert case_diff.change == "newly_passed" + assert case_diff.score_delta.status == "available" + assert case_diff.score_delta.value == pytest.approx(1.0) + assert case_diff.is_hard is True + assert case_diff.is_critical is False + assert case_diff.severe_regression is False + assert diff.newly_passed_count == 1 + + +def test_compare_evaluations_marks_critical_severe_regression_and_unavailable_as_incomparable(): + passing_snapshot = _snapshot( + _case_run( + run_id=1, + status=EvalStatus.PASSED, + score=1.0, + actual_text='{"route":"order_lookup"}', + ) + ) + baseline = attribute_evaluation(standardize_snapshot(passing_snapshot)) + failing_snapshot = _snapshot(_case_run(run_id=1, status=EvalStatus.FAILED, score=0.0)).model_copy( + update={"phase": "candidate", "average_score": 0.0} + ) + candidate = attribute_evaluation(standardize_snapshot(failing_snapshot)) + + regression = compare_evaluations( + baseline, + candidate, + hard_case_ids=set(), + critical_case_ids={"case-order"}, + severe_case_score_drop=0.2, + ).cases[0] + assert regression.change == "newly_failed" + assert regression.is_critical is True + assert regression.severe_regression is True + + unavailable_snapshot = _snapshot( + _case_run( + run_id=1, + status=EvalStatus.NOT_EVALUATED, + score=None, + error_message="candidate metric unavailable", + ) + ).model_copy(update={"phase": "candidate", "average_score": None}) + unavailable = attribute_evaluation(standardize_snapshot(unavailable_snapshot)) + incomparable = compare_evaluations( + baseline, + unavailable, + hard_case_ids=set(), + critical_case_ids=set(), + severe_case_score_drop=0.2, + ).cases[0] + assert incomparable.change == "incomparable" + assert incomparable.score_delta.status == "unavailable" + assert incomparable.severe_regression is False + + +def test_compare_evaluations_rejects_case_and_metric_threshold_mismatch(): + baseline = attribute_evaluation( + standardize_snapshot( + _snapshot(_case_run(run_id=1, status=EvalStatus.FAILED, score=0.0)) + ) + ) + candidate = attribute_evaluation( + standardize_snapshot( + _snapshot( + _case_run( + run_id=1, + status=EvalStatus.PASSED, + score=1.0, + metrics=[_metric(score=1.0, status=EvalStatus.PASSED, threshold=0.5)], + ) + ).model_copy(update={"phase": "candidate"}) + ) + ) + with pytest.raises(EvaluationAnalysisError, match="threshold changed"): + compare_evaluations( + baseline, + candidate, + hard_case_ids=set(), + critical_case_ids=set(), + severe_case_score_drop=0.2, + ) + + different_case = candidate.model_copy( + update={"cases": [candidate.cases[0].model_copy(update={"eval_id": "other-case"})]} + ) + with pytest.raises(EvaluationAnalysisError, match="case ids do not match"): + compare_evaluations( + baseline, + different_case, + hard_case_ids=set(), + critical_case_ids=set(), + severe_case_score_drop=0.2, + ) + + +@pytest.mark.parametrize( + ( + "scenario", + "train_newly_passed", + "validation_newly_passed", + "validation_newly_failed", + "validation_unchanged", + "overfit_status", + ), + [ + ("improve", 2, 2, 0, 1, "not_detected"), + ("no_improvement", 0, 0, 0, 3, "not_detected"), + ("overfit", 2, 0, 1, 2, "detected"), + ], +) +@pytest.mark.asyncio +async def test_stage3a_scenario_analysis_matrix( + tmp_path: Path, + scenario: str, + train_newly_passed: int, + validation_newly_passed: int, + validation_newly_failed: int, + validation_unchanged: int, + overfit_status: str, +): + root = _copy_example(tmp_path, scenario) + prepared = prepare_run(root / "pipeline.json", run_id=f"stage3a_{scenario}") + + result = await run_fake_stage(prepared, scenario=scenario) # type: ignore[arg-type] + + analysis = result.analysis + assert analysis.train_diff.newly_passed_count == train_newly_passed + assert analysis.validation_diff.newly_passed_count == validation_newly_passed + assert analysis.validation_diff.newly_failed_count == validation_newly_failed + assert analysis.validation_diff.unchanged_count == validation_unchanged + assert analysis.overfit_status == overfit_status + + if scenario == "overfit": + refund = next( + case for case in analysis.validation_diff.cases if case.eval_id == "val_refund_route" + ) + assert refund.change == "newly_failed" + assert refund.is_critical is True + assert refund.severe_regression is True + assert refund.candidate_attribution is not None + assert refund.candidate_attribution.primary_category == "routing_error" + + restored = FakeStageResult.model_validate_json(result.model_dump_json()) + assert restored.analysis == analysis + + +@pytest.mark.asyncio +async def test_stage3a_does_not_hide_unexpected_value_errors(tmp_path: Path, monkeypatch): + root = _copy_example(tmp_path, "unexpected_analysis_error") + prepared = prepare_run(root / "pipeline.json", run_id="unexpected_analysis_error") + + def fail_analysis(**_kwargs): + raise ValueError("injected programming error") + + monkeypatch.setattr(pipeline_module, "build_evaluation_analysis", fail_analysis) + + with pytest.raises(ValueError, match="injected programming error"): + await run_fake_stage(prepared, scenario="improve") From 92590c0709901b4edc6a83e107748966367699a0 Mon Sep 17 00:00:00 2001 From: guocfu Date: Thu, 16 Jul 2026 15:06:37 +0800 Subject: [PATCH 04/18] =?UTF-8?q?feat(evaluation):=20=E5=AE=9E=E7=8E=B0?= =?UTF-8?q?=E7=8B=AC=E7=AB=8B=20Gate=20=E4=B8=8E=E9=A2=84=E7=AE=97?= =?UTF-8?q?=E5=86=B3=E7=AD=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...65\350\267\257\347\272\277\345\233\276.md" | 4 +- .../optimization/eval_optimize_loop/README.md | 27 +- .../optimization/eval_optimize_loop/gate.py | 382 +++++++++++ .../eval_optimize_loop/pipeline.py | 34 + .../eval_optimize_loop/run_pipeline.py | 20 +- .../eval_optimize_loop/schemas.py | 48 +- .../test_eval_optimize_loop_stage2.py | 2 +- .../test_eval_optimize_loop_stage3b.py | 624 ++++++++++++++++++ 8 files changed, 1126 insertions(+), 15 deletions(-) create mode 100644 examples/optimization/eval_optimize_loop/gate.py create mode 100644 tests/evaluation/test_eval_optimize_loop_stage3b.py diff --git "a/analysis/evaluation optimization pipeline/\345\256\236\346\226\275\351\230\266\346\256\265\350\267\257\347\272\277\345\233\276.md" "b/analysis/evaluation optimization pipeline/\345\256\236\346\226\275\351\230\266\346\256\265\350\267\257\347\272\277\345\233\276.md" index bf7b8787..29d905d4 100644 --- "a/analysis/evaluation optimization pipeline/\345\256\236\346\226\275\351\230\266\346\256\265\350\267\257\347\272\277\345\233\276.md" +++ "b/analysis/evaluation optimization pipeline/\345\256\236\346\226\275\351\230\266\346\256\265\350\267\257\347\272\277\345\233\276.md" @@ -14,7 +14,7 @@ | 1. 输入准备与 Prompt 工作区 | 已完成 | 校验配置和输入，创建可复现、隔离的 Prompt 工作副本 | | 2. 确定性离线评测闭环 | 已完成 | 使用 fake agent/provider 完成 baseline 和 candidate 的四次完整评测 | | 3a. 评测标准化、失败归因与 Case Diff | 已完成 | 统一评测数据，解释失败并比较候选变化 | -| 3b. 独立 Gate | 待完成 | 根据 diff、关键 case 和预算规则给出接受或拒绝决策 | +| 3b. 独立 Gate | 已完成 | 根据 diff、关键 case 和预算规则给出接受或拒绝决策 | | 4. 真实优化器与安全写回 | 待完成 | 接入 AgentOptimizer，审计候选，并在 Gate 接受后安全更新源 Prompt | | 5. 报告、产物与资源观测 | 待完成 | 输出结构化报告、可读报告和完整审计产物 | | 6. 离线模式与端到端验收 | 待完成 | 补齐 fake judge/trace 场景、示例输出、文档和完整验收测试 | @@ -59,7 +59,7 @@ ## 3b. 独立 Gate -**状态：`[ ]` 待完成** +**状态：`[x]` 已完成** - 实现与优化器解耦的 Gate，消费阶段 3a 生成的评测与 diff 数据。 - 执行验证集最小提升、通过率不得下降、hard/critical case、severe regression 和必需 metric 规则。 diff --git a/examples/optimization/eval_optimize_loop/README.md b/examples/optimization/eval_optimize_loop/README.md index 61980376..10d9b59c 100644 --- a/examples/optimization/eval_optimize_loop/README.md +++ b/examples/optimization/eval_optimize_loop/README.md @@ -1,32 +1,35 @@ -# Evaluation + Optimization Loop — Stage 3a +# Evaluation + Optimization Loop — Stage 3b This example now provides the deterministic offline foundation for an auditable evaluation and prompt-optimization loop. Stage 1 validates inputs and creates an isolated prompt workspace. Stage 2 evaluates that baseline on train and validation data, generates a fake prompt candidate, and evaluates the candidate on both datasets again. Stage 3a normalizes the four SDK result sets, attributes -failures from available evidence, and builds train/validation case diffs. +failures from available evidence, and builds train/validation case diffs. Stage +3b applies an independent Gate to those diffs and returns a complete acceptance +decision without writing the source prompt. No model, API key, judge, or optimizer is used. The fake agent reads explicit capability rules from the current working prompt, while the fake candidate provider can generate an improving, behaviorally equivalent, or overfit candidate. The returned result retains both the raw SDK outputs and a serializable analysis with metric deltas, hard/critical labels, severe -regressions, and overfit status. Gate, reporting, and source writeback remain -later-stage work. +regressions, and overfit status. The Gate checks validation improvement and pass +rate, protected cases, required metrics, overfitting, and configured budgets. +Reporting and source writeback remain later-stage work. Run the deterministic stage from the repository root: ```bash python examples/optimization/eval_optimize_loop/run_pipeline.py \ - --run-id local_stage3a \ + --run-id local_stage3b \ --scenario improve ``` The command creates: ```text -runs/local_stage3a/ +runs/local_stage3b/ └── workspace/ └── prompts/ └── 01_system_prompt.md @@ -46,11 +49,19 @@ Attribution scans evidence from every run and invocation, retains all matching evidence, and chooses one primary category using the fixed priority in `attribution.py`; the other distinct matches become secondary categories. -Run the stage-one through stage-3a tests with: +Gate always evaluates every rule so a rejection retains all reasons. In fake +mode elapsed duration is measured, while monetary cost and token usage are +explicitly `unavailable` rather than recorded as zero. Configured unavailable +budgets follow `budget.on_unavailable`; budgets without a limit are skipped. +The built-in scenarios produce ACCEPT for `improve` and REJECT for both +`no_improvement` and `overfit`. + +Run the stage-one through stage-3b tests with: ```bash pytest -q \ tests/evaluation/test_eval_optimize_loop_stage1.py \ tests/evaluation/test_eval_optimize_loop_stage2.py \ - tests/evaluation/test_eval_optimize_loop_stage3a.py + tests/evaluation/test_eval_optimize_loop_stage3a.py \ + tests/evaluation/test_eval_optimize_loop_stage3b.py ``` diff --git a/examples/optimization/eval_optimize_loop/gate.py b/examples/optimization/eval_optimize_loop/gate.py new file mode 100644 index 00000000..1e4f7b49 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/gate.py @@ -0,0 +1,382 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under the Apache License, Version 2.0. +"""Deterministic acceptance Gate for stage 3b.""" + +from __future__ import annotations + +from collections.abc import Sequence +from typing import Literal + +from .config import BudgetConfig +from .config import GateConfig +from .schemas import CaseDiff +from .schemas import CaseEvaluation +from .schemas import EvaluationAnalysis +from .schemas import GateDecision +from .schemas import GateRuleId +from .schemas import GateRuleResult +from .schemas import ObservableValue +from .schemas import ResourceMeasurements + + +class GateEvaluationError(ValueError): + """Stage 3a analysis is structurally unsafe for Gate evaluation.""" + + +def _case_ids( + cases: Sequence[CaseEvaluation | CaseDiff], + *, + context: str, +) -> set[str]: + ids = [case.eval_id for case in cases] + if len(ids) != len(set(ids)): + raise GateEvaluationError(f"{context} contains duplicate case ids") + return set(ids) + + +def _validate_analysis(analysis: EvaluationAnalysis) -> None: + if analysis.train_diff.split != "train": + raise GateEvaluationError("train_diff.split must be 'train'") + if analysis.validation_diff.split != "validation": + raise GateEvaluationError("validation_diff.split must be 'validation'") + + evaluations = ( + ("baseline_train", analysis.baseline_train, "baseline", "train"), + ("baseline_validation", analysis.baseline_validation, "baseline", "validation"), + ("candidate_train", analysis.candidate_train, "candidate", "train"), + ("candidate_validation", analysis.candidate_validation, "candidate", "validation"), + ) + for label, evaluation, expected_phase, expected_split in evaluations: + if evaluation.phase != expected_phase or evaluation.split != expected_split: + raise GateEvaluationError(f"{label} has an unexpected phase or split") + _case_ids(evaluation.cases, context=label) + for case in evaluation.cases: + metric_names = [metric.metric_name for metric in case.metrics] + if len(metric_names) != len(set(metric_names)): + raise GateEvaluationError( + f"{label} case {case.eval_id!r} contains duplicate metric names" + ) + + train_diff_ids = _case_ids(analysis.train_diff.cases, context="train_diff") + validation_diff_ids = _case_ids( + analysis.validation_diff.cases, + context="validation_diff", + ) + candidate_train_ids = {case.eval_id for case in analysis.candidate_train.cases} + candidate_validation_ids = { + case.eval_id for case in analysis.candidate_validation.cases + } + if train_diff_ids != candidate_train_ids: + raise GateEvaluationError("train diff and candidate evaluation case ids do not match") + if validation_diff_ids != candidate_validation_ids: + raise GateEvaluationError( + "validation diff and candidate evaluation case ids do not match" + ) + + +def _evaluation_completeness(analysis: EvaluationAnalysis) -> GateRuleResult: + incomplete_case_ids: set[str] = set() + incomplete_metric_names: set[str] = set() + evaluations = ( + analysis.baseline_train, + analysis.baseline_validation, + analysis.candidate_train, + analysis.candidate_validation, + ) + complete = True + for evaluation in evaluations: + if not evaluation.cases or evaluation.average_score.status != "available": + complete = False + for case in evaluation.cases: + if ( + case.status == "not_evaluated" + or case.average_score.status != "available" + or not case.metrics + ): + complete = False + incomplete_case_ids.add(case.eval_id) + for metric in case.metrics: + if metric.status == "not_evaluated" or metric.score.status != "available": + complete = False + incomplete_case_ids.add(case.eval_id) + incomplete_metric_names.add(metric.metric_name) + return GateRuleResult( + rule_id="evaluation_completeness", + outcome="pass" if complete else "reject", + message=( + "All four evaluations contain complete case and metric results." + if complete + else "One or more evaluation cases or metrics are incomplete." + ), + case_ids=sorted(incomplete_case_ids), + metric_names=sorted(incomplete_metric_names), + ) + + +def _minimum_validation_score_delta( + analysis: EvaluationAnalysis, + config: GateConfig, +) -> GateRuleResult: + delta = analysis.validation_diff.score_delta + passed = ( + delta.status == "available" + and float(delta.value) >= config.min_validation_score_delta + ) + return GateRuleResult( + rule_id="minimum_validation_score_delta", + outcome="pass" if passed else "reject", + message=( + "Validation score improvement meets the configured minimum." + if passed + else "Validation score improvement is unavailable or below the configured minimum." + ), + observed={"validation_score_delta": delta}, + threshold=config.min_validation_score_delta, + ) + + +def _validation_pass_rate(analysis: EvaluationAnalysis, config: GateConfig) -> GateRuleResult: + if not config.reject_on_validation_pass_rate_drop: + return GateRuleResult( + rule_id="validation_pass_rate_non_decrease", + outcome="skipped", + message="Validation pass-rate protection is disabled.", + ) + baseline_total = len(analysis.baseline_validation.cases) + candidate_total = len(analysis.candidate_validation.cases) + if baseline_total == 0 or candidate_total == 0: + return GateRuleResult( + rule_id="validation_pass_rate_non_decrease", + outcome="reject", + message="Validation pass rate is unavailable because an evaluation has no cases.", + ) + baseline_rate = analysis.baseline_validation.passed_case_count / baseline_total + candidate_rate = analysis.candidate_validation.passed_case_count / candidate_total + passed = candidate_rate >= baseline_rate + return GateRuleResult( + rule_id="validation_pass_rate_non_decrease", + outcome="pass" if passed else "reject", + message=( + "Validation pass rate did not decrease." + if passed + else "Validation pass rate decreased from baseline." + ), + observed={ + "baseline_validation_pass_rate": ObservableValue( + status="available", value=baseline_rate, unit="ratio" + ), + "candidate_validation_pass_rate": ObservableValue( + status="available", value=candidate_rate, unit="ratio" + ), + }, + ) + + +def _all_case_diffs(analysis: EvaluationAnalysis) -> list[CaseDiff]: + return sorted( + [*analysis.train_diff.cases, *analysis.validation_diff.cases], + key=lambda case: (case.split, case.eval_id), + ) + + +def _new_hard_failures(analysis: EvaluationAnalysis, config: GateConfig) -> GateRuleResult: + if not config.reject_new_hard_fail: + return GateRuleResult( + rule_id="no_new_hard_fail", + outcome="skipped", + message="New hard-failure protection is disabled.", + ) + case_ids = sorted( + case.eval_id + for case in _all_case_diffs(analysis) + if case.is_hard and case.change == "newly_failed" + ) + return GateRuleResult( + rule_id="no_new_hard_fail", + outcome="reject" if case_ids else "pass", + message=( + "New hard failures were found." + if case_ids + else "No new hard failures were found." + ), + case_ids=case_ids, + ) + + +def _critical_regressions(analysis: EvaluationAnalysis, config: GateConfig) -> GateRuleResult: + if not config.reject_critical_regression: + return GateRuleResult( + rule_id="no_critical_regression", + outcome="skipped", + message="Critical-case regression protection is disabled.", + ) + case_ids = sorted( + case.eval_id + for case in _all_case_diffs(analysis) + if case.is_critical and case.change in {"newly_failed", "regressed"} + ) + return GateRuleResult( + rule_id="no_critical_regression", + outcome="reject" if case_ids else "pass", + message=( + "Critical-case regressions were found." + if case_ids + else "No critical-case regressions were found." + ), + case_ids=case_ids, + ) + + +def _severe_regressions(analysis: EvaluationAnalysis) -> GateRuleResult: + case_ids = sorted( + case.eval_id for case in _all_case_diffs(analysis) if case.severe_regression + ) + return GateRuleResult( + rule_id="no_severe_regression", + outcome="reject" if case_ids else "pass", + message=( + "Severe case regressions were found." + if case_ids + else "No severe case regressions were found." + ), + case_ids=case_ids, + ) + + +def _required_metrics(analysis: EvaluationAnalysis, config: GateConfig) -> GateRuleResult: + failed_case_ids: set[str] = set() + failed_metric_names: set[str] = set() + for evaluation in (analysis.candidate_train, analysis.candidate_validation): + for case in evaluation.cases: + metric_map = {metric.metric_name: metric for metric in case.metrics} + if config.required_metrics == "all": + required_names = sorted(metric_map) + if not required_names: + failed_case_ids.add(case.eval_id) + continue + else: + required_names = sorted(config.required_metrics) + for name in required_names: + metric = metric_map.get(name) + if ( + metric is None + or metric.status != "passed" + or metric.score.status != "available" + ): + failed_case_ids.add(case.eval_id) + failed_metric_names.add(name) + return GateRuleResult( + rule_id="required_metrics", + outcome="reject" if failed_case_ids else "pass", + message=( + "Required metrics are missing, unavailable, or below threshold." + if failed_case_ids + else "All required candidate metrics are available and passed." + ), + case_ids=sorted(failed_case_ids), + metric_names=sorted(failed_metric_names), + ) + + +def _overfitting(analysis: EvaluationAnalysis) -> GateRuleResult: + passed = analysis.overfit_status == "not_detected" + return GateRuleResult( + rule_id="no_overfitting", + outcome="pass" if passed else "reject", + message=( + "No train-improvement/validation-regression pattern was detected." + if passed + else f"Overfit status is {analysis.overfit_status!r}: {analysis.overfit_reason}" + ), + ) + + +def _budget_result( + rule_id: GateRuleId, + measurement_name: str, + measurement: ObservableValue, + limit: float | int | None, + on_unavailable: Literal["reject", "warning"], +) -> GateRuleResult: + if limit is None: + return GateRuleResult( + rule_id=rule_id, + outcome="skipped", + message=f"{measurement_name} budget is not configured.", + ) + if measurement.status != "available": + return GateRuleResult( + rule_id=rule_id, + outcome=on_unavailable, + message=( + f"{measurement_name} is unavailable; policy is {on_unavailable}." + ), + observed={measurement_name: measurement}, + threshold=float(limit), + ) + passed = float(measurement.value) <= float(limit) + return GateRuleResult( + rule_id=rule_id, + outcome="pass" if passed else "reject", + message=( + f"{measurement_name} is within the configured budget." + if passed + else f"{measurement_name} exceeds the configured budget." + ), + observed={measurement_name: measurement}, + threshold=float(limit), + ) + + +def evaluate_gate( + analysis: EvaluationAnalysis, + gate_config: GateConfig, + budget_config: BudgetConfig, + measurements: ResourceMeasurements, +) -> GateDecision: + """Evaluate every configured rule and return one complete decision.""" + _validate_analysis(analysis) + quality_results = [ + _evaluation_completeness(analysis), + _minimum_validation_score_delta(analysis, gate_config), + _validation_pass_rate(analysis, gate_config), + _new_hard_failures(analysis, gate_config), + _critical_regressions(analysis, gate_config), + _severe_regressions(analysis), + _required_metrics(analysis, gate_config), + _overfitting(analysis), + ] + results = quality_results + [ + _budget_result( + "cost_budget", + "cost_usd", + measurements.cost_usd, + budget_config.max_cost_usd, + budget_config.on_unavailable, + ), + _budget_result( + "token_budget", + "total_tokens", + measurements.total_tokens, + budget_config.max_tokens, + budget_config.on_unavailable, + ), + _budget_result( + "duration_budget", + "duration_seconds", + measurements.duration_seconds, + budget_config.max_duration_seconds, + budget_config.on_unavailable, + ), + ] + rejection_reasons = [result.message for result in results if result.outcome == "reject"] + warnings = [result.message for result in results if result.outcome == "warning"] + return GateDecision( + decision="reject" if rejection_reasons else "accept", + rule_results=results, + rejection_reasons=rejection_reasons, + warnings=warnings, + ) diff --git a/examples/optimization/eval_optimize_loop/pipeline.py b/examples/optimization/eval_optimize_loop/pipeline.py index 2de7f040..88529de5 100644 --- a/examples/optimization/eval_optimize_loop/pipeline.py +++ b/examples/optimization/eval_optimize_loop/pipeline.py @@ -14,6 +14,7 @@ from datetime import timezone from hashlib import sha256 from pathlib import Path +from time import perf_counter from typing import Literal from uuid import uuid4 @@ -31,6 +32,8 @@ from .evaluation_adapter import EvaluationAnalysisError from .fake import DeterministicFakeAgent from .fake import DeterministicFakeCandidateProvider +from .gate import evaluate_gate +from .gate import GateEvaluationError from .prompt_workspace import PromptWorkspaceError from .prompt_workspace import resolve_inside_example_root from .prompt_workspace import stage_prompt_workspace @@ -39,6 +42,8 @@ from .schemas import FakeCandidateScenario from .schemas import FakeEvaluationSnapshot from .schemas import FakeStageResult +from .schemas import ObservableValue +from .schemas import ResourceMeasurements from .schemas import WorkspaceSnapshot @@ -351,6 +356,7 @@ async def run_fake_stage( "the example uses deterministic final-response matching" ) + started_at = perf_counter() selected_scenario = scenario or prepared.config.execution.fake_candidate_scenario train_evalset = _reload_prepared_evalset( Path(prepared.input_snapshot.train_evalset_path), @@ -432,6 +438,32 @@ async def run_fake_stage( ) except EvaluationAnalysisError as exc: raise FakeStageExecutionError(f"stage 3a analysis failed: {exc}") from exc + measurements = ResourceMeasurements( + cost_usd=ObservableValue( + status="unavailable", + unit="USD", + reason="Fake mode does not report monetary cost.", + ), + total_tokens=ObservableValue( + status="unavailable", + unit="tokens", + reason="Fake mode does not report token usage.", + ), + duration_seconds=ObservableValue( + status="available", + value=perf_counter() - started_at, + unit="seconds", + ), + ) + try: + gate_decision = evaluate_gate( + analysis, + prepared.config.gate, + prepared.config.budget, + measurements, + ) + except GateEvaluationError as exc: + raise FakeStageExecutionError(f"stage 3b gate failed: {exc}") from exc return FakeStageResult( scenario=selected_scenario, candidate=candidate, @@ -440,4 +472,6 @@ async def run_fake_stage( candidate_train=candidate_train, candidate_validation=candidate_validation, analysis=analysis, + measurements=measurements, + gate_decision=gate_decision, ) diff --git a/examples/optimization/eval_optimize_loop/run_pipeline.py b/examples/optimization/eval_optimize_loop/run_pipeline.py index 473c2e7b..5ce4d808 100644 --- a/examples/optimization/eval_optimize_loop/run_pipeline.py +++ b/examples/optimization/eval_optimize_loop/run_pipeline.py @@ -3,7 +3,7 @@ # Copyright (C) 2026 Tencent. All rights reserved. # # tRPC-Agent-Python is licensed under the Apache License, Version 2.0. -"""Run the deterministic offline evaluation/candidate stage.""" +"""Run the deterministic offline evaluation, candidate, analysis, and Gate stage.""" from __future__ import annotations @@ -34,7 +34,9 @@ def _format_snapshot(label: str, snapshot: object) -> str: def main() -> None: - parser = argparse.ArgumentParser(description="Run the deterministic offline eval/optimization stage.") + parser = argparse.ArgumentParser( + description="Run the deterministic offline evaluation and Gate stage." + ) parser.add_argument("--config", type=Path, default=_HERE / "pipeline.json") parser.add_argument("--run-id", help="Optional reproducible run identifier.") parser.add_argument( @@ -52,7 +54,19 @@ def main() -> None: print(_format_snapshot("Baseline validation", result.baseline_validation)) print(_format_snapshot("Candidate train", result.candidate_train)) print(_format_snapshot("Candidate validation", result.candidate_validation)) - print("Stage 2 does not run Gate, write reports, or update source prompts.") + print(f"Gate decision: {result.gate_decision.decision.upper()}") + rejected_rules = [ + rule for rule in result.gate_decision.rule_results if rule.outcome == "reject" + ] + if rejected_rules: + print("Rejection reasons:") + for rule in rejected_rules: + print(f"- [{rule.rule_id}] {rule.message}") + if result.gate_decision.warnings: + print("Warnings:") + for warning in result.gate_decision.warnings: + print(f"- {warning}") + print("Stage 3b does not write reports or update source prompts.") if __name__ == "__main__": diff --git a/examples/optimization/eval_optimize_loop/schemas.py b/examples/optimization/eval_optimize_loop/schemas.py index bc559418..0cd29529 100644 --- a/examples/optimization/eval_optimize_loop/schemas.py +++ b/examples/optimization/eval_optimize_loop/schemas.py @@ -45,6 +45,21 @@ "incomparable", ] OverfitStatus = Literal["detected", "not_detected", "unavailable"] +GateRuleId = Literal[ + "evaluation_completeness", + "minimum_validation_score_delta", + "validation_pass_rate_non_decrease", + "no_new_hard_fail", + "no_critical_regression", + "no_severe_regression", + "required_metrics", + "no_overfitting", + "cost_budget", + "token_budget", + "duration_budget", +] +GateRuleOutcome = Literal["pass", "reject", "warning", "skipped"] +GateDecisionValue = Literal["accept", "reject"] class ObservableValue(EvalBaseModel): @@ -284,8 +299,37 @@ class EvaluationAnalysis(EvalBaseModel): overfit_reason: str +class ResourceMeasurements(EvalBaseModel): + """Resource observations available when Gate evaluates a candidate.""" + + cost_usd: ObservableValue + total_tokens: ObservableValue + duration_seconds: ObservableValue + + +class GateRuleResult(EvalBaseModel): + """One deterministic policy result with evidence for later reporting.""" + + rule_id: GateRuleId + outcome: GateRuleOutcome + message: str + case_ids: list[str] = Field(default_factory=list) + metric_names: list[str] = Field(default_factory=list) + observed: dict[str, ObservableValue] = Field(default_factory=dict) + threshold: Optional[float] = None + + +class GateDecision(EvalBaseModel): + """The complete, auditable acceptance decision produced by Gate.""" + + decision: GateDecisionValue + rule_results: list[GateRuleResult] + rejection_reasons: list[str] = Field(default_factory=list) + warnings: list[str] = Field(default_factory=list) + + class FakeStageResult(EvalBaseModel): - """The four full evaluations and candidate metadata produced in stage two.""" + """Evaluation, candidate, analysis, and Gate outputs from the offline stage.""" scenario: FakeCandidateScenario candidate: FakeCandidateProposal @@ -294,3 +338,5 @@ class FakeStageResult(EvalBaseModel): candidate_train: FakeEvaluationSnapshot candidate_validation: FakeEvaluationSnapshot analysis: EvaluationAnalysis + measurements: ResourceMeasurements + gate_decision: GateDecision diff --git a/tests/evaluation/test_eval_optimize_loop_stage2.py b/tests/evaluation/test_eval_optimize_loop_stage2.py index a34b92da..96d7719b 100644 --- a/tests/evaluation/test_eval_optimize_loop_stage2.py +++ b/tests/evaluation/test_eval_optimize_loop_stage2.py @@ -330,4 +330,4 @@ def test_stage_two_cli_smoke(tmp_path: Path): assert "Baseline validation: 1/3 passed" in completed.stdout assert "Candidate train: 3/3 passed" in completed.stdout assert "Candidate validation: 0/3 passed" in completed.stdout - assert "does not run Gate" in completed.stdout + assert "Gate decision: REJECT" in completed.stdout diff --git a/tests/evaluation/test_eval_optimize_loop_stage3b.py b/tests/evaluation/test_eval_optimize_loop_stage3b.py new file mode 100644 index 00000000..64901f3f --- /dev/null +++ b/tests/evaluation/test_eval_optimize_loop_stage3b.py @@ -0,0 +1,624 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under the Apache License, Version 2.0. +"""Tests for stage 3b deterministic Gate decisions.""" + +from __future__ import annotations + +import shutil +import subprocess +import sys +from pathlib import Path +from statistics import mean + +import pytest + +from examples.optimization.eval_optimize_loop import pipeline as pipeline_module +from examples.optimization.eval_optimize_loop.case_diff import compare_evaluations +from examples.optimization.eval_optimize_loop.config import BudgetConfig +from examples.optimization.eval_optimize_loop.config import GateConfig +from examples.optimization.eval_optimize_loop.pipeline import prepare_run +from examples.optimization.eval_optimize_loop.pipeline import FakeStageExecutionError +from examples.optimization.eval_optimize_loop.pipeline import run_fake_stage +from examples.optimization.eval_optimize_loop.schemas import CaseEvaluation +from examples.optimization.eval_optimize_loop.schemas import EvaluationAnalysis +from examples.optimization.eval_optimize_loop.schemas import FakeStageResult +from examples.optimization.eval_optimize_loop.schemas import MetricOutcome +from examples.optimization.eval_optimize_loop.schemas import ObservableValue +from examples.optimization.eval_optimize_loop.schemas import ResourceMeasurements +from examples.optimization.eval_optimize_loop.schemas import StandardizedEvaluation + + +_REPO_ROOT = Path(__file__).resolve().parents[2] +_EXAMPLE = _REPO_ROOT / "examples" / "optimization" / "eval_optimize_loop" + + +def _copy_example(tmp_path: Path, name: str) -> Path: + target = tmp_path / name + shutil.copytree(_EXAMPLE, target, ignore=shutil.ignore_patterns("runs", "__pycache__")) + return target + + +def _available(value: float, unit: str | None = None) -> ObservableValue: + return ObservableValue(status="available", value=value, unit=unit) + + +def _case( + eval_id: str, + *, + status: str, + score: float, + metric_name: str = "final_response_avg_score", +) -> CaseEvaluation: + return CaseEvaluation( + eval_id=eval_id, + status=status, + average_score=_available(score), + metrics=[ + MetricOutcome( + metric_name=metric_name, + threshold=0.5, + status=status, + score=_available(score), + ) + ], + runs=[], + ) + + +def _evaluation( + phase: str, + split: str, + cases: list[CaseEvaluation], +) -> StandardizedEvaluation: + return StandardizedEvaluation( + phase=phase, + split=split, + eval_set_id=f"{split}-set", + cases=cases, + passed_case_count=sum(case.status == "passed" for case in cases), + failed_case_count=sum(case.status == "failed" for case in cases), + not_evaluated_case_count=sum(case.status == "not_evaluated" for case in cases), + average_score=_available(mean(float(case.average_score.value) for case in cases)), + ) + + +def _analysis( + *, + baseline_train: list[CaseEvaluation] | None = None, + candidate_train: list[CaseEvaluation] | None = None, + baseline_validation: list[CaseEvaluation] | None = None, + candidate_validation: list[CaseEvaluation] | None = None, + hard_case_ids: set[str] | None = None, + critical_case_ids: set[str] | None = None, + overfit_status: str = "not_detected", +) -> EvaluationAnalysis: + normalized_baseline_train = _evaluation( + "baseline", + "train", + baseline_train or [_case("train_case", status="failed", score=0.0)], + ) + normalized_candidate_train = _evaluation( + "candidate", + "train", + candidate_train or [_case("train_case", status="passed", score=1.0)], + ) + normalized_baseline_validation = _evaluation( + "baseline", + "validation", + baseline_validation or [_case("validation_case", status="failed", score=0.0)], + ) + normalized_candidate_validation = _evaluation( + "candidate", + "validation", + candidate_validation or [_case("validation_case", status="passed", score=1.0)], + ) + train_diff = compare_evaluations( + normalized_baseline_train, + normalized_candidate_train, + hard_case_ids=hard_case_ids or set(), + critical_case_ids=critical_case_ids or set(), + severe_case_score_drop=0.2, + ) + validation_diff = compare_evaluations( + normalized_baseline_validation, + normalized_candidate_validation, + hard_case_ids=hard_case_ids or set(), + critical_case_ids=critical_case_ids or set(), + severe_case_score_drop=0.2, + ) + return EvaluationAnalysis( + baseline_train=normalized_baseline_train, + baseline_validation=normalized_baseline_validation, + candidate_train=normalized_candidate_train, + candidate_validation=normalized_candidate_validation, + train_diff=train_diff, + validation_diff=validation_diff, + overfit_status=overfit_status, + overfit_reason="test overfit status", + ) + + +def _measurements() -> ResourceMeasurements: + return ResourceMeasurements( + cost_usd=ObservableValue(status="unavailable", reason="not observed"), + total_tokens=ObservableValue(status="unavailable", reason="not observed"), + duration_seconds=_available(1.0, "seconds"), + ) + + +def _rule(decision, rule_id: str): + return next(result for result in decision.rule_results if result.rule_id == rule_id) + + +def test_gate_models_are_serializable_and_auditable(): + from examples.optimization.eval_optimize_loop.schemas import GateDecision + from examples.optimization.eval_optimize_loop.schemas import GateRuleResult + from examples.optimization.eval_optimize_loop.schemas import ObservableValue + from examples.optimization.eval_optimize_loop.schemas import ResourceMeasurements + + measurements = ResourceMeasurements( + cost_usd=ObservableValue(status="unavailable", reason="fake mode"), + total_tokens=ObservableValue(status="available", value=12, unit="tokens"), + duration_seconds=ObservableValue(status="available", value=0.5, unit="seconds"), + ) + decision = GateDecision( + decision="reject", + rule_results=[ + GateRuleResult( + rule_id="minimum_validation_score_delta", + outcome="reject", + message="Validation score improvement is below the configured minimum.", + observed={"validation_score_delta": ObservableValue(status="available", value=0)}, + threshold=0.05, + ) + ], + rejection_reasons=["Validation score improvement is below the configured minimum."], + warnings=[], + ) + + restored_measurements = ResourceMeasurements.model_validate_json( + measurements.model_dump_json() + ) + restored_decision = GateDecision.model_validate_json(decision.model_dump_json()) + + assert restored_measurements == measurements + assert restored_decision == decision + assert restored_decision.rule_results[0].case_ids == [] + assert restored_decision.rule_results[0].metric_names == [] + + +def test_gate_accepts_candidate_and_returns_every_rule_in_stable_order(): + from examples.optimization.eval_optimize_loop.gate import evaluate_gate + + decision = evaluate_gate( + _analysis(), + GateConfig(), + BudgetConfig(max_duration_seconds=10.0, on_unavailable="warning"), + _measurements(), + ) + + assert decision.decision == "accept" + assert decision.rejection_reasons == [] + assert decision.warnings == [] + assert [result.rule_id for result in decision.rule_results] == [ + "evaluation_completeness", + "minimum_validation_score_delta", + "validation_pass_rate_non_decrease", + "no_new_hard_fail", + "no_critical_regression", + "no_severe_regression", + "required_metrics", + "no_overfitting", + "cost_budget", + "token_budget", + "duration_budget", + ] + assert [result.outcome for result in decision.rule_results[-3:]] == [ + "skipped", + "skipped", + "pass", + ] + + +def test_gate_rejects_incomplete_evaluation_and_unavailable_validation_delta(): + from examples.optimization.eval_optimize_loop.gate import evaluate_gate + + analysis = _analysis() + unavailable = ObservableValue(status="unavailable", reason="metric unavailable") + candidate_case = analysis.candidate_validation.cases[0] + incomplete_case = candidate_case.model_copy( + update={ + "status": "not_evaluated", + "average_score": unavailable, + "metrics": [ + candidate_case.metrics[0].model_copy( + update={"status": "not_evaluated", "score": unavailable} + ) + ], + } + ) + incomplete_candidate = analysis.candidate_validation.model_copy( + update={ + "cases": [incomplete_case], + "passed_case_count": 0, + "not_evaluated_case_count": 1, + "average_score": unavailable, + } + ) + incomplete_diff = analysis.validation_diff.model_copy( + update={"candidate_average_score": unavailable, "score_delta": unavailable} + ) + incomplete_analysis = analysis.model_copy( + update={ + "candidate_validation": incomplete_candidate, + "validation_diff": incomplete_diff, + } + ) + + decision = evaluate_gate( + incomplete_analysis, + GateConfig(), + BudgetConfig(), + _measurements(), + ) + + assert decision.decision == "reject" + assert _rule(decision, "evaluation_completeness").outcome == "reject" + assert _rule(decision, "minimum_validation_score_delta").outcome == "reject" + assert len(decision.rejection_reasons) >= 2 + + +def test_minimum_validation_delta_accepts_exact_threshold(): + from examples.optimization.eval_optimize_loop.gate import evaluate_gate + + analysis = _analysis( + baseline_validation=[_case("validation_case", status="failed", score=0.0)], + candidate_validation=[_case("validation_case", status="failed", score=0.05)], + ) + + decision = evaluate_gate( + analysis, + GateConfig(min_validation_score_delta=0.05), + BudgetConfig(), + _measurements(), + ) + + assert _rule(decision, "minimum_validation_score_delta").outcome == "pass" + + +def test_validation_pass_rate_drop_rejects_unless_rule_is_disabled(): + from examples.optimization.eval_optimize_loop.gate import evaluate_gate + + analysis = _analysis( + baseline_validation=[ + _case("validation_a", status="passed", score=1.0), + _case("validation_b", status="failed", score=0.0), + ], + candidate_validation=[ + _case("validation_a", status="failed", score=0.0), + _case("validation_b", status="failed", score=0.0), + ], + ) + + rejected = evaluate_gate(analysis, GateConfig(), BudgetConfig(), _measurements()) + disabled = evaluate_gate( + analysis, + GateConfig(reject_on_validation_pass_rate_drop=False), + BudgetConfig(), + _measurements(), + ) + + result = _rule(rejected, "validation_pass_rate_non_decrease") + assert result.outcome == "reject" + assert result.observed["baseline_validation_pass_rate"].value == 0.5 + assert result.observed["candidate_validation_pass_rate"].value == 0.0 + assert _rule(disabled, "validation_pass_rate_non_decrease").outcome == "skipped" + + +def test_gate_collects_hard_critical_and_severe_train_regressions(): + from examples.optimization.eval_optimize_loop.gate import evaluate_gate + + analysis = _analysis( + baseline_train=[ + _case("hard_case", status="passed", score=1.0), + _case("critical_case", status="passed", score=1.0), + _case("severe_case", status="passed", score=1.0), + ], + candidate_train=[ + _case("hard_case", status="failed", score=0.0), + _case("critical_case", status="passed", score=0.9), + _case("severe_case", status="passed", score=0.7), + ], + hard_case_ids={"hard_case"}, + critical_case_ids={"critical_case"}, + ) + + decision = evaluate_gate(analysis, GateConfig(), BudgetConfig(), _measurements()) + + assert _rule(decision, "no_new_hard_fail").outcome == "reject" + assert _rule(decision, "no_new_hard_fail").case_ids == ["hard_case"] + assert _rule(decision, "no_critical_regression").outcome == "reject" + assert _rule(decision, "no_critical_regression").case_ids == ["critical_case"] + assert _rule(decision, "no_severe_regression").outcome == "reject" + assert _rule(decision, "no_severe_regression").case_ids == ["hard_case", "severe_case"] + + disabled = evaluate_gate( + analysis, + GateConfig(reject_new_hard_fail=False, reject_critical_regression=False), + BudgetConfig(), + _measurements(), + ) + assert _rule(disabled, "no_new_hard_fail").outcome == "skipped" + assert _rule(disabled, "no_critical_regression").outcome == "skipped" + assert _rule(disabled, "no_severe_regression").outcome == "reject" + + +@pytest.mark.parametrize("required_metrics", [["required_metric"], "all"]) +def test_required_metrics_reject_missing_or_empty_candidate_metrics(required_metrics): + from examples.optimization.eval_optimize_loop.gate import evaluate_gate + + analysis = _analysis() + if required_metrics == "all": + empty_case = analysis.candidate_train.cases[0].model_copy(update={"metrics": []}) + analysis = analysis.model_copy( + update={ + "candidate_train": analysis.candidate_train.model_copy( + update={"cases": [empty_case]} + ) + } + ) + + decision = evaluate_gate( + analysis, + GateConfig(required_metrics=required_metrics), + BudgetConfig(), + _measurements(), + ) + + result = _rule(decision, "required_metrics") + assert result.outcome == "reject" + if required_metrics == ["required_metric"]: + assert result.metric_names == ["required_metric"] + assert result.case_ids == ["train_case", "validation_case"] + else: + assert result.case_ids == ["train_case"] + + +def test_required_metrics_reject_failed_and_unavailable_candidate_metrics(): + from examples.optimization.eval_optimize_loop.gate import evaluate_gate + + failed_case = _case("train_case", status="failed", score=0.0) + unavailable = ObservableValue(status="unavailable", reason="judge failed") + unavailable_case = _case("validation_case", status="passed", score=1.0) + unavailable_case = unavailable_case.model_copy( + update={ + "metrics": [ + unavailable_case.metrics[0].model_copy( + update={"status": "not_evaluated", "score": unavailable} + ) + ] + } + ) + analysis = _analysis( + candidate_train=[failed_case], + candidate_validation=[unavailable_case], + ) + + decision = evaluate_gate(analysis, GateConfig(), BudgetConfig(), _measurements()) + + result = _rule(decision, "required_metrics") + assert result.outcome == "reject" + assert result.case_ids == ["train_case", "validation_case"] + assert result.metric_names == ["final_response_avg_score"] + + +@pytest.mark.parametrize("overfit_status", ["detected", "unavailable"]) +def test_overfit_detected_or_unavailable_rejects(overfit_status: str): + from examples.optimization.eval_optimize_loop.gate import evaluate_gate + + decision = evaluate_gate( + _analysis(overfit_status=overfit_status), + GateConfig(), + BudgetConfig(), + _measurements(), + ) + + assert _rule(decision, "no_overfitting").outcome == "reject" + + +def test_budget_rules_handle_limits_unavailable_policy_and_disabled_limits(): + from examples.optimization.eval_optimize_loop.gate import evaluate_gate + + measurements = ResourceMeasurements( + cost_usd=_available(2.0, "USD"), + total_tokens=ObservableValue(status="unavailable", reason="usage missing"), + duration_seconds=_available(1.0, "seconds"), + ) + warning_decision = evaluate_gate( + _analysis(), + GateConfig(), + BudgetConfig(max_cost_usd=1.0, max_tokens=100, on_unavailable="warning"), + measurements, + ) + + assert _rule(warning_decision, "cost_budget").outcome == "reject" + assert _rule(warning_decision, "token_budget").outcome == "warning" + assert _rule(warning_decision, "duration_budget").outcome == "skipped" + assert len(warning_decision.warnings) == 1 + + reject_decision = evaluate_gate( + _analysis(), + GateConfig(), + BudgetConfig(max_tokens=100, max_duration_seconds=1.0, on_unavailable="reject"), + measurements, + ) + assert _rule(reject_decision, "cost_budget").outcome == "skipped" + assert _rule(reject_decision, "token_budget").outcome == "reject" + assert _rule(reject_decision, "duration_budget").outcome == "pass" + + +def test_gate_rejects_structurally_inconsistent_analysis(): + from examples.optimization.eval_optimize_loop.gate import GateEvaluationError + from examples.optimization.eval_optimize_loop.gate import evaluate_gate + + analysis = _analysis() + wrong_split = analysis.model_copy( + update={"train_diff": analysis.train_diff.model_copy(update={"split": "validation"})} + ) + with pytest.raises(GateEvaluationError, match="train_diff.split"): + evaluate_gate(wrong_split, GateConfig(), BudgetConfig(), _measurements()) + + missing_diff_case = analysis.model_copy( + update={ + "train_diff": analysis.train_diff.model_copy(update={"cases": []}), + } + ) + with pytest.raises(GateEvaluationError, match="case ids"): + evaluate_gate(missing_diff_case, GateConfig(), BudgetConfig(), _measurements()) + + candidate_case = analysis.candidate_train.cases[0] + duplicate_metric_case = candidate_case.model_copy( + update={"metrics": [candidate_case.metrics[0], candidate_case.metrics[0]]} + ) + duplicate_metric_analysis = analysis.model_copy( + update={ + "candidate_train": analysis.candidate_train.model_copy( + update={"cases": [duplicate_metric_case]} + ) + } + ) + with pytest.raises(GateEvaluationError, match="duplicate metric"): + evaluate_gate( + duplicate_metric_analysis, + GateConfig(), + BudgetConfig(), + _measurements(), + ) + + +@pytest.mark.asyncio +async def test_fake_stage_returns_measurements_and_accepts_improving_candidate(tmp_path: Path): + root = _copy_example(tmp_path, "improve") + source = root / "prompts" / "system.md" + baseline_source = source.read_text(encoding="utf-8") + prepared = prepare_run(root / "pipeline.json", run_id="stage3b_improve") + + result = await run_fake_stage(prepared, scenario="improve") + + assert result.gate_decision.decision == "accept" + assert result.gate_decision.rejection_reasons == [] + assert result.measurements.duration_seconds.status == "available" + assert result.measurements.duration_seconds.value >= 0 + assert result.measurements.cost_usd.status == "unavailable" + assert result.measurements.total_tokens.status == "unavailable" + assert source.read_text(encoding="utf-8") == baseline_source + assert "deterministic-fake-candidate:start" in ( + await prepared.working_target.read_all() + )["system_prompt"] + + +@pytest.mark.parametrize( + ("scenario", "expected_decision", "expected_reject_rules"), + [ + ("improve", "accept", []), + ( + "no_improvement", + "reject", + ["minimum_validation_score_delta", "required_metrics"], + ), + ( + "overfit", + "reject", + [ + "minimum_validation_score_delta", + "validation_pass_rate_non_decrease", + "no_critical_regression", + "no_severe_regression", + "required_metrics", + "no_overfitting", + ], + ), + ], +) +@pytest.mark.asyncio +async def test_stage3b_scenario_gate_matrix( + tmp_path: Path, + scenario: str, + expected_decision: str, + expected_reject_rules: list[str], +): + root = _copy_example(tmp_path, f"matrix_{scenario}") + source = root / "prompts" / "system.md" + baseline_source = source.read_text(encoding="utf-8") + prepared = prepare_run(root / "pipeline.json", run_id=f"stage3b_{scenario}") + + result = await run_fake_stage(prepared, scenario=scenario) # type: ignore[arg-type] + + assert result.gate_decision.decision == expected_decision + assert [ + rule.rule_id + for rule in result.gate_decision.rule_results + if rule.outcome == "reject" + ] == expected_reject_rules + assert source.read_text(encoding="utf-8") == baseline_source + assert FakeStageResult.model_validate_json(result.model_dump_json()) == result + + +@pytest.mark.asyncio +async def test_fake_stage_wraps_gate_structure_errors_only(tmp_path: Path, monkeypatch): + from examples.optimization.eval_optimize_loop.gate import GateEvaluationError + + root = _copy_example(tmp_path, "gate_structure_error") + prepared = prepare_run(root / "pipeline.json", run_id="gate_structure_error") + + def fail_gate(*_args, **_kwargs): + raise GateEvaluationError("injected unsafe analysis") + + monkeypatch.setattr(pipeline_module, "evaluate_gate", fail_gate) + + with pytest.raises(FakeStageExecutionError, match="stage 3b gate failed"): + await run_fake_stage(prepared, scenario="improve") + + +@pytest.mark.asyncio +async def test_fake_stage_does_not_hide_unexpected_gate_value_errors(tmp_path: Path, monkeypatch): + root = _copy_example(tmp_path, "gate_programming_error") + prepared = prepare_run(root / "pipeline.json", run_id="gate_programming_error") + + def fail_gate(*_args, **_kwargs): + raise ValueError("injected programming error") + + monkeypatch.setattr(pipeline_module, "evaluate_gate", fail_gate) + + with pytest.raises(ValueError, match="injected programming error"): + await run_fake_stage(prepared, scenario="improve") + + +def test_stage3b_cli_prints_gate_decision_and_rejection_reasons(tmp_path: Path): + root = _copy_example(tmp_path, "cli_overfit") + + completed = subprocess.run( + [ + sys.executable, + str(_EXAMPLE / "run_pipeline.py"), + "--config", + str(root / "pipeline.json"), + "--run-id", + "stage3b_cli", + "--scenario", + "overfit", + ], + cwd=_REPO_ROOT, + check=True, + capture_output=True, + text=True, + timeout=180, + ) + + assert "Gate decision: REJECT" in completed.stdout + assert "Rejection reasons:" in completed.stdout + assert "[no_overfitting]" in completed.stdout + assert "Stage 3b does not write reports or update source prompts." in completed.stdout From 3942dbbe456214c90f43ba36d543dc08fe34afd2 Mon Sep 17 00:00:00 2001 From: guocfu Date: Sat, 18 Jul 2026 15:31:17 +0800 Subject: [PATCH 05/18] =?UTF-8?q?feat(evaluation):=20=E6=8E=A5=E5=85=A5?= =?UTF-8?q?=E7=9C=9F=E5=AE=9E=E4=BC=98=E5=8C=96=E5=99=A8=E4=B8=8E=E5=AE=89?= =?UTF-8?q?=E5=85=A8=E5=86=99=E5=9B=9E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../optimization/eval_optimize_loop/README.md | 104 ++-- .../eval_optimize_loop/candidate_provider.py | 153 +++++ .../optimization/eval_optimize_loop/config.py | 6 + .../eval_optimize_loop/pipeline.py | 261 +++++++- .../eval_optimize_loop/run_pipeline.py | 17 +- .../eval_optimize_loop/schemas.py | 72 ++- .../eval_optimize_loop/writeback.py | 166 +++++ .../test_eval_optimize_loop_stage3b.py | 3 +- .../test_eval_optimize_loop_stage4.py | 586 ++++++++++++++++++ 9 files changed, 1287 insertions(+), 81 deletions(-) create mode 100644 examples/optimization/eval_optimize_loop/candidate_provider.py create mode 100644 examples/optimization/eval_optimize_loop/writeback.py create mode 100644 tests/evaluation/test_eval_optimize_loop_stage4.py diff --git a/examples/optimization/eval_optimize_loop/README.md b/examples/optimization/eval_optimize_loop/README.md index 10d9b59c..3ec5397a 100644 --- a/examples/optimization/eval_optimize_loop/README.md +++ b/examples/optimization/eval_optimize_loop/README.md @@ -1,67 +1,77 @@ -# Evaluation + Optimization Loop — Stage 3b +# Evaluation + Optimization Loop — Stage 4 -This example now provides the deterministic offline foundation for an auditable -evaluation and prompt-optimization loop. Stage 1 validates inputs and creates -an isolated prompt workspace. Stage 2 evaluates that baseline on train and -validation data, generates a fake prompt candidate, and evaluates the candidate -on both datasets again. Stage 3a normalizes the four SDK result sets, attributes -failures from available evidence, and builds train/validation case diffs. Stage -3b applies an independent Gate to those diffs and returns a complete acceptance -decision without writing the source prompt. +This example provides an auditable evaluation and prompt-optimization loop. +Stages 1–3 prepare an isolated prompt workspace, run baseline and candidate +evaluations on both train and validation datasets, normalize failures, build +case diffs, detect overfitting, and apply an independent Gate. Stage 4 adds a +common Candidate Provider boundary, an `AgentOptimizer` adapter, and guarded +source-prompt writeback. -No model, API key, judge, or optimizer is used. The fake agent reads explicit -capability rules from the current working prompt, while the fake candidate -provider can generate an improving, behaviorally equivalent, or overfit -candidate. The returned result retains both the raw SDK outputs and a -serializable analysis with metric deltas, hard/critical labels, severe -regressions, and overfit status. The Gate checks validation improvement and pass -rate, protected cases, required metrics, overfitting, and configured budgets. -Reporting and source writeback remain later-stage work. - -Run the deterministic stage from the repository root: +The deterministic fake mode still runs without a model, API key, judge, or +optimizer. Its built-in scenarios produce ACCEPT for `improve` and REJECT for +both `no_improvement` and `overfit`: ```bash python examples/optimization/eval_optimize_loop/run_pipeline.py \ - --run-id local_stage3b \ + --run-id local_stage4 \ --scenario improve ``` -The command creates: +The CLI is intentionally self-contained and supports fake mode only. Real mode +is a Python integration point because the pipeline cannot construct an +application-specific agent. Set `execution.mode` to `real`, prepare the run, +and inject an async SDK-compatible `call_agent`: + +```python +import asyncio + +from examples.optimization.eval_optimize_loop.pipeline import prepare_run +from examples.optimization.eval_optimize_loop.pipeline import run_real_stage -```text -runs/local_stage3b/ -└── workspace/ - └── prompts/ - └── 01_system_prompt.md +prepared = prepare_run("examples/optimization/eval_optimize_loop/pipeline.json") +agent = MyBusinessAgent(target_prompt=prepared.working_target) +result = asyncio.run( + run_real_stage(prepared, call_agent=agent.call_agent) +) ``` -All paths in `pipeline.json` are relative to the example directory. Prompt -sources are never written. The candidate remains in the isolated run workspace -after evaluation so it can be inspected. `optimizer.json` remains the native -SDK optimizer configuration, while orchestration settings stay in -`pipeline.json`. +The injected agent must reread `prepared.working_target` on every call. The +same callable is used for baseline regression, optimizer search, and final +candidate regression. Optimizer minibatches never replace the four complete +pipeline evaluations: baseline train/validation and candidate train/validation. + +`AgentOptimizer` always receives the isolated working target and +`update_source=False`. Its native `result.json`, round records, prompt snapshots, +scores, and configuration snapshot are retained under `runs//optimizer/` +when `artifacts.retain_optimizer_native_artifacts` is enabled. + +Source prompts are updated only when all of these conditions hold: + +- Gate returns ACCEPT; +- `writeback.enabled` is true; +- every source Prompt still matches its preparation-time SHA-256 hash; +- the write succeeds and an exact readback matches the accepted candidate. -For cases with multiple runs or invocations, normalization keeps every -invocation's expected/actual response, tool calls, and metric results. Case -status and score are aggregated from the SDK's overall metrics; invocation -metrics are evidence and do not receive extra weight in that aggregate. -Attribution scans evidence from every run and invocation, retains all matching -evidence, and chooses one primary category using the fixed priority in -`attribution.py`; the other distinct matches become secondary categories. +Writeback returns `written`, `skipped`, `blocked`, or `failed` with an auditable +reason. Gate rejection and disabled writeback are skipped; concurrent source +edits are blocked; recoverable write/readback failures are rolled back and +returned as failed. If rollback integrity cannot be proven, the pipeline raises +an error instead of claiming the source is safe. The checked-in configuration +keeps writeback disabled by default. -Gate always evaluates every rule so a rejection retains all reasons. In fake -mode elapsed duration is measured, while monetary cost and token usage are -explicitly `unavailable` rather than recorded as zero. Configured unavailable -budgets follow `budget.on_unavailable`; budgets without a limit are skipped. -The built-in scenarios produce ACCEPT for `improve` and REJECT for both -`no_improvement` and `overfit`. +Elapsed duration is observable. Full monetary cost and token usage remain +`unavailable` because an injected business agent may make calls that the SDK +optimizer does not account for; native optimizer resource fields are retained +without treating missing values as zero. JSON/Markdown reports and an artifact +index are Stage 5 work. -Run the stage-one through stage-3b tests with: +Run the Stage 1–4 tests with: ```bash -pytest -q \ +.venv/bin/pytest -q \ tests/evaluation/test_eval_optimize_loop_stage1.py \ tests/evaluation/test_eval_optimize_loop_stage2.py \ tests/evaluation/test_eval_optimize_loop_stage3a.py \ - tests/evaluation/test_eval_optimize_loop_stage3b.py + tests/evaluation/test_eval_optimize_loop_stage3b.py \ + tests/evaluation/test_eval_optimize_loop_stage4.py ``` diff --git a/examples/optimization/eval_optimize_loop/candidate_provider.py b/examples/optimization/eval_optimize_loop/candidate_provider.py new file mode 100644 index 00000000..bcfe3e1c --- /dev/null +++ b/examples/optimization/eval_optimize_loop/candidate_provider.py @@ -0,0 +1,153 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under the Apache License, Version 2.0. +"""Candidate provider boundary for fake and AgentOptimizer-backed proposals.""" + +from __future__ import annotations + +import json +import shutil +from dataclasses import dataclass +from hashlib import sha256 +from pathlib import Path +from typing import Protocol + +from trpc_agent_sdk.evaluation import AgentOptimizer +from trpc_agent_sdk.evaluation import CallAgent +from trpc_agent_sdk.evaluation import OptimizeResult +from trpc_agent_sdk.evaluation import TargetPrompt + +from .fake.candidate_provider import DeterministicFakeCandidateProvider +from .schemas import CandidateProposal +from .schemas import FakeCandidateScenario +from .schemas import OptimizerCandidateProposal + + +class CandidateProviderError(RuntimeError): + """A provider could not produce a safe, complete candidate.""" + + +def prompt_mapping_sha256(prompts: dict[str, str]) -> str: + """Hash a complete prompt mapping using a stable JSON representation.""" + canonical = json.dumps( + prompts, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ) + return sha256(canonical.encode("utf-8")).hexdigest() + + +@dataclass(frozen=True) +class CandidateRequest: + """Validated inputs handed to one candidate provider.""" + + current_prompts: dict[str, str] + target_prompt: TargetPrompt + optimizer_config_path: Path + train_evalset_path: Path + validation_evalset_path: Path + output_dir: Path + seed: int + retain_native_artifacts: bool = True + + +@dataclass(frozen=True) +class CandidateGeneration: + """A normalized proposal plus an optional native optimizer result.""" + + proposal: CandidateProposal + optimize_result: OptimizeResult | None = None + + +class CandidateProvider(Protocol): + """Asynchronous candidate generation used by the pipeline orchestrator.""" + + async def propose(self, request: CandidateRequest) -> CandidateGeneration: + """Return one complete proposal without updating source prompts.""" + + +class FakeCandidateProviderAdapter: + """Lift the pure synchronous fake provider into the common async boundary.""" + + def __init__(self, scenario: FakeCandidateScenario) -> None: + self._scenario = scenario + + async def propose(self, request: CandidateRequest) -> CandidateGeneration: + proposal = DeterministicFakeCandidateProvider().propose( + request.current_prompts, + scenario=self._scenario, + seed=request.seed, + ) + return CandidateGeneration(proposal=proposal) + + +class AgentOptimizerCandidateProvider: + """Adapt AgentOptimizer to the pipeline's review-before-write contract.""" + + def __init__(self, call_agent: CallAgent) -> None: + self._call_agent = call_agent + + async def propose(self, request: CandidateRequest) -> CandidateGeneration: + try: + result = await AgentOptimizer.optimize( + config_path=str(request.optimizer_config_path), + call_agent=self._call_agent, + target_prompt=request.target_prompt, + train_dataset_path=str(request.train_evalset_path), + validation_dataset_path=str(request.validation_evalset_path), + output_dir=str(request.output_dir), + update_source=False, + verbose=0, + ) + except Exception as exc: + raise CandidateProviderError(f"AgentOptimizer failed: {exc}") from exc + + if result.status != "SUCCEEDED": + raise CandidateProviderError( + f"AgentOptimizer returned {result.status}: {result.error_message or result.finish_reason}" + ) + expected_fields = set(request.current_prompts) + if set(result.baseline_prompts) != expected_fields: + raise CandidateProviderError("optimizer baseline prompt fields do not match the prepared target") + if result.baseline_prompts != request.current_prompts: + raise CandidateProviderError("optimizer baseline prompts do not match the prepared working prompts") + if set(result.best_prompts) != expected_fields: + raise CandidateProviderError("optimizer best prompt fields do not match the prepared target") + if any(not isinstance(value, str) for value in result.best_prompts.values()): + raise CandidateProviderError("optimizer best prompts must contain only strings") + + parent_hash = prompt_mapping_sha256(request.current_prompts) + candidate_hash = prompt_mapping_sha256(result.best_prompts) + changed_fields = [ + name + for name in request.current_prompts + if request.current_prompts[name] != result.best_prompts[name] + ] + retained_output_dir = str(request.output_dir) if request.retain_native_artifacts else None + proposal = OptimizerCandidateProposal( + prompts=dict(result.best_prompts), + changed_fields=changed_fields, + rationale=( + f"AgentOptimizer selected the best candidate after {result.total_rounds} rounds " + f"with finish_reason={result.finish_reason}." + ), + parent_prompt_sha256=parent_hash, + candidate_prompt_sha256=candidate_hash, + candidate_id=f"optimizer-{candidate_hash[:12]}", + finish_reason=result.finish_reason, + stop_reason=result.stop_reason, + baseline_pass_rate=result.baseline_pass_rate, + best_pass_rate=result.best_pass_rate, + optimizer_output_dir=retained_output_dir, + ) + if not request.retain_native_artifacts: + try: + shutil.rmtree(request.output_dir) + except OSError as exc: + raise CandidateProviderError( + f"failed to discard optimizer artifacts: {exc}" + ) from exc + return CandidateGeneration(proposal=proposal, optimize_result=result) diff --git a/examples/optimization/eval_optimize_loop/config.py b/examples/optimization/eval_optimize_loop/config.py index 1c02e63d..406d6a8e 100644 --- a/examples/optimization/eval_optimize_loop/config.py +++ b/examples/optimization/eval_optimize_loop/config.py @@ -165,6 +165,12 @@ class WritebackConfig(EvalBaseModel): enabled: bool = False require_source_hash_match: bool = True + @model_validator(mode="after") + def _require_hash_guard_when_enabled(self) -> "WritebackConfig": + if self.enabled and not self.require_source_hash_match: + raise ValueError("enabled writeback requires require_source_hash_match=true") + return self + class PipelineConfig(EvalBaseModel): """The complete, example-local ``pipeline.json`` schema (version 1).""" diff --git a/examples/optimization/eval_optimize_loop/pipeline.py b/examples/optimization/eval_optimize_loop/pipeline.py index 88529de5..b0933318 100644 --- a/examples/optimization/eval_optimize_loop/pipeline.py +++ b/examples/optimization/eval_optimize_loop/pipeline.py @@ -3,7 +3,7 @@ # Copyright (C) 2026 Tencent. All rights reserved. # # tRPC-Agent-Python is licensed under the Apache License, Version 2.0. -"""Pipeline preparation and the deterministic offline stage-two loop.""" +"""Preparation, candidate regression, Gate, and guarded writeback orchestration.""" from __future__ import annotations @@ -19,6 +19,7 @@ from uuid import uuid4 from trpc_agent_sdk.evaluation import AgentEvaluator +from trpc_agent_sdk.evaluation import CallAgent from trpc_agent_sdk.evaluation import EvalCaseResult from trpc_agent_sdk.evaluation import EvalSet from trpc_agent_sdk.evaluation import EvalStatus @@ -27,11 +28,14 @@ from trpc_agent_sdk.evaluation import load_optimize_config from .analysis import build_evaluation_analysis +from .candidate_provider import AgentOptimizerCandidateProvider +from .candidate_provider import CandidateProviderError +from .candidate_provider import CandidateRequest +from .candidate_provider import FakeCandidateProviderAdapter from .config import PipelineConfig from .config import load_pipeline_config from .evaluation_adapter import EvaluationAnalysisError from .fake import DeterministicFakeAgent -from .fake import DeterministicFakeCandidateProvider from .gate import evaluate_gate from .gate import GateEvaluationError from .prompt_workspace import PromptWorkspaceError @@ -44,15 +48,21 @@ from .schemas import FakeStageResult from .schemas import ObservableValue from .schemas import ResourceMeasurements +from .schemas import RealStageResult from .schemas import WorkspaceSnapshot +from .writeback import perform_writeback class PipelinePreparationError(ValueError): """The example cannot safely prepare an evaluation/optimization run.""" -class FakeStageExecutionError(RuntimeError): - """The deterministic stage-two loop could not complete safely.""" +class PipelineExecutionError(RuntimeError): + """A prepared pipeline run could not complete safely.""" + + +# Compatibility for callers and tests written before the real-mode stage. +FakeStageExecutionError = PipelineExecutionError @dataclass(frozen=True) @@ -134,6 +144,19 @@ def _file_sha256(path: Path) -> str: return sha256(path.read_bytes()).hexdigest() +def _verify_prepared_file(path: Path, *, label: str, expected_sha256: str) -> None: + """Reject an input whose bytes changed after ``prepare_run``.""" + try: + actual_sha256 = _file_sha256(path) + except OSError as exc: + raise FakeStageExecutionError(f"failed to reload prepared {label}: {path}: {exc}") from exc + if actual_sha256 != expected_sha256: + raise FakeStageExecutionError( + f"{label} changed after prepare_run: {path}; " + f"expected sha256 {expected_sha256}, got {actual_sha256}" + ) + + def _reload_prepared_evalset( path: Path, *, @@ -246,7 +269,7 @@ def prepare_run(pipeline_config_path: str | Path, *, run_id: str | None = None) raise -def _summarize_fake_results( +def _summarize_results( eval_results_by_eval_id: dict[str, list[EvalCaseResult]], ) -> tuple[int, int, float | None]: passed_cases = 0 @@ -262,7 +285,7 @@ def _summarize_fake_results( return passed_cases, len(eval_results_by_eval_id), average_score -def _validate_fake_results( +def _validate_results( *, eval_set: EvalSet, eval_results_by_eval_id: dict[str, list[EvalCaseResult]], @@ -289,11 +312,11 @@ def _validate_fake_results( ) -async def _evaluate_fake_split( +async def _evaluate_split( *, prepared: PreparedRun, eval_set: EvalSet, - agent: DeterministicFakeAgent, + call_agent: CallAgent, phase: Literal["baseline", "candidate"], split: Literal["train", "validation"], ) -> FakeEvaluationSnapshot: @@ -302,7 +325,7 @@ async def _evaluate_fake_split( failed_summary, details_lines, result_lines, eval_results_by_eval_id = ( await AgentEvaluator.evaluate_eval_set( eval_set, - call_agent=agent.call_agent, + call_agent=call_agent, eval_config=prepared.optimizer_config.evaluate, num_runs=num_runs, print_detailed_results=False, @@ -313,14 +336,14 @@ async def _evaluate_fake_split( except Exception as exc: raise FakeStageExecutionError(f"{phase} {split} evaluation failed: {exc}") from exc - _validate_fake_results( + _validate_results( eval_set=eval_set, eval_results_by_eval_id=eval_results_by_eval_id, num_runs=num_runs, phase=phase, split=split, ) - passed_cases, total_cases, average_score = _summarize_fake_results(eval_results_by_eval_id) + passed_cases, total_cases, average_score = _summarize_results(eval_results_by_eval_id) return FakeEvaluationSnapshot( phase=phase, split=split, @@ -335,6 +358,24 @@ async def _evaluate_fake_split( ) +async def _restore_working_baseline( + prepared: PreparedRun, + baseline_prompts: dict[str, str], +) -> bool: + """Restore optimizer leftovers and prove the isolated baseline is present.""" + try: + current = await prepared.working_target.read_all() + was_modified = current != baseline_prompts + if was_modified: + await prepared.working_target.write_all(baseline_prompts) + restored = await prepared.working_target.read_all() + except Exception as exc: + raise FakeStageExecutionError(f"failed to restore optimizer working prompts: {exc}") from exc + if restored != baseline_prompts: + raise FakeStageExecutionError("optimizer working prompts did not match baseline after restoration") + return was_modified + + async def run_fake_stage( prepared: PreparedRun, *, @@ -380,27 +421,33 @@ async def run_fake_stage( raise FakeStageExecutionError("working prompts no longer match the prepared baseline snapshot") agent = DeterministicFakeAgent(prepared.working_target) - baseline_train = await _evaluate_fake_split( + baseline_train = await _evaluate_split( prepared=prepared, eval_set=train_evalset, - agent=agent, + call_agent=agent.call_agent, phase="baseline", split="train", ) - baseline_validation = await _evaluate_fake_split( + baseline_validation = await _evaluate_split( prepared=prepared, eval_set=validation_evalset, - agent=agent, + call_agent=agent.call_agent, phase="baseline", split="validation", ) + request = CandidateRequest( + current_prompts=baseline_prompts, + target_prompt=prepared.working_target, + optimizer_config_path=Path(prepared.input_snapshot.optimizer_config_path), + train_evalset_path=Path(prepared.input_snapshot.train_evalset_path), + validation_evalset_path=Path(prepared.input_snapshot.validation_evalset_path), + output_dir=Path(prepared.workspace.run_dir) / "fake_provider", + seed=prepared.input_snapshot.seed, + ) try: - candidate = DeterministicFakeCandidateProvider().propose( - baseline_prompts, - scenario=selected_scenario, - seed=prepared.input_snapshot.seed, - ) + generated = await FakeCandidateProviderAdapter(selected_scenario).propose(request) + candidate = generated.proposal except Exception as exc: raise FakeStageExecutionError(f"fake candidate generation failed: {exc}") from exc @@ -412,17 +459,17 @@ async def run_fake_stage( if written_prompts != candidate.prompts: raise FakeStageExecutionError("candidate prompt readback did not match the generated proposal") - candidate_train = await _evaluate_fake_split( + candidate_train = await _evaluate_split( prepared=prepared, eval_set=train_evalset, - agent=agent, + call_agent=agent.call_agent, phase="candidate", split="train", ) - candidate_validation = await _evaluate_fake_split( + candidate_validation = await _evaluate_split( prepared=prepared, eval_set=validation_evalset, - agent=agent, + call_agent=agent.call_agent, phase="candidate", split="validation", ) @@ -464,6 +511,13 @@ async def run_fake_stage( ) except GateEvaluationError as exc: raise FakeStageExecutionError(f"stage 3b gate failed: {exc}") from exc + writeback = await perform_writeback( + decision=gate_decision, + config=prepared.config.writeback, + snapshots=prepared.input_snapshot.prompt_snapshots, + source_target=prepared.source_target, + candidate=candidate, + ) return FakeStageResult( scenario=selected_scenario, candidate=candidate, @@ -474,4 +528,163 @@ async def run_fake_stage( analysis=analysis, measurements=measurements, gate_decision=gate_decision, + writeback=writeback, + ) + + +async def run_real_stage( + prepared: PreparedRun, + *, + call_agent: CallAgent, +) -> RealStageResult: + """Generate a real optimizer candidate and run the full guarded regression.""" + if prepared.config.execution.mode != "real": + raise FakeStageExecutionError( + f"run_real_stage requires execution.mode='real', got {prepared.config.execution.mode!r}" + ) + if prepared.config.execution.use_fake_judge: + raise FakeStageExecutionError("execution.use_fake_judge=true is not supported in real mode") + + started_at = perf_counter() + _verify_prepared_file( + Path(prepared.input_snapshot.optimizer_config_path), + label="optimizer_config", + expected_sha256=prepared.input_snapshot.optimizer_config_sha256, + ) + train_evalset = _reload_prepared_evalset( + Path(prepared.input_snapshot.train_evalset_path), + label="train_evalset", + expected_sha256=prepared.input_snapshot.train_evalset_sha256, + ) + validation_evalset = _reload_prepared_evalset( + Path(prepared.input_snapshot.validation_evalset_path), + label="validation_evalset", + expected_sha256=prepared.input_snapshot.validation_evalset_sha256, + ) + try: + baseline_prompts = await prepared.working_target.read_all() + except Exception as exc: + raise FakeStageExecutionError(f"failed to read prepared working prompts: {exc}") from exc + expected_baseline = { + snapshot.field_name: snapshot.content for snapshot in prepared.input_snapshot.prompt_snapshots + } + if baseline_prompts != expected_baseline: + raise FakeStageExecutionError("working prompts no longer match the prepared baseline snapshot") + + baseline_train = await _evaluate_split( + prepared=prepared, + eval_set=train_evalset, + call_agent=call_agent, + phase="baseline", + split="train", + ) + baseline_validation = await _evaluate_split( + prepared=prepared, + eval_set=validation_evalset, + call_agent=call_agent, + phase="baseline", + split="validation", + ) + + request = CandidateRequest( + current_prompts=baseline_prompts, + target_prompt=prepared.working_target, + optimizer_config_path=Path(prepared.input_snapshot.optimizer_config_path), + train_evalset_path=Path(prepared.input_snapshot.train_evalset_path), + validation_evalset_path=Path(prepared.input_snapshot.validation_evalset_path), + output_dir=Path(prepared.workspace.run_dir) / "optimizer", + seed=prepared.input_snapshot.seed, + retain_native_artifacts=prepared.config.artifacts.retain_optimizer_native_artifacts, + ) + try: + generated = await AgentOptimizerCandidateProvider(call_agent).propose(request) + except CandidateProviderError as exc: + await _restore_working_baseline(prepared, baseline_prompts) + raise FakeStageExecutionError(f"real candidate generation failed: {exc}") from exc + + if await _restore_working_baseline(prepared, baseline_prompts): + raise FakeStageExecutionError("optimizer did not restore working prompts after update_source=False") + + candidate = generated.proposal + try: + await prepared.working_target.write_all(candidate.prompts) + written_prompts = await prepared.working_target.read_all() + except Exception as exc: + raise FakeStageExecutionError(f"candidate prompt write failed: {exc}") from exc + if written_prompts != candidate.prompts: + raise FakeStageExecutionError("candidate prompt readback did not match the generated proposal") + + candidate_train = await _evaluate_split( + prepared=prepared, + eval_set=train_evalset, + call_agent=call_agent, + phase="candidate", + split="train", + ) + candidate_validation = await _evaluate_split( + prepared=prepared, + eval_set=validation_evalset, + call_agent=call_agent, + phase="candidate", + split="validation", + ) + try: + analysis = build_evaluation_analysis( + baseline_train=baseline_train, + baseline_validation=baseline_validation, + candidate_train=candidate_train, + candidate_validation=candidate_validation, + hard_case_ids=set(prepared.config.case_labels.hard_case_ids), + critical_case_ids=set(prepared.config.case_labels.critical_case_ids), + severe_case_score_drop=prepared.config.gate.severe_case_score_drop, + ) + except EvaluationAnalysisError as exc: + raise FakeStageExecutionError(f"stage 3a analysis failed: {exc}") from exc + measurements = ResourceMeasurements( + cost_usd=ObservableValue( + status="unavailable", + unit="USD", + reason="The injected agent's full pipeline cost is not observable.", + ), + total_tokens=ObservableValue( + status="unavailable", + unit="tokens", + reason="The injected agent's full pipeline token usage is not observable.", + ), + duration_seconds=ObservableValue( + status="available", + value=perf_counter() - started_at, + unit="seconds", + ), + ) + try: + gate_decision = evaluate_gate( + analysis, + prepared.config.gate, + prepared.config.budget, + measurements, + ) + except GateEvaluationError as exc: + raise FakeStageExecutionError(f"stage 3b gate failed: {exc}") from exc + + writeback = await perform_writeback( + decision=gate_decision, + config=prepared.config.writeback, + snapshots=prepared.input_snapshot.prompt_snapshots, + source_target=prepared.source_target, + candidate=candidate, + ) + + assert generated.optimize_result is not None + return RealStageResult( + candidate=candidate, + optimize_result=generated.optimize_result, + baseline_train=baseline_train, + baseline_validation=baseline_validation, + candidate_train=candidate_train, + candidate_validation=candidate_validation, + analysis=analysis, + measurements=measurements, + gate_decision=gate_decision, + writeback=writeback, ) diff --git a/examples/optimization/eval_optimize_loop/run_pipeline.py b/examples/optimization/eval_optimize_loop/run_pipeline.py index 5ce4d808..9c9cfccc 100644 --- a/examples/optimization/eval_optimize_loop/run_pipeline.py +++ b/examples/optimization/eval_optimize_loop/run_pipeline.py @@ -19,7 +19,9 @@ sys.path.insert(0, str(_REPO_ROOT)) from examples.optimization.eval_optimize_loop.pipeline import prepare_run from examples.optimization.eval_optimize_loop.pipeline import run_fake_stage + from examples.optimization.eval_optimize_loop.config import load_pipeline_config else: + from .config import load_pipeline_config from .pipeline import prepare_run from .pipeline import run_fake_stage @@ -46,6 +48,15 @@ def main() -> None: ) args = parser.parse_args() + config = load_pipeline_config(args.config) + if config.execution.mode == "real": + parser.error( + "real mode requires an injected business agent; use the Python API " + "run_real_stage(prepared, call_agent=...)" + ) + if config.execution.mode != "fake": + parser.error("this CLI currently supports execution.mode='fake' only") + prepared = prepare_run(args.config, run_id=args.run_id) result = asyncio.run(run_fake_stage(prepared, scenario=args.scenario)) print(f"Completed deterministic pipeline stage: {prepared.workspace.run_dir}") @@ -66,7 +77,11 @@ def main() -> None: print("Warnings:") for warning in result.gate_decision.warnings: print(f"- {warning}") - print("Stage 3b does not write reports or update source prompts.") + print( + f"Writeback: {result.writeback.status.upper()} " + f"({result.writeback.reason})" + ) + print("Stage 4 does not write optimization reports; reporting is added in Stage 5.") if __name__ == "__main__": diff --git a/examples/optimization/eval_optimize_loop/schemas.py b/examples/optimization/eval_optimize_loop/schemas.py index 0cd29529..7b440b4a 100644 --- a/examples/optimization/eval_optimize_loop/schemas.py +++ b/examples/optimization/eval_optimize_loop/schemas.py @@ -21,6 +21,7 @@ from trpc_agent_sdk.evaluation import EvalBaseModel from trpc_agent_sdk.evaluation import EvalCaseResult +from trpc_agent_sdk.evaluation import OptimizeResult FakeCandidateScenario = Literal["improve", "no_improvement", "overfit"] @@ -60,6 +61,15 @@ ] GateRuleOutcome = Literal["pass", "reject", "warning", "skipped"] GateDecisionValue = Literal["accept", "reject"] +WritebackStatus = Literal["skipped", "written", "blocked", "failed"] +WritebackReason = Literal[ + "gate_rejected", + "disabled", + "source_drift", + "write_error", + "readback_mismatch", + "written", +] class ObservableValue(EvalBaseModel): @@ -113,19 +123,40 @@ class WorkspaceSnapshot(EvalBaseModel): prompts_dir: str -class FakeCandidateProposal(EvalBaseModel): - """One deterministic prompt proposal produced without a real optimizer.""" +class CandidateProposal(EvalBaseModel): + """Common, serializable identity and prompt payload for any provider.""" - scenario: FakeCandidateScenario + provider: Literal["fake", "agent_optimizer"] prompts: dict[str, str] changed_fields: list[str] rationale: str - seed: int parent_prompt_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") candidate_prompt_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + candidate_id: str + + +class FakeCandidateProposal(CandidateProposal): + """One deterministic prompt proposal produced without a real optimizer.""" + + provider: Literal["fake"] = "fake" + scenario: FakeCandidateScenario + seed: int candidate_id: str = Field(pattern=r"^fake-(improve|no_improvement|overfit)-[0-9a-f]{12}$") +class OptimizerCandidateProposal(CandidateProposal): + """Best candidate returned by a successful real AgentOptimizer run.""" + + provider: Literal["agent_optimizer"] = "agent_optimizer" + optimizer_status: Literal["SUCCEEDED"] = "SUCCEEDED" + finish_reason: str + stop_reason: Optional[str] = None + baseline_pass_rate: float = Field(ge=0.0, le=1.0) + best_pass_rate: float = Field(ge=0.0, le=1.0) + optimizer_output_dir: Optional[str] = None + candidate_id: str = Field(pattern=r"^optimizer-[0-9a-f]{12}$") + + class FakeEvaluationSnapshot(EvalBaseModel): """Complete SDK outputs from one fake-agent dataset evaluation.""" @@ -328,11 +359,21 @@ class GateDecision(EvalBaseModel): warnings: list[str] = Field(default_factory=list) -class FakeStageResult(EvalBaseModel): - """Evaluation, candidate, analysis, and Gate outputs from the offline stage.""" +class WritebackResult(EvalBaseModel): + """Auditable outcome of the post-Gate source prompt operation.""" + + status: WritebackStatus + reason: WritebackReason + attempted: bool = False + changed_fields: list[str] = Field(default_factory=list) + source_hashes_before: dict[str, str] = Field(default_factory=dict) + source_hashes_after: dict[str, str] = Field(default_factory=dict) + error_message: Optional[str] = None + + +class PipelineStageResult(EvalBaseModel): + """Evaluation, analysis, Gate, and writeback fields shared by all modes.""" - scenario: FakeCandidateScenario - candidate: FakeCandidateProposal baseline_train: FakeEvaluationSnapshot baseline_validation: FakeEvaluationSnapshot candidate_train: FakeEvaluationSnapshot @@ -340,3 +381,18 @@ class FakeStageResult(EvalBaseModel): analysis: EvaluationAnalysis measurements: ResourceMeasurements gate_decision: GateDecision + writeback: WritebackResult + + +class FakeStageResult(PipelineStageResult): + """Full deterministic fake-mode pipeline result.""" + + scenario: FakeCandidateScenario + candidate: FakeCandidateProposal + + +class RealStageResult(PipelineStageResult): + """Full regression and Gate result for an AgentOptimizer proposal.""" + + candidate: OptimizerCandidateProposal + optimize_result: OptimizeResult diff --git a/examples/optimization/eval_optimize_loop/writeback.py b/examples/optimization/eval_optimize_loop/writeback.py new file mode 100644 index 00000000..302d1f90 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/writeback.py @@ -0,0 +1,166 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under the Apache License, Version 2.0. +"""Post-Gate source prompt writeback with drift detection and verification.""" + +from __future__ import annotations + +from hashlib import sha256 + +from trpc_agent_sdk.evaluation import TargetPrompt + +from .candidate_provider import prompt_mapping_sha256 +from .config import WritebackConfig +from .prompt_workspace import SourcePromptDriftError +from .prompt_workspace import verify_source_hashes +from .schemas import CandidateProposal +from .schemas import GateDecision +from .schemas import PromptSnapshot +from .schemas import WritebackResult + + +class WritebackIntegrityError(RuntimeError): + """The pipeline cannot prove that source prompts remain in a safe state.""" + + +def _field_hashes(prompts: dict[str, str]) -> dict[str, str]: + return { + name: sha256(content.encode("utf-8")).hexdigest() + for name, content in prompts.items() + } + + +async def _blocked_for_drift( + source_target: TargetPrompt, + message: str, +) -> WritebackResult: + try: + observed = await source_target.read_all() + except Exception: + observed = {} + return WritebackResult( + status="blocked", + reason="source_drift", + source_hashes_before=_field_hashes(observed), + error_message=message, + ) + + +async def _restore_and_verify( + source_target: TargetPrompt, + baseline: dict[str, str], +) -> dict[str, str]: + """Restore only when needed, then prove the exact baseline is present.""" + try: + current = await source_target.read_all() + except Exception: + current = None + if current != baseline: + try: + await source_target.write_all(baseline) + except Exception as exc: + raise WritebackIntegrityError(f"source prompt rollback failed: {exc}") from exc + try: + restored = await source_target.read_all() + except Exception as exc: + raise WritebackIntegrityError(f"failed to verify source prompt rollback: {exc}") from exc + if restored != baseline: + raise WritebackIntegrityError("source prompts do not match the pre-write snapshot after rollback") + return restored + + +async def perform_writeback( + *, + decision: GateDecision, + config: WritebackConfig, + snapshots: list[PromptSnapshot], + source_target: TargetPrompt, + candidate: CandidateProposal, +) -> WritebackResult: + """Apply a candidate only after ACCEPT and return a structured outcome.""" + if decision.decision == "reject": + return WritebackResult(status="skipped", reason="gate_rejected") + if not config.enabled: + return WritebackResult(status="skipped", reason="disabled") + if not config.require_source_hash_match: + raise WritebackIntegrityError("enabled writeback requires source hash verification") + if prompt_mapping_sha256(candidate.prompts) != candidate.candidate_prompt_sha256: + raise WritebackIntegrityError("candidate prompt hash does not match its prompt payload") + + try: + verify_source_hashes(snapshots) + except SourcePromptDriftError as exc: + return await _blocked_for_drift(source_target, str(exc)) + + try: + baseline = await source_target.read_all() + except Exception as exc: + return WritebackResult( + status="failed", + reason="write_error", + error_message=f"failed to read source prompts before writeback: {exc}", + ) + expected_baseline = {snapshot.field_name: snapshot.content for snapshot in snapshots} + if baseline != expected_baseline: + return await _blocked_for_drift( + source_target, + "source prompts changed after the initial hash check", + ) + hashes_before = _field_hashes(baseline) + + # This synchronous check is intentionally adjacent to the path-backed + # write. It narrows the compare/write window after the awaited read above. + try: + verify_source_hashes(snapshots) + except SourcePromptDriftError as exc: + return await _blocked_for_drift(source_target, str(exc)) + + try: + await source_target.write_all(candidate.prompts) + except Exception as exc: + restored = await _restore_and_verify(source_target, baseline) + return WritebackResult( + status="failed", + reason="write_error", + attempted=True, + changed_fields=list(candidate.changed_fields), + source_hashes_before=hashes_before, + source_hashes_after=_field_hashes(restored), + error_message=str(exc), + ) + + try: + written = await source_target.read_all() + except Exception as exc: + restored = await _restore_and_verify(source_target, baseline) + return WritebackResult( + status="failed", + reason="readback_mismatch", + attempted=True, + changed_fields=list(candidate.changed_fields), + source_hashes_before=hashes_before, + source_hashes_after=_field_hashes(restored), + error_message=f"failed to read source prompts after writeback: {exc}", + ) + if written != candidate.prompts: + restored = await _restore_and_verify(source_target, baseline) + return WritebackResult( + status="failed", + reason="readback_mismatch", + attempted=True, + changed_fields=list(candidate.changed_fields), + source_hashes_before=hashes_before, + source_hashes_after=_field_hashes(restored), + error_message="source prompt readback did not match the accepted candidate", + ) + + return WritebackResult( + status="written", + reason="written", + attempted=True, + changed_fields=list(candidate.changed_fields), + source_hashes_before=hashes_before, + source_hashes_after=_field_hashes(written), + ) diff --git a/tests/evaluation/test_eval_optimize_loop_stage3b.py b/tests/evaluation/test_eval_optimize_loop_stage3b.py index 64901f3f..0e15fa87 100644 --- a/tests/evaluation/test_eval_optimize_loop_stage3b.py +++ b/tests/evaluation/test_eval_optimize_loop_stage3b.py @@ -621,4 +621,5 @@ def test_stage3b_cli_prints_gate_decision_and_rejection_reasons(tmp_path: Path): assert "Gate decision: REJECT" in completed.stdout assert "Rejection reasons:" in completed.stdout assert "[no_overfitting]" in completed.stdout - assert "Stage 3b does not write reports or update source prompts." in completed.stdout + assert "Writeback: SKIPPED (gate_rejected)" in completed.stdout + assert "Stage 4 does not write optimization reports" in completed.stdout diff --git a/tests/evaluation/test_eval_optimize_loop_stage4.py b/tests/evaluation/test_eval_optimize_loop_stage4.py new file mode 100644 index 00000000..b746096f --- /dev/null +++ b/tests/evaluation/test_eval_optimize_loop_stage4.py @@ -0,0 +1,586 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under the Apache License, Version 2.0. +"""Tests for Stage 4 real optimization and guarded prompt writeback.""" + +from __future__ import annotations + +import json +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest + +from examples.optimization.eval_optimize_loop import candidate_provider as candidate_provider_module +from examples.optimization.eval_optimize_loop import writeback as writeback_module +from examples.optimization.eval_optimize_loop.candidate_provider import AgentOptimizerCandidateProvider +from examples.optimization.eval_optimize_loop.candidate_provider import CandidateProviderError +from examples.optimization.eval_optimize_loop.candidate_provider import CandidateRequest +from examples.optimization.eval_optimize_loop.config import WritebackConfig +from examples.optimization.eval_optimize_loop.fake import DeterministicFakeAgent +from examples.optimization.eval_optimize_loop.fake import DeterministicFakeCandidateProvider +from examples.optimization.eval_optimize_loop.pipeline import prepare_run +from examples.optimization.eval_optimize_loop.pipeline import FakeStageExecutionError +from examples.optimization.eval_optimize_loop.pipeline import run_fake_stage +from examples.optimization.eval_optimize_loop.pipeline import run_real_stage +from examples.optimization.eval_optimize_loop.schemas import GateDecision +from examples.optimization.eval_optimize_loop.schemas import RealStageResult +from examples.optimization.eval_optimize_loop.writeback import perform_writeback +from trpc_agent_sdk.evaluation import OptimizeResult + + +_REPO_ROOT = Path(__file__).resolve().parents[2] +_EXAMPLE = _REPO_ROOT / "examples" / "optimization" / "eval_optimize_loop" + + +def _copy_example(tmp_path: Path, name: str = "eval_optimize_loop") -> Path: + target = tmp_path / name + shutil.copytree(_EXAMPLE, target, ignore=shutil.ignore_patterns("runs", "__pycache__")) + return target + + +def _set_real_mode(root: Path, *, writeback: bool = False) -> None: + path = root / "pipeline.json" + payload = json.loads(path.read_text(encoding="utf-8")) + payload["execution"]["mode"] = "real" + payload["writeback"]["enabled"] = writeback + path.write_text(json.dumps(payload), encoding="utf-8") + + +def _optimize_result( + baseline_prompts: dict[str, str], + best_prompts: dict[str, str], + *, + status: str = "SUCCEEDED", +) -> OptimizeResult: + return OptimizeResult( + algorithm="gepa_reflective", + status=status, + finish_reason="completed" if status == "SUCCEEDED" else "error", + error_message="" if status == "SUCCEEDED" else "optimizer failed", + baseline_pass_rate=1 / 3, + best_pass_rate=1.0, + pass_rate_improvement=2 / 3, + baseline_prompts=baseline_prompts, + best_prompts=best_prompts, + total_rounds=0, + rounds=[], + total_reflection_lm_calls=1, + total_llm_cost=0.25, + total_token_usage={"prompt": 10, "completion": 5, "total": 15}, + duration_seconds=0.5, + started_at="2026-07-17T00:00:00Z", + finished_at="2026-07-17T00:00:01Z", + ) + + +def _install_optimizer_result( + monkeypatch: pytest.MonkeyPatch, + baseline: dict[str, str], + best: dict[str, str], +) -> None: + async def fake_optimize(**kwargs): + output_dir = Path(str(kwargs["output_dir"])) + output_dir.mkdir(parents=True) + result = _optimize_result(baseline, best) + result.dump_to(str(output_dir / "result.json")) + return result + + monkeypatch.setattr( + "examples.optimization.eval_optimize_loop.candidate_provider.AgentOptimizer.optimize", + fake_optimize, + ) + + +@pytest.mark.asyncio +async def test_real_candidate_provider_uses_isolated_target_and_never_updates_source( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + root = _copy_example(tmp_path) + prepared = prepare_run(root / "pipeline.json", run_id="stage4_provider") + baseline = await prepared.working_target.read_all() + best = {name: f"{content}\noptimized" for name, content in baseline.items()} + captured: dict[str, object] = {} + + async def call_agent(query: str) -> str: + return query + + async def fake_optimize(**kwargs): + captured.update(kwargs) + output_dir = Path(str(kwargs["output_dir"])) + output_dir.mkdir(parents=True) + result = _optimize_result(baseline, best) + result.dump_to(str(output_dir / "result.json")) + return result + + monkeypatch.setattr( + "examples.optimization.eval_optimize_loop.candidate_provider.AgentOptimizer.optimize", + fake_optimize, + ) + request = CandidateRequest( + current_prompts=baseline, + target_prompt=prepared.working_target, + optimizer_config_path=Path(prepared.input_snapshot.optimizer_config_path), + train_evalset_path=Path(prepared.input_snapshot.train_evalset_path), + validation_evalset_path=Path(prepared.input_snapshot.validation_evalset_path), + output_dir=Path(prepared.workspace.run_dir) / "optimizer", + seed=prepared.input_snapshot.seed, + ) + + generated = await AgentOptimizerCandidateProvider(call_agent).propose(request) + + assert captured["target_prompt"] is prepared.working_target + assert captured["update_source"] is False + assert captured["verbose"] == 0 + assert generated.proposal.provider == "agent_optimizer" + assert generated.proposal.prompts == best + assert generated.proposal.changed_fields == ["system_prompt"] + assert generated.proposal.candidate_id.startswith("optimizer-") + assert generated.optimize_result == _optimize_result(baseline, best) + assert (request.output_dir / "result.json").is_file() + assert await prepared.working_target.read_all() == baseline + + +@pytest.mark.asyncio +async def test_real_stage_runs_full_regression_and_skips_disabled_writeback( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + root = _copy_example(tmp_path, "real_stage") + _set_real_mode(root) + source_path = root / "prompts" / "system.md" + source_before = source_path.read_text(encoding="utf-8") + prepared = prepare_run(root / "pipeline.json", run_id="stage4_real") + baseline = await prepared.working_target.read_all() + best = DeterministicFakeCandidateProvider().propose( + baseline, + scenario="improve", + seed=42, + ).prompts + + async def fake_optimize(**kwargs): + output_dir = Path(str(kwargs["output_dir"])) + output_dir.mkdir(parents=True) + result = _optimize_result(baseline, best) + result.dump_to(str(output_dir / "result.json")) + return result + + monkeypatch.setattr( + "examples.optimization.eval_optimize_loop.candidate_provider.AgentOptimizer.optimize", + fake_optimize, + ) + agent = DeterministicFakeAgent(prepared.working_target) + + result = await run_real_stage(prepared, call_agent=agent.call_agent) + + assert isinstance(result, RealStageResult) + assert result.baseline_train.passed_case_count == 1 + assert result.baseline_validation.passed_case_count == 1 + assert result.candidate_train.passed_case_count == 3 + assert result.candidate_validation.passed_case_count == 3 + assert result.gate_decision.decision == "accept" + assert result.writeback.status == "skipped" + assert result.writeback.reason == "disabled" + assert result.optimize_result.best_prompts == best + assert source_path.read_text(encoding="utf-8") == source_before + assert await prepared.working_target.read_all() == best + assert RealStageResult.model_validate_json(result.model_dump_json()) == result + + +@pytest.mark.asyncio +async def test_real_stage_writes_accepted_candidate_after_hash_check( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + root = _copy_example(tmp_path, "write_accepted") + _set_real_mode(root, writeback=True) + prepared = prepare_run(root / "pipeline.json", run_id="stage4_write") + baseline = await prepared.working_target.read_all() + best = DeterministicFakeCandidateProvider().propose( + baseline, + scenario="improve", + seed=42, + ).prompts + _install_optimizer_result(monkeypatch, baseline, best) + + result = await run_real_stage( + prepared, + call_agent=DeterministicFakeAgent(prepared.working_target).call_agent, + ) + + assert result.gate_decision.decision == "accept" + assert result.writeback.status == "written" + assert result.writeback.reason == "written" + assert result.writeback.attempted is True + assert result.writeback.changed_fields == ["system_prompt"] + assert await prepared.source_target.read_all() == best + assert result.writeback.source_hashes_before != result.writeback.source_hashes_after + + +@pytest.mark.asyncio +async def test_real_stage_blocks_writeback_when_source_drifted( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + root = _copy_example(tmp_path, "source_drift") + _set_real_mode(root, writeback=True) + prepared = prepare_run(root / "pipeline.json", run_id="stage4_drift") + baseline = await prepared.working_target.read_all() + best = DeterministicFakeCandidateProvider().propose( + baseline, + scenario="improve", + seed=42, + ).prompts + _install_optimizer_result(monkeypatch, baseline, best) + source_path = root / "prompts" / "system.md" + source_path.write_text("concurrent user edit", encoding="utf-8") + + result = await run_real_stage( + prepared, + call_agent=DeterministicFakeAgent(prepared.working_target).call_agent, + ) + + assert result.gate_decision.decision == "accept" + assert result.writeback.status == "blocked" + assert result.writeback.reason == "source_drift" + assert result.writeback.attempted is False + assert source_path.read_text(encoding="utf-8") == "concurrent user edit" + + +@pytest.mark.asyncio +async def test_real_stage_returns_failed_writeback_after_verified_rollback( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + root = _copy_example(tmp_path, "write_failure") + _set_real_mode(root, writeback=True) + prepared = prepare_run(root / "pipeline.json", run_id="stage4_write_failure") + baseline = await prepared.working_target.read_all() + best = DeterministicFakeCandidateProvider().propose( + baseline, + scenario="improve", + seed=42, + ).prompts + _install_optimizer_result(monkeypatch, baseline, best) + + async def fail_write(prompts: dict[str, str]) -> None: + raise OSError("simulated source write failure") + + monkeypatch.setattr(prepared.source_target, "write_all", fail_write) + + result = await run_real_stage( + prepared, + call_agent=DeterministicFakeAgent(prepared.working_target).call_agent, + ) + + assert result.writeback.status == "failed" + assert result.writeback.reason == "write_error" + assert result.writeback.attempted is True + assert "simulated source write failure" in (result.writeback.error_message or "") + assert await prepared.source_target.read_all() == baseline + + +@pytest.mark.asyncio +async def test_fake_stage_uses_the_same_guarded_writeback_path(tmp_path: Path): + root = _copy_example(tmp_path, "fake_writeback") + path = root / "pipeline.json" + payload = json.loads(path.read_text(encoding="utf-8")) + payload["writeback"]["enabled"] = True + path.write_text(json.dumps(payload), encoding="utf-8") + prepared = prepare_run(root / "pipeline.json", run_id="stage4_fake_write") + + result = await run_fake_stage(prepared, scenario="improve") + + assert result.gate_decision.decision == "accept" + assert result.writeback.status == "written" + assert await prepared.source_target.read_all() == result.candidate.prompts + + +@pytest.mark.asyncio +async def test_fake_stage_never_writes_a_gate_rejected_candidate(tmp_path: Path): + root = _copy_example(tmp_path, "fake_reject") + path = root / "pipeline.json" + payload = json.loads(path.read_text(encoding="utf-8")) + payload["writeback"]["enabled"] = True + path.write_text(json.dumps(payload), encoding="utf-8") + prepared = prepare_run(root / "pipeline.json", run_id="stage4_fake_reject") + source_before = await prepared.source_target.read_all() + + result = await run_fake_stage(prepared, scenario="no_improvement") + + assert result.gate_decision.decision == "reject" + assert result.writeback.status == "skipped" + assert result.writeback.reason == "gate_rejected" + assert await prepared.source_target.read_all() == source_before + + +@pytest.mark.asyncio +async def test_real_candidate_provider_rejects_failed_optimizer_result( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + root = _copy_example(tmp_path, "optimizer_failed") + prepared = prepare_run(root / "pipeline.json", run_id="stage4_optimizer_failed") + baseline = await prepared.working_target.read_all() + + async def failed_optimize(**kwargs): + return _optimize_result(baseline, baseline, status="FAILED") + + monkeypatch.setattr( + "examples.optimization.eval_optimize_loop.candidate_provider.AgentOptimizer.optimize", + failed_optimize, + ) + request = CandidateRequest( + current_prompts=baseline, + target_prompt=prepared.working_target, + optimizer_config_path=Path(prepared.input_snapshot.optimizer_config_path), + train_evalset_path=Path(prepared.input_snapshot.train_evalset_path), + validation_evalset_path=Path(prepared.input_snapshot.validation_evalset_path), + output_dir=Path(prepared.workspace.run_dir) / "optimizer", + seed=42, + ) + + with pytest.raises(CandidateProviderError, match="returned FAILED"): + await AgentOptimizerCandidateProvider(DeterministicFakeAgent(prepared.working_target).call_agent).propose( + request + ) + + +@pytest.mark.asyncio +async def test_real_stage_rejects_optimizer_config_drift_before_optimization( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + root = _copy_example(tmp_path, "optimizer_drift") + _set_real_mode(root) + prepared = prepare_run(root / "pipeline.json", run_id="stage4_optimizer_drift") + optimizer_path = root / "optimizer.json" + optimizer_path.write_text( + optimizer_path.read_text(encoding="utf-8") + "\n", + encoding="utf-8", + ) + called = False + + async def should_not_optimize(**kwargs): + nonlocal called + called = True + raise AssertionError("optimizer must not run after config drift") + + monkeypatch.setattr( + "examples.optimization.eval_optimize_loop.candidate_provider.AgentOptimizer.optimize", + should_not_optimize, + ) + + with pytest.raises(FakeStageExecutionError, match="optimizer_config changed"): + await run_real_stage( + prepared, + call_agent=DeterministicFakeAgent(prepared.working_target).call_agent, + ) + assert called is False + + +@pytest.mark.asyncio +async def test_writeback_readback_mismatch_restores_source( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + root = _copy_example(tmp_path, "readback_mismatch") + prepared = prepare_run(root / "pipeline.json", run_id="stage4_readback") + baseline = await prepared.source_target.read_all() + candidate = DeterministicFakeCandidateProvider().propose( + baseline, + scenario="improve", + seed=42, + ) + original_read_all = prepared.source_target.read_all + read_count = 0 + + async def mismatched_readback(): + nonlocal read_count + read_count += 1 + actual = await original_read_all() + if read_count == 2: + return {"system_prompt": "unexpected readback"} + return actual + + monkeypatch.setattr(prepared.source_target, "read_all", mismatched_readback) + + result = await perform_writeback( + decision=GateDecision(decision="accept", rule_results=[]), + config=WritebackConfig(enabled=True), + snapshots=prepared.input_snapshot.prompt_snapshots, + source_target=prepared.source_target, + candidate=candidate, + ) + + assert result.status == "failed" + assert result.reason == "readback_mismatch" + assert await original_read_all() == baseline + + +def test_enabled_writeback_cannot_disable_source_hash_guard(): + with pytest.raises(ValueError, match="requires require_source_hash_match=true"): + WritebackConfig(enabled=True, require_source_hash_match=False) + + +@pytest.mark.asyncio +async def test_real_stage_restores_working_prompt_after_optimizer_exception( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + root = _copy_example(tmp_path, "optimizer_exception") + _set_real_mode(root) + prepared = prepare_run(root / "pipeline.json", run_id="stage4_optimizer_exception") + baseline = await prepared.working_target.read_all() + source_before = await prepared.source_target.read_all() + changed = {name: f"{value}\nleft by failed optimizer" for name, value in baseline.items()} + + async def mutating_failure(**kwargs): + await kwargs["target_prompt"].write_all(changed) + raise RuntimeError("simulated optimizer crash") + + monkeypatch.setattr( + "examples.optimization.eval_optimize_loop.candidate_provider.AgentOptimizer.optimize", + mutating_failure, + ) + + with pytest.raises(FakeStageExecutionError, match="real candidate generation failed"): + await run_real_stage( + prepared, + call_agent=DeterministicFakeAgent(prepared.working_target).call_agent, + ) + + assert await prepared.working_target.read_all() == baseline + assert await prepared.source_target.read_all() == source_before + + +@pytest.mark.asyncio +async def test_real_candidate_provider_can_discard_native_artifacts_after_extraction( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + root = _copy_example(tmp_path, "discard_artifacts") + prepared = prepare_run(root / "pipeline.json", run_id="stage4_discard_artifacts") + baseline = await prepared.working_target.read_all() + best = {name: f"{content}\noptimized" for name, content in baseline.items()} + _install_optimizer_result(monkeypatch, baseline, best) + output_dir = Path(prepared.workspace.run_dir) / "optimizer" + request = CandidateRequest( + current_prompts=baseline, + target_prompt=prepared.working_target, + optimizer_config_path=Path(prepared.input_snapshot.optimizer_config_path), + train_evalset_path=Path(prepared.input_snapshot.train_evalset_path), + validation_evalset_path=Path(prepared.input_snapshot.validation_evalset_path), + output_dir=output_dir, + seed=42, + retain_native_artifacts=False, + ) + + generated = await AgentOptimizerCandidateProvider( + DeterministicFakeAgent(prepared.working_target).call_agent + ).propose(request) + + assert generated.optimize_result is not None + assert generated.proposal.optimizer_output_dir is None + assert not output_dir.exists() + + +def test_cli_explains_that_real_mode_requires_injected_call_agent(tmp_path: Path): + root = _copy_example(tmp_path, "real_cli") + _set_real_mode(root) + + completed = subprocess.run( + [ + sys.executable, + str(root / "run_pipeline.py"), + "--config", + str(root / "pipeline.json"), + "--run-id", + "stage4_real_cli", + ], + cwd=_REPO_ROOT, + capture_output=True, + text=True, + check=False, + ) + + assert completed.returncode == 2 + assert "run_real_stage(prepared, call_agent=...)" in completed.stderr + assert not (root / "runs" / "stage4_real_cli").exists() + + +@pytest.mark.asyncio +async def test_writeback_blocks_drift_introduced_after_initial_hash_check( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + root = _copy_example(tmp_path, "late_source_drift") + prepared = prepare_run(root / "pipeline.json", run_id="stage4_late_drift") + baseline = await prepared.source_target.read_all() + candidate = DeterministicFakeCandidateProvider().propose( + baseline, + scenario="improve", + seed=42, + ) + source_path = root / "prompts" / "system.md" + original_verify = writeback_module.verify_source_hashes + checks = 0 + + def verify_then_drift(snapshots): + nonlocal checks + checks += 1 + original_verify(snapshots) + if checks == 1: + source_path.write_text("late concurrent edit", encoding="utf-8") + + monkeypatch.setattr(writeback_module, "verify_source_hashes", verify_then_drift) + + result = await perform_writeback( + decision=GateDecision(decision="accept", rule_results=[]), + config=WritebackConfig(enabled=True), + snapshots=prepared.input_snapshot.prompt_snapshots, + source_target=prepared.source_target, + candidate=candidate, + ) + + assert result.status == "blocked" + assert result.reason == "source_drift" + assert result.attempted is False + assert source_path.read_text(encoding="utf-8") == "late concurrent edit" + + +@pytest.mark.asyncio +async def test_real_candidate_provider_wraps_artifact_cleanup_failure( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + root = _copy_example(tmp_path, "artifact_cleanup_failure") + prepared = prepare_run(root / "pipeline.json", run_id="stage4_cleanup_failure") + baseline = await prepared.working_target.read_all() + best = {name: f"{content}\noptimized" for name, content in baseline.items()} + _install_optimizer_result(monkeypatch, baseline, best) + request = CandidateRequest( + current_prompts=baseline, + target_prompt=prepared.working_target, + optimizer_config_path=Path(prepared.input_snapshot.optimizer_config_path), + train_evalset_path=Path(prepared.input_snapshot.train_evalset_path), + validation_evalset_path=Path(prepared.input_snapshot.validation_evalset_path), + output_dir=Path(prepared.workspace.run_dir) / "optimizer", + seed=42, + retain_native_artifacts=False, + ) + + def fail_cleanup(path: Path, **kwargs) -> None: + raise OSError("simulated cleanup failure") + + monkeypatch.setattr(candidate_provider_module.shutil, "rmtree", fail_cleanup) + + with pytest.raises(CandidateProviderError, match="discard optimizer artifacts"): + await AgentOptimizerCandidateProvider( + DeterministicFakeAgent(prepared.working_target).call_agent + ).propose(request) From 26e79c6ec1a615d05e09bf6274402f1ab865530c Mon Sep 17 00:00:00 2001 From: guocfu Date: Sat, 18 Jul 2026 17:19:56 +0800 Subject: [PATCH 06/18] =?UTF-8?q?test(evaluation):=20=E5=A2=9E=E5=8A=A0?= =?UTF-8?q?=E7=9C=9F=E5=AE=9E=E5=8F=8C=E6=A8=A1=E5=9E=8B=E9=9B=86=E6=88=90?= =?UTF-8?q?=E5=85=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../optimization/eval_optimize_loop/README.md | 33 +- .../eval_optimize_loop/candidate_provider.py | 80 +++- .../eval_optimize_loop/pipeline.py | 4 + .../eval_optimize_loop/pipeline.real.json | 53 +++ .../eval_optimize_loop/real_agent.py | 112 +++++ .../run_real_integration.py | 140 +++++++ .../eval_optimize_loop/schemas.py | 20 + ...est_eval_optimize_loop_real_integration.py | 388 ++++++++++++++++++ 8 files changed, 824 insertions(+), 6 deletions(-) create mode 100644 examples/optimization/eval_optimize_loop/pipeline.real.json create mode 100644 examples/optimization/eval_optimize_loop/real_agent.py create mode 100644 examples/optimization/eval_optimize_loop/run_real_integration.py create mode 100644 tests/evaluation/test_eval_optimize_loop_real_integration.py diff --git a/examples/optimization/eval_optimize_loop/README.md b/examples/optimization/eval_optimize_loop/README.md index 3ec5397a..658bc84a 100644 --- a/examples/optimization/eval_optimize_loop/README.md +++ b/examples/optimization/eval_optimize_loop/README.md @@ -17,10 +17,32 @@ python examples/optimization/eval_optimize_loop/run_pipeline.py \ --scenario improve ``` -The CLI is intentionally self-contained and supports fake mode only. Real mode -is a Python integration point because the pipeline cannot construct an -application-specific agent. Set `execution.mode` to `real`, prepare the run, -and inject an async SDK-compatible `call_agent`: +The offline CLI remains self-contained and supports fake mode only. For an +explicit real integration smoke run, configure the OpenAI-compatible business +model connection in the environment: + +```bash +export TRPC_AGENT_API_KEY=... +export TRPC_AGENT_BASE_URL=... +export TRPC_AGENT_MODEL_NAME=... + +.venv/bin/python examples/optimization/eval_optimize_loop/run_real_integration.py \ + --run-real \ + --optimizer-model-name mimo-v2.5 \ + --max-candidate-proposals 1 +``` + +The business model uses the three environment values above. The reflection +optimizer model is selected explicitly with CLI arguments; its endpoint and +credential reuse the environment without writing their resolved values to run +artifacts. `--run-real` is mandatory so an accidental command cannot spend API +quota. This entry always uses `pipeline.real.json`, where source writeback is +disabled, and reports ACCEPT or REJECT without treating REJECT as a process +failure. + +Applications with a custom agent can still use the Python integration point. +Set `execution.mode` to `real`, prepare the run, and inject an async +SDK-compatible `call_agent`: ```python import asyncio @@ -73,5 +95,6 @@ Run the Stage 1–4 tests with: tests/evaluation/test_eval_optimize_loop_stage2.py \ tests/evaluation/test_eval_optimize_loop_stage3a.py \ tests/evaluation/test_eval_optimize_loop_stage3b.py \ - tests/evaluation/test_eval_optimize_loop_stage4.py + tests/evaluation/test_eval_optimize_loop_stage4.py \ + tests/evaluation/test_eval_optimize_loop_real_integration.py ``` diff --git a/examples/optimization/eval_optimize_loop/candidate_provider.py b/examples/optimization/eval_optimize_loop/candidate_provider.py index bcfe3e1c..4a395e30 100644 --- a/examples/optimization/eval_optimize_loop/candidate_provider.py +++ b/examples/optimization/eval_optimize_loop/candidate_provider.py @@ -23,6 +23,7 @@ from .schemas import CandidateProposal from .schemas import FakeCandidateScenario from .schemas import OptimizerCandidateProposal +from .schemas import OptimizerRuntimeParameters class CandidateProviderError(RuntimeError): @@ -52,6 +53,8 @@ class CandidateRequest: output_dir: Path seed: int retain_native_artifacts: bool = True + runtime_parameters: OptimizerRuntimeParameters | None = None + expected_optimizer_sha256: str | None = None @dataclass(frozen=True) @@ -90,10 +93,85 @@ class AgentOptimizerCandidateProvider: def __init__(self, call_agent: CallAgent) -> None: self._call_agent = call_agent + @staticmethod + def _replace_persisted_connection_values(value: object) -> object: + """递归将可能被 SDK 复制到产物的连接值替换为环境占位符。""" + if isinstance(value, list): + return [ + AgentOptimizerCandidateProvider._replace_persisted_connection_values(item) + for item in value + ] + if not isinstance(value, dict): + return value + placeholders = { + "api_key": "${TRPC_AGENT_API_KEY}", + "apiKey": "${TRPC_AGENT_API_KEY}", + "base_url": "${TRPC_AGENT_BASE_URL}", + "baseUrl": "${TRPC_AGENT_BASE_URL}", + } + return { + key: placeholders.get( + key, + AgentOptimizerCandidateProvider._replace_persisted_connection_values(item), + ) + for key, item in value.items() + } + + @staticmethod + def _prepare_runtime_config(request: CandidateRequest) -> Path: + """由已校验模板生成无明文凭据的本次运行配置。""" + if request.runtime_parameters is None: + return request.optimizer_config_path + + try: + raw = request.optimizer_config_path.read_bytes() + if ( + request.expected_optimizer_sha256 is not None + and sha256(raw).hexdigest() != request.expected_optimizer_sha256 + ): + raise CandidateProviderError("optimizer config changed after preparation") + payload = AgentOptimizerCandidateProvider._replace_persisted_connection_values( + json.loads(raw.decode("utf-8")) + ) + algorithm = payload["optimize"]["algorithm"] + except CandidateProviderError: + raise + except (OSError, UnicodeDecodeError, json.JSONDecodeError, KeyError, TypeError) as exc: + raise CandidateProviderError(f"failed to prepare optimizer runtime config: {exc}") from exc + + parameters = request.runtime_parameters + reflection_lm: dict[str, object] = { + "provider_name": parameters.provider_name, + "model_name": parameters.model_name, + "variant": parameters.variant, + "base_url": "${TRPC_AGENT_BASE_URL}", + "api_key": "${TRPC_AGENT_API_KEY}", + "generation_config": { + "temperature": parameters.temperature, + "max_tokens": parameters.max_tokens, + }, + } + if parameters.think is not None: + reflection_lm["think"] = parameters.think + algorithm["reflection_lm"] = reflection_lm + algorithm["max_candidate_proposals"] = parameters.max_candidate_proposals + + runtime_path = request.output_dir.parent / "optimizer.runtime.json" + try: + runtime_path.parent.mkdir(parents=True, exist_ok=True) + runtime_path.write_text( + json.dumps(payload, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + except OSError as exc: + raise CandidateProviderError(f"failed to write optimizer runtime config: {exc}") from exc + return runtime_path + async def propose(self, request: CandidateRequest) -> CandidateGeneration: + runtime_config_path = self._prepare_runtime_config(request) try: result = await AgentOptimizer.optimize( - config_path=str(request.optimizer_config_path), + config_path=str(runtime_config_path), call_agent=self._call_agent, target_prompt=request.target_prompt, train_dataset_path=str(request.train_evalset_path), diff --git a/examples/optimization/eval_optimize_loop/pipeline.py b/examples/optimization/eval_optimize_loop/pipeline.py index b0933318..3d7cc0a7 100644 --- a/examples/optimization/eval_optimize_loop/pipeline.py +++ b/examples/optimization/eval_optimize_loop/pipeline.py @@ -47,6 +47,7 @@ from .schemas import FakeEvaluationSnapshot from .schemas import FakeStageResult from .schemas import ObservableValue +from .schemas import OptimizerRuntimeParameters from .schemas import ResourceMeasurements from .schemas import RealStageResult from .schemas import WorkspaceSnapshot @@ -536,6 +537,7 @@ async def run_real_stage( prepared: PreparedRun, *, call_agent: CallAgent, + optimizer_parameters: OptimizerRuntimeParameters | None = None, ) -> RealStageResult: """Generate a real optimizer candidate and run the full guarded regression.""" if prepared.config.execution.mode != "real": @@ -595,6 +597,8 @@ async def run_real_stage( output_dir=Path(prepared.workspace.run_dir) / "optimizer", seed=prepared.input_snapshot.seed, retain_native_artifacts=prepared.config.artifacts.retain_optimizer_native_artifacts, + runtime_parameters=optimizer_parameters, + expected_optimizer_sha256=prepared.input_snapshot.optimizer_config_sha256, ) try: generated = await AgentOptimizerCandidateProvider(call_agent).propose(request) diff --git a/examples/optimization/eval_optimize_loop/pipeline.real.json b/examples/optimization/eval_optimize_loop/pipeline.real.json new file mode 100644 index 00000000..1493cec0 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline.real.json @@ -0,0 +1,53 @@ +{ + "config_version": 1, + "execution": { + "mode": "real", + "fake_candidate_scenario": "improve", + "use_fake_judge": false + }, + "inputs": { + "train_evalset": "data/train.evalset.json", + "validation_evalset": "data/val.evalset.json", + "optimizer_config": "optimizer.json" + }, + "prompts": [ + { + "name": "system_prompt", + "path": "prompts/system.md" + } + ], + "run": { + "runs_dir": "runs", + "seed": 42 + }, + "case_labels": { + "hard_case_ids": ["val_knowledge_recall"], + "critical_case_ids": ["val_refund_route"] + }, + "gate": { + "min_validation_score_delta": 0.05, + "reject_on_validation_pass_rate_drop": true, + "reject_new_hard_fail": true, + "reject_critical_regression": true, + "severe_case_score_drop": 0.2, + "required_metrics": ["final_response_avg_score"] + }, + "budget": { + "max_duration_seconds": 180, + "on_unavailable": "warning" + }, + "reporting": { + "write_json": true, + "write_markdown": true, + "include_case_evidence": true + }, + "artifacts": { + "copy_input_files": true, + "retain_optimizer_native_artifacts": true, + "audit_all_candidates": false + }, + "writeback": { + "enabled": false, + "require_source_hash_match": true + } +} diff --git a/examples/optimization/eval_optimize_loop/real_agent.py b/examples/optimization/eval_optimize_loop/real_agent.py new file mode 100644 index 00000000..013ee11e --- /dev/null +++ b/examples/optimization/eval_optimize_loop/real_agent.py @@ -0,0 +1,112 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under the Apache License, Version 2.0. +"""真实业务模型适配器：每个评测 case 都读取当前工作 Prompt。""" + +from __future__ import annotations + +import os +import uuid +from dataclasses import dataclass +from typing import Mapping + +from trpc_agent_sdk.agents import LlmAgent +from trpc_agent_sdk.evaluation import TargetPrompt +from trpc_agent_sdk.models import OpenAIModel +from trpc_agent_sdk.runners import Runner +from trpc_agent_sdk.sessions import InMemorySessionService +from trpc_agent_sdk.types import Content +from trpc_agent_sdk.types import GenerateContentConfig +from trpc_agent_sdk.types import Part + + +@dataclass(frozen=True) +class BusinessModelConfig: + """来自环境变量的业务模型连接信息。""" + + api_key: str + base_url: str + model_name: str + + +def load_business_model_config( + environ: Mapping[str, str] | None = None, +) -> BusinessModelConfig: + """读取业务模型环境变量，缺失时一次性报告全部字段。""" + values = os.environ if environ is None else environ + names = ( + "TRPC_AGENT_API_KEY", + "TRPC_AGENT_BASE_URL", + "TRPC_AGENT_MODEL_NAME", + ) + missing = [name for name in names if not values.get(name, "").strip()] + if missing: + raise ValueError(f"missing required environment variables: {', '.join(missing)}") + return BusinessModelConfig( + api_key=values["TRPC_AGENT_API_KEY"].strip(), + base_url=values["TRPC_AGENT_BASE_URL"].strip(), + model_name=values["TRPC_AGENT_MODEL_NAME"].strip(), + ) + + +def _render_instruction(prompts: dict[str, str]) -> str: + """稳定拼接一个或多个工作 Prompt 字段。""" + if len(prompts) == 1: + return next(iter(prompts.values())) + return "\n\n".join(f"## {name}\n{content}" for name, content in prompts.items()) + + +class RealBusinessAgent: + """以真实模型执行评测，并确保 case 与 Prompt 版本相互隔离。""" + + def __init__(self, target_prompt: TargetPrompt, config: BusinessModelConfig) -> None: + self._target_prompt = target_prompt + self._config = config + + async def call_agent(self, query: str) -> str: + """重新读取工作 Prompt，运行独立 session，只返回正式最终文本。""" + prompts = await self._target_prompt.read_all() + model = OpenAIModel( + model_name=self._config.model_name, + api_key=self._config.api_key, + base_url=self._config.base_url, + ) + root_agent = LlmAgent( + name="eval_optimize_real_agent", + description="真实模型驱动的评测与 Prompt 优化示例 Agent。", + model=model, + instruction=_render_instruction(prompts), + generate_content_config=GenerateContentConfig( + temperature=0.0, + max_output_tokens=512, + ), + ) + session_service = InMemorySessionService() + runner = Runner( + app_name="eval_optimize_real_integration", + agent=root_agent, + session_service=session_service, + ) + session_id = str(uuid.uuid4()) + user_id = "real-integration" + await session_service.create_session( + app_name="eval_optimize_real_integration", + user_id=user_id, + session_id=session_id, + state={}, + ) + message = Content(role="user", parts=[Part.from_text(text=query)]) + final_text = "" + async for event in runner.run_async( + user_id=user_id, + session_id=session_id, + new_message=message, + ): + if not event.is_final_response() or not event.content or not event.content.parts: + continue + for part in event.content.parts: + if not part.thought and part.text: + final_text += part.text + return final_text.strip() diff --git a/examples/optimization/eval_optimize_loop/run_real_integration.py b/examples/optimization/eval_optimize_loop/run_real_integration.py new file mode 100644 index 00000000..9a3c605b --- /dev/null +++ b/examples/optimization/eval_optimize_loop/run_real_integration.py @@ -0,0 +1,140 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under the Apache License, Version 2.0. +"""显式运行真实业务模型与真实反思模型的完整回归闭环。""" + +from __future__ import annotations + +import argparse +import asyncio +import sys +from pathlib import Path + +from pydantic import ValidationError + +_HERE = Path(__file__).resolve().parent +if __package__ in (None, ""): + _REPO_ROOT = _HERE.parents[2] + if str(_REPO_ROOT) not in sys.path: + sys.path.insert(0, str(_REPO_ROOT)) + from examples.optimization.eval_optimize_loop.config import load_pipeline_config + from examples.optimization.eval_optimize_loop.pipeline import prepare_run + from examples.optimization.eval_optimize_loop.pipeline import run_real_stage + from examples.optimization.eval_optimize_loop.real_agent import BusinessModelConfig + from examples.optimization.eval_optimize_loop.real_agent import RealBusinessAgent + from examples.optimization.eval_optimize_loop.real_agent import load_business_model_config + from examples.optimization.eval_optimize_loop.schemas import OptimizerRuntimeParameters +else: + from .config import load_pipeline_config + from .pipeline import prepare_run + from .pipeline import run_real_stage + from .real_agent import BusinessModelConfig + from .real_agent import RealBusinessAgent + from .real_agent import load_business_model_config + from .schemas import OptimizerRuntimeParameters + + +def _think_value(value: str) -> bool | None: + return {"auto": None, "on": True, "off": False}[value] + + +def _format_snapshot(label: str, snapshot: object) -> str: + score = getattr(snapshot, "average_score", None) + score_text = "unavailable" if score is None else f"{score:.3f}" + return ( + f"{label}: {snapshot.passed_case_count}/{snapshot.total_case_count} passed, " + f"average score={score_text}" + ) + + +def _build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Run the explicitly enabled real-model pipeline.") + parser.add_argument("--run-real", action="store_true", help="Confirm that real API calls are intended.") + parser.add_argument("--config", type=Path, default=_HERE / "pipeline.real.json") + parser.add_argument("--run-id") + parser.add_argument("--optimizer-model-name", required=True) + parser.add_argument("--optimizer-provider-name", default="openai") + parser.add_argument("--optimizer-temperature", type=float, default=0.8) + parser.add_argument("--optimizer-max-tokens", type=int, default=4096) + parser.add_argument("--optimizer-think", choices=("auto", "on", "off"), default="auto") + parser.add_argument("--max-candidate-proposals", type=int, default=1) + return parser + + +async def _run( + args: argparse.Namespace, + business_config: BusinessModelConfig, + parameters: OptimizerRuntimeParameters, +): + prepared = prepare_run(args.config, run_id=args.run_id) + source_before = await prepared.source_target.read_all() + agent = RealBusinessAgent(prepared.working_target, business_config) + try: + result = await run_real_stage( + prepared, + call_agent=agent.call_agent, + optimizer_parameters=parameters, + ) + except Exception as exc: + source_after = await prepared.source_target.read_all() + if source_after != source_before: + raise RuntimeError( + "source Prompt changed during a failed real integration run" + ) from exc + raise + source_after = await prepared.source_target.read_all() + if source_after != source_before: + raise RuntimeError("source Prompt changed even though real integration writeback is disabled") + return prepared, result + + +def main() -> int: + parser = _build_parser() + args = parser.parse_args() + if not args.run_real: + parser.error("real API calls require explicit --run-real confirmation") + try: + business_config = load_business_model_config() + parameters = OptimizerRuntimeParameters( + provider_name=args.optimizer_provider_name, + model_name=args.optimizer_model_name, + temperature=args.optimizer_temperature, + max_tokens=args.optimizer_max_tokens, + think=_think_value(args.optimizer_think), + max_candidate_proposals=args.max_candidate_proposals, + ) + config = load_pipeline_config(args.config) + except (OSError, ValueError, ValidationError) as exc: + parser.error(str(exc)) + if config.execution.mode != "real": + parser.error("real integration requires execution.mode='real'") + if config.writeback.enabled: + parser.error("real integration requires writeback.enabled=false") + + try: + prepared, result = asyncio.run(_run(args, business_config, parameters)) + except Exception as exc: + print(f"Real integration failed: {exc}", file=sys.stderr) + return 1 + + print(f"Completed real-model pipeline: {prepared.workspace.run_dir}") + print(_format_snapshot("Baseline train", result.baseline_train)) + print(_format_snapshot("Baseline validation", result.baseline_validation)) + print(_format_snapshot("Candidate train", result.candidate_train)) + print(_format_snapshot("Candidate validation", result.candidate_validation)) + print( + f"Optimizer: {result.optimize_result.status}, " + f"rounds={result.optimize_result.total_rounds}, candidate={result.candidate.candidate_id}" + ) + print(f"Gate decision: {result.gate_decision.decision.upper()}") + for reason in result.gate_decision.rejection_reasons: + print(f"- {reason}") + print(f"Writeback: {result.writeback.status.upper()} ({result.writeback.reason})") + print("Source Prompt unchanged: yes") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/optimization/eval_optimize_loop/schemas.py b/examples/optimization/eval_optimize_loop/schemas.py index 7b440b4a..9e7de77b 100644 --- a/examples/optimization/eval_optimize_loop/schemas.py +++ b/examples/optimization/eval_optimize_loop/schemas.py @@ -17,6 +17,7 @@ from typing import Optional from pydantic import Field +from pydantic import field_validator from pydantic import model_validator from trpc_agent_sdk.evaluation import EvalBaseModel @@ -72,6 +73,25 @@ ] +class OptimizerRuntimeParameters(EvalBaseModel): + """命令行显式传入的反思优化模型参数，不包含任何凭据。""" + + provider_name: str = "openai" + model_name: str + variant: str = "" + temperature: float = Field(default=0.8, ge=0.0, allow_inf_nan=False) + max_tokens: int = Field(default=4096, gt=0) + think: Optional[bool] = None + max_candidate_proposals: int = Field(default=1, gt=0) + + @field_validator("provider_name", "model_name") + @classmethod + def _require_non_empty_model_identity(cls, value: str) -> str: + if not value.strip(): + raise ValueError("model identity must not be empty") + return value.strip() + + class ObservableValue(EvalBaseModel): """A measurement whose absence is explicit rather than silently zero.""" diff --git a/tests/evaluation/test_eval_optimize_loop_real_integration.py b/tests/evaluation/test_eval_optimize_loop_real_integration.py new file mode 100644 index 00000000..fe258b0d --- /dev/null +++ b/tests/evaluation/test_eval_optimize_loop_real_integration.py @@ -0,0 +1,388 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under the Apache License, Version 2.0. +"""真实业务 Agent 与真实反思模型集成入口的离线契约测试。""" + +from __future__ import annotations + +import json +import shutil +import subprocess +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from examples.optimization.eval_optimize_loop.candidate_provider import ( + AgentOptimizerCandidateProvider, +) +from examples.optimization.eval_optimize_loop.candidate_provider import CandidateRequest +from examples.optimization.eval_optimize_loop.pipeline import prepare_run +from examples.optimization.eval_optimize_loop.real_agent import BusinessModelConfig +from examples.optimization.eval_optimize_loop.real_agent import RealBusinessAgent +from examples.optimization.eval_optimize_loop.real_agent import load_business_model_config +from examples.optimization.eval_optimize_loop import run_real_integration +from examples.optimization.eval_optimize_loop.schemas import OptimizerRuntimeParameters +from trpc_agent_sdk.evaluation import OptimizeResult + + +_REPO_ROOT = Path(__file__).resolve().parents[2] +_EXAMPLE = _REPO_ROOT / "examples" / "optimization" / "eval_optimize_loop" + + +def _copy_example(tmp_path: Path) -> Path: + target = tmp_path / "eval_optimize_loop" + shutil.copytree(_EXAMPLE, target, ignore=shutil.ignore_patterns("runs", "__pycache__")) + return target + + +def _result(baseline: dict[str, str]) -> OptimizeResult: + best = {name: f"{value}\noptimized" for name, value in baseline.items()} + return OptimizeResult( + algorithm="gepa_reflective", + status="SUCCEEDED", + finish_reason="completed", + baseline_pass_rate=0.3, + best_pass_rate=0.7, + pass_rate_improvement=0.4, + baseline_prompts=baseline, + best_prompts=best, + total_rounds=1, + rounds=[], + total_reflection_lm_calls=1, + total_llm_cost=0.0, + total_token_usage={"prompt": 0, "completion": 0, "total": 0}, + duration_seconds=0.1, + started_at="2026-07-18T00:00:00Z", + finished_at="2026-07-18T00:00:01Z", + ) + + +def test_optimizer_runtime_parameters_are_serializable_and_validated(): + parameters = OptimizerRuntimeParameters( + model_name="mimo-v2.5", + temperature=0.2, + max_tokens=2048, + think=False, + max_candidate_proposals=1, + ) + + assert OptimizerRuntimeParameters.model_validate_json(parameters.model_dump_json()) == parameters + with pytest.raises(ValueError): + OptimizerRuntimeParameters(model_name="", max_candidate_proposals=1) + with pytest.raises(ValueError): + OptimizerRuntimeParameters(model_name="model", max_candidate_proposals=0) + with pytest.raises(ValueError): + OptimizerRuntimeParameters(model_name="model", temperature=float("inf")) + + +def test_business_model_config_requires_all_three_environment_values(): + with pytest.raises(ValueError) as exc_info: + load_business_model_config({}) + + message = str(exc_info.value) + assert "TRPC_AGENT_API_KEY" in message + assert "TRPC_AGENT_BASE_URL" in message + assert "TRPC_AGENT_MODEL_NAME" in message + + normalized = load_business_model_config( + { + "TRPC_AGENT_API_KEY": " key ", + "TRPC_AGENT_BASE_URL": " https://example.test ", + "TRPC_AGENT_MODEL_NAME": " model ", + } + ) + assert normalized == BusinessModelConfig( + api_key="key", + base_url="https://example.test", + model_name="model", + ) + + +@pytest.mark.asyncio +async def test_real_business_agent_rereads_prompts_and_returns_only_final_non_thought_text( + monkeypatch: pytest.MonkeyPatch, +): + prompt_versions = [ + {"system_prompt": "baseline instruction"}, + {"system_prompt": "candidate instruction"}, + ] + + class FakeTarget: + async def read_all(self): + return prompt_versions.pop(0) + + class FakePart: + def __init__(self, text: str, *, thought: bool = False): + self.text = text + self.thought = thought + + class FakeEvent: + def __init__(self, final: bool, parts: list[FakePart]): + self._final = final + self.content = type("Content", (), {"parts": parts})() + + def is_final_response(self): + return self._final + + captured_instructions: list[str] = [] + + class FakeLlmAgent: + def __init__(self, **kwargs): + captured_instructions.append(kwargs["instruction"]) + + class FakeSessionService: + async def create_session(self, **kwargs): + return None + + class FakeRunner: + def __init__(self, **kwargs): + pass + + async def run_async(self, **kwargs): + yield FakeEvent(False, [FakePart("intermediate")]) + yield FakeEvent(True, [FakePart("thinking", thought=True), FakePart("answer")]) + + monkeypatch.setattr("examples.optimization.eval_optimize_loop.real_agent.OpenAIModel", lambda **kwargs: object()) + monkeypatch.setattr("examples.optimization.eval_optimize_loop.real_agent.LlmAgent", FakeLlmAgent) + monkeypatch.setattr( + "examples.optimization.eval_optimize_loop.real_agent.InMemorySessionService", + FakeSessionService, + ) + monkeypatch.setattr("examples.optimization.eval_optimize_loop.real_agent.Runner", FakeRunner) + agent = RealBusinessAgent( + FakeTarget(), + BusinessModelConfig(api_key="secret", base_url="https://example.test", model_name="model"), + ) + + assert await agent.call_agent("first") == "answer" + assert await agent.call_agent("second") == "answer" + assert captured_instructions == ["baseline instruction", "candidate instruction"] + + +@pytest.mark.asyncio +async def test_provider_writes_runtime_config_without_persisting_secrets( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + root = _copy_example(tmp_path) + prepared = prepare_run(root / "pipeline.json", run_id="runtime_config") + baseline = await prepared.working_target.read_all() + original_config = Path(prepared.input_snapshot.optimizer_config_path).read_text(encoding="utf-8") + captured: dict[str, object] = {} + + async def fake_optimize(**kwargs): + captured.update(kwargs) + output_dir = Path(str(kwargs["output_dir"])) + output_dir.mkdir(parents=True, exist_ok=True) + shutil.copyfile(str(kwargs["config_path"]), output_dir / "config.snapshot.json") + return _result(baseline) + + monkeypatch.setattr( + "examples.optimization.eval_optimize_loop.candidate_provider.AgentOptimizer.optimize", + fake_optimize, + ) + request = CandidateRequest( + current_prompts=baseline, + target_prompt=prepared.working_target, + optimizer_config_path=Path(prepared.input_snapshot.optimizer_config_path), + train_evalset_path=Path(prepared.input_snapshot.train_evalset_path), + validation_evalset_path=Path(prepared.input_snapshot.validation_evalset_path), + output_dir=Path(prepared.workspace.run_dir) / "optimizer", + seed=42, + runtime_parameters=OptimizerRuntimeParameters( + provider_name="openai", + model_name="mimo-v2.5", + variant="chat_completions", + temperature=0.25, + max_tokens=1024, + think=True, + max_candidate_proposals=1, + ), + ) + + template = json.loads(request.optimizer_config_path.read_text(encoding="utf-8")) + template["evaluate"]["credential_probe"] = { + "api_key": "judge-secret", + "base_url": "https://judge.example.test", + } + request.optimizer_config_path.write_text(json.dumps(template), encoding="utf-8") + + await AgentOptimizerCandidateProvider(lambda query: query).propose(request) + + runtime_path = Path(str(captured["config_path"])) + runtime = json.loads(runtime_path.read_text(encoding="utf-8")) + reflection = runtime["optimize"]["algorithm"]["reflection_lm"] + assert runtime_path == Path(prepared.workspace.run_dir) / "optimizer.runtime.json" + assert reflection == { + "provider_name": "openai", + "model_name": "mimo-v2.5", + "variant": "chat_completions", + "base_url": "${TRPC_AGENT_BASE_URL}", + "api_key": "${TRPC_AGENT_API_KEY}", + "generation_config": {"temperature": 0.25, "max_tokens": 1024}, + "think": True, + } + assert runtime["optimize"]["algorithm"]["max_candidate_proposals"] == 1 + assert runtime["evaluate"]["credential_probe"] == { + "api_key": "${TRPC_AGENT_API_KEY}", + "base_url": "${TRPC_AGENT_BASE_URL}", + } + runtime_text = runtime_path.read_text(encoding="utf-8") + snapshot_text = (request.output_dir / "config.snapshot.json").read_text(encoding="utf-8") + assert "judge-secret" not in runtime_text + assert "judge-secret" not in snapshot_text + assert original_config != request.optimizer_config_path.read_text(encoding="utf-8") + + +@pytest.mark.asyncio +async def test_run_checks_source_prompt_even_when_pipeline_fails(monkeypatch: pytest.MonkeyPatch): + class SourceTarget: + reads = 0 + + async def read_all(self): + self.reads += 1 + return {"system_prompt": "before" if self.reads == 1 else "drifted"} + + prepared = SimpleNamespace( + source_target=SourceTarget(), + working_target=object(), + ) + + monkeypatch.setattr(run_real_integration, "prepare_run", lambda *args, **kwargs: prepared) + monkeypatch.setattr(run_real_integration, "RealBusinessAgent", lambda *args, **kwargs: object()) + + async def fail_stage(*args, **kwargs): + raise RuntimeError("pipeline failed") + + monkeypatch.setattr(run_real_integration, "run_real_stage", fail_stage) + args = SimpleNamespace(config=Path("pipeline.real.json"), run_id="failure") + + with pytest.raises(RuntimeError, match="source Prompt changed"): + await run_real_integration._run( + args, + BusinessModelConfig(api_key="key", base_url="url", model_name="model"), + OptimizerRuntimeParameters(model_name="model"), + ) + + +def test_real_cli_complete_gate_reject_still_exits_zero( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +): + snapshot = SimpleNamespace(passed_case_count=1, total_case_count=3, average_score=1 / 3) + result = SimpleNamespace( + baseline_train=snapshot, + baseline_validation=snapshot, + candidate_train=snapshot, + candidate_validation=snapshot, + optimize_result=SimpleNamespace(status="SUCCEEDED", total_rounds=1), + candidate=SimpleNamespace(candidate_id="optimizer-123456789abc"), + gate_decision=SimpleNamespace(decision="reject", rejection_reasons=["no improvement"]), + writeback=SimpleNamespace(status="skipped", reason="gate_rejected"), + ) + prepared = SimpleNamespace(workspace=SimpleNamespace(run_dir="runs/offline")) + + async def complete(*args, **kwargs): + return prepared, result + + monkeypatch.setenv("TRPC_AGENT_API_KEY", "key") + monkeypatch.setenv("TRPC_AGENT_BASE_URL", "https://example.test") + monkeypatch.setenv("TRPC_AGENT_MODEL_NAME", "business-model") + monkeypatch.setattr(run_real_integration, "_run", complete) + monkeypatch.setattr( + sys, + "argv", + ["run_real_integration.py", "--run-real", "--optimizer-model-name", "optimizer-model"], + ) + + assert run_real_integration.main() == 0 + output = capsys.readouterr().out + assert "Gate decision: REJECT" in output + assert "Baseline train" in output + assert "Baseline validation" in output + assert "Candidate train" in output + assert "Candidate validation" in output + + +def test_real_cli_pipeline_error_exits_one( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +): + async def fail(*args, **kwargs): + raise RuntimeError("offline failure") + + monkeypatch.setenv("TRPC_AGENT_API_KEY", "key") + monkeypatch.setenv("TRPC_AGENT_BASE_URL", "https://example.test") + monkeypatch.setenv("TRPC_AGENT_MODEL_NAME", "business-model") + monkeypatch.setattr(run_real_integration, "_run", fail) + monkeypatch.setattr( + sys, + "argv", + ["run_real_integration.py", "--run-real", "--optimizer-model-name", "optimizer-model"], + ) + + assert run_real_integration.main() == 1 + assert "offline failure" in capsys.readouterr().err + + +def test_real_cli_requires_explicit_confirmation_before_creating_workspace(tmp_path: Path): + root = _copy_example(tmp_path) + completed = subprocess.run( + [ + sys.executable, + str(root / "run_real_integration.py"), + "--config", + str(root / "pipeline.real.json"), + "--optimizer-model-name", + "mimo-v2.5", + "--run-id", + "must_not_exist", + ], + cwd=_REPO_ROOT, + capture_output=True, + text=True, + check=False, + ) + + assert completed.returncode == 2 + assert "--run-real" in completed.stderr + assert not (root / "runs" / "must_not_exist").exists() + + +def test_real_cli_requires_business_model_environment_before_creating_workspace( + tmp_path: Path, +): + root = _copy_example(tmp_path) + env = { + key: value + for key, value in __import__("os").environ.items() + if key not in {"TRPC_AGENT_API_KEY", "TRPC_AGENT_BASE_URL", "TRPC_AGENT_MODEL_NAME"} + } + completed = subprocess.run( + [ + sys.executable, + str(root / "run_real_integration.py"), + "--run-real", + "--config", + str(root / "pipeline.real.json"), + "--optimizer-model-name", + "mimo-v2.5", + "--run-id", + "missing_env", + ], + cwd=_REPO_ROOT, + env=env, + capture_output=True, + text=True, + check=False, + ) + + assert completed.returncode == 2 + assert "TRPC_AGENT_API_KEY" in completed.stderr + assert "TRPC_AGENT_BASE_URL" in completed.stderr + assert "TRPC_AGENT_MODEL_NAME" in completed.stderr + assert not (root / "runs" / "missing_env").exists() From a790aa8f27d4cab124efc99077915705992f1dd4 Mon Sep 17 00:00:00 2001 From: guocfu Date: Sat, 18 Jul 2026 20:40:42 +0800 Subject: [PATCH 07/18] =?UTF-8?q?chore:=20=E5=BF=BD=E7=95=A5=E6=9C=AC?= =?UTF-8?q?=E5=9C=B0=E5=BC=80=E5=8F=91=E5=B7=A5=E4=BD=9C=E6=A0=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 8c703b63..2f95285f 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,6 @@ pyrightconfig.json # Generated artifacts for the evaluation/optimization loop example. examples/optimization/eval_optimize_loop/runs/ + +# Local isolated development worktrees. +/.worktrees/ From 21cd6cabe5bed72e25aad5d8d556e808953e905b Mon Sep 17 00:00:00 2001 From: guocfu Date: Sat, 18 Jul 2026 21:02:06 +0800 Subject: [PATCH 08/18] =?UTF-8?q?feat(evaluation):=20=E5=A2=9E=E5=8A=A0?= =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=8A=A5=E5=91=8A=E6=A8=A1=E5=9E=8B=E4=B8=8E?= =?UTF-8?q?=E6=B8=B2=E6=9F=93?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../eval_optimize_loop/report_builder.py | 99 ++++++++++++ .../eval_optimize_loop/schemas.py | 108 +++++++++++++ ...val_optimize_loop_stage5_report_builder.py | 150 ++++++++++++++++++ 3 files changed, 357 insertions(+) create mode 100644 examples/optimization/eval_optimize_loop/report_builder.py create mode 100644 tests/evaluation/test_eval_optimize_loop_stage5_report_builder.py diff --git a/examples/optimization/eval_optimize_loop/report_builder.py b/examples/optimization/eval_optimize_loop/report_builder.py new file mode 100644 index 00000000..d4b3b11f --- /dev/null +++ b/examples/optimization/eval_optimize_loop/report_builder.py @@ -0,0 +1,99 @@ +"""Pure builders for serializable optimization reports.""" + +from __future__ import annotations + +from datetime import datetime +from typing import TYPE_CHECKING + +from .schemas import FailureReport, OptimizerResourceObservation, OptimizationReport +from .schemas import PipelineStageResult, RealStageResult, ReportProgress + +if TYPE_CHECKING: + from .pipeline import PreparedRun + +_OPTIMIZER_SCOPE = ( + "Optimizer-only observation; excludes complete business Agent evaluation usage." +) + +def _optimizer_resources(result: PipelineStageResult) -> OptimizerResourceObservation: + if not isinstance(result, RealStageResult): + return OptimizerResourceObservation( + status="not_applicable", scope_note="Fake mode does not run AgentOptimizer.", + ) + native = result.optimize_result + return OptimizerResourceObservation( + status="available", scope_note=_OPTIMIZER_SCOPE, total_rounds=native.total_rounds, + reflection_lm_calls=native.total_reflection_lm_calls, cost_usd=native.total_llm_cost, + token_usage=dict(native.total_token_usage), duration_seconds=native.duration_seconds, + ) + +def build_optimization_report( + prepared: PreparedRun, result: PipelineStageResult, *, progress: ReportProgress, finished_at: datetime, +) -> OptimizationReport: + return OptimizationReport( + run_id=prepared.workspace.run_id, execution_mode=prepared.config.execution.mode, + seed=prepared.input_snapshot.seed, started_at=progress.started_at, finished_at=finished_at, + input_snapshot=prepared.input_snapshot, candidate=result.candidate, + baseline_train=result.baseline_train, baseline_validation=result.baseline_validation, + candidate_train=result.candidate_train, candidate_validation=result.candidate_validation, + analysis=result.analysis, pipeline_resources=result.measurements, + optimizer_resources=_optimizer_resources(result), gate_decision=result.gate_decision, + writeback=result.writeback, + ) + +def build_failure_report( + prepared: PreparedRun, *, progress: ReportProgress, error: Exception, + source_prompt_hashes: dict[str, str], existing_artifacts: list[str], generated_at: datetime, +) -> FailureReport: + return FailureReport( + run_id=prepared.workspace.run_id, execution_mode=prepared.config.execution.mode, + failed_phase=progress.current_phase, exception_type=type(error).__name__, + error_message=str(error), generated_at=generated_at, input_snapshot=prepared.input_snapshot, + source_prompt_hashes=dict(sorted(source_prompt_hashes.items())), + completed_phases=progress.completed_phases, existing_artifacts=sorted(existing_artifacts), + ) + + +def render_optimization_markdown(report: OptimizationReport) -> str: + decision = report.gate_decision.decision.upper() + lines = [ + "# Optimization Report", + "", + f"- Run: `{report.run_id}`", + f"- Mode: `{report.execution_mode}`", + f"- Gate decision: {decision}", + f"- Candidate: `{report.candidate.candidate_id}`", + "", + "## Full Evaluations", + "", + ] + for label, snapshot in ( + ("Baseline train", report.baseline_train), + ("Baseline validation", report.baseline_validation), + ("Candidate train", report.candidate_train), + ("Candidate validation", report.candidate_validation), + ): + score = snapshot.average_score if snapshot.average_score is not None else "unavailable" + lines.append( + f"- {label}: {snapshot.passed_case_count}/{snapshot.total_case_count} passed; " + f"average score={score}" + ) + lines.extend(["", "## Gate", ""]) + lines.extend(f"- Rejection: {reason}" for reason in report.gate_decision.rejection_reasons) + lines.extend(f"- Warning: {warning}" for warning in report.gate_decision.warnings) + if not report.gate_decision.rejection_reasons and not report.gate_decision.warnings: + lines.append("- No rejection reasons or warnings.") + lines.extend(["", "## Candidate Changes", ""]) + changed = report.candidate.changed_fields or ["none"] + lines.extend(f"- {field}" for field in changed) + lines.extend(["", "## Overfit", f"- Status: {report.analysis.overfit_status}", + f"- Reason: {report.analysis.overfit_reason}", "", "## Writeback", + f"- Status: {report.writeback.status}", f"- Reason: {report.writeback.reason}", + "", "## Pipeline Observations", + f"- Cost: {report.pipeline_resources.cost_usd.status}", + f"- Tokens: {report.pipeline_resources.total_tokens.status}", + f"- Duration: {report.pipeline_resources.duration_seconds.status}", + "", "## Optimizer Scope", + f"- Status: {report.optimizer_resources.status}", + f"- {report.optimizer_resources.scope_note}"]) + return "\n".join(lines) + "\n" diff --git a/examples/optimization/eval_optimize_loop/schemas.py b/examples/optimization/eval_optimize_loop/schemas.py index 9e7de77b..fa1c1b37 100644 --- a/examples/optimization/eval_optimize_loop/schemas.py +++ b/examples/optimization/eval_optimize_loop/schemas.py @@ -12,9 +12,11 @@ from __future__ import annotations +from datetime import datetime from typing import Any from typing import Literal from typing import Optional +from typing import Union from pydantic import Field from pydantic import field_validator @@ -416,3 +418,109 @@ class RealStageResult(PipelineStageResult): candidate: OptimizerCandidateProposal optimize_result: OptimizeResult + +ReportPhase = Literal[ + "baseline_train", "baseline_validation", "candidate_generation", "candidate_train", + "candidate_validation", "analysis", "gate", "writeback", "reporting", +] + +class ReportProgress(EvalBaseModel): + started_at: datetime + current_phase: ReportPhase + completed_phases: list[ReportPhase] = Field(default_factory=list) + + @model_validator(mode="after") + def _validate_phases(self) -> "ReportProgress": + if len(self.completed_phases) != len(set(self.completed_phases)): + raise ValueError("completed phases must not contain duplicates") + if self.current_phase in self.completed_phases: + raise ValueError("completed phases must not include current phase") + return self + +class OptimizerResourceObservation(EvalBaseModel): + status: Literal["available", "unavailable", "not_applicable"] + scope_note: str + total_rounds: Optional[int] = Field(default=None, ge=0) + reflection_lm_calls: Optional[int] = Field(default=None, ge=0) + cost_usd: Optional[float] = Field(default=None, ge=0.0) + token_usage: Optional[dict[str, int]] = None + duration_seconds: Optional[float] = Field(default=None, ge=0.0) + + @model_validator(mode="after") + def _validate_status(self) -> "OptimizerResourceObservation": + values = (self.total_rounds, self.reflection_lm_calls, self.cost_usd, self.token_usage, self.duration_seconds) + if self.status == "available" and any(value is None for value in values): + raise ValueError("available optimizer observations require all resource values") + if self.status != "available" and any(value is not None for value in values): + raise ValueError("non-available optimizer observations must not carry resource values") + return self + +class ArtifactReference(EvalBaseModel): + artifact_id: str + artifact_type: Literal["input", "prompt", "evaluation", "candidate", "optimizer_native", "report"] + relative_path: Optional[str] = None + required: bool + produced_by: ReportPhase + status: Literal["available", "unavailable"] + size_bytes: Optional[int] = Field(default=None, ge=0) + sha256: Optional[str] = Field(default=None, pattern=r"^[0-9a-f]{64}$") + unavailable_reason: Optional[str] = None + + @model_validator(mode="after") + def _validate_status(self) -> "ArtifactReference": + if self.status == "available": + if self.relative_path is None or self.size_bytes is None or self.sha256 is None or self.unavailable_reason is not None: + raise ValueError("available artifacts require path, size, hash, and no unavailable reason") + elif self.unavailable_reason is None or self.size_bytes is not None or self.sha256 is not None: + raise ValueError("unavailable artifacts require a reason and no size or hash") + return self + +class ArtifactIndex(EvalBaseModel): + schema_version: Literal[1] = 1 + run_id: str + generated_at: datetime + artifacts: list[ArtifactReference] + + @model_validator(mode="after") + def _validate_artifacts(self) -> "ArtifactIndex": + artifact_ids = [artifact.artifact_id for artifact in self.artifacts] + paths = [artifact.relative_path for artifact in self.artifacts if artifact.relative_path is not None] + if len(artifact_ids) != len(set(artifact_ids)): + raise ValueError("artifact IDs must be unique") + if len(paths) != len(set(paths)): + raise ValueError("artifact relative paths must be unique") + return self + +class OptimizationReport(EvalBaseModel): + schema_version: Literal[1] = 1 + status: Literal["completed"] = "completed" + run_id: str + execution_mode: Literal["fake", "real"] + seed: int + started_at: datetime + finished_at: datetime + input_snapshot: InputSnapshot + candidate: Union[FakeCandidateProposal, OptimizerCandidateProposal] + baseline_train: FakeEvaluationSnapshot + baseline_validation: FakeEvaluationSnapshot + candidate_train: FakeEvaluationSnapshot + candidate_validation: FakeEvaluationSnapshot + analysis: EvaluationAnalysis + pipeline_resources: ResourceMeasurements + optimizer_resources: OptimizerResourceObservation + gate_decision: GateDecision + writeback: WritebackResult + +class FailureReport(EvalBaseModel): + schema_version: Literal[1] = 1 + status: Literal["failed"] = "failed" + run_id: str + execution_mode: Literal["fake", "real"] + failed_phase: ReportPhase + exception_type: str + error_message: str + generated_at: datetime + input_snapshot: InputSnapshot + source_prompt_hashes: dict[str, str] + completed_phases: list[ReportPhase] + existing_artifacts: list[str] diff --git a/tests/evaluation/test_eval_optimize_loop_stage5_report_builder.py b/tests/evaluation/test_eval_optimize_loop_stage5_report_builder.py new file mode 100644 index 00000000..020d47d2 --- /dev/null +++ b/tests/evaluation/test_eval_optimize_loop_stage5_report_builder.py @@ -0,0 +1,150 @@ +"""Tests for Stage 5 serializable optimization reports.""" + +from datetime import datetime, timezone +import shutil +from pathlib import Path + +import pytest + +from examples.optimization.eval_optimize_loop.pipeline import prepare_run, run_fake_stage +from examples.optimization.eval_optimize_loop.report_builder import ( + build_failure_report, build_optimization_report, render_optimization_markdown, +) +from examples.optimization.eval_optimize_loop.schemas import ( + ArtifactIndex, ArtifactReference, OptimizationReport, OptimizerResourceObservation, ReportProgress, +) +from examples.optimization.eval_optimize_loop.schemas import OptimizerCandidateProposal, RealStageResult +from trpc_agent_sdk.evaluation import OptimizeResult + +_ROOT = Path(__file__).resolve().parents[2] +_EXAMPLE = _ROOT / "examples" / "optimization" / "eval_optimize_loop" + +def _copy_example(tmp_path: Path, name: str = "eval_optimize_loop") -> Path: + target = tmp_path / name + shutil.copytree(_EXAMPLE, target, ignore=shutil.ignore_patterns("runs", "__pycache__")) + return target + +def _progress() -> ReportProgress: + return ReportProgress( + started_at=datetime(2026, 7, 18, tzinfo=timezone.utc), current_phase="reporting", + completed_phases=[ + "baseline_train", "baseline_validation", "candidate_generation", + "candidate_train", "candidate_validation", "analysis", "gate", "writeback", + ], + ) + + +def _optimize_result(baseline_prompts: dict[str, str], best_prompts: dict[str, str]) -> OptimizeResult: + return OptimizeResult( + algorithm="gepa_reflective", status="SUCCEEDED", finish_reason="completed", error_message="", + baseline_pass_rate=1 / 3, best_pass_rate=1.0, pass_rate_improvement=2 / 3, + baseline_prompts=baseline_prompts, best_prompts=best_prompts, total_rounds=1, rounds=[], + total_reflection_lm_calls=1, total_llm_cost=0.25, + total_token_usage={"prompt": 10, "completion": 5, "total": 15}, duration_seconds=0.5, + started_at="2026-07-17T00:00:00Z", finished_at="2026-07-17T00:00:01Z", + ) + +@pytest.mark.asyncio +async def test_fake_report_is_serializable_and_marks_optimizer_resources_not_applicable(tmp_path): + root = _copy_example(tmp_path) + prepared = prepare_run(root / "pipeline.json", run_id="report_fake") + report = build_optimization_report( + prepared, await run_fake_stage(prepared, scenario="improve"), progress=_progress(), + finished_at=datetime(2026, 7, 18, 0, 1, tzinfo=timezone.utc), + ) + assert report.status == "completed" + assert report.execution_mode == "fake" + assert report.optimizer_resources.status == "not_applicable" + assert report.pipeline_resources.total_tokens.status == "unavailable" + assert OptimizationReport.model_validate_json(report.model_dump_json()) == report + + +@pytest.mark.asyncio +async def test_real_report_includes_optimizer_resource_observations(tmp_path): + root = _copy_example(tmp_path) + prepared = prepare_run(root / "pipeline.json", run_id="report_real") + fake_result = await run_fake_stage(prepared, scenario="improve") + candidate = OptimizerCandidateProposal.model_validate({ + **fake_result.candidate.model_dump(exclude={"scenario", "seed"}), "provider": "agent_optimizer", + "optimizer_status": "SUCCEEDED", "finish_reason": "completed", + "baseline_pass_rate": 1 / 3, "best_pass_rate": 1.0, + "candidate_id": f"optimizer-{fake_result.candidate.candidate_id[-12:]}", + }) + result = RealStageResult( + **fake_result.model_dump(exclude={"scenario", "candidate"}), candidate=candidate, + optimize_result=_optimize_result(await prepared.source_target.read_all(), candidate.prompts), + ) + report = build_optimization_report( + prepared, result, progress=_progress(), + finished_at=datetime(2026, 7, 18, 0, 1, tzinfo=timezone.utc), + ) + assert report.optimizer_resources.status == "available" + assert report.optimizer_resources.total_rounds == 1 + assert report.optimizer_resources.reflection_lm_calls == 1 + assert report.optimizer_resources.scope_note == ( + "Optimizer-only observation; excludes complete business Agent evaluation usage." + ) + +@pytest.mark.asyncio +async def test_markdown_includes_accept_and_reject_report_evidence(tmp_path): + root = _copy_example(tmp_path) + prepared = prepare_run(root / "pipeline.json", run_id="report_markdown") + accepted = build_optimization_report( + prepared, await run_fake_stage(prepared, scenario="improve"), progress=_progress(), + finished_at=datetime(2026, 7, 18, 0, 1, tzinfo=timezone.utc), + ) + rejected_root = _copy_example(tmp_path, "eval_optimize_loop_reject") + rejected_prepared = prepare_run(rejected_root / "pipeline.json", run_id="report_markdown_reject") + rejected = build_optimization_report( + rejected_prepared, await run_fake_stage(rejected_prepared, scenario="no_improvement"), progress=_progress(), + finished_at=datetime(2026, 7, 18, 0, 1, tzinfo=timezone.utc), + ) + markdown = render_optimization_markdown(accepted) + assert "# Optimization Report" in markdown + assert "Gate decision: ACCEPT" in markdown + assert "Baseline train" in markdown + assert "Candidate validation" in markdown + assert "Writeback" in markdown + assert "unavailable" in markdown + assert "Gate decision: REJECT" in render_optimization_markdown(rejected) + +@pytest.mark.asyncio +async def test_failure_report_is_deterministic_and_records_only_error_identity(tmp_path): + root = _copy_example(tmp_path) + prepared = prepare_run(root / "pipeline.json", run_id="report_failure") + report = build_failure_report( + prepared, progress=_progress(), error=RuntimeError("pipeline failed"), + source_prompt_hashes={"z": "last", "a": "first"}, existing_artifacts=["z.json", "a.json"], + generated_at=datetime(2026, 7, 18, 0, 1, tzinfo=timezone.utc), + ) + assert report.status == "failed" + assert report.failed_phase == "reporting" + assert report.exception_type == "RuntimeError" + assert report.error_message == "pipeline failed" + assert list(report.source_prompt_hashes) == ["a", "z"] + assert report.existing_artifacts == ["a.json", "z.json"] + +def test_report_dto_invariants_reject_inconsistent_resource_and_artifact_states(): + with pytest.raises(ValueError, match="available optimizer observations"): + OptimizerResourceObservation(status="available", scope_note="missing values") + with pytest.raises(ValueError, match="non-available optimizer observations"): + OptimizerResourceObservation(status="unavailable", scope_note="none", total_rounds=1) + with pytest.raises(ValueError, match="available artifacts require"): + ArtifactReference( + artifact_id="input", artifact_type="input", required=True, + produced_by="baseline_train", status="available", + ) + with pytest.raises(ValueError, match="artifact IDs must be unique"): + ArtifactIndex( + run_id="report", generated_at=datetime(2026, 7, 18, tzinfo=timezone.utc), + artifacts=[ + ArtifactReference( + artifact_id="input", artifact_type="input", required=True, + produced_by="baseline_train", status="unavailable", unavailable_reason="not found", + ), + ArtifactReference( + artifact_id="input", artifact_type="prompt", required=True, + produced_by="candidate_generation", status="unavailable", unavailable_reason="not found", + ), + ], + ) From 746fc7b8f6fb8f2de7b6b22670ec129ac16881d1 Mon Sep 17 00:00:00 2001 From: guocfu Date: Sat, 18 Jul 2026 21:43:13 +0800 Subject: [PATCH 09/18] =?UTF-8?q?fix(evaluation):=20=E4=BF=AE=E6=AD=A3?= =?UTF-8?q?=E6=8A=A5=E5=91=8A=E9=81=A5=E6=B5=8B=E4=B8=8E=E9=94=99=E8=AF=AF?= =?UTF-8?q?=E8=84=B1=E6=95=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../eval_optimize_loop/report_builder.py | 112 ++++++++++++++-- .../eval_optimize_loop/schemas.py | 43 ++++-- ...val_optimize_loop_stage5_report_builder.py | 126 ++++++++++++++++-- 3 files changed, 249 insertions(+), 32 deletions(-) diff --git a/examples/optimization/eval_optimize_loop/report_builder.py b/examples/optimization/eval_optimize_loop/report_builder.py index d4b3b11f..cc99c984 100644 --- a/examples/optimization/eval_optimize_loop/report_builder.py +++ b/examples/optimization/eval_optimize_loop/report_builder.py @@ -3,9 +3,12 @@ from __future__ import annotations from datetime import datetime +import os +import re from typing import TYPE_CHECKING -from .schemas import FailureReport, OptimizerResourceObservation, OptimizationReport +from .schemas import FailureReport, OptimizerResourceObservation, OptimizerResourceValue +from .schemas import OptimizationReport from .schemas import PipelineStageResult, RealStageResult, ReportProgress if TYPE_CHECKING: @@ -14,17 +17,90 @@ _OPTIMIZER_SCOPE = ( "Optimizer-only observation; excludes complete business Agent evaluation usage." ) +_FAKE_OPTIMIZER_REASON = "Fake mode does not run AgentOptimizer." +_MISSING_COST_REASON = ( + "Reflection LM calls were observed but optimizer cost was not reported." +) +_MISSING_TOKEN_REASON = ( + "Reflection LM calls were observed but optimizer token usage was not reported." +) +_REDACTED = "[REDACTED]" +_SENSITIVE_ENV_NAMES = ("TRPC_AGENT_API_KEY", "TRPC_AGENT_BASE_URL") +_SENSITIVE_KEY_VALUE = re.compile( + r"(?P[\"']?(?:api_?key|base_?url|authorization)[\"']?\s*[:=]\s*)" + r"(?P[\"'][^\"']*[\"']|(?:(?:bearer|basic|token)\s+)?[^\s,;}\]]+)", + re.IGNORECASE, +) +_BEARER_VALUE = re.compile( + r"\bbearer(?:\s+|\s*[:=]\s*)[\"']?[^\s,;}\]\"']+[\"']?", + re.IGNORECASE, +) + + +def _not_applicable_optimizer_value( + unit: str, +) -> OptimizerResourceValue[object]: + return OptimizerResourceValue[object]( + status="not_applicable", + unit=unit, + reason=_FAKE_OPTIMIZER_REASON, + ) + + +def _redact_error_message(error: Exception) -> str: + message = str(error) + environment_values = { + os.environ.get(name, "") + for name in _SENSITIVE_ENV_NAMES + if os.environ.get(name, "") + } + for sensitive_value in sorted(environment_values, key=len, reverse=True): + message = message.replace(sensitive_value, _REDACTED) + message = _SENSITIVE_KEY_VALUE.sub( + lambda match: f"{match.group('prefix')}{_REDACTED}", + message, + ) + return _BEARER_VALUE.sub(f"Bearer {_REDACTED}", message) + def _optimizer_resources(result: PipelineStageResult) -> OptimizerResourceObservation: if not isinstance(result, RealStageResult): return OptimizerResourceObservation( - status="not_applicable", scope_note="Fake mode does not run AgentOptimizer.", + scope_note=_FAKE_OPTIMIZER_REASON, + total_rounds=_not_applicable_optimizer_value("rounds"), + reflection_lm_calls=_not_applicable_optimizer_value("calls"), + cost_usd=_not_applicable_optimizer_value("USD"), + token_usage=_not_applicable_optimizer_value("tokens"), + duration_seconds=_not_applicable_optimizer_value("seconds"), ) native = result.optimize_result + reflection_calls = native.total_reflection_lm_calls + cost_missing = reflection_calls > 0 and native.total_llm_cost <= 0 + token_usage = dict(native.total_token_usage) + tokens_missing = reflection_calls > 0 and token_usage.get("total", 0) <= 0 return OptimizerResourceObservation( - status="available", scope_note=_OPTIMIZER_SCOPE, total_rounds=native.total_rounds, - reflection_lm_calls=native.total_reflection_lm_calls, cost_usd=native.total_llm_cost, - token_usage=dict(native.total_token_usage), duration_seconds=native.duration_seconds, + scope_note=_OPTIMIZER_SCOPE, + total_rounds=OptimizerResourceValue[int]( + status="available", value=native.total_rounds, unit="rounds", + ), + reflection_lm_calls=OptimizerResourceValue[int]( + status="available", value=reflection_calls, unit="calls", + ), + cost_usd=OptimizerResourceValue[float]( + status="unavailable" if cost_missing else "available", + value=None if cost_missing else native.total_llm_cost, + unit="USD", + reason=_MISSING_COST_REASON if cost_missing else None, + ), + token_usage=OptimizerResourceValue[dict[str, int]]( + status="unavailable" if tokens_missing else "available", + value=None if tokens_missing else token_usage, + unit="tokens", + reason=_MISSING_TOKEN_REASON if tokens_missing else None, + ), + duration_seconds=OptimizerResourceValue[float]( + status="available", value=native.duration_seconds, unit="seconds", + ), ) def build_optimization_report( @@ -48,7 +124,8 @@ def build_failure_report( return FailureReport( run_id=prepared.workspace.run_id, execution_mode=prepared.config.execution.mode, failed_phase=progress.current_phase, exception_type=type(error).__name__, - error_message=str(error), generated_at=generated_at, input_snapshot=prepared.input_snapshot, + error_message=_redact_error_message(error), generated_at=generated_at, + input_snapshot=prepared.input_snapshot, source_prompt_hashes=dict(sorted(source_prompt_hashes.items())), completed_phases=progress.completed_phases, existing_artifacts=sorted(existing_artifacts), ) @@ -93,7 +170,24 @@ def render_optimization_markdown(report: OptimizationReport) -> str: f"- Cost: {report.pipeline_resources.cost_usd.status}", f"- Tokens: {report.pipeline_resources.total_tokens.status}", f"- Duration: {report.pipeline_resources.duration_seconds.status}", - "", "## Optimizer Scope", - f"- Status: {report.optimizer_resources.status}", - f"- {report.optimizer_resources.scope_note}"]) + "", "## Optimizer Resources"]) + for label, observation in ( + ("Rounds", report.optimizer_resources.total_rounds), + ("Reflection calls", report.optimizer_resources.reflection_lm_calls), + ("Cost", report.optimizer_resources.cost_usd), + ("Token usage", report.optimizer_resources.token_usage), + ("Duration", report.optimizer_resources.duration_seconds), + ): + line = f"- {label}: {observation.status}; unit={observation.unit}" + if observation.value is not None: + value = observation.value + if isinstance(value, dict): + value = ", ".join( + f"{key}={item}" for key, item in sorted(value.items()) + ) + line += f"; value={value}" + if observation.reason is not None: + line += f"; reason={observation.reason}" + lines.append(line) + lines.extend(["", "## Optimizer Scope", f"- {report.optimizer_resources.scope_note}"]) return "\n".join(lines) + "\n" diff --git a/examples/optimization/eval_optimize_loop/schemas.py b/examples/optimization/eval_optimize_loop/schemas.py index fa1c1b37..cb1d955c 100644 --- a/examples/optimization/eval_optimize_loop/schemas.py +++ b/examples/optimization/eval_optimize_loop/schemas.py @@ -14,8 +14,10 @@ from datetime import datetime from typing import Any +from typing import Generic from typing import Literal from typing import Optional +from typing import TypeVar from typing import Union from pydantic import Field @@ -437,24 +439,39 @@ def _validate_phases(self) -> "ReportProgress": raise ValueError("completed phases must not include current phase") return self -class OptimizerResourceObservation(EvalBaseModel): + +OptimizerResourceValueT = TypeVar("OptimizerResourceValueT") + + +class OptimizerResourceValue(EvalBaseModel, Generic[OptimizerResourceValueT]): status: Literal["available", "unavailable", "not_applicable"] - scope_note: str - total_rounds: Optional[int] = Field(default=None, ge=0) - reflection_lm_calls: Optional[int] = Field(default=None, ge=0) - cost_usd: Optional[float] = Field(default=None, ge=0.0) - token_usage: Optional[dict[str, int]] = None - duration_seconds: Optional[float] = Field(default=None, ge=0.0) + value: Optional[OptimizerResourceValueT] = None + unit: str = Field(min_length=1) + reason: Optional[str] = None @model_validator(mode="after") - def _validate_status(self) -> "OptimizerResourceObservation": - values = (self.total_rounds, self.reflection_lm_calls, self.cost_usd, self.token_usage, self.duration_seconds) - if self.status == "available" and any(value is None for value in values): - raise ValueError("available optimizer observations require all resource values") - if self.status != "available" and any(value is not None for value in values): - raise ValueError("non-available optimizer observations must not carry resource values") + def _validate_status(self) -> "OptimizerResourceValue[OptimizerResourceValueT]": + if self.status == "available": + if self.value is None: + raise ValueError("available optimizer resource values require a value") + if isinstance(self.value, (int, float)) and not self.value >= 0: + raise ValueError("optimizer numeric resource values must be non-negative") + else: + if self.value is not None: + raise ValueError("non-available optimizer resource values must not carry a value") + if self.reason is None or not self.reason.strip(): + raise ValueError("non-available optimizer resource values require a reason") return self + +class OptimizerResourceObservation(EvalBaseModel): + scope_note: str + total_rounds: OptimizerResourceValue[int] + reflection_lm_calls: OptimizerResourceValue[int] + cost_usd: OptimizerResourceValue[float] + token_usage: OptimizerResourceValue[dict[str, int]] + duration_seconds: OptimizerResourceValue[float] + class ArtifactReference(EvalBaseModel): artifact_id: str artifact_type: Literal["input", "prompt", "evaluation", "candidate", "optimizer_native", "report"] diff --git a/tests/evaluation/test_eval_optimize_loop_stage5_report_builder.py b/tests/evaluation/test_eval_optimize_loop_stage5_report_builder.py index 020d47d2..a58ee8f0 100644 --- a/tests/evaluation/test_eval_optimize_loop_stage5_report_builder.py +++ b/tests/evaluation/test_eval_optimize_loop_stage5_report_builder.py @@ -11,7 +11,7 @@ build_failure_report, build_optimization_report, render_optimization_markdown, ) from examples.optimization.eval_optimize_loop.schemas import ( - ArtifactIndex, ArtifactReference, OptimizationReport, OptimizerResourceObservation, ReportProgress, + ArtifactIndex, ArtifactReference, OptimizationReport, OptimizerResourceValue, ReportProgress, ) from examples.optimization.eval_optimize_loop.schemas import OptimizerCandidateProposal, RealStageResult from trpc_agent_sdk.evaluation import OptimizeResult @@ -54,7 +54,17 @@ async def test_fake_report_is_serializable_and_marks_optimizer_resources_not_app ) assert report.status == "completed" assert report.execution_mode == "fake" - assert report.optimizer_resources.status == "not_applicable" + optimizer_resources = report.optimizer_resources + for observation in ( + optimizer_resources.total_rounds, + optimizer_resources.reflection_lm_calls, + optimizer_resources.cost_usd, + optimizer_resources.token_usage, + optimizer_resources.duration_seconds, + ): + assert observation.status == "not_applicable" + assert observation.value is None + assert observation.reason == "Fake mode does not run AgentOptimizer." assert report.pipeline_resources.total_tokens.status == "unavailable" assert OptimizationReport.model_validate_json(report.model_dump_json()) == report @@ -78,13 +88,67 @@ async def test_real_report_includes_optimizer_resource_observations(tmp_path): prepared, result, progress=_progress(), finished_at=datetime(2026, 7, 18, 0, 1, tzinfo=timezone.utc), ) - assert report.optimizer_resources.status == "available" - assert report.optimizer_resources.total_rounds == 1 - assert report.optimizer_resources.reflection_lm_calls == 1 + assert report.optimizer_resources.total_rounds.status == "available" + assert report.optimizer_resources.total_rounds.value == 1 + assert report.optimizer_resources.total_rounds.unit == "rounds" + assert report.optimizer_resources.reflection_lm_calls.status == "available" + assert report.optimizer_resources.reflection_lm_calls.value == 1 + assert report.optimizer_resources.cost_usd.status == "available" + assert report.optimizer_resources.token_usage.status == "available" + assert report.optimizer_resources.duration_seconds.status == "available" assert report.optimizer_resources.scope_note == ( "Optimizer-only observation; excludes complete business Agent evaluation usage." ) + +@pytest.mark.asyncio +async def test_real_report_marks_incomplete_cost_and_token_telemetry_unavailable(tmp_path): + root = _copy_example(tmp_path) + prepared = prepare_run(root / "pipeline.json", run_id="report_real_incomplete") + fake_result = await run_fake_stage(prepared, scenario="improve") + candidate = OptimizerCandidateProposal.model_validate({ + **fake_result.candidate.model_dump(exclude={"scenario", "seed"}), "provider": "agent_optimizer", + "optimizer_status": "SUCCEEDED", "finish_reason": "completed", + "baseline_pass_rate": 1 / 3, "best_pass_rate": 1.0, + "candidate_id": f"optimizer-{fake_result.candidate.candidate_id[-12:]}", + }) + native = _optimize_result( + await prepared.source_target.read_all(), candidate.prompts, + ).model_copy(update={ + "total_reflection_lm_calls": 2, + "total_llm_cost": 0.0, + "total_token_usage": {"prompt": 0, "completion": 0, "total": 0}, + }) + result = RealStageResult( + **fake_result.model_dump(exclude={"scenario", "candidate"}), candidate=candidate, + optimize_result=native, + ) + + report = build_optimization_report( + prepared, result, progress=_progress(), + finished_at=datetime(2026, 7, 18, 0, 1, tzinfo=timezone.utc), + ) + + assert report.optimizer_resources.total_rounds.status == "available" + assert report.optimizer_resources.total_rounds.value == 1 + assert report.optimizer_resources.reflection_lm_calls.status == "available" + assert report.optimizer_resources.reflection_lm_calls.value == 2 + assert report.optimizer_resources.duration_seconds.status == "available" + assert report.optimizer_resources.duration_seconds.value == 0.5 + assert report.optimizer_resources.cost_usd.status == "unavailable" + assert report.optimizer_resources.cost_usd.value is None + assert report.optimizer_resources.cost_usd.reason + assert report.optimizer_resources.token_usage.status == "unavailable" + assert report.optimizer_resources.token_usage.value is None + assert report.optimizer_resources.token_usage.reason + + markdown = render_optimization_markdown(report) + assert "Rounds: available" in markdown + assert "Reflection calls: available" in markdown + assert "Cost: unavailable" in markdown + assert "Token usage: unavailable" in markdown + assert "Duration: available" in markdown + @pytest.mark.asyncio async def test_markdown_includes_accept_and_reject_report_evidence(tmp_path): root = _copy_example(tmp_path) @@ -124,11 +188,46 @@ async def test_failure_report_is_deterministic_and_records_only_error_identity(t assert list(report.source_prompt_hashes) == ["a", "z"] assert report.existing_artifacts == ["a.json", "z.json"] -def test_report_dto_invariants_reject_inconsistent_resource_and_artifact_states(): - with pytest.raises(ValueError, match="available optimizer observations"): - OptimizerResourceObservation(status="available", scope_note="missing values") - with pytest.raises(ValueError, match="non-available optimizer observations"): - OptimizerResourceObservation(status="unavailable", scope_note="none", total_rounds=1) + +@pytest.mark.asyncio +async def test_failure_report_redacts_environment_values_and_common_secret_forms(tmp_path, monkeypatch): + root = _copy_example(tmp_path) + prepared = prepare_run(root / "pipeline.json", run_id="report_failure_redacted") + api_key = "env-api-key-secret" + base_url = "https://env-base-url-secret.example/v1" + monkeypatch.setenv("TRPC_AGENT_API_KEY", api_key) + monkeypatch.setenv("TRPC_AGENT_BASE_URL", base_url) + error = RuntimeError( + f"timeout contacting {base_url} with {api_key}; " + "api_key=snake-secret apiKey:'camel-secret' " + "Authorization: Bearer authorization-secret Bearer loose-bearer-secret " + "base_url=https://snake-url-secret.example/v1 " + 'baseUrl="https://camel-url-secret.example/v1"; retryable=true' + ) + + report = build_failure_report( + prepared, progress=_progress(), error=error, + source_prompt_hashes={}, existing_artifacts=[], + generated_at=datetime(2026, 7, 18, 0, 1, tzinfo=timezone.utc), + ) + + for secret in ( + api_key, + base_url, + "snake-secret", + "camel-secret", + "authorization-secret", + "loose-bearer-secret", + "https://snake-url-secret.example/v1", + "https://camel-url-secret.example/v1", + ): + assert secret not in report.error_message + assert report.error_message.count("[REDACTED]") >= 8 + assert "timeout contacting" in report.error_message + assert "retryable=true" in report.error_message + + +def test_report_dto_invariants_reject_inconsistent_artifact_states(): with pytest.raises(ValueError, match="available artifacts require"): ArtifactReference( artifact_id="input", artifact_type="input", required=True, @@ -148,3 +247,10 @@ def test_report_dto_invariants_reject_inconsistent_resource_and_artifact_states( ), ], ) + + +def test_optimizer_resource_value_rejects_negative_numeric_value(): + with pytest.raises(ValueError, match="non-negative"): + OptimizerResourceValue[int]( + status="available", value=-1, unit="calls", + ) From e8ecbfd8304bb65bad1f35e9a1eddb2838d05066 Mon Sep 17 00:00:00 2001 From: guocfu Date: Sun, 19 Jul 2026 11:41:22 +0800 Subject: [PATCH 10/18] =?UTF-8?q?fix(evaluation):=20=E6=94=B6=E7=B4=A7?= =?UTF-8?q?=E6=8A=A5=E5=91=8A=E8=84=B1=E6=95=8F=E4=B8=8E=E9=81=A5=E6=B5=8B?= =?UTF-8?q?=E5=AE=8C=E6=95=B4=E6=80=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../eval_optimize_loop/report_builder.py | 32 ++++++- ...val_optimize_loop_stage5_report_builder.py | 94 +++++++++++++++++++ 2 files changed, 121 insertions(+), 5 deletions(-) diff --git a/examples/optimization/eval_optimize_loop/report_builder.py b/examples/optimization/eval_optimize_loop/report_builder.py index cc99c984..d3190a06 100644 --- a/examples/optimization/eval_optimize_loop/report_builder.py +++ b/examples/optimization/eval_optimize_loop/report_builder.py @@ -24,13 +24,15 @@ _MISSING_TOKEN_REASON = ( "Reflection LM calls were observed but optimizer token usage was not reported." ) +_INVALID_TOKEN_REASON = "Optimizer token usage was malformed or inconsistent." _REDACTED = "[REDACTED]" _SENSITIVE_ENV_NAMES = ("TRPC_AGENT_API_KEY", "TRPC_AGENT_BASE_URL") _SENSITIVE_KEY_VALUE = re.compile( - r"(?P[\"']?(?:api_?key|base_?url|authorization)[\"']?\s*[:=]\s*)" + r"(?P[\"']?(?:api[_-]?key|base[_-]?url|authorization)[\"']?\s*[:=]\s*)" r"(?P[\"'][^\"']*[\"']|(?:(?:bearer|basic|token)\s+)?[^\s,;}\]]+)", re.IGNORECASE, ) +_HTTP_URL = re.compile(r"https?://[^\s,;}\]<>\"']+", re.IGNORECASE) _BEARER_VALUE = re.compile( r"\bbearer(?:\s+|\s*[:=]\s*)[\"']?[^\s,;}\]\"']+[\"']?", re.IGNORECASE, @@ -60,7 +62,19 @@ def _redact_error_message(error: Exception) -> str: lambda match: f"{match.group('prefix')}{_REDACTED}", message, ) - return _BEARER_VALUE.sub(f"Bearer {_REDACTED}", message) + message = _BEARER_VALUE.sub(f"Bearer {_REDACTED}", message) + return _HTTP_URL.sub(_REDACTED, message) + + +def _is_complete_token_usage(value: object) -> bool: + if not isinstance(value, dict): + return False + required = ("prompt", "completion", "total") + if not all(key in value for key in required): + return False + if not all(type(value[key]) is int and value[key] >= 0 for key in required): + return False + return value["total"] == value["prompt"] + value["completion"] def _optimizer_resources(result: PipelineStageResult) -> OptimizerResourceObservation: @@ -76,8 +90,12 @@ def _optimizer_resources(result: PipelineStageResult) -> OptimizerResourceObserv native = result.optimize_result reflection_calls = native.total_reflection_lm_calls cost_missing = reflection_calls > 0 and native.total_llm_cost <= 0 - token_usage = dict(native.total_token_usage) - tokens_missing = reflection_calls > 0 and token_usage.get("total", 0) <= 0 + token_usage = native.total_token_usage + token_usage_valid = _is_complete_token_usage(token_usage) + tokens_missing = ( + not token_usage_valid + or (reflection_calls > 0 and token_usage["total"] <= 0) + ) return OptimizerResourceObservation( scope_note=_OPTIMIZER_SCOPE, total_rounds=OptimizerResourceValue[int]( @@ -96,7 +114,11 @@ def _optimizer_resources(result: PipelineStageResult) -> OptimizerResourceObserv status="unavailable" if tokens_missing else "available", value=None if tokens_missing else token_usage, unit="tokens", - reason=_MISSING_TOKEN_REASON if tokens_missing else None, + reason=( + _INVALID_TOKEN_REASON + if tokens_missing and not token_usage_valid + else _MISSING_TOKEN_REASON if tokens_missing else None + ), ), duration_seconds=OptimizerResourceValue[float]( status="available", value=native.duration_seconds, unit="seconds", diff --git a/tests/evaluation/test_eval_optimize_loop_stage5_report_builder.py b/tests/evaluation/test_eval_optimize_loop_stage5_report_builder.py index a58ee8f0..7ff42c82 100644 --- a/tests/evaluation/test_eval_optimize_loop_stage5_report_builder.py +++ b/tests/evaluation/test_eval_optimize_loop_stage5_report_builder.py @@ -227,6 +227,100 @@ async def test_failure_report_redacts_environment_values_and_common_secret_forms assert "retryable=true" in report.error_message +def test_failure_report_redacts_hyphenated_keys_and_bare_urls(tmp_path): + root = _copy_example(tmp_path) + prepared = prepare_run(root / "pipeline.json", run_id="report_failure_hyphenated") + error = RuntimeError( + "X-API-Key: hyphen-api-secret x-api-key=lower-hyphen-secret; " + "endpoint https://private.example/v1?token=secret and http://10.0.0.8:8080/run" + ) + + report = build_failure_report( + prepared, progress=_progress(), error=error, + source_prompt_hashes={}, existing_artifacts=[], + generated_at=datetime(2026, 7, 18, 0, 1, tzinfo=timezone.utc), + ) + + assert "hyphen-api-secret" not in report.error_message + assert "lower-hyphen-secret" not in report.error_message + assert "https://private.example/v1?token=secret" not in report.error_message + assert "http://10.0.0.8:8080/run" not in report.error_message + assert "endpoint [REDACTED]" in report.error_message + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "token_usage", + [ + {}, + {"prompt": 1, "completion": 2}, + {"prompt": 1, "completion": "2", "total": 3}, + {"prompt": True, "completion": 2, "total": 3}, + {"prompt": -1, "completion": 2, "total": 1}, + {"prompt": 1, "completion": 2, "total": 4}, + ["not", "a", "dict"], + ], +) +async def test_real_report_marks_malformed_token_usage_unavailable(tmp_path, token_usage): + root = _copy_example(tmp_path) + prepared = prepare_run(root / "pipeline.json", run_id="report_real_bad_tokens") + fake_result = await run_fake_stage(prepared, scenario="improve") + candidate = OptimizerCandidateProposal.model_validate({ + **fake_result.candidate.model_dump(exclude={"scenario", "seed"}), "provider": "agent_optimizer", + "optimizer_status": "SUCCEEDED", "finish_reason": "completed", + "baseline_pass_rate": 1 / 3, "best_pass_rate": 1.0, + "candidate_id": f"optimizer-{fake_result.candidate.candidate_id[-12:]}", + }) + native = _optimize_result( + await prepared.source_target.read_all(), candidate.prompts, + ).model_copy(update={"total_token_usage": token_usage}) + result = RealStageResult( + **fake_result.model_dump(exclude={"scenario", "candidate"}), candidate=candidate, + optimize_result=native, + ) + + report = build_optimization_report( + prepared, result, progress=_progress(), + finished_at=datetime(2026, 7, 18, 0, 1, tzinfo=timezone.utc), + ) + + assert report.optimizer_resources.token_usage.status == "unavailable" + assert report.optimizer_resources.token_usage.value is None + + +@pytest.mark.asyncio +async def test_real_report_allows_consistent_zero_token_usage_without_reflection_calls(tmp_path): + root = _copy_example(tmp_path) + prepared = prepare_run(root / "pipeline.json", run_id="report_real_zero_tokens") + fake_result = await run_fake_stage(prepared, scenario="improve") + candidate = OptimizerCandidateProposal.model_validate({ + **fake_result.candidate.model_dump(exclude={"scenario", "seed"}), "provider": "agent_optimizer", + "optimizer_status": "SUCCEEDED", "finish_reason": "completed", + "baseline_pass_rate": 1 / 3, "best_pass_rate": 1.0, + "candidate_id": f"optimizer-{fake_result.candidate.candidate_id[-12:]}", + }) + native = _optimize_result( + await prepared.source_target.read_all(), candidate.prompts, + ).model_copy(update={ + "total_reflection_lm_calls": 0, + "total_token_usage": {"prompt": 0, "completion": 0, "total": 0, "cached": 7}, + }) + result = RealStageResult( + **fake_result.model_dump(exclude={"scenario", "candidate"}), candidate=candidate, + optimize_result=native, + ) + + report = build_optimization_report( + prepared, result, progress=_progress(), + finished_at=datetime(2026, 7, 18, 0, 1, tzinfo=timezone.utc), + ) + + assert report.optimizer_resources.token_usage.status == "available" + assert report.optimizer_resources.token_usage.value == { + "prompt": 0, "completion": 0, "total": 0, "cached": 7, + } + + def test_report_dto_invariants_reject_inconsistent_artifact_states(): with pytest.raises(ValueError, match="available artifacts require"): ArtifactReference( From cc54f82b2618bd80fc0e7b86a3932397abe17d45 Mon Sep 17 00:00:00 2001 From: guocfu Date: Sun, 19 Jul 2026 11:57:23 +0800 Subject: [PATCH 11/18] =?UTF-8?q?feat(evaluation):=20=E5=8E=9F=E5=AD=90?= =?UTF-8?q?=E5=8F=91=E5=B8=83=E6=8A=A5=E5=91=8A=E4=B8=8E=E5=AE=A1=E8=AE=A1?= =?UTF-8?q?=E7=B4=A2=E5=BC=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../eval_optimize_loop/artifact_writer.py | 466 ++++++++++++++++++ ...est_eval_optimize_loop_stage5_artifacts.py | 310 ++++++++++++ 2 files changed, 776 insertions(+) create mode 100644 examples/optimization/eval_optimize_loop/artifact_writer.py create mode 100644 tests/evaluation/test_eval_optimize_loop_stage5_artifacts.py diff --git a/examples/optimization/eval_optimize_loop/artifact_writer.py b/examples/optimization/eval_optimize_loop/artifact_writer.py new file mode 100644 index 00000000..468f8027 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/artifact_writer.py @@ -0,0 +1,466 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under the Apache License, Version 2.0. +"""Safe and atomic materialization of optimization report artifacts.""" + +from __future__ import annotations + +import hashlib +import os +from pathlib import Path +import shutil +from typing import Literal, TypeAlias +from uuid import uuid4 + +from pydantic import BaseModel + +from .report_builder import render_optimization_markdown +from .schemas import ArtifactIndex, ArtifactReference, FailureReport +from .schemas import OptimizationReport, ReportPhase + + +ArtifactType: TypeAlias = Literal[ + "input", + "prompt", + "evaluation", + "candidate", + "optimizer_native", + "report", +] + +_INPUT_COPY_DISABLED = "artifacts.copy_input_files=false" + + +class ArtifactWriteError(RuntimeError): + """Raised when an artifact cannot be safely materialized or discovered.""" + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _resolved_run_dir(run_dir: Path) -> Path: + if run_dir.is_symlink(): + raise ArtifactWriteError(f"run directory must not be a symbolic link: {run_dir}") + try: + root = run_dir.resolve(strict=True) + except OSError as exc: + raise ArtifactWriteError(f"run directory is unavailable: {run_dir}: {exc}") from exc + if not root.is_dir(): + raise ArtifactWriteError(f"run directory must be a directory: {run_dir}") + return root + + +def _inside_run(run_dir: Path, path: Path) -> Path: + root = run_dir.resolve(strict=True) + lexical = path if path.is_absolute() else root / path + try: + relative = lexical.relative_to(root) + except ValueError as exc: + raise ArtifactWriteError(f"artifact escapes run directory: {path}") from exc + + current = root + for component in relative.parts: + current /= component + if current.is_symlink(): + raise ArtifactWriteError(f"artifact must not be a symbolic link: {path}") + + try: + resolved = lexical.resolve(strict=True) + except OSError as exc: + raise ArtifactWriteError(f"artifact is unavailable: {path}: {exc}") from exc + if not resolved.is_relative_to(root): + raise ArtifactWriteError(f"artifact escapes run directory: {path}") + if not resolved.is_file(): + raise ArtifactWriteError(f"artifact must be a regular file: {path}") + return resolved + + +def discover_run_artifacts(run_dir: Path) -> list[str]: + """Return regular files below a run without ever accepting symlinks.""" + root = _resolved_run_dir(run_dir) + paths: list[str] = [] + for directory, directory_names, file_names in os.walk(root, followlinks=False): + current = Path(directory) + directory_names.sort() + file_names.sort() + + retained_directories = [] + for name in directory_names: + path = current / name + if path.is_symlink(): + raise ArtifactWriteError( + f"artifact must not be a symbolic link: {path}" + ) + relative = path.relative_to(root).as_posix() + if ".report.tmp-" not in relative: + retained_directories.append(name) + directory_names[:] = retained_directories + + for name in file_names: + path = current / name + if path.is_symlink(): + raise ArtifactWriteError( + f"artifact must not be a symbolic link: {path}" + ) + relative = path.relative_to(root).as_posix() + if name == "failure_report.json" or ".report.tmp-" in relative: + continue + if path.is_file(): + paths.append(relative) + return sorted(paths) + + +def _write_text(path: Path, content: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + + +def _json_text(model: BaseModel) -> str: + return model.model_dump_json(by_alias=False, indent=2) + "\n" + + +def _published_relative_path(root: Path, path: Path) -> str: + relative = path.relative_to(root) + if relative.parts and relative.parts[0].startswith(".report.tmp-"): + relative = Path("report", *relative.parts[1:]) + return relative.as_posix() + + +def _available_reference( + run_dir: Path, + path: Path, + *, + artifact_id: str, + artifact_type: ArtifactType, + required: bool, + produced_by: ReportPhase, +) -> ArtifactReference: + root = run_dir.resolve(strict=True) + resolved = _inside_run(root, path) + return ArtifactReference( + artifact_id=artifact_id, + artifact_type=artifact_type, + relative_path=_published_relative_path(root, path), + required=required, + produced_by=produced_by, + status="available", + size_bytes=resolved.stat().st_size, + sha256=_sha256(resolved), + ) + + +def _unavailable_input_reference( + *, artifact_id: str, produced_by: ReportPhase +) -> ArtifactReference: + return ArtifactReference( + artifact_id=artifact_id, + artifact_type="input", + required=True, + produced_by=produced_by, + status="unavailable", + unavailable_reason=_INPUT_COPY_DISABLED, + ) + + +def _safe_prompt_name(field_name: str) -> str: + safe = "".join( + character if character.isalnum() or character in "._-" else "_" + for character in field_name + ) + return safe if safe not in {"", ".", ".."} else "prompt" + + +def _validate_available_references( + root: Path, staging: Path, index: ArtifactIndex +) -> None: + for reference in index.artifacts: + if reference.status != "available": + continue + if reference.relative_path is None: + raise ArtifactWriteError( + f"available artifact has no relative path: {reference.artifact_id}" + ) + relative = Path(reference.relative_path) + if relative.is_absolute() or ".." in relative.parts: + raise ArtifactWriteError( + f"artifact path is not run-relative: {reference.relative_path}" + ) + if relative.parts and relative.parts[0] == "report": + path = staging.joinpath(*relative.parts[1:]) + else: + path = root / relative + resolved = _inside_run(root, path) + if resolved.stat().st_size != reference.size_bytes: + raise ArtifactWriteError( + f"artifact size changed during staging: {reference.relative_path}" + ) + if _sha256(resolved) != reference.sha256: + raise ArtifactWriteError( + f"artifact hash changed during staging: {reference.relative_path}" + ) + + +def _copy_input( + *, + root: Path, + staging: Path, + source: Path, + expected_sha256: str, + destination_name: str, + artifact_id: str, + produced_by: ReportPhase, +) -> ArtifactReference: + if source.is_symlink(): + raise ArtifactWriteError(f"input must not be a symbolic link: {source}") + try: + actual_sha256 = _sha256(source) + except OSError as exc: + raise ArtifactWriteError(f"failed to read input {source}: {exc}") from exc + if actual_sha256 != expected_sha256: + raise ArtifactWriteError(f"input hash mismatch: {source}") + + destination = staging / "inputs" / destination_name + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source, destination) + if _sha256(destination) != expected_sha256: + raise ArtifactWriteError(f"input hash changed while copying: {source}") + return _available_reference( + root, + destination, + artifact_id=artifact_id, + artifact_type="input", + required=True, + produced_by=produced_by, + ) + + +def publish_report_bundle( + report: OptimizationReport, + *, + run_dir: Path, + copy_input_files: bool, +) -> ArtifactIndex: + """Build a complete report in staging and atomically publish its directory.""" + staging: Path | None = None + try: + root = _resolved_run_dir(run_dir) + target = root / "report" + if target.exists() or target.is_symlink(): + raise ArtifactWriteError(f"report directory already exists: {target}") + + existing_paths = discover_run_artifacts(root) + native_paths = [ + relative + for relative in existing_paths + if relative.startswith("optimizer/") + or ("/" not in relative and relative.endswith(".runtime.json")) + ] + + staging = root / f".report.tmp-{uuid4().hex}" + staging.mkdir() + references: list[ArtifactReference] = [] + + report_json = staging / "optimization_report.json" + _write_text(report_json, _json_text(report)) + references.append( + _available_reference( + root, + report_json, + artifact_id="report.optimization_json", + artifact_type="report", + required=True, + produced_by="reporting", + ) + ) + + report_markdown = staging / "optimization_report.md" + _write_text(report_markdown, render_optimization_markdown(report)) + references.append( + _available_reference( + root, + report_markdown, + artifact_id="report.optimization_markdown", + artifact_type="report", + required=True, + produced_by="reporting", + ) + ) + + evaluations = ( + ("baseline_train", report.baseline_train, "baseline_train"), + ("baseline_validation", report.baseline_validation, "baseline_validation"), + ("candidate_train", report.candidate_train, "candidate_train"), + ( + "candidate_validation", + report.candidate_validation, + "candidate_validation", + ), + ) + for name, evaluation, produced_by in evaluations: + path = staging / "evaluations" / f"{name}.json" + _write_text(path, _json_text(evaluation)) + references.append( + _available_reference( + root, + path, + artifact_id=f"evaluation.{name}", + artifact_type="evaluation", + required=True, + produced_by=produced_by, + ) + ) + + for index, snapshot in enumerate(report.input_snapshot.prompt_snapshots): + path = ( + staging + / "prompts" + / "baseline" + / f"{index:03d}-{_safe_prompt_name(snapshot.field_name)}.md" + ) + _write_text(path, snapshot.content) + references.append( + _available_reference( + root, + path, + artifact_id=f"prompt.baseline.{snapshot.field_name}", + artifact_type="prompt", + required=True, + produced_by="baseline_train", + ) + ) + + for index, (field_name, content) in enumerate(report.candidate.prompts.items()): + path = ( + staging + / "prompts" + / "candidate" + / f"{index:03d}-{_safe_prompt_name(field_name)}.md" + ) + _write_text(path, content) + references.append( + _available_reference( + root, + path, + artifact_id=f"prompt.candidate.{field_name}", + artifact_type="prompt", + required=True, + produced_by="candidate_generation", + ) + ) + + input_specs = ( + ( + "input.pipeline_config", + Path(report.input_snapshot.pipeline_config_path), + report.input_snapshot.pipeline_config_sha256, + "pipeline_config.json", + "baseline_train", + ), + ( + "input.optimizer_config", + Path(report.input_snapshot.optimizer_config_path), + report.input_snapshot.optimizer_config_sha256, + "optimizer_config.json", + "candidate_generation", + ), + ( + "input.train_evalset", + Path(report.input_snapshot.train_evalset_path), + report.input_snapshot.train_evalset_sha256, + "train_evalset.json", + "baseline_train", + ), + ( + "input.validation_evalset", + Path(report.input_snapshot.validation_evalset_path), + report.input_snapshot.validation_evalset_sha256, + "validation_evalset.json", + "baseline_validation", + ), + ) + for artifact_id, source, expected_hash, destination_name, produced_by in input_specs: + if copy_input_files: + references.append( + _copy_input( + root=root, + staging=staging, + source=source, + expected_sha256=expected_hash, + destination_name=destination_name, + artifact_id=artifact_id, + produced_by=produced_by, + ) + ) + else: + references.append( + _unavailable_input_reference( + artifact_id=artifact_id, + produced_by=produced_by, + ) + ) + + for relative in native_paths: + native_path = root / relative + references.append( + _available_reference( + root, + native_path, + artifact_id=f"optimizer_native.{relative}", + artifact_type="optimizer_native", + required=False, + produced_by="candidate_generation", + ) + ) + + index = ArtifactIndex( + run_id=report.run_id, + generated_at=report.finished_at, + artifacts=references, + ) + index_path = staging / "artifact_index.json" + _write_text(index_path, _json_text(index)) + + OptimizationReport.model_validate_json(report_json.read_text(encoding="utf-8")) + validated_index = ArtifactIndex.model_validate_json( + index_path.read_text(encoding="utf-8") + ) + _validate_available_references(root, staging, validated_index) + + staging.rename(target) + staging = None + return validated_index + except Exception as exc: + if staging is not None: + shutil.rmtree(staging, ignore_errors=True) + raise ArtifactWriteError(f"failed to publish report bundle: {exc}") from exc + + +def write_failure_report(report: FailureReport, *, run_dir: Path) -> Path: + """Atomically write first-failure evidence without allowing replacement.""" + temporary: Path | None = None + try: + root = _resolved_run_dir(run_dir) + target = root / "failure_report.json" + if target.exists() or target.is_symlink(): + raise ArtifactWriteError(f"failure report already exists: {target}") + temporary = root / f".failure_report.tmp-{uuid4().hex}" + _write_text(temporary, _json_text(report)) + FailureReport.model_validate_json(temporary.read_text(encoding="utf-8")) + try: + os.link(temporary, target) + except FileExistsError as exc: + raise ArtifactWriteError(f"failure report already exists: {target}") from exc + temporary.unlink() + temporary = None + return target + except Exception as exc: + if temporary is not None: + temporary.unlink(missing_ok=True) + raise ArtifactWriteError(f"failed to write failure report: {exc}") from exc diff --git a/tests/evaluation/test_eval_optimize_loop_stage5_artifacts.py b/tests/evaluation/test_eval_optimize_loop_stage5_artifacts.py new file mode 100644 index 00000000..a97dfadf --- /dev/null +++ b/tests/evaluation/test_eval_optimize_loop_stage5_artifacts.py @@ -0,0 +1,310 @@ +"""Tests for safe and atomic Stage 5 report artifact publication.""" + +from datetime import datetime, timezone +from hashlib import sha256 +from pathlib import Path +import shutil + +import pytest + +from examples.optimization.eval_optimize_loop import artifact_writer +from examples.optimization.eval_optimize_loop.artifact_writer import ( + ArtifactWriteError, + discover_run_artifacts, + publish_report_bundle, + write_failure_report, +) +from examples.optimization.eval_optimize_loop.pipeline import prepare_run, run_fake_stage +from examples.optimization.eval_optimize_loop.report_builder import ( + build_failure_report, + build_optimization_report, +) +from examples.optimization.eval_optimize_loop.schemas import FailureReport, ReportProgress + + +_ROOT = Path(__file__).resolve().parents[2] +_EXAMPLE = _ROOT / "examples" / "optimization" / "eval_optimize_loop" + + +def _copy_example(tmp_path: Path) -> Path: + target = tmp_path / "eval_optimize_loop" + shutil.copytree(_EXAMPLE, target, ignore=shutil.ignore_patterns("runs", "__pycache__")) + return target + + +async def _build_fake_report(tmp_path: Path, run_id: str): + root = _copy_example(tmp_path) + prepared = prepare_run(root / "pipeline.json", run_id=run_id) + result = await run_fake_stage(prepared, scenario="improve") + shutil.rmtree(Path(prepared.workspace.run_dir) / "report", ignore_errors=True) + progress = ReportProgress( + started_at=datetime(2026, 7, 18, tzinfo=timezone.utc), + current_phase="reporting", + completed_phases=[ + "baseline_train", + "baseline_validation", + "candidate_generation", + "candidate_train", + "candidate_validation", + "analysis", + "gate", + "writeback", + ], + ) + report = build_optimization_report( + prepared, + result, + progress=progress, + finished_at=datetime(2026, 7, 18, 0, 1, tzinfo=timezone.utc), + ) + return prepared, report + + +@pytest.mark.asyncio +async def test_publish_report_bundle_materializes_and_indexes_required_files(tmp_path): + prepared, report = await _build_fake_report(tmp_path, "artifact_complete") + run_dir = Path(prepared.workspace.run_dir) + + index = publish_report_bundle(report, run_dir=run_dir, copy_input_files=True) + + report_dir = run_dir / "report" + assert (report_dir / "optimization_report.json").is_file() + assert (report_dir / "optimization_report.md").is_file() + assert (report_dir / "artifact_index.json").is_file() + for name in ( + "baseline_train", + "baseline_validation", + "candidate_train", + "candidate_validation", + ): + assert (report_dir / "evaluations" / f"{name}.json").is_file() + assert all(ref.relative_path != "report/artifact_index.json" for ref in index.artifacts) + + available = [ref for ref in index.artifacts if ref.status == "available"] + assert available + for ref in available: + assert ref.relative_path is not None + relative_path = Path(ref.relative_path) + assert not relative_path.is_absolute() + assert ".." not in relative_path.parts + path = run_dir / relative_path + assert path.is_file() + assert ref.sha256 == sha256(path.read_bytes()).hexdigest() + assert ref.size_bytes == path.stat().st_size + + +@pytest.mark.asyncio +async def test_publish_rejects_input_hash_drift(tmp_path): + prepared, report = await _build_fake_report(tmp_path, "artifact_drift") + train_path = Path(prepared.input_snapshot.train_evalset_path) + train_path.write_text( + train_path.read_text(encoding="utf-8") + "\n", + encoding="utf-8", + ) + run_dir = Path(prepared.workspace.run_dir) + + with pytest.raises(ArtifactWriteError, match="input hash"): + publish_report_bundle(report, run_dir=run_dir, copy_input_files=True) + + assert not (run_dir / "report").exists() + assert list(run_dir.glob(".report.tmp-*")) == [] + + +@pytest.mark.asyncio +async def test_publish_rejects_existing_report_directory(tmp_path): + prepared, report = await _build_fake_report(tmp_path, "artifact_existing") + report_dir = Path(prepared.workspace.run_dir) / "report" + report_dir.mkdir() + + with pytest.raises(ArtifactWriteError, match="already exists"): + publish_report_bundle( + report, + run_dir=Path(prepared.workspace.run_dir), + copy_input_files=True, + ) + + +@pytest.mark.parametrize("target_inside_run", [False, True]) +def test_discovery_rejects_any_file_symlink(tmp_path, target_inside_run): + run_dir = tmp_path / "run" + run_dir.mkdir() + target_root = run_dir if target_inside_run else tmp_path + target = target_root / "target.txt" + target.write_text("target", encoding="utf-8") + (run_dir / "link.txt").symlink_to(target) + + with pytest.raises(ArtifactWriteError, match="symbolic link"): + discover_run_artifacts(run_dir) + + +def test_discovery_rejects_directory_symlink_without_following_it(tmp_path): + run_dir = tmp_path / "run" + run_dir.mkdir() + outside = tmp_path / "outside" + outside.mkdir() + (outside / "artifact.txt").write_text("outside", encoding="utf-8") + (run_dir / "linked-directory").symlink_to(outside, target_is_directory=True) + + with pytest.raises(ArtifactWriteError, match="symbolic link"): + discover_run_artifacts(run_dir) + + +@pytest.mark.asyncio +async def test_publish_rejects_symlink_in_optimizer_artifacts(tmp_path): + prepared, report = await _build_fake_report(tmp_path, "artifact_optimizer_link") + run_dir = Path(prepared.workspace.run_dir) + optimizer_dir = run_dir / "optimizer" + optimizer_dir.mkdir() + outside = tmp_path / "optimizer-result.json" + outside.write_text("{}\n", encoding="utf-8") + (optimizer_dir / "result.json").symlink_to(outside) + + with pytest.raises(ArtifactWriteError, match="symbolic link"): + publish_report_bundle(report, run_dir=run_dir, copy_input_files=True) + + assert not (run_dir / "report").exists() + assert list(run_dir.glob(".report.tmp-*")) == [] + + +@pytest.mark.asyncio +async def test_publish_failure_removes_staging_directory(tmp_path, monkeypatch): + prepared, report = await _build_fake_report(tmp_path, "artifact_partial") + calls = 0 + original = artifact_writer._write_text + + def fail_second_write(path, content): + nonlocal calls + calls += 1 + if calls == 2: + raise OSError("simulated write failure") + original(path, content) + + monkeypatch.setattr(artifact_writer, "_write_text", fail_second_write) + run_dir = Path(prepared.workspace.run_dir) + + with pytest.raises(ArtifactWriteError, match="simulated write failure"): + publish_report_bundle(report, run_dir=run_dir, copy_input_files=True) + + assert not (run_dir / "report").exists() + assert list(run_dir.glob(".report.tmp-*")) == [] + + +@pytest.mark.asyncio +async def test_failure_report_is_atomic_and_round_trips(tmp_path): + prepared, _ = await _build_fake_report(tmp_path, "artifact_failure") + progress = ReportProgress( + started_at=datetime(2026, 7, 18, tzinfo=timezone.utc), + current_phase="analysis", + completed_phases=[ + "baseline_train", + "baseline_validation", + "candidate_generation", + "candidate_train", + "candidate_validation", + ], + ) + failure = build_failure_report( + prepared, + progress=progress, + error=RuntimeError("simulated analysis failure"), + source_prompt_hashes={ + snapshot.field_name: snapshot.sha256 + for snapshot in prepared.input_snapshot.prompt_snapshots + }, + existing_artifacts=[], + generated_at=datetime(2026, 7, 18, 0, 1, tzinfo=timezone.utc), + ) + + path = write_failure_report(failure, run_dir=Path(prepared.workspace.run_dir)) + + assert FailureReport.model_validate_json(path.read_text(encoding="utf-8")) == failure + assert list(path.parent.glob(".failure_report.tmp-*")) == [] + + +@pytest.mark.asyncio +async def test_failure_report_rejects_existing_first_failure_evidence(tmp_path): + prepared, _ = await _build_fake_report(tmp_path, "artifact_failure_existing") + run_dir = Path(prepared.workspace.run_dir) + target = run_dir / "failure_report.json" + target.write_text('{"first": true}\n', encoding="utf-8") + original = target.read_bytes() + progress = ReportProgress( + started_at=datetime(2026, 7, 18, tzinfo=timezone.utc), + current_phase="reporting", + completed_phases=[], + ) + failure = build_failure_report( + prepared, + progress=progress, + error=RuntimeError("second failure"), + source_prompt_hashes={}, + existing_artifacts=[], + generated_at=datetime(2026, 7, 18, 0, 1, tzinfo=timezone.utc), + ) + + with pytest.raises(ArtifactWriteError, match="already exists"): + write_failure_report(failure, run_dir=run_dir) + + assert target.read_bytes() == original + assert list(run_dir.glob(".failure_report.tmp-*")) == [] + + +@pytest.mark.asyncio +async def test_failure_report_does_not_overwrite_target_created_during_write( + tmp_path, monkeypatch +): + prepared, _ = await _build_fake_report(tmp_path, "artifact_failure_race") + run_dir = Path(prepared.workspace.run_dir) + target = run_dir / "failure_report.json" + first_failure = b'{"first": true}\n' + original_write = artifact_writer._write_text + + def create_first_failure_after_temporary_write(path, content): + original_write(path, content) + target.write_bytes(first_failure) + + monkeypatch.setattr( + artifact_writer, + "_write_text", + create_first_failure_after_temporary_write, + ) + progress = ReportProgress( + started_at=datetime(2026, 7, 18, tzinfo=timezone.utc), + current_phase="reporting", + completed_phases=[], + ) + failure = build_failure_report( + prepared, + progress=progress, + error=RuntimeError("racing failure"), + source_prompt_hashes={}, + existing_artifacts=[], + generated_at=datetime(2026, 7, 18, 0, 1, tzinfo=timezone.utc), + ) + + with pytest.raises(ArtifactWriteError, match="already exists"): + write_failure_report(failure, run_dir=run_dir) + + assert target.read_bytes() == first_failure + assert list(run_dir.glob(".failure_report.tmp-*")) == [] + + +@pytest.mark.asyncio +async def test_copy_input_files_false_records_unavailable_inputs(tmp_path): + prepared, report = await _build_fake_report(tmp_path, "artifact_no_inputs") + + index = publish_report_bundle( + report, + run_dir=Path(prepared.workspace.run_dir), + copy_input_files=False, + ) + + input_refs = [ + ref for ref in index.artifacts if ref.artifact_id.startswith("input.") + ] + assert input_refs + assert {ref.status for ref in input_refs} == {"unavailable"} + assert all( + ref.unavailable_reason == "artifacts.copy_input_files=false" + for ref in input_refs + ) From 94618bc9a5d0411eb5d2d5fa7f9762d6df32b74a Mon Sep 17 00:00:00 2001 From: guocfu Date: Sun, 19 Jul 2026 12:31:15 +0800 Subject: [PATCH 12/18] =?UTF-8?q?fix(evaluation):=20=E5=8A=A0=E5=9B=BA?= =?UTF-8?q?=E6=8A=A5=E5=91=8A=E5=8F=91=E5=B8=83=E4=B8=8E=E9=85=8D=E7=BD=AE?= =?UTF-8?q?=E5=BF=AB=E7=85=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../eval_optimize_loop/artifact_writer.py | 112 ++++++++++++++- ...est_eval_optimize_loop_stage5_artifacts.py | 127 ++++++++++++++++-- 2 files changed, 228 insertions(+), 11 deletions(-) diff --git a/examples/optimization/eval_optimize_loop/artifact_writer.py b/examples/optimization/eval_optimize_loop/artifact_writer.py index 468f8027..d39baa8b 100644 --- a/examples/optimization/eval_optimize_loop/artifact_writer.py +++ b/examples/optimization/eval_optimize_loop/artifact_writer.py @@ -7,11 +7,15 @@ from __future__ import annotations +import ctypes +import errno import hashlib +import json import os from pathlib import Path import shutil -from typing import Literal, TypeAlias +import sys +from typing import Callable, Literal, TypeAlias from uuid import uuid4 from pydantic import BaseModel @@ -31,6 +35,20 @@ ] _INPUT_COPY_DISABLED = "artifacts.copy_input_files=false" +_SENSITIVE_CONFIG_KEYS = {"apikey", "authorization", "baseurl"} +_APPROVED_SENSITIVE_VALUES = { + "", + "${TRPC_AGENT_API_KEY}", + "${TRPC_AGENT_BASE_URL}", + "fake-not-used-in-offline-mode", +} +_AT_FDCWD = -100 +_RENAME_NOREPLACE = 1 +_RENAMEAT2_UNAVAILABLE = { + errno.ENOSYS, + errno.EINVAL, + getattr(errno, "EOPNOTSUPP", errno.ENOTSUP), +} class ArtifactWriteError(RuntimeError): @@ -126,6 +144,87 @@ def _json_text(model: BaseModel) -> str: return model.model_dump_json(by_alias=False, indent=2) + "\n" +def _normalized_sensitive_key(key: str) -> str: + return key.replace("_", "").replace("-", "").casefold() + + +def _validate_sensitive_config_values(value: object, *, path: str = "$") -> None: + if isinstance(value, list): + for index, item in enumerate(value): + _validate_sensitive_config_values(item, path=f"{path}[{index}]") + return + if not isinstance(value, dict): + return + for key, item in value.items(): + item_path = f"{path}.{key}" + if _normalized_sensitive_key(key) in _SENSITIVE_CONFIG_KEYS: + if not isinstance(item, str) or item not in _APPROVED_SENSITIVE_VALUES: + raise ArtifactWriteError( + "sensitive optimizer config value is not an approved " + f"placeholder: {item_path}" + ) + else: + _validate_sensitive_config_values(item, path=item_path) + + +def _validate_optimizer_config_for_copy(path: Path) -> None: + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ArtifactWriteError( + f"failed to parse optimizer config snapshot: {path}: {exc}" + ) from exc + _validate_sensitive_config_values(payload) + + +def _target_exists(path: Path) -> bool: + return path.exists() or path.is_symlink() + + +def _rename_directory_no_replace(source: Path, target: Path) -> None: + """Atomically publish a directory without replacing an existing target. + + Linux uses renameat2 with RENAME_NOREPLACE. Platforms without that primitive + use a controlled fallback that rechecks the target immediately before rename. + """ + if sys.platform.startswith("linux"): + try: + libc = ctypes.CDLL(None, use_errno=True) + renameat2 = libc.renameat2 + except (AttributeError, OSError): + renameat2 = None + if renameat2 is not None: + renameat2.argtypes = [ + ctypes.c_int, + ctypes.c_char_p, + ctypes.c_int, + ctypes.c_char_p, + ctypes.c_uint, + ] + renameat2.restype = ctypes.c_int + ctypes.set_errno(0) + result = renameat2( + _AT_FDCWD, + os.fsencode(source), + _AT_FDCWD, + os.fsencode(target), + _RENAME_NOREPLACE, + ) + if result == 0: + return + error_number = ctypes.get_errno() + if error_number == errno.EEXIST: + raise ArtifactWriteError( + f"report directory already exists: {target}" + ) + if error_number not in _RENAMEAT2_UNAVAILABLE: + raise OSError(error_number, os.strerror(error_number), target) + + if _target_exists(target): + raise ArtifactWriteError(f"report directory already exists: {target}") + source.rename(target) + + def _published_relative_path(root: Path, path: Path) -> str: relative = path.relative_to(root) if relative.parts and relative.parts[0].startswith(".report.tmp-"): @@ -216,6 +315,7 @@ def _copy_input( destination_name: str, artifact_id: str, produced_by: ReportPhase, + content_validator: Callable[[Path], None] | None = None, ) -> ArtifactReference: if source.is_symlink(): raise ArtifactWriteError(f"input must not be a symbolic link: {source}") @@ -225,6 +325,8 @@ def _copy_input( raise ArtifactWriteError(f"failed to read input {source}: {exc}") from exc if actual_sha256 != expected_sha256: raise ArtifactWriteError(f"input hash mismatch: {source}") + if content_validator is not None: + content_validator(source) destination = staging / "inputs" / destination_name destination.parent.mkdir(parents=True, exist_ok=True) @@ -387,6 +489,11 @@ def publish_report_bundle( ) for artifact_id, source, expected_hash, destination_name, produced_by in input_specs: if copy_input_files: + content_validator = ( + _validate_optimizer_config_for_copy + if artifact_id == "input.optimizer_config" + else None + ) references.append( _copy_input( root=root, @@ -396,6 +503,7 @@ def publish_report_bundle( destination_name=destination_name, artifact_id=artifact_id, produced_by=produced_by, + content_validator=content_validator, ) ) else: @@ -433,7 +541,7 @@ def publish_report_bundle( ) _validate_available_references(root, staging, validated_index) - staging.rename(target) + _rename_directory_no_replace(staging, target) staging = None return validated_index except Exception as exc: diff --git a/tests/evaluation/test_eval_optimize_loop_stage5_artifacts.py b/tests/evaluation/test_eval_optimize_loop_stage5_artifacts.py index a97dfadf..d02f75ae 100644 --- a/tests/evaluation/test_eval_optimize_loop_stage5_artifacts.py +++ b/tests/evaluation/test_eval_optimize_loop_stage5_artifacts.py @@ -2,6 +2,7 @@ from datetime import datetime, timezone from hashlib import sha256 +import json from pathlib import Path import shutil @@ -32,8 +33,7 @@ def _copy_example(tmp_path: Path) -> Path: return target -async def _build_fake_report(tmp_path: Path, run_id: str): - root = _copy_example(tmp_path) +async def _build_fake_report_for_root(root: Path, run_id: str): prepared = prepare_run(root / "pipeline.json", run_id=run_id) result = await run_fake_stage(prepared, scenario="improve") shutil.rmtree(Path(prepared.workspace.run_dir) / "report", ignore_errors=True) @@ -60,6 +60,10 @@ async def _build_fake_report(tmp_path: Path, run_id: str): return prepared, report +async def _build_fake_report(tmp_path: Path, run_id: str): + return await _build_fake_report_for_root(_copy_example(tmp_path), run_id) + + @pytest.mark.asyncio async def test_publish_report_bundle_materializes_and_indexes_required_files(tmp_path): prepared, report = await _build_fake_report(tmp_path, "artifact_complete") @@ -71,15 +75,62 @@ async def test_publish_report_bundle_materializes_and_indexes_required_files(tmp assert (report_dir / "optimization_report.json").is_file() assert (report_dir / "optimization_report.md").is_file() assert (report_dir / "artifact_index.json").is_file() - for name in ( - "baseline_train", - "baseline_validation", - "candidate_train", - "candidate_validation", - ): - assert (report_dir / "evaluations" / f"{name}.json").is_file() assert all(ref.relative_path != "report/artifact_index.json" for ref in index.artifacts) + input_paths = { + ref.artifact_id: ref.relative_path + for ref in index.artifacts + if ref.artifact_type == "input" + } + assert input_paths == { + "input.pipeline_config": "report/inputs/pipeline_config.json", + "input.optimizer_config": "report/inputs/optimizer_config.json", + "input.train_evalset": "report/inputs/train_evalset.json", + "input.validation_evalset": "report/inputs/validation_evalset.json", + } + assert { + ref.artifact_id: ref.relative_path + for ref in index.artifacts + if ref.artifact_type == "prompt" + } == { + "prompt.baseline.system_prompt": ( + "report/prompts/baseline/000-system_prompt.md" + ), + "prompt.candidate.system_prompt": ( + "report/prompts/candidate/000-system_prompt.md" + ), + } + baseline_prompt = report_dir / "prompts" / "baseline" / "000-system_prompt.md" + candidate_prompt = report_dir / "prompts" / "candidate" / "000-system_prompt.md" + assert baseline_prompt.read_text(encoding="utf-8") == ( + report.input_snapshot.prompt_snapshots[0].content + ) + assert candidate_prompt.read_text(encoding="utf-8") == ( + report.candidate.prompts["system_prompt"] + ) + evaluation_paths = { + ref.artifact_id: ref.relative_path + for ref in index.artifacts + if ref.artifact_type == "evaluation" + } + assert evaluation_paths == { + f"evaluation.{name}": f"report/evaluations/{name}.json" + for name in ( + "baseline_train", + "baseline_validation", + "candidate_train", + "candidate_validation", + ) + } + assert { + path.name for path in (report_dir / "evaluations").iterdir() + } == { + "baseline_train.json", + "baseline_validation.json", + "candidate_train.json", + "candidate_validation.json", + } + available = [ref for ref in index.artifacts if ref.status == "available"] assert available for ref in available: @@ -110,6 +161,36 @@ async def test_publish_rejects_input_hash_drift(tmp_path): assert list(run_dir.glob(".report.tmp-*")) == [] +@pytest.mark.asyncio +async def test_publish_rejects_plaintext_secret_in_optimizer_config_without_leaking_it( + tmp_path, +): + root = _copy_example(tmp_path) + optimizer_path = root / "optimizer.json" + payload = json.loads(optimizer_path.read_text(encoding="utf-8")) + secret = "sk-real-sentinel-secret-must-not-be-copied" + payload["optimize"]["algorithm"]["reflection_lm"]["api_key"] = secret + optimizer_path.write_text( + json.dumps(payload, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + prepared, report = await _build_fake_report_for_root(root, "artifact_secret") + run_dir = Path(prepared.workspace.run_dir) + + with pytest.raises(ArtifactWriteError, match="sensitive optimizer config"): + publish_report_bundle(report, run_dir=run_dir, copy_input_files=True) + + assert secret in optimizer_path.read_text(encoding="utf-8") + assert not (run_dir / "report").exists() + assert list(run_dir.glob(".report.tmp-*")) == [] + secret_bytes = secret.encode("utf-8") + assert all( + secret_bytes not in path.read_bytes() + for path in run_dir.rglob("*") + if path.is_file() + ) + + @pytest.mark.asyncio async def test_publish_rejects_existing_report_directory(tmp_path): prepared, report = await _build_fake_report(tmp_path, "artifact_existing") @@ -124,6 +205,34 @@ async def test_publish_rejects_existing_report_directory(tmp_path): ) +@pytest.mark.asyncio +async def test_publish_does_not_replace_report_directory_created_during_staging( + tmp_path, monkeypatch +): + prepared, report = await _build_fake_report(tmp_path, "artifact_report_race") + run_dir = Path(prepared.workspace.run_dir) + report_dir = run_dir / "report" + original_write = artifact_writer._write_text + + def create_competing_report_before_publish(path, content): + original_write(path, content) + if path.name == "artifact_index.json": + report_dir.mkdir() + + monkeypatch.setattr( + artifact_writer, + "_write_text", + create_competing_report_before_publish, + ) + + with pytest.raises(ArtifactWriteError, match="already exists"): + publish_report_bundle(report, run_dir=run_dir, copy_input_files=True) + + assert report_dir.is_dir() + assert list(report_dir.iterdir()) == [] + assert list(run_dir.glob(".report.tmp-*")) == [] + + @pytest.mark.parametrize("target_inside_run", [False, True]) def test_discovery_rejects_any_file_symlink(tmp_path, target_inside_run): run_dir = tmp_path / "run" From a59723792cbc681fc12435196d19f695b4f0cfa8 Mon Sep 17 00:00:00 2001 From: guocfu Date: Sun, 19 Jul 2026 13:00:15 +0800 Subject: [PATCH 13/18] =?UTF-8?q?fix(evaluation):=20=E7=A6=81=E6=AD=A2?= =?UTF-8?q?=E9=9D=9E=E5=8E=9F=E5=AD=90=E6=8A=A5=E5=91=8A=E5=8F=91=E5=B8=83?= =?UTF-8?q?=E5=9B=9E=E9=80=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../eval_optimize_loop/artifact_writer.py | 101 ++++++++++++------ ...est_eval_optimize_loop_stage5_artifacts.py | 20 ++++ 2 files changed, 86 insertions(+), 35 deletions(-) diff --git a/examples/optimization/eval_optimize_loop/artifact_writer.py b/examples/optimization/eval_optimize_loop/artifact_writer.py index d39baa8b..eeebfa74 100644 --- a/examples/optimization/eval_optimize_loop/artifact_writer.py +++ b/examples/optimization/eval_optimize_loop/artifact_writer.py @@ -44,6 +44,7 @@ } _AT_FDCWD = -100 _RENAME_NOREPLACE = 1 +_RENAME_EXCL = 0x4 _RENAMEAT2_UNAVAILABLE = { errno.ENOSYS, errno.EINVAL, @@ -177,52 +178,82 @@ def _validate_optimizer_config_for_copy(path: Path) -> None: _validate_sensitive_config_values(payload) -def _target_exists(path: Path) -> bool: - return path.exists() or path.is_symlink() - - def _rename_directory_no_replace(source: Path, target: Path) -> None: """Atomically publish a directory without replacing an existing target. - Linux uses renameat2 with RENAME_NOREPLACE. Platforms without that primitive - use a controlled fallback that rechecks the target immediately before rename. + Each supported platform uses an atomic no-replace primitive. Platforms + without that primitive fail closed rather than risking a replacement race. """ if sys.platform.startswith("linux"): try: libc = ctypes.CDLL(None, use_errno=True) renameat2 = libc.renameat2 except (AttributeError, OSError): - renameat2 = None - if renameat2 is not None: - renameat2.argtypes = [ - ctypes.c_int, - ctypes.c_char_p, - ctypes.c_int, - ctypes.c_char_p, - ctypes.c_uint, - ] - renameat2.restype = ctypes.c_int - ctypes.set_errno(0) - result = renameat2( - _AT_FDCWD, - os.fsencode(source), - _AT_FDCWD, - os.fsencode(target), - _RENAME_NOREPLACE, + raise ArtifactWriteError( + "atomic no-replace unavailable: Linux renameat2 is unavailable" ) - if result == 0: - return - error_number = ctypes.get_errno() - if error_number == errno.EEXIST: - raise ArtifactWriteError( - f"report directory already exists: {target}" - ) - if error_number not in _RENAMEAT2_UNAVAILABLE: - raise OSError(error_number, os.strerror(error_number), target) + renameat2.argtypes = [ + ctypes.c_int, + ctypes.c_char_p, + ctypes.c_int, + ctypes.c_char_p, + ctypes.c_uint, + ] + renameat2.restype = ctypes.c_int + ctypes.set_errno(0) + result = renameat2( + _AT_FDCWD, + os.fsencode(source), + _AT_FDCWD, + os.fsencode(target), + _RENAME_NOREPLACE, + ) + if result == 0: + return + error_number = ctypes.get_errno() + if error_number == errno.EEXIST: + raise ArtifactWriteError(f"report directory already exists: {target}") + if error_number in _RENAMEAT2_UNAVAILABLE: + raise ArtifactWriteError( + "atomic no-replace unavailable: Linux renameat2 does not support " + f"RENAME_NOREPLACE ({os.strerror(error_number)})" + ) + raise OSError(error_number, os.strerror(error_number), target) + + if sys.platform.startswith("win"): + try: + os.rename(source, target) + except FileExistsError as exc: + raise ArtifactWriteError(f"report directory already exists: {target}") from exc + return + + if sys.platform == "darwin": + try: + libc = ctypes.CDLL(None, use_errno=True) + renamex_np = libc.renamex_np + except (AttributeError, OSError): + raise ArtifactWriteError( + "atomic no-replace unavailable: Darwin renamex_np is unavailable" + ) + renamex_np.argtypes = [ctypes.c_char_p, ctypes.c_char_p, ctypes.c_uint] + renamex_np.restype = ctypes.c_int + ctypes.set_errno(0) + result = renamex_np(os.fsencode(source), os.fsencode(target), _RENAME_EXCL) + if result == 0: + return + error_number = ctypes.get_errno() + if error_number == errno.EEXIST: + raise ArtifactWriteError(f"report directory already exists: {target}") + if error_number in _RENAMEAT2_UNAVAILABLE: + raise ArtifactWriteError( + "atomic no-replace unavailable: Darwin renamex_np does not support " + f"RENAME_EXCL ({os.strerror(error_number)})" + ) + raise OSError(error_number, os.strerror(error_number), target) - if _target_exists(target): - raise ArtifactWriteError(f"report directory already exists: {target}") - source.rename(target) + raise ArtifactWriteError( + f"atomic no-replace unavailable: unsupported platform {sys.platform}" + ) def _published_relative_path(root: Path, path: Path) -> str: diff --git a/tests/evaluation/test_eval_optimize_loop_stage5_artifacts.py b/tests/evaluation/test_eval_optimize_loop_stage5_artifacts.py index d02f75ae..9febb31a 100644 --- a/tests/evaluation/test_eval_optimize_loop_stage5_artifacts.py +++ b/tests/evaluation/test_eval_optimize_loop_stage5_artifacts.py @@ -233,6 +233,26 @@ def create_competing_report_before_publish(path, content): assert list(run_dir.glob(".report.tmp-*")) == [] +def test_publish_fails_closed_when_atomic_no_replace_is_unsupported( + tmp_path, monkeypatch +): + source = tmp_path / "staging" + target = tmp_path / "report" + source.mkdir() + + def unexpected_rename(self, destination): + raise AssertionError("普通 Path.rename 不得作为 no-replace 回退") + + monkeypatch.setattr(artifact_writer.sys, "platform", "freebsd") + monkeypatch.setattr(Path, "rename", unexpected_rename) + + with pytest.raises(ArtifactWriteError, match="atomic no-replace unavailable"): + artifact_writer._rename_directory_no_replace(source, target) + + assert source.is_dir() + assert not target.exists() + + @pytest.mark.parametrize("target_inside_run", [False, True]) def test_discovery_rejects_any_file_symlink(tmp_path, target_inside_run): run_dir = tmp_path / "run" From 1504625803200588c6183071486405fb5ea9b778 Mon Sep 17 00:00:00 2001 From: guocfu Date: Sun, 19 Jul 2026 13:26:20 +0800 Subject: [PATCH 14/18] =?UTF-8?q?fix(evaluation):=20=E9=98=BB=E6=AD=A2?= =?UTF-8?q?=E6=89=A9=E5=B1=95=E9=85=8D=E7=BD=AE=E6=B3=84=E9=9C=B2=E5=87=AD?= =?UTF-8?q?=E6=8D=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../eval_optimize_loop/artifact_writer.py | 20 ++++++++++++++++++- ...est_eval_optimize_loop_stage5_artifacts.py | 10 ++++++++-- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/examples/optimization/eval_optimize_loop/artifact_writer.py b/examples/optimization/eval_optimize_loop/artifact_writer.py index eeebfa74..e3eef9f3 100644 --- a/examples/optimization/eval_optimize_loop/artifact_writer.py +++ b/examples/optimization/eval_optimize_loop/artifact_writer.py @@ -35,7 +35,25 @@ ] _INPUT_COPY_DISABLED = "artifacts.copy_input_files=false" -_SENSITIVE_CONFIG_KEYS = {"apikey", "authorization", "baseurl"} +_SENSITIVE_CONFIG_KEYS = { + "accesstoken", + "apikey", + "auth", + "authorization", + "authtoken", + "baseurl", + "bearertoken", + "clientsecret", + "credential", + "credentials", + "password", + "passwd", + "privatekey", + "secret", + "secretkey", + "token", + "xapikey", +} _APPROVED_SENSITIVE_VALUES = { "", "${TRPC_AGENT_API_KEY}", diff --git a/tests/evaluation/test_eval_optimize_loop_stage5_artifacts.py b/tests/evaluation/test_eval_optimize_loop_stage5_artifacts.py index 9febb31a..c801fe46 100644 --- a/tests/evaluation/test_eval_optimize_loop_stage5_artifacts.py +++ b/tests/evaluation/test_eval_optimize_loop_stage5_artifacts.py @@ -161,15 +161,21 @@ async def test_publish_rejects_input_hash_drift(tmp_path): assert list(run_dir.glob(".report.tmp-*")) == [] +@pytest.mark.parametrize( + "sensitive_key", + ["api_key", "access_token", "client_secret", "password", "x-api-key"], +) @pytest.mark.asyncio async def test_publish_rejects_plaintext_secret_in_optimizer_config_without_leaking_it( - tmp_path, + tmp_path, sensitive_key ): root = _copy_example(tmp_path) optimizer_path = root / "optimizer.json" payload = json.loads(optimizer_path.read_text(encoding="utf-8")) secret = "sk-real-sentinel-secret-must-not-be-copied" - payload["optimize"]["algorithm"]["reflection_lm"]["api_key"] = secret + payload["optimize"]["algorithm"]["reflection_lm"]["extra_fields"] = { + sensitive_key: secret + } optimizer_path.write_text( json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8", From 2e5eb6cde4b3796b5f042e9e41f17e4c49841257 Mon Sep 17 00:00:00 2001 From: guocfu Date: Sun, 19 Jul 2026 20:10:32 +0800 Subject: [PATCH 15/18] =?UTF-8?q?feat(evaluation):=20=E5=AE=8C=E6=88=90?= =?UTF-8?q?=E7=AC=AC=E4=BA=94=E9=98=B6=E6=AE=B5=E6=8A=A5=E5=91=8A=E4=B8=8E?= =?UTF-8?q?=E5=AE=A1=E8=AE=A1=E9=97=AD=E7=8E=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../optimization/eval_optimize_loop/README.md | 68 ++- .../eval_optimize_loop/artifact_writer.py | 32 +- .../eval_optimize_loop/pipeline.py | 203 +++++++- .../eval_optimize_loop/run_pipeline.py | 5 +- .../run_real_integration.py | 4 + .../test_eval_optimize_loop_stage3b.py | 4 +- ...est_eval_optimize_loop_stage5_artifacts.py | 54 ++- ...test_eval_optimize_loop_stage5_pipeline.py | 442 ++++++++++++++++++ 8 files changed, 792 insertions(+), 20 deletions(-) create mode 100644 tests/evaluation/test_eval_optimize_loop_stage5_pipeline.py diff --git a/examples/optimization/eval_optimize_loop/README.md b/examples/optimization/eval_optimize_loop/README.md index 658bc84a..be980a1a 100644 --- a/examples/optimization/eval_optimize_loop/README.md +++ b/examples/optimization/eval_optimize_loop/README.md @@ -1,11 +1,12 @@ -# Evaluation + Optimization Loop — Stage 4 +# Evaluation + Optimization Loop — Stage 5 This example provides an auditable evaluation and prompt-optimization loop. Stages 1–3 prepare an isolated prompt workspace, run baseline and candidate evaluations on both train and validation datasets, normalize failures, build case diffs, detect overfitting, and apply an independent Gate. Stage 4 adds a common Candidate Provider boundary, an `AgentOptimizer` adapter, and guarded -source-prompt writeback. +source-prompt writeback. Stage 5 publishes a complete JSON/Markdown report and +artifact index, or preserves a standalone failure report when a run fails. The deterministic fake mode still runs without a model, API key, judge, or optimizer. Its built-in scenarios produce ACCEPT for `improve` and REJECT for @@ -13,7 +14,7 @@ both `no_improvement` and `overfit`: ```bash python examples/optimization/eval_optimize_loop/run_pipeline.py \ - --run-id local_stage4 \ + --run-id local_stage5 \ --scenario improve ``` @@ -38,7 +39,8 @@ credential reuse the environment without writing their resolved values to run artifacts. `--run-real` is mandatory so an accidental command cannot spend API quota. This entry always uses `pipeline.real.json`, where source writeback is disabled, and reports ACCEPT or REJECT without treating REJECT as a process -failure. +failure. Both CLIs print the paths of the JSON report, Markdown report, and +artifact index after a completed run. Applications with a custom agent can still use the Python integration point. Set `execution.mode` to `real`, prepare the run, and inject an async @@ -67,6 +69,45 @@ pipeline evaluations: baseline train/validation and candidate train/validation. scores, and configuration snapshot are retained under `runs//optimizer/` when `artifacts.retain_optimizer_native_artifacts` is enabled. +## Report artifacts + +A completed run atomically publishes the formal bundle at +`runs//report/`: + +```text +report/ +├── optimization_report.json +├── optimization_report.md +├── artifact_index.json +├── inputs/ +│ ├── pipeline_config.json +│ ├── optimizer_config.json +│ ├── train_evalset.json +│ └── validation_evalset.json +├── evaluations/ +│ ├── baseline_train.json +│ ├── baseline_validation.json +│ ├── candidate_train.json +│ └── candidate_validation.json +└── prompts/ + ├── baseline/ + └── candidate/ +``` + +The formal `report/` directory is visible only after all required artifacts +have been written and validated. If any post-preparation phase fails, the +pipeline does not leave a partial formal report; it atomically writes +`runs//failure_report.json` with the failed phase, completed phases, +sanitized error information, source Prompt hashes, and already existing +artifacts. A failure-report write error is surfaced together with the original +pipeline error. + +`artifact_index.json` records each artifact's relative path, SHA-256 hash, byte +size, producer phase, and availability. Consumers can use those fields to +verify that an artifact has not drifted since publication. Input copies are +validated against the preparation snapshot, and sensitive resolved credentials +are not accepted into the report bundle. + Source prompts are updated only when all of these conditions hold: - Gate returns ACCEPT; @@ -81,13 +122,15 @@ returned as failed. If rollback integrity cannot be proven, the pipeline raises an error instead of claiming the source is safe. The checked-in configuration keeps writeback disabled by default. -Elapsed duration is observable. Full monetary cost and token usage remain -`unavailable` because an injected business agent may make calls that the SDK -optimizer does not account for; native optimizer resource fields are retained -without treating missing values as zero. JSON/Markdown reports and an artifact -index are Stage 5 work. +The report separates whole-pipeline resources from optimizer-only observations. +Pipeline duration is observable, while full pipeline monetary cost and token +usage remain `unavailable` because business-agent calls may not expose complete +telemetry. In real mode, optimizer rounds, reflection calls, duration, cost, and +token usage are reported independently from the native optimizer result; an +unreliable or incomplete field stays `unavailable` instead of being treated as +zero. In fake mode, optimizer-only fields are `not_applicable`. -Run the Stage 1–4 tests with: +Run the Stage 1–5 tests with: ```bash .venv/bin/pytest -q \ @@ -96,5 +139,8 @@ Run the Stage 1–4 tests with: tests/evaluation/test_eval_optimize_loop_stage3a.py \ tests/evaluation/test_eval_optimize_loop_stage3b.py \ tests/evaluation/test_eval_optimize_loop_stage4.py \ - tests/evaluation/test_eval_optimize_loop_real_integration.py + tests/evaluation/test_eval_optimize_loop_real_integration.py \ + tests/evaluation/test_eval_optimize_loop_stage5_report_builder.py \ + tests/evaluation/test_eval_optimize_loop_stage5_artifacts.py \ + tests/evaluation/test_eval_optimize_loop_stage5_pipeline.py ``` diff --git a/examples/optimization/eval_optimize_loop/artifact_writer.py b/examples/optimization/eval_optimize_loop/artifact_writer.py index e3eef9f3..341e0abe 100644 --- a/examples/optimization/eval_optimize_loop/artifact_writer.py +++ b/examples/optimization/eval_optimize_loop/artifact_writer.py @@ -54,6 +54,21 @@ "token", "xapikey", } +_SENSITIVE_CONFIG_KEY_SUFFIXES = { + "accesstoken", + "apikey", + "authtoken", + "baseurl", + "bearertoken", + "clientsecret", + "credential", + "credentials", + "endpointurl", + "password", + "passwd", + "privatekey", + "secretkey", +} _APPROVED_SENSITIVE_VALUES = { "", "${TRPC_AGENT_API_KEY}", @@ -167,7 +182,22 @@ def _normalized_sensitive_key(key: str) -> str: return key.replace("_", "").replace("-", "").casefold() +def _is_sensitive_config_key(key: str) -> bool: + normalized = _normalized_sensitive_key(key) + return normalized in _SENSITIVE_CONFIG_KEYS or any( + normalized.endswith(suffix) + for suffix in _SENSITIVE_CONFIG_KEY_SUFFIXES + ) + + def _validate_sensitive_config_values(value: object, *, path: str = "$") -> None: + if isinstance(value, str): + if value.strip().casefold().startswith(("http://", "https://")): + raise ArtifactWriteError( + "sensitive optimizer config value is not an approved " + f"placeholder: {path}" + ) + return if isinstance(value, list): for index, item in enumerate(value): _validate_sensitive_config_values(item, path=f"{path}[{index}]") @@ -176,7 +206,7 @@ def _validate_sensitive_config_values(value: object, *, path: str = "$") -> None return for key, item in value.items(): item_path = f"{path}.{key}" - if _normalized_sensitive_key(key) in _SENSITIVE_CONFIG_KEYS: + if _is_sensitive_config_key(key): if not isinstance(item, str) or item not in _APPROVED_SENSITIVE_VALUES: raise ArtifactWriteError( "sensitive optimizer config value is not an approved " diff --git a/examples/optimization/eval_optimize_loop/pipeline.py b/examples/optimization/eval_optimize_loop/pipeline.py index 3d7cc0a7..bbb26672 100644 --- a/examples/optimization/eval_optimize_loop/pipeline.py +++ b/examples/optimization/eval_optimize_loop/pipeline.py @@ -10,6 +10,7 @@ import re import shutil from dataclasses import dataclass +from dataclasses import field from datetime import datetime from datetime import timezone from hashlib import sha256 @@ -28,6 +29,9 @@ from trpc_agent_sdk.evaluation import load_optimize_config from .analysis import build_evaluation_analysis +from .artifact_writer import discover_run_artifacts +from .artifact_writer import publish_report_bundle +from .artifact_writer import write_failure_report from .candidate_provider import AgentOptimizerCandidateProvider from .candidate_provider import CandidateProviderError from .candidate_provider import CandidateRequest @@ -42,6 +46,8 @@ from .prompt_workspace import resolve_inside_example_root from .prompt_workspace import stage_prompt_workspace from .prompt_workspace import validate_prompt_sources +from .report_builder import build_failure_report +from .report_builder import build_optimization_report from .schemas import InputSnapshot from .schemas import FakeCandidateScenario from .schemas import FakeEvaluationSnapshot @@ -50,6 +56,8 @@ from .schemas import OptimizerRuntimeParameters from .schemas import ResourceMeasurements from .schemas import RealStageResult +from .schemas import ReportPhase +from .schemas import ReportProgress from .schemas import WorkspaceSnapshot from .writeback import perform_writeback @@ -79,6 +87,111 @@ class PreparedRun: example_root: Path +@dataclass +class _MutableReportProgress: + """Track the active report phase without marking it complete too early.""" + + started_at: datetime + current_phase: ReportPhase = "baseline_train" + completed_phases: list[ReportPhase] = field(default_factory=list) + + def enter(self, phase: ReportPhase) -> None: + if self.current_phase not in self.completed_phases and self.current_phase != phase: + self.completed_phases.append(self.current_phase) + self.current_phase = phase + + def snapshot(self) -> ReportProgress: + return ReportProgress( + started_at=self.started_at, + current_phase=self.current_phase, + completed_phases=list(self.completed_phases), + ) + + +async def _source_prompt_hashes(prepared: PreparedRun) -> dict[str, str]: + try: + prompts = await prepared.source_target.read_all() + except Exception: + # Failure evidence must remain writable even when the source itself is + # unavailable. An empty mapping means the final source state could not + # be observed; it must never be replaced with stale snapshot hashes. + return {} + return { + name: sha256(value.encode("utf-8")).hexdigest() + for name, value in sorted(prompts.items()) + } + + +async def _record_failure( + prepared: PreparedRun, + progress: _MutableReportProgress, + error: Exception, +) -> None: + run_dir = Path(prepared.workspace.run_dir) + existing = discover_run_artifacts(run_dir) + report = build_failure_report( + prepared, + progress=progress.snapshot(), + error=error, + source_prompt_hashes=await _source_prompt_hashes(prepared), + existing_artifacts=existing, + generated_at=datetime.now(timezone.utc), + ) + write_failure_report(report, run_dir=run_dir) + + +async def _rollback_written_source( + prepared: PreparedRun, + result: FakeStageResult | RealStageResult, +) -> None: + """Restore the prepared source Prompt if success reporting cannot publish.""" + if result.writeback.status != "written": + return + baseline = { + snapshot.field_name: snapshot.content + for snapshot in prepared.input_snapshot.prompt_snapshots + } + current = await prepared.source_target.read_all() + if current != result.candidate.prompts: + raise PipelineExecutionError( + "source Prompt changed after writeback; refusing reporting-failure rollback" + ) + # Path-backed TargetPrompt.write_all performs its atomic replacements + # synchronously, so this task does not yield between the adjacent check and + # write. Callback-backed sources retain the caller's documented atomicity + # responsibility, as they do for the normal writeback path. + await prepared.source_target.write_all(baseline) + restored = await prepared.source_target.read_all() + if restored != baseline: + raise PipelineExecutionError( + "source Prompt rollback after reporting failure could not be verified" + ) + + +async def _handle_stage_failure( + prepared: PreparedRun, + progress: _MutableReportProgress, + error: Exception, + result: FakeStageResult | RealStageResult | None, +) -> None: + failure_error: Exception = error + if progress.current_phase == "reporting" and result is not None: + try: + await _rollback_written_source(prepared, result) + except Exception as rollback_exc: + failure_error = PipelineExecutionError( + f"{error}; additionally failed to roll back source Prompt: {rollback_exc}" + ) + try: + await _record_failure(prepared, progress, failure_error) + except Exception as report_exc: + raise PipelineExecutionError( + f"{failure_error}; additionally failed to write failure report: {report_exc}" + ) from error + if failure_error is not error: + raise failure_error from error + + _RUN_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_-]*$") @@ -377,10 +490,11 @@ async def _restore_working_baseline( return was_modified -async def run_fake_stage( +async def _execute_fake_stage( prepared: PreparedRun, *, scenario: FakeCandidateScenario | None = None, + progress: _MutableReportProgress, ) -> FakeStageResult: """Run four deterministic evaluations without a model, judge, or optimizer. @@ -388,6 +502,7 @@ async def run_fake_stage( the isolated working target on success or candidate-evaluation failure so the run can be inspected later. """ + progress.enter("baseline_train") if prepared.config.execution.mode != "fake": raise FakeStageExecutionError( f"run_fake_stage requires execution.mode='fake', got {prepared.config.execution.mode!r}" @@ -422,6 +537,7 @@ async def run_fake_stage( raise FakeStageExecutionError("working prompts no longer match the prepared baseline snapshot") agent = DeterministicFakeAgent(prepared.working_target) + progress.enter("baseline_train") baseline_train = await _evaluate_split( prepared=prepared, eval_set=train_evalset, @@ -429,6 +545,7 @@ async def run_fake_stage( phase="baseline", split="train", ) + progress.enter("baseline_validation") baseline_validation = await _evaluate_split( prepared=prepared, eval_set=validation_evalset, @@ -437,6 +554,7 @@ async def run_fake_stage( split="validation", ) + progress.enter("candidate_generation") request = CandidateRequest( current_prompts=baseline_prompts, target_prompt=prepared.working_target, @@ -460,6 +578,7 @@ async def run_fake_stage( if written_prompts != candidate.prompts: raise FakeStageExecutionError("candidate prompt readback did not match the generated proposal") + progress.enter("candidate_train") candidate_train = await _evaluate_split( prepared=prepared, eval_set=train_evalset, @@ -467,6 +586,7 @@ async def run_fake_stage( phase="candidate", split="train", ) + progress.enter("candidate_validation") candidate_validation = await _evaluate_split( prepared=prepared, eval_set=validation_evalset, @@ -474,6 +594,7 @@ async def run_fake_stage( phase="candidate", split="validation", ) + progress.enter("analysis") try: analysis = build_evaluation_analysis( baseline_train=baseline_train, @@ -503,6 +624,7 @@ async def run_fake_stage( unit="seconds", ), ) + progress.enter("gate") try: gate_decision = evaluate_gate( analysis, @@ -512,6 +634,7 @@ async def run_fake_stage( ) except GateEvaluationError as exc: raise FakeStageExecutionError(f"stage 3b gate failed: {exc}") from exc + progress.enter("writeback") writeback = await perform_writeback( decision=gate_decision, config=prepared.config.writeback, @@ -533,13 +656,15 @@ async def run_fake_stage( ) -async def run_real_stage( +async def _execute_real_stage( prepared: PreparedRun, *, call_agent: CallAgent, optimizer_parameters: OptimizerRuntimeParameters | None = None, + progress: _MutableReportProgress, ) -> RealStageResult: """Generate a real optimizer candidate and run the full guarded regression.""" + progress.enter("baseline_train") if prepared.config.execution.mode != "real": raise FakeStageExecutionError( f"run_real_stage requires execution.mode='real', got {prepared.config.execution.mode!r}" @@ -573,6 +698,7 @@ async def run_real_stage( if baseline_prompts != expected_baseline: raise FakeStageExecutionError("working prompts no longer match the prepared baseline snapshot") + progress.enter("baseline_train") baseline_train = await _evaluate_split( prepared=prepared, eval_set=train_evalset, @@ -580,6 +706,7 @@ async def run_real_stage( phase="baseline", split="train", ) + progress.enter("baseline_validation") baseline_validation = await _evaluate_split( prepared=prepared, eval_set=validation_evalset, @@ -588,6 +715,7 @@ async def run_real_stage( split="validation", ) + progress.enter("candidate_generation") request = CandidateRequest( current_prompts=baseline_prompts, target_prompt=prepared.working_target, @@ -618,6 +746,7 @@ async def run_real_stage( if written_prompts != candidate.prompts: raise FakeStageExecutionError("candidate prompt readback did not match the generated proposal") + progress.enter("candidate_train") candidate_train = await _evaluate_split( prepared=prepared, eval_set=train_evalset, @@ -625,6 +754,7 @@ async def run_real_stage( phase="candidate", split="train", ) + progress.enter("candidate_validation") candidate_validation = await _evaluate_split( prepared=prepared, eval_set=validation_evalset, @@ -632,6 +762,7 @@ async def run_real_stage( phase="candidate", split="validation", ) + progress.enter("analysis") try: analysis = build_evaluation_analysis( baseline_train=baseline_train, @@ -661,6 +792,7 @@ async def run_real_stage( unit="seconds", ), ) + progress.enter("gate") try: gate_decision = evaluate_gate( analysis, @@ -671,6 +803,7 @@ async def run_real_stage( except GateEvaluationError as exc: raise FakeStageExecutionError(f"stage 3b gate failed: {exc}") from exc + progress.enter("writeback") writeback = await perform_writeback( decision=gate_decision, config=prepared.config.writeback, @@ -692,3 +825,69 @@ async def run_real_stage( gate_decision=gate_decision, writeback=writeback, ) + + +async def run_fake_stage( + prepared: PreparedRun, + *, + scenario: FakeCandidateScenario | None = None, +) -> FakeStageResult: + """Run fake regression and atomically publish its audit report.""" + progress = _MutableReportProgress(started_at=datetime.now(timezone.utc)) + result: FakeStageResult | None = None + try: + result = await _execute_fake_stage( + prepared, + scenario=scenario, + progress=progress, + ) + progress.enter("reporting") + report = build_optimization_report( + prepared, + result, + progress=progress.snapshot(), + finished_at=datetime.now(timezone.utc), + ) + publish_report_bundle( + report, + run_dir=Path(prepared.workspace.run_dir), + copy_input_files=prepared.config.artifacts.copy_input_files, + ) + return result + except Exception as exc: + await _handle_stage_failure(prepared, progress, exc, result) + raise + + +async def run_real_stage( + prepared: PreparedRun, + *, + call_agent: CallAgent, + optimizer_parameters: OptimizerRuntimeParameters | None = None, +) -> RealStageResult: + """Run real optimization and atomically publish its audit report.""" + progress = _MutableReportProgress(started_at=datetime.now(timezone.utc)) + result: RealStageResult | None = None + try: + result = await _execute_real_stage( + prepared, + call_agent=call_agent, + optimizer_parameters=optimizer_parameters, + progress=progress, + ) + progress.enter("reporting") + report = build_optimization_report( + prepared, + result, + progress=progress.snapshot(), + finished_at=datetime.now(timezone.utc), + ) + publish_report_bundle( + report, + run_dir=Path(prepared.workspace.run_dir), + copy_input_files=prepared.config.artifacts.copy_input_files, + ) + return result + except Exception as exc: + await _handle_stage_failure(prepared, progress, exc, result) + raise diff --git a/examples/optimization/eval_optimize_loop/run_pipeline.py b/examples/optimization/eval_optimize_loop/run_pipeline.py index 9c9cfccc..b1e3c333 100644 --- a/examples/optimization/eval_optimize_loop/run_pipeline.py +++ b/examples/optimization/eval_optimize_loop/run_pipeline.py @@ -81,7 +81,10 @@ def main() -> None: f"Writeback: {result.writeback.status.upper()} " f"({result.writeback.reason})" ) - print("Stage 4 does not write optimization reports; reporting is added in Stage 5.") + report_dir = Path(prepared.workspace.run_dir) / "report" + print(f"JSON report: {report_dir / 'optimization_report.json'}") + print(f"Markdown report: {report_dir / 'optimization_report.md'}") + print(f"Artifact index: {report_dir / 'artifact_index.json'}") if __name__ == "__main__": diff --git a/examples/optimization/eval_optimize_loop/run_real_integration.py b/examples/optimization/eval_optimize_loop/run_real_integration.py index 9a3c605b..03d0d5e3 100644 --- a/examples/optimization/eval_optimize_loop/run_real_integration.py +++ b/examples/optimization/eval_optimize_loop/run_real_integration.py @@ -133,6 +133,10 @@ def main() -> int: print(f"- {reason}") print(f"Writeback: {result.writeback.status.upper()} ({result.writeback.reason})") print("Source Prompt unchanged: yes") + report_dir = Path(prepared.workspace.run_dir) / "report" + print(f"JSON report: {report_dir / 'optimization_report.json'}") + print(f"Markdown report: {report_dir / 'optimization_report.md'}") + print(f"Artifact index: {report_dir / 'artifact_index.json'}") return 0 diff --git a/tests/evaluation/test_eval_optimize_loop_stage3b.py b/tests/evaluation/test_eval_optimize_loop_stage3b.py index 0e15fa87..c1516ab5 100644 --- a/tests/evaluation/test_eval_optimize_loop_stage3b.py +++ b/tests/evaluation/test_eval_optimize_loop_stage3b.py @@ -622,4 +622,6 @@ def test_stage3b_cli_prints_gate_decision_and_rejection_reasons(tmp_path: Path): assert "Rejection reasons:" in completed.stdout assert "[no_overfitting]" in completed.stdout assert "Writeback: SKIPPED (gate_rejected)" in completed.stdout - assert "Stage 4 does not write optimization reports" in completed.stdout + assert "optimization_report.json" in completed.stdout + assert "optimization_report.md" in completed.stdout + assert "artifact_index.json" in completed.stdout diff --git a/tests/evaluation/test_eval_optimize_loop_stage5_artifacts.py b/tests/evaluation/test_eval_optimize_loop_stage5_artifacts.py index c801fe46..2cff48b1 100644 --- a/tests/evaluation/test_eval_optimize_loop_stage5_artifacts.py +++ b/tests/evaluation/test_eval_optimize_loop_stage5_artifacts.py @@ -9,13 +9,14 @@ import pytest from examples.optimization.eval_optimize_loop import artifact_writer +from examples.optimization.eval_optimize_loop import pipeline as pipeline_module from examples.optimization.eval_optimize_loop.artifact_writer import ( ArtifactWriteError, discover_run_artifacts, publish_report_bundle, write_failure_report, ) -from examples.optimization.eval_optimize_loop.pipeline import prepare_run, run_fake_stage +from examples.optimization.eval_optimize_loop.pipeline import prepare_run from examples.optimization.eval_optimize_loop.report_builder import ( build_failure_report, build_optimization_report, @@ -35,8 +36,14 @@ def _copy_example(tmp_path: Path) -> Path: async def _build_fake_report_for_root(root: Path, run_id: str): prepared = prepare_run(root / "pipeline.json", run_id=run_id) - result = await run_fake_stage(prepared, scenario="improve") - shutil.rmtree(Path(prepared.workspace.run_dir) / "report", ignore_errors=True) + execution_progress = pipeline_module._MutableReportProgress( + started_at=datetime(2026, 7, 18, tzinfo=timezone.utc) + ) + result = await pipeline_module._execute_fake_stage( + prepared, + scenario="improve", + progress=execution_progress, + ) progress = ReportProgress( started_at=datetime(2026, 7, 18, tzinfo=timezone.utc), current_phase="reporting", @@ -163,7 +170,16 @@ async def test_publish_rejects_input_hash_drift(tmp_path): @pytest.mark.parametrize( "sensitive_key", - ["api_key", "access_token", "client_secret", "password", "x-api-key"], + [ + "api_key", + "access_token", + "client_secret", + "password", + "x-api-key", + "openai_api_key", + "provider_api_key", + "endpoint_url", + ], ) @pytest.mark.asyncio async def test_publish_rejects_plaintext_secret_in_optimizer_config_without_leaking_it( @@ -197,6 +213,36 @@ async def test_publish_rejects_plaintext_secret_in_optimizer_config_without_leak ) +@pytest.mark.asyncio +async def test_publish_rejects_literal_url_hidden_under_provider_specific_key( + tmp_path, +): + root = _copy_example(tmp_path) + optimizer_path = root / "optimizer.json" + payload = json.loads(optimizer_path.read_text(encoding="utf-8")) + endpoint = "https://private-provider.example.test/v1" + payload["optimize"]["algorithm"]["reflection_lm"]["extra_fields"] = { + "service_location": endpoint + } + optimizer_path.write_text( + json.dumps(payload, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + prepared, report = await _build_fake_report_for_root(root, "artifact_url") + run_dir = Path(prepared.workspace.run_dir) + + with pytest.raises(ArtifactWriteError, match="sensitive optimizer config"): + publish_report_bundle(report, run_dir=run_dir, copy_input_files=True) + + assert not (run_dir / "report").exists() + endpoint_bytes = endpoint.encode("utf-8") + assert all( + endpoint_bytes not in path.read_bytes() + for path in run_dir.rglob("*") + if path.is_file() + ) + + @pytest.mark.asyncio async def test_publish_rejects_existing_report_directory(tmp_path): prepared, report = await _build_fake_report(tmp_path, "artifact_existing") diff --git a/tests/evaluation/test_eval_optimize_loop_stage5_pipeline.py b/tests/evaluation/test_eval_optimize_loop_stage5_pipeline.py new file mode 100644 index 00000000..f913cc3a --- /dev/null +++ b/tests/evaluation/test_eval_optimize_loop_stage5_pipeline.py @@ -0,0 +1,442 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under the Apache License, Version 2.0. +"""Stage 5 integration tests for success and failure report orchestration.""" + +from __future__ import annotations + +import json +from pathlib import Path +import shutil +import subprocess +import sys +import time +from types import SimpleNamespace + +import pytest + +from examples.optimization.eval_optimize_loop import pipeline as pipeline_module +from examples.optimization.eval_optimize_loop import run_real_integration +from examples.optimization.eval_optimize_loop.fake import DeterministicFakeAgent +from examples.optimization.eval_optimize_loop.fake import DeterministicFakeCandidateProvider +from examples.optimization.eval_optimize_loop.pipeline import PipelineExecutionError +from examples.optimization.eval_optimize_loop.pipeline import prepare_run +from examples.optimization.eval_optimize_loop.pipeline import run_fake_stage +from examples.optimization.eval_optimize_loop.pipeline import run_real_stage +from examples.optimization.eval_optimize_loop.schemas import ArtifactIndex +from examples.optimization.eval_optimize_loop.schemas import FailureReport +from examples.optimization.eval_optimize_loop.schemas import OptimizationReport +from examples.optimization.eval_optimize_loop.schemas import OptimizerRuntimeParameters +from trpc_agent_sdk.evaluation import OptimizeResult + + +_REPO_ROOT = Path(__file__).resolve().parents[2] +_EXAMPLE = _REPO_ROOT / "examples" / "optimization" / "eval_optimize_loop" + + +def _copy_example(tmp_path: Path, name: str = "eval_optimize_loop") -> Path: + target = tmp_path / name + shutil.copytree( + _EXAMPLE, + target, + ignore=shutil.ignore_patterns("runs", "__pycache__"), + ) + return target + + +def _set_real_mode(root: Path) -> None: + path = root / "pipeline.json" + payload = json.loads(path.read_text(encoding="utf-8")) + payload["execution"]["mode"] = "real" + path.write_text(json.dumps(payload), encoding="utf-8") + + +def _optimize_result( + baseline_prompts: dict[str, str], + best_prompts: dict[str, str], +) -> OptimizeResult: + return OptimizeResult( + algorithm="gepa_reflective", + status="SUCCEEDED", + finish_reason="completed", + error_message="", + baseline_pass_rate=1 / 3, + best_pass_rate=1.0, + pass_rate_improvement=2 / 3, + baseline_prompts=baseline_prompts, + best_prompts=best_prompts, + total_rounds=1, + rounds=[], + total_reflection_lm_calls=1, + total_llm_cost=0.25, + total_token_usage={"prompt": 10, "completion": 5, "total": 15}, + duration_seconds=0.5, + started_at="2026-07-18T00:00:00Z", + finished_at="2026-07-18T00:00:01Z", + ) + + +@pytest.mark.parametrize("scenario", ["improve", "no_improvement", "overfit"]) +@pytest.mark.asyncio +async def test_fake_stage_publishes_complete_report_bundle( + tmp_path: Path, + scenario: str, +) -> None: + root = _copy_example(tmp_path) + prepared = prepare_run(root / "pipeline.json", run_id=f"stage5_{scenario}") + + result = await run_fake_stage(prepared, scenario=scenario) + + run_dir = Path(prepared.workspace.run_dir) + report_dir = run_dir / "report" + report = OptimizationReport.model_validate_json( + (report_dir / "optimization_report.json").read_text(encoding="utf-8") + ) + assert report.gate_decision == result.gate_decision + assert report.candidate == result.candidate + assert (report_dir / "artifact_index.json").is_file() + assert not (run_dir / "failure_report.json").exists() + + +@pytest.mark.asyncio +async def test_real_stage_reports_optimizer_native_artifacts( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + root = _copy_example(tmp_path) + _set_real_mode(root) + prepared = prepare_run(root / "pipeline.json", run_id="stage5_real") + baseline = await prepared.working_target.read_all() + best = DeterministicFakeCandidateProvider().propose( + baseline, + scenario="improve", + seed=prepared.input_snapshot.seed, + ).prompts + + async def fake_optimize(**kwargs: object) -> OptimizeResult: + output_dir = Path(str(kwargs["output_dir"])) + rounds_dir = output_dir / "rounds" + rounds_dir.mkdir(parents=True) + result = _optimize_result(baseline, best) + result.dump_to(str(output_dir / "result.json")) + (rounds_dir / "round_001.json").write_text("{}\n", encoding="utf-8") + return result + + monkeypatch.setattr( + "examples.optimization.eval_optimize_loop.candidate_provider.AgentOptimizer.optimize", + fake_optimize, + ) + + result = await run_real_stage( + prepared, + call_agent=DeterministicFakeAgent(prepared.working_target).call_agent, + optimizer_parameters=OptimizerRuntimeParameters( + model_name="stage5-reflection-model", + max_candidate_proposals=1, + ), + ) + + report_dir = Path(prepared.workspace.run_dir) / "report" + report = OptimizationReport.model_validate_json( + (report_dir / "optimization_report.json").read_text(encoding="utf-8") + ) + index = ArtifactIndex.model_validate_json( + (report_dir / "artifact_index.json").read_text(encoding="utf-8") + ) + native_paths = { + reference.relative_path + for reference in index.artifacts + if reference.artifact_type == "optimizer_native" + } + assert native_paths == { + "optimizer.runtime.json", + "optimizer/result.json", + "optimizer/rounds/round_001.json", + } + assert report.candidate == result.candidate + assert report.optimizer_resources.total_rounds.status == "available" + assert report.optimizer_resources.reflection_lm_calls.status == "available" + assert report.optimizer_resources.duration_seconds.status == "available" + + +_FAILURE_CASES = [ + ("_evaluate_split", "baseline_train"), + ("FakeCandidateProviderAdapter.propose", "candidate_generation"), + ("build_evaluation_analysis", "analysis"), + ("evaluate_gate", "gate"), + ("perform_writeback", "writeback"), + ("publish_report_bundle", "reporting"), +] + + +def _install_stage_failure( + monkeypatch: pytest.MonkeyPatch, + target: str, +) -> None: + def fail_sync(*args: object, **kwargs: object) -> None: + raise RuntimeError("simulated stage failure") + + async def fail_async(*args: object, **kwargs: object) -> None: + raise RuntimeError("simulated stage failure") + + if target == "FakeCandidateProviderAdapter.propose": + monkeypatch.setattr( + pipeline_module.FakeCandidateProviderAdapter, + "propose", + fail_async, + ) + elif target in {"_evaluate_split", "perform_writeback"}: + monkeypatch.setattr(pipeline_module, target, fail_async) + else: + monkeypatch.setattr(pipeline_module, target, fail_sync, raising=False) + + +@pytest.mark.parametrize(("target", "expected_phase"), _FAILURE_CASES) +@pytest.mark.asyncio +async def test_fake_stage_writes_phase_specific_failure_report( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + target: str, + expected_phase: str, +) -> None: + root = _copy_example(tmp_path) + prepared = prepare_run( + root / "pipeline.json", + run_id=f"stage5_failure_{expected_phase}", + ) + _install_stage_failure(monkeypatch, target) + + with pytest.raises(Exception, match="simulated stage failure"): + await run_fake_stage(prepared, scenario="improve") + + run_dir = Path(prepared.workspace.run_dir) + failure = FailureReport.model_validate_json( + (run_dir / "failure_report.json").read_text(encoding="utf-8") + ) + assert failure.failed_phase == expected_phase + assert failure.exception_type in { + "RuntimeError", + "PipelineExecutionError", + "ArtifactWriteError", + } + assert "simulated stage failure" in failure.error_message + assert not (run_dir / "report").exists() + + +@pytest.mark.asyncio +async def test_failure_report_error_preserves_original_exception_as_cause( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + root = _copy_example(tmp_path) + prepared = prepare_run(root / "pipeline.json", run_id="stage5_double_failure") + + async def fail_evaluation(*args: object, **kwargs: object) -> None: + raise RuntimeError("simulated stage failure") + + def fail_failure_report(*args: object, **kwargs: object) -> None: + raise RuntimeError("failed to write failure report") + + monkeypatch.setattr(pipeline_module, "_evaluate_split", fail_evaluation) + monkeypatch.setattr( + pipeline_module, + "write_failure_report", + fail_failure_report, + raising=False, + ) + + with pytest.raises(PipelineExecutionError) as raised: + await run_fake_stage(prepared, scenario="improve") + + assert "simulated stage failure" in str(raised.value) + assert "failed to write failure report" in str(raised.value) + assert isinstance(raised.value.__cause__, RuntimeError) + assert str(raised.value.__cause__) == "simulated stage failure" + + +@pytest.mark.asyncio +async def test_reporting_failure_rolls_back_successful_source_writeback( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + root = _copy_example(tmp_path) + config_path = root / "pipeline.json" + payload = json.loads(config_path.read_text(encoding="utf-8")) + payload["writeback"]["enabled"] = True + config_path.write_text(json.dumps(payload), encoding="utf-8") + prepared = prepare_run(config_path, run_id="stage5_reporting_rollback") + source_before = await prepared.source_target.read_all() + + def fail_report(*args: object, **kwargs: object) -> None: + raise RuntimeError("simulated reporting failure") + + monkeypatch.setattr(pipeline_module, "publish_report_bundle", fail_report) + + with pytest.raises(RuntimeError, match="simulated reporting failure"): + await run_fake_stage(prepared, scenario="improve") + + assert await prepared.source_target.read_all() == source_before + failure = FailureReport.model_validate_json( + (Path(prepared.workspace.run_dir) / "failure_report.json").read_text( + encoding="utf-8" + ) + ) + assert failure.failed_phase == "reporting" + + +@pytest.mark.asyncio +async def test_reporting_and_writeback_rollback_failure_are_both_exposed( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + root = _copy_example(tmp_path) + config_path = root / "pipeline.json" + payload = json.loads(config_path.read_text(encoding="utf-8")) + payload["writeback"]["enabled"] = True + config_path.write_text(json.dumps(payload), encoding="utf-8") + prepared = prepare_run(config_path, run_id="stage5_reporting_rollback_failure") + + def fail_report(*args: object, **kwargs: object) -> None: + raise RuntimeError("simulated reporting failure") + + async def fail_rollback(*args: object, **kwargs: object) -> None: + raise RuntimeError("simulated source rollback failure") + + monkeypatch.setattr(pipeline_module, "publish_report_bundle", fail_report) + monkeypatch.setattr(pipeline_module, "_rollback_written_source", fail_rollback) + + with pytest.raises(PipelineExecutionError) as raised: + await run_fake_stage(prepared, scenario="improve") + + assert "simulated reporting failure" in str(raised.value) + assert "simulated source rollback failure" in str(raised.value) + assert isinstance(raised.value.__cause__, RuntimeError) + assert str(raised.value.__cause__) == "simulated reporting failure" + + +@pytest.mark.asyncio +async def test_failure_report_remains_writable_when_source_prompt_is_unreadable( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + root = _copy_example(tmp_path) + prepared = prepare_run(root / "pipeline.json", run_id="stage5_source_unreadable") + progress = pipeline_module._MutableReportProgress( + started_at=pipeline_module.datetime.now(pipeline_module.timezone.utc) + ) + + async def fail_source_read() -> dict[str, str]: + raise OSError("simulated unreadable source Prompt") + + monkeypatch.setattr(prepared.source_target, "read_all", fail_source_read) + + await pipeline_module._record_failure( + prepared, + progress, + RuntimeError("simulated pipeline failure"), + ) + + failure = FailureReport.model_validate_json( + (Path(prepared.workspace.run_dir) / "failure_report.json").read_text( + encoding="utf-8" + ) + ) + assert failure.source_prompt_hashes == {} + + +def test_fake_cli_prints_report_paths(tmp_path: Path) -> None: + root = _copy_example(tmp_path) + + completed = subprocess.run( + [ + sys.executable, + str(root / "run_pipeline.py"), + "--config", + str(root / "pipeline.json"), + "--run-id", + "stage5_cli", + ], + cwd=_REPO_ROOT, + capture_output=True, + text=True, + check=False, + ) + + assert completed.returncode == 0, completed.stderr + assert "optimization_report.json" in completed.stdout + assert "optimization_report.md" in completed.stdout + assert "artifact_index.json" in completed.stdout + + +@pytest.mark.asyncio +async def test_all_fake_scenarios_duration_is_under_three_minutes_with_reports( + tmp_path: Path, +) -> None: + started_at = time.perf_counter() + + for scenario in ("improve", "no_improvement", "overfit"): + root = _copy_example(tmp_path, name=f"eval_optimize_loop_{scenario}") + prepared = prepare_run(root / "pipeline.json", run_id=f"stage5_{scenario}") + + await run_fake_stage(prepared, scenario=scenario) + + report_dir = Path(prepared.workspace.run_dir) / "report" + assert (report_dir / "optimization_report.json").is_file() + assert (report_dir / "optimization_report.md").is_file() + assert (report_dir / "artifact_index.json").is_file() + + assert time.perf_counter() - started_at < 180.0 + + +def test_real_cli_gate_reject_prints_report_paths_and_exits_zero( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + snapshot = SimpleNamespace( + passed_case_count=1, + total_case_count=3, + average_score=1 / 3, + ) + result = SimpleNamespace( + baseline_train=snapshot, + baseline_validation=snapshot, + candidate_train=snapshot, + candidate_validation=snapshot, + optimize_result=SimpleNamespace(status="SUCCEEDED", total_rounds=1), + candidate=SimpleNamespace(candidate_id="optimizer-stage5"), + gate_decision=SimpleNamespace( + decision="reject", + rejection_reasons=["no improvement"], + ), + writeback=SimpleNamespace(status="skipped", reason="gate_rejected"), + ) + prepared = SimpleNamespace( + workspace=SimpleNamespace(run_dir="runs/stage5_real_cli") + ) + + async def complete(*args: object, **kwargs: object) -> tuple[object, object]: + return prepared, result + + monkeypatch.setenv("TRPC_AGENT_API_KEY", "key") + monkeypatch.setenv("TRPC_AGENT_BASE_URL", "https://example.test") + monkeypatch.setenv("TRPC_AGENT_MODEL_NAME", "business-model") + monkeypatch.setattr(run_real_integration, "_run", complete) + monkeypatch.setattr( + sys, + "argv", + [ + "run_real_integration.py", + "--run-real", + "--optimizer-model-name", + "optimizer-model", + ], + ) + + assert run_real_integration.main() == 0 + output = capsys.readouterr().out + assert "Gate decision: REJECT" in output + assert "optimization_report.json" in output + assert "optimization_report.md" in output + assert "artifact_index.json" in output From 38fac753096f0fd0a28362564dfb87d72a6043c6 Mon Sep 17 00:00:00 2001 From: guocfu Date: Tue, 21 Jul 2026 17:45:08 +0800 Subject: [PATCH 16/18] =?UTF-8?q?feat(evaluation):=20=E5=AE=8C=E6=88=90?= =?UTF-8?q?=E7=A6=BB=E7=BA=BF=E6=A8=A1=E5=9E=8B=E4=B8=8E=20Trace=20?= =?UTF-8?q?=E5=9B=9E=E6=94=BE=E9=AA=8C=E6=94=B6=E9=93=BE=E8=B7=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../optimization/eval_optimize_loop/DESIGN.md | 9 + .../optimization/eval_optimize_loop/README.md | 28 +- .../eval_optimize_loop/analysis.py | 10 +- .../eval_optimize_loop/artifact_writer.py | 30 +- .../eval_optimize_loop/business_agent.py | 102 + .../eval_optimize_loop/candidate_provider.py | 4 +- .../optimization/eval_optimize_loop/config.py | 76 +- .../data/traces/baseline.train.evalset.json | 1 + .../traces/baseline.validation.evalset.json | 1 + .../data/traces/improve.train.evalset.json | 1 + .../traces/improve.validation.evalset.json | 1 + .../traces/no_improvement.train.evalset.json | 1 + .../no_improvement.validation.evalset.json | 1 + .../data/traces/overfit.train.evalset.json | 1 + .../traces/overfit.validation.evalset.json | 1 + .../data/traces/prompts/improve.md | 5 + .../data/traces/prompts/no_improvement.md | 1 + .../data/traces/prompts/overfit.md | 5 + .../eval_optimize_loop/evaluation_adapter.py | 4 +- .../eval_optimize_loop/fake/__init__.py | 9 +- .../eval_optimize_loop/fake/agent.py | 127 - .../fake/candidate_provider.py | 8 +- .../eval_optimize_loop/fake/model.py | 189 + .../eval_optimize_loop/pipeline.json | 5 +- .../eval_optimize_loop/pipeline.py | 371 +- .../eval_optimize_loop/pipeline.real.json | 3 +- .../eval_optimize_loop/pipeline.trace.json | 46 + .../eval_optimize_loop/real_agent.py | 73 +- .../eval_optimize_loop/report_builder.py | 26 +- .../eval_optimize_loop/run_pipeline.py | 17 +- .../sample_output/artifact_index.json | 139 + .../evaluations/baseline_train.json | 500 +++ .../evaluations/baseline_validation.json | 500 +++ .../evaluations/candidate_train.json | 457 ++ .../evaluations/candidate_validation.json | 457 ++ .../inputs/optimizer_config.json | 36 + .../sample_output/inputs/pipeline_config.json | 52 + .../sample_output/inputs/train_evalset.json | 40 + .../inputs/validation_evalset.json | 40 + .../sample_output/optimization_report.json | 3666 +++++++++++++++++ .../sample_output/optimization_report.md | 44 + .../prompts/baseline/000-system_prompt.md | 5 + .../prompts/candidate/000-system_prompt.md | 13 + .../eval_optimize_loop/schemas.py | 77 +- ...est_eval_optimize_loop_real_integration.py | 12 +- .../test_eval_optimize_loop_stage1.py | 22 +- .../test_eval_optimize_loop_stage2.py | 64 +- .../test_eval_optimize_loop_stage3a.py | 16 +- .../test_eval_optimize_loop_stage3b.py | 18 +- .../test_eval_optimize_loop_stage4.py | 44 +- ...est_eval_optimize_loop_stage5_artifacts.py | 2 +- ...test_eval_optimize_loop_stage5_pipeline.py | 30 +- ...val_optimize_loop_stage5_report_builder.py | 22 +- ...st_eval_optimize_loop_stage6_acceptance.py | 60 + .../test_eval_optimize_loop_stage6_offline.py | 143 + .../test_eval_optimize_loop_stage6_trace.py | 99 + 56 files changed, 7308 insertions(+), 406 deletions(-) create mode 100644 examples/optimization/eval_optimize_loop/DESIGN.md create mode 100644 examples/optimization/eval_optimize_loop/business_agent.py create mode 100644 examples/optimization/eval_optimize_loop/data/traces/baseline.train.evalset.json create mode 100644 examples/optimization/eval_optimize_loop/data/traces/baseline.validation.evalset.json create mode 100644 examples/optimization/eval_optimize_loop/data/traces/improve.train.evalset.json create mode 100644 examples/optimization/eval_optimize_loop/data/traces/improve.validation.evalset.json create mode 100644 examples/optimization/eval_optimize_loop/data/traces/no_improvement.train.evalset.json create mode 100644 examples/optimization/eval_optimize_loop/data/traces/no_improvement.validation.evalset.json create mode 100644 examples/optimization/eval_optimize_loop/data/traces/overfit.train.evalset.json create mode 100644 examples/optimization/eval_optimize_loop/data/traces/overfit.validation.evalset.json create mode 100644 examples/optimization/eval_optimize_loop/data/traces/prompts/improve.md create mode 100644 examples/optimization/eval_optimize_loop/data/traces/prompts/no_improvement.md create mode 100644 examples/optimization/eval_optimize_loop/data/traces/prompts/overfit.md delete mode 100644 examples/optimization/eval_optimize_loop/fake/agent.py create mode 100644 examples/optimization/eval_optimize_loop/fake/model.py create mode 100644 examples/optimization/eval_optimize_loop/pipeline.trace.json create mode 100644 examples/optimization/eval_optimize_loop/sample_output/artifact_index.json create mode 100644 examples/optimization/eval_optimize_loop/sample_output/evaluations/baseline_train.json create mode 100644 examples/optimization/eval_optimize_loop/sample_output/evaluations/baseline_validation.json create mode 100644 examples/optimization/eval_optimize_loop/sample_output/evaluations/candidate_train.json create mode 100644 examples/optimization/eval_optimize_loop/sample_output/evaluations/candidate_validation.json create mode 100644 examples/optimization/eval_optimize_loop/sample_output/inputs/optimizer_config.json create mode 100644 examples/optimization/eval_optimize_loop/sample_output/inputs/pipeline_config.json create mode 100644 examples/optimization/eval_optimize_loop/sample_output/inputs/train_evalset.json create mode 100644 examples/optimization/eval_optimize_loop/sample_output/inputs/validation_evalset.json create mode 100644 examples/optimization/eval_optimize_loop/sample_output/optimization_report.json create mode 100644 examples/optimization/eval_optimize_loop/sample_output/optimization_report.md create mode 100644 examples/optimization/eval_optimize_loop/sample_output/prompts/baseline/000-system_prompt.md create mode 100644 examples/optimization/eval_optimize_loop/sample_output/prompts/candidate/000-system_prompt.md create mode 100644 tests/evaluation/test_eval_optimize_loop_stage6_acceptance.py create mode 100644 tests/evaluation/test_eval_optimize_loop_stage6_offline.py create mode 100644 tests/evaluation/test_eval_optimize_loop_stage6_trace.py diff --git a/examples/optimization/eval_optimize_loop/DESIGN.md b/examples/optimization/eval_optimize_loop/DESIGN.md new file mode 100644 index 00000000..b4a6f708 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/DESIGN.md @@ -0,0 +1,9 @@ +# 设计边界 + +本示例把运行环境与替代组件分开描述。`offline`、`real`、`trace` 是三种 Pipeline 运行模式。`offline` 仍使用 SDK 的 `LlmAgent`、Runner 和独立 Session，只把 Agent 内部模型替换成 `DeterministicFakeModel`，并用确定性 Candidate Provider 代替真实优化器。它用于验证 Prompt 改变是否经过真实 Agent 编排影响输出。 + +`real` 使用真实业务模型生成回复，并由 `AgentOptimizer` 调用真实反思模型产生 Prompt 候选；只有 Gate 接受、源 Prompt 哈希未漂移且显式启用写回时，才允许更新源文件。 + +`trace` 直接评测预录制的 `actual_conversation`，不再次运行 Agent、Model 或 Candidate Provider。它适合复现工具轨迹和生产故障，但只能证明候选版本与轨迹的关联，不能证明 Prompt 导致了该轨迹。因此 Trace 即使获得 ACCEPT，也固定跳过源 Prompt 写回。 + +确定性 metric 负责精确匹配等硬规则。LLM Judge 若需要，应作为带 rubric 的评测指标显式配置；本示例不提供容易混淆职责的 `use_fake_judge` 开关。 diff --git a/examples/optimization/eval_optimize_loop/README.md b/examples/optimization/eval_optimize_loop/README.md index be980a1a..3fc547d1 100644 --- a/examples/optimization/eval_optimize_loop/README.md +++ b/examples/optimization/eval_optimize_loop/README.md @@ -1,4 +1,12 @@ -# Evaluation + Optimization Loop — Stage 5 +# Evaluation + Optimization Loop — Stage 6 + +## 三种运行模式 + +- `offline`：使用真实 SDK `LlmAgent` 和 `Runner`，Agent 内部注入 `DeterministicFakeModel`，候选由确定性 Candidate Provider 生成。无需 API Key。 +- `trace`：使用 SDK 原生 `eval_mode="trace"` 回放 baseline/candidate 的 train 与 validation 轨迹，不运行 Agent、Model 或 Candidate Provider。即使 Gate 接受，也不会写回源 Prompt。 +- `real`：业务 Agent 和反思优化模型均使用真实模型，由显式入口 `run_real_integration.py --run-real` 启用。 + +Fake Model 是 `offline` 模式中的模型实现，不是独立运行模式；Fake Candidate Provider 只替代候选生成，不替代业务 Agent。精确匹配等硬规则由 `optimizer.json` 的确定性 metric 执行。需要语义判断时，应显式配置带 rubric 的 LLM Judge，而不是使用布尔 Fake Judge 开关。详细边界见 [DESIGN.md](DESIGN.md)。 This example provides an auditable evaluation and prompt-optimization loop. Stages 1–3 prepare an isolated prompt workspace, run baseline and candidate @@ -8,17 +16,25 @@ common Candidate Provider boundary, an `AgentOptimizer` adapter, and guarded source-prompt writeback. Stage 5 publishes a complete JSON/Markdown report and artifact index, or preserves a standalone failure report when a run fails. -The deterministic fake mode still runs without a model, API key, judge, or -optimizer. Its built-in scenarios produce ACCEPT for `improve` and REJECT for +The deterministic offline mode runs through the SDK Agent without an API key, +judge, or real optimizer. Its built-in scenarios produce ACCEPT for `improve` and REJECT for both `no_improvement` and `overfit`: ```bash python examples/optimization/eval_optimize_loop/run_pipeline.py \ - --run-id local_stage5 \ + --run-id local_stage6 \ --scenario improve ``` -The offline CLI remains self-contained and supports fake mode only. For an +The same CLI also supports trace replay: + +```bash +python examples/optimization/eval_optimize_loop/run_pipeline.py \ + --config examples/optimization/eval_optimize_loop/pipeline.trace.json \ + --scenario overfit +``` + +For an explicit real integration smoke run, configure the OpenAI-compatible business model connection in the environment: @@ -128,7 +144,7 @@ usage remain `unavailable` because business-agent calls may not expose complete telemetry. In real mode, optimizer rounds, reflection calls, duration, cost, and token usage are reported independently from the native optimizer result; an unreliable or incomplete field stays `unavailable` instead of being treated as -zero. In fake mode, optimizer-only fields are `not_applicable`. +zero. In offline mode, optimizer-only fields are `not_applicable`. Run the Stage 1–5 tests with: diff --git a/examples/optimization/eval_optimize_loop/analysis.py b/examples/optimization/eval_optimize_loop/analysis.py index 2e4ab48b..9210ae3a 100644 --- a/examples/optimization/eval_optimize_loop/analysis.py +++ b/examples/optimization/eval_optimize_loop/analysis.py @@ -11,7 +11,7 @@ from .case_diff import compare_evaluations from .evaluation_adapter import standardize_snapshot from .schemas import EvaluationAnalysis -from .schemas import FakeEvaluationSnapshot +from .schemas import EvaluationSnapshot from .schemas import ObservableValue from .schemas import OverfitStatus @@ -39,10 +39,10 @@ def _overfit_status( def build_evaluation_analysis( *, - baseline_train: FakeEvaluationSnapshot, - baseline_validation: FakeEvaluationSnapshot, - candidate_train: FakeEvaluationSnapshot, - candidate_validation: FakeEvaluationSnapshot, + baseline_train: EvaluationSnapshot, + baseline_validation: EvaluationSnapshot, + candidate_train: EvaluationSnapshot, + candidate_validation: EvaluationSnapshot, hard_case_ids: set[str], critical_case_ids: set[str], severe_case_score_drop: float, diff --git a/examples/optimization/eval_optimize_loop/artifact_writer.py b/examples/optimization/eval_optimize_loop/artifact_writer.py index 341e0abe..2353177c 100644 --- a/examples/optimization/eval_optimize_loop/artifact_writer.py +++ b/examples/optimization/eval_optimize_loop/artifact_writer.py @@ -23,6 +23,7 @@ from .report_builder import render_optimization_markdown from .schemas import ArtifactIndex, ArtifactReference, FailureReport from .schemas import OptimizationReport, ReportPhase +from .schemas import TraceCandidateProposal ArtifactType: TypeAlias = Literal[ @@ -536,7 +537,7 @@ def publish_report_bundle( ) ) - input_specs = ( + input_specs = [ ( "input.pipeline_config", Path(report.input_snapshot.pipeline_config_path), @@ -565,7 +566,32 @@ def publish_report_bundle( "validation_evalset.json", "baseline_validation", ), - ) + ] + if ( + isinstance(report.candidate, TraceCandidateProposal) + and report.input_snapshot.trace_inputs is not None + ): + trace = report.input_snapshot.trace_inputs.scenarios[ + report.candidate.scenario + ] + input_specs.extend( + [ + ( + "input.trace.candidate_train", + Path(trace.train_evalset_path), + trace.train_evalset_sha256, + "candidate_train_trace.json", + "candidate_train", + ), + ( + "input.trace.candidate_validation", + Path(trace.validation_evalset_path), + trace.validation_evalset_sha256, + "candidate_validation_trace.json", + "candidate_validation", + ), + ] + ) for artifact_id, source, expected_hash, destination_name, produced_by in input_specs: if copy_input_files: content_validator = ( diff --git a/examples/optimization/eval_optimize_loop/business_agent.py b/examples/optimization/eval_optimize_loop/business_agent.py new file mode 100644 index 00000000..e849e7e7 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/business_agent.py @@ -0,0 +1,102 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under the Apache License, Version 2.0. +"""共享的 SDK 业务 Agent：模型可注入，Prompt 每次重新读取。""" + +from __future__ import annotations + +from collections.abc import Callable +from uuid import uuid4 + +from trpc_agent_sdk.agents import LlmAgent +from trpc_agent_sdk.evaluation import TargetPrompt +from trpc_agent_sdk.models import LLMModel +from trpc_agent_sdk.runners import Runner +from trpc_agent_sdk.sessions import InMemorySessionService +from trpc_agent_sdk.types import Content +from trpc_agent_sdk.types import GenerateContentConfig +from trpc_agent_sdk.types import Part + + +ModelFactory = Callable[[], LLMModel] + + +def render_instruction(prompts: dict[str, str]) -> str: + """稳定拼接一个或多个工作 Prompt 字段。""" + if len(prompts) == 1: + return next(iter(prompts.values())) + return "\n\n".join( + f"## {name}\n{content}" for name, content in prompts.items() + ) + + +class BusinessAgent: + """以独立 SDK Session 执行一次 Prompt 敏感的业务请求。""" + + def __init__( + self, + target_prompt: TargetPrompt, + model_factory: ModelFactory, + *, + agent_name: str, + app_name: str, + user_id: str, + ) -> None: + self._target_prompt = target_prompt + self._model_factory = model_factory + self._agent_name = agent_name + self._app_name = app_name + self._user_id = user_id + + async def call_agent(self, query: str) -> str: + """重新读取 Prompt，运行独立 Session，并返回最终可见文本。""" + if not isinstance(query, str): + raise TypeError("query must be a string") + + prompts = await self._target_prompt.read_all() + model = self._model_factory() + root_agent = LlmAgent( + name=self._agent_name, + description="Evaluation and prompt optimization business agent.", + model=model, + instruction=render_instruction(prompts), + generate_content_config=GenerateContentConfig( + temperature=0.0, + max_output_tokens=512, + ), + ) + session_service = InMemorySessionService() + runner = Runner( + app_name=self._app_name, + agent=root_agent, + session_service=session_service, + ) + session_id = str(uuid4()) + await session_service.create_session( + app_name=self._app_name, + user_id=self._user_id, + session_id=session_id, + state={}, + ) + message = Content( + role="user", + parts=[Part.from_text(text=query)], + ) + final_text = "" + async for event in runner.run_async( + user_id=self._user_id, + session_id=session_id, + new_message=message, + ): + if ( + not event.is_final_response() + or not event.content + or not event.content.parts + ): + continue + for part in event.content.parts: + if not part.thought and part.text: + final_text += part.text + return final_text.strip() diff --git a/examples/optimization/eval_optimize_loop/candidate_provider.py b/examples/optimization/eval_optimize_loop/candidate_provider.py index 4a395e30..69b65822 100644 --- a/examples/optimization/eval_optimize_loop/candidate_provider.py +++ b/examples/optimization/eval_optimize_loop/candidate_provider.py @@ -21,7 +21,7 @@ from .fake.candidate_provider import DeterministicFakeCandidateProvider from .schemas import CandidateProposal -from .schemas import FakeCandidateScenario +from .schemas import CandidateScenario from .schemas import OptimizerCandidateProposal from .schemas import OptimizerRuntimeParameters @@ -75,7 +75,7 @@ async def propose(self, request: CandidateRequest) -> CandidateGeneration: class FakeCandidateProviderAdapter: """Lift the pure synchronous fake provider into the common async boundary.""" - def __init__(self, scenario: FakeCandidateScenario) -> None: + def __init__(self, scenario: CandidateScenario) -> None: self._scenario = scenario async def propose(self, request: CandidateRequest) -> CandidateGeneration: diff --git a/examples/optimization/eval_optimize_loop/config.py b/examples/optimization/eval_optimize_loop/config.py index 406d6a8e..20536963 100644 --- a/examples/optimization/eval_optimize_loop/config.py +++ b/examples/optimization/eval_optimize_loop/config.py @@ -30,11 +30,29 @@ class ExecutionConfig(EvalBaseModel): - """How a later phase obtains an optimization candidate.""" + """Pipeline execution mode and deterministic candidate scenario.""" - mode: Literal["fake", "real", "trace"] = "fake" - fake_candidate_scenario: Literal["improve", "no_improvement", "overfit"] = "improve" - use_fake_judge: bool = False + mode: Literal["offline", "real", "trace"] = "offline" + candidate_scenario: Literal["improve", "no_improvement", "overfit"] = "improve" + + @model_validator(mode="before") + @classmethod + def _reject_removed_execution_options(cls, value: object) -> object: + if not isinstance(value, dict): + return value + if value.get("mode") == "fake": + raise ValueError("execution.mode='fake' was renamed to 'offline'") + if "use_fake_judge" in value: + raise ValueError( + "execution.use_fake_judge was removed; configure evaluation " + "metrics or rubric explicitly in optimizer.json" + ) + if "fake_candidate_scenario" in value: + raise ValueError( + "execution.fake_candidate_scenario was renamed to " + "execution.candidate_scenario" + ) + return value class InputPathsConfig(EvalBaseModel): @@ -72,6 +90,37 @@ def _require_non_empty_relative_path(cls, value: str) -> str: return value +class TraceCandidateInputsConfig(EvalBaseModel): + """一个 Trace 候选版本的评测集和 Prompt 快照路径。""" + + train_evalset: str + validation_evalset: str + prompts: list[PromptFieldConfig] = Field(min_length=1) + + @field_validator("train_evalset", "validation_evalset") + @classmethod + def _require_relative_trace_path(cls, value: str) -> str: + if not value.strip() or Path(value).is_absolute(): + raise ValueError("trace evalset path must be a non-empty relative path") + return value + + +class TraceInputsConfig(EvalBaseModel): + """三个确定性候选场景的 Trace 输入。""" + + candidates: dict[ + Literal["improve", "no_improvement", "overfit"], + TraceCandidateInputsConfig, + ] + + @model_validator(mode="after") + def _require_all_scenarios(self) -> "TraceInputsConfig": + required = {"improve", "no_improvement", "overfit"} + if set(self.candidates) != required: + raise ValueError("trace_inputs must define improve, no_improvement, and overfit") + return self + + class RunConfig(EvalBaseModel): """Reproducibility and workspace location settings.""" @@ -186,12 +235,31 @@ class PipelineConfig(EvalBaseModel): reporting: ReportingConfig = Field(default_factory=ReportingConfig) artifacts: ArtifactConfig = Field(default_factory=ArtifactConfig) writeback: WritebackConfig = Field(default_factory=WritebackConfig) + trace_inputs: Optional[TraceInputsConfig] = None @model_validator(mode="after") def _require_unique_prompt_names(self) -> "PipelineConfig": names = [prompt.name for prompt in self.prompts] if len(names) != len(set(names)): raise ValueError("prompts must not contain duplicate field names") + if self.execution.mode == "trace": + if self.trace_inputs is None: + raise ValueError("trace mode requires trace_inputs") + if self.writeback.enabled: + raise ValueError("trace mode does not allow source Prompt writeback") + expected = set(names) + for scenario, inputs in self.trace_inputs.candidates.items(): + candidate_names = [prompt.name for prompt in inputs.prompts] + if len(candidate_names) != len(set(candidate_names)): + raise ValueError( + f"trace candidate {scenario} has duplicate prompt names" + ) + if set(candidate_names) != expected: + raise ValueError( + f"trace candidate {scenario} prompt fields must match baseline" + ) + elif self.trace_inputs is not None: + raise ValueError("trace_inputs is only allowed in trace mode") return self diff --git a/examples/optimization/eval_optimize_loop/data/traces/baseline.train.evalset.json b/examples/optimization/eval_optimize_loop/data/traces/baseline.train.evalset.json new file mode 100644 index 00000000..eef80beb --- /dev/null +++ b/examples/optimization/eval_optimize_loop/data/traces/baseline.train.evalset.json @@ -0,0 +1 @@ +{"eval_set_id":"trace_train","eval_cases":[{"eval_id":"train_output_format","eval_mode":"trace","conversation":[{"invocation_id":"train_output_format","user_content":{"role":"user","parts":[{"text":"How can I update my email address?"}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"account\",\"message\":\"Open profile settings to update your email.\"}"}]}}],"actual_conversation":[{"invocation_id":"train_output_format","user_content":{"role":"user","parts":[{"text":"How can I update my email address?"}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"account\",\"message\":\"Open profile settings to update your email.\"}"}]}}]},{"eval_id":"train_tool_choice","eval_mode":"trace","conversation":[{"invocation_id":"train_tool_choice","user_content":{"role":"user","parts":[{"text":"Check the status of order A100."}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"order_lookup\",\"message\":\"Checking order A100.\"}"}]}}],"actual_conversation":[{"invocation_id":"train_tool_choice","user_content":{"role":"user","parts":[{"text":"Check the status of order A100."}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"general_support\",\"message\":\"Please provide more details so I can route your request.\"}"}]}}]},{"eval_id":"train_tool_arguments","eval_mode":"trace","conversation":[{"invocation_id":"train_tool_arguments","user_content":{"role":"user","parts":[{"text":"Look up order B-204 for customer 17."}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"order_lookup\",\"message\":\"Checking order B-204 for customer 17.\"}"}]}}],"actual_conversation":[{"invocation_id":"train_tool_arguments","user_content":{"role":"user","parts":[{"text":"Look up order B-204 for customer 17."}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"general_support\",\"message\":\"Please provide more details so I can route your request.\"}"}]}}]}]} diff --git a/examples/optimization/eval_optimize_loop/data/traces/baseline.validation.evalset.json b/examples/optimization/eval_optimize_loop/data/traces/baseline.validation.evalset.json new file mode 100644 index 00000000..c3dfdf1f --- /dev/null +++ b/examples/optimization/eval_optimize_loop/data/traces/baseline.validation.evalset.json @@ -0,0 +1 @@ +{"eval_set_id":"trace_validation","eval_cases":[{"eval_id":"val_paraphrase","eval_mode":"trace","conversation":[{"invocation_id":"val_paraphrase","user_content":{"role":"user","parts":[{"text":"Where do I change the address tied to my account?"}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"account\",\"message\":\"Open profile settings to update your address.\"}"}]}}],"actual_conversation":[{"invocation_id":"val_paraphrase","user_content":{"role":"user","parts":[{"text":"Where do I change the address tied to my account?"}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"general_support\",\"message\":\"Please provide more details so I can route your request.\"}"}]}}]},{"eval_id":"val_knowledge_recall","eval_mode":"trace","conversation":[{"invocation_id":"val_knowledge_recall","user_content":{"role":"user","parts":[{"text":"How long does standard shipping usually take?"}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"shipping_policy\",\"message\":\"Standard shipping normally takes 3-5 business days.\"}"}]}}],"actual_conversation":[{"invocation_id":"val_knowledge_recall","user_content":{"role":"user","parts":[{"text":"How long does standard shipping usually take?"}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"general_support\",\"message\":\"Please provide more details so I can route your request.\"}"}]}}]},{"eval_id":"val_refund_route","eval_mode":"trace","conversation":[{"invocation_id":"val_refund_route","user_content":{"role":"user","parts":[{"text":"I was charged twice and need the duplicate payment refunded."}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"billing_refund\",\"message\":\"I will route this duplicate charge for refund review.\"}"}]}}],"actual_conversation":[{"invocation_id":"val_refund_route","user_content":{"role":"user","parts":[{"text":"I was charged twice and need the duplicate payment refunded."}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"billing_refund\",\"message\":\"I will route this duplicate charge for refund review.\"}"}]}}]}]} diff --git a/examples/optimization/eval_optimize_loop/data/traces/improve.train.evalset.json b/examples/optimization/eval_optimize_loop/data/traces/improve.train.evalset.json new file mode 100644 index 00000000..0ad61dae --- /dev/null +++ b/examples/optimization/eval_optimize_loop/data/traces/improve.train.evalset.json @@ -0,0 +1 @@ +{"eval_set_id":"trace_train","eval_cases":[{"eval_id":"train_output_format","eval_mode":"trace","conversation":[{"invocation_id":"train_output_format","user_content":{"role":"user","parts":[{"text":"How can I update my email address?"}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"account\",\"message\":\"Open profile settings to update your email.\"}"}]}}],"actual_conversation":[{"invocation_id":"train_output_format","user_content":{"role":"user","parts":[{"text":"How can I update my email address?"}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"account\",\"message\":\"Open profile settings to update your email.\"}"}]}}]},{"eval_id":"train_tool_choice","eval_mode":"trace","conversation":[{"invocation_id":"train_tool_choice","user_content":{"role":"user","parts":[{"text":"Check the status of order A100."}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"order_lookup\",\"message\":\"Checking order A100.\"}"}]}}],"actual_conversation":[{"invocation_id":"train_tool_choice","user_content":{"role":"user","parts":[{"text":"Check the status of order A100."}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"order_lookup\",\"message\":\"Checking order A100.\"}"}]}}]},{"eval_id":"train_tool_arguments","eval_mode":"trace","conversation":[{"invocation_id":"train_tool_arguments","user_content":{"role":"user","parts":[{"text":"Look up order B-204 for customer 17."}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"order_lookup\",\"message\":\"Checking order B-204 for customer 17.\"}"}]}}],"actual_conversation":[{"invocation_id":"train_tool_arguments","user_content":{"role":"user","parts":[{"text":"Look up order B-204 for customer 17."}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"order_lookup\",\"message\":\"Checking order B-204 for customer 17.\"}"}]}}]}]} diff --git a/examples/optimization/eval_optimize_loop/data/traces/improve.validation.evalset.json b/examples/optimization/eval_optimize_loop/data/traces/improve.validation.evalset.json new file mode 100644 index 00000000..4f9a9fde --- /dev/null +++ b/examples/optimization/eval_optimize_loop/data/traces/improve.validation.evalset.json @@ -0,0 +1 @@ +{"eval_set_id":"trace_validation","eval_cases":[{"eval_id":"val_paraphrase","eval_mode":"trace","conversation":[{"invocation_id":"val_paraphrase","user_content":{"role":"user","parts":[{"text":"Where do I change the address tied to my account?"}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"account\",\"message\":\"Open profile settings to update your address.\"}"}]}}],"actual_conversation":[{"invocation_id":"val_paraphrase","user_content":{"role":"user","parts":[{"text":"Where do I change the address tied to my account?"}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"account\",\"message\":\"Open profile settings to update your address.\"}"}]}}]},{"eval_id":"val_knowledge_recall","eval_mode":"trace","conversation":[{"invocation_id":"val_knowledge_recall","user_content":{"role":"user","parts":[{"text":"How long does standard shipping usually take?"}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"shipping_policy\",\"message\":\"Standard shipping normally takes 3-5 business days.\"}"}]}}],"actual_conversation":[{"invocation_id":"val_knowledge_recall","user_content":{"role":"user","parts":[{"text":"How long does standard shipping usually take?"}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"shipping_policy\",\"message\":\"Standard shipping normally takes 3-5 business days.\"}"}]}}]},{"eval_id":"val_refund_route","eval_mode":"trace","conversation":[{"invocation_id":"val_refund_route","user_content":{"role":"user","parts":[{"text":"I was charged twice and need the duplicate payment refunded."}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"billing_refund\",\"message\":\"I will route this duplicate charge for refund review.\"}"}]}}],"actual_conversation":[{"invocation_id":"val_refund_route","user_content":{"role":"user","parts":[{"text":"I was charged twice and need the duplicate payment refunded."}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"billing_refund\",\"message\":\"I will route this duplicate charge for refund review.\"}"}]}}]}]} diff --git a/examples/optimization/eval_optimize_loop/data/traces/no_improvement.train.evalset.json b/examples/optimization/eval_optimize_loop/data/traces/no_improvement.train.evalset.json new file mode 100644 index 00000000..eef80beb --- /dev/null +++ b/examples/optimization/eval_optimize_loop/data/traces/no_improvement.train.evalset.json @@ -0,0 +1 @@ +{"eval_set_id":"trace_train","eval_cases":[{"eval_id":"train_output_format","eval_mode":"trace","conversation":[{"invocation_id":"train_output_format","user_content":{"role":"user","parts":[{"text":"How can I update my email address?"}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"account\",\"message\":\"Open profile settings to update your email.\"}"}]}}],"actual_conversation":[{"invocation_id":"train_output_format","user_content":{"role":"user","parts":[{"text":"How can I update my email address?"}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"account\",\"message\":\"Open profile settings to update your email.\"}"}]}}]},{"eval_id":"train_tool_choice","eval_mode":"trace","conversation":[{"invocation_id":"train_tool_choice","user_content":{"role":"user","parts":[{"text":"Check the status of order A100."}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"order_lookup\",\"message\":\"Checking order A100.\"}"}]}}],"actual_conversation":[{"invocation_id":"train_tool_choice","user_content":{"role":"user","parts":[{"text":"Check the status of order A100."}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"general_support\",\"message\":\"Please provide more details so I can route your request.\"}"}]}}]},{"eval_id":"train_tool_arguments","eval_mode":"trace","conversation":[{"invocation_id":"train_tool_arguments","user_content":{"role":"user","parts":[{"text":"Look up order B-204 for customer 17."}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"order_lookup\",\"message\":\"Checking order B-204 for customer 17.\"}"}]}}],"actual_conversation":[{"invocation_id":"train_tool_arguments","user_content":{"role":"user","parts":[{"text":"Look up order B-204 for customer 17."}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"general_support\",\"message\":\"Please provide more details so I can route your request.\"}"}]}}]}]} diff --git a/examples/optimization/eval_optimize_loop/data/traces/no_improvement.validation.evalset.json b/examples/optimization/eval_optimize_loop/data/traces/no_improvement.validation.evalset.json new file mode 100644 index 00000000..c3dfdf1f --- /dev/null +++ b/examples/optimization/eval_optimize_loop/data/traces/no_improvement.validation.evalset.json @@ -0,0 +1 @@ +{"eval_set_id":"trace_validation","eval_cases":[{"eval_id":"val_paraphrase","eval_mode":"trace","conversation":[{"invocation_id":"val_paraphrase","user_content":{"role":"user","parts":[{"text":"Where do I change the address tied to my account?"}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"account\",\"message\":\"Open profile settings to update your address.\"}"}]}}],"actual_conversation":[{"invocation_id":"val_paraphrase","user_content":{"role":"user","parts":[{"text":"Where do I change the address tied to my account?"}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"general_support\",\"message\":\"Please provide more details so I can route your request.\"}"}]}}]},{"eval_id":"val_knowledge_recall","eval_mode":"trace","conversation":[{"invocation_id":"val_knowledge_recall","user_content":{"role":"user","parts":[{"text":"How long does standard shipping usually take?"}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"shipping_policy\",\"message\":\"Standard shipping normally takes 3-5 business days.\"}"}]}}],"actual_conversation":[{"invocation_id":"val_knowledge_recall","user_content":{"role":"user","parts":[{"text":"How long does standard shipping usually take?"}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"general_support\",\"message\":\"Please provide more details so I can route your request.\"}"}]}}]},{"eval_id":"val_refund_route","eval_mode":"trace","conversation":[{"invocation_id":"val_refund_route","user_content":{"role":"user","parts":[{"text":"I was charged twice and need the duplicate payment refunded."}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"billing_refund\",\"message\":\"I will route this duplicate charge for refund review.\"}"}]}}],"actual_conversation":[{"invocation_id":"val_refund_route","user_content":{"role":"user","parts":[{"text":"I was charged twice and need the duplicate payment refunded."}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"billing_refund\",\"message\":\"I will route this duplicate charge for refund review.\"}"}]}}]}]} diff --git a/examples/optimization/eval_optimize_loop/data/traces/overfit.train.evalset.json b/examples/optimization/eval_optimize_loop/data/traces/overfit.train.evalset.json new file mode 100644 index 00000000..0ad61dae --- /dev/null +++ b/examples/optimization/eval_optimize_loop/data/traces/overfit.train.evalset.json @@ -0,0 +1 @@ +{"eval_set_id":"trace_train","eval_cases":[{"eval_id":"train_output_format","eval_mode":"trace","conversation":[{"invocation_id":"train_output_format","user_content":{"role":"user","parts":[{"text":"How can I update my email address?"}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"account\",\"message\":\"Open profile settings to update your email.\"}"}]}}],"actual_conversation":[{"invocation_id":"train_output_format","user_content":{"role":"user","parts":[{"text":"How can I update my email address?"}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"account\",\"message\":\"Open profile settings to update your email.\"}"}]}}]},{"eval_id":"train_tool_choice","eval_mode":"trace","conversation":[{"invocation_id":"train_tool_choice","user_content":{"role":"user","parts":[{"text":"Check the status of order A100."}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"order_lookup\",\"message\":\"Checking order A100.\"}"}]}}],"actual_conversation":[{"invocation_id":"train_tool_choice","user_content":{"role":"user","parts":[{"text":"Check the status of order A100."}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"order_lookup\",\"message\":\"Checking order A100.\"}"}]}}]},{"eval_id":"train_tool_arguments","eval_mode":"trace","conversation":[{"invocation_id":"train_tool_arguments","user_content":{"role":"user","parts":[{"text":"Look up order B-204 for customer 17."}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"order_lookup\",\"message\":\"Checking order B-204 for customer 17.\"}"}]}}],"actual_conversation":[{"invocation_id":"train_tool_arguments","user_content":{"role":"user","parts":[{"text":"Look up order B-204 for customer 17."}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"order_lookup\",\"message\":\"Checking order B-204 for customer 17.\"}"}]}}]}]} diff --git a/examples/optimization/eval_optimize_loop/data/traces/overfit.validation.evalset.json b/examples/optimization/eval_optimize_loop/data/traces/overfit.validation.evalset.json new file mode 100644 index 00000000..2e7a3653 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/data/traces/overfit.validation.evalset.json @@ -0,0 +1 @@ +{"eval_set_id":"trace_validation","eval_cases":[{"eval_id":"val_paraphrase","eval_mode":"trace","conversation":[{"invocation_id":"val_paraphrase","user_content":{"role":"user","parts":[{"text":"Where do I change the address tied to my account?"}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"account\",\"message\":\"Open profile settings to update your address.\"}"}]}}],"actual_conversation":[{"invocation_id":"val_paraphrase","user_content":{"role":"user","parts":[{"text":"Where do I change the address tied to my account?"}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"general_support\",\"message\":\"Please provide more details so I can route your request.\"}"}]}}]},{"eval_id":"val_knowledge_recall","eval_mode":"trace","conversation":[{"invocation_id":"val_knowledge_recall","user_content":{"role":"user","parts":[{"text":"How long does standard shipping usually take?"}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"shipping_policy\",\"message\":\"Standard shipping normally takes 3-5 business days.\"}"}]}}],"actual_conversation":[{"invocation_id":"val_knowledge_recall","user_content":{"role":"user","parts":[{"text":"How long does standard shipping usually take?"}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"general_support\",\"message\":\"Please provide more details so I can route your request.\"}"}]}}]},{"eval_id":"val_refund_route","eval_mode":"trace","conversation":[{"invocation_id":"val_refund_route","user_content":{"role":"user","parts":[{"text":"I was charged twice and need the duplicate payment refunded."}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"billing_refund\",\"message\":\"I will route this duplicate charge for refund review.\"}"}]}}],"actual_conversation":[{"invocation_id":"val_refund_route","user_content":{"role":"user","parts":[{"text":"I was charged twice and need the duplicate payment refunded."}]},"final_response":{"role":"model","parts":[{"text":"{\"route\":\"general_support\",\"message\":\"Please provide more details so I can route your request.\"}"}]}}]}]} diff --git a/examples/optimization/eval_optimize_loop/data/traces/prompts/improve.md b/examples/optimization/eval_optimize_loop/data/traces/prompts/improve.md new file mode 100644 index 00000000..602dee3b --- /dev/null +++ b/examples/optimization/eval_optimize_loop/data/traces/prompts/improve.md @@ -0,0 +1,5 @@ +Customer support routing candidate for trace replay. + + + + diff --git a/examples/optimization/eval_optimize_loop/data/traces/prompts/no_improvement.md b/examples/optimization/eval_optimize_loop/data/traces/prompts/no_improvement.md new file mode 100644 index 00000000..6fcc4d80 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/data/traces/prompts/no_improvement.md @@ -0,0 +1 @@ +Customer support routing candidate with wording-only changes for trace replay. diff --git a/examples/optimization/eval_optimize_loop/data/traces/prompts/overfit.md b/examples/optimization/eval_optimize_loop/data/traces/prompts/overfit.md new file mode 100644 index 00000000..56d2fe69 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/data/traces/prompts/overfit.md @@ -0,0 +1,5 @@ +Narrow customer support routing candidate for trace replay. + + + + diff --git a/examples/optimization/eval_optimize_loop/evaluation_adapter.py b/examples/optimization/eval_optimize_loop/evaluation_adapter.py index f0f70391..e32ee9b1 100644 --- a/examples/optimization/eval_optimize_loop/evaluation_adapter.py +++ b/examples/optimization/eval_optimize_loop/evaluation_adapter.py @@ -20,7 +20,7 @@ from .schemas import CaseEvaluation from .schemas import CaseRunOutcome from .schemas import EvaluationStatus -from .schemas import FakeEvaluationSnapshot +from .schemas import EvaluationSnapshot from .schemas import InvocationEvidence from .schemas import MetricOutcome from .schemas import ObservableValue @@ -202,7 +202,7 @@ def _case_evaluation( ) -def standardize_snapshot(snapshot: FakeEvaluationSnapshot) -> StandardizedEvaluation: +def standardize_snapshot(snapshot: EvaluationSnapshot) -> StandardizedEvaluation: """Normalize one complete SDK snapshot without discarding raw evidence.""" cases = [ _case_evaluation( diff --git a/examples/optimization/eval_optimize_loop/fake/__init__.py b/examples/optimization/eval_optimize_loop/fake/__init__.py index c6895fa9..6524f8f7 100644 --- a/examples/optimization/eval_optimize_loop/fake/__init__.py +++ b/examples/optimization/eval_optimize_loop/fake/__init__.py @@ -5,7 +5,12 @@ # tRPC-Agent-Python is licensed under the Apache License, Version 2.0. """Deterministic offline components for the eval/optimization loop.""" -from .agent import DeterministicFakeAgent from .candidate_provider import DeterministicFakeCandidateProvider +from .model import DeterministicFakeModel +from .model import deterministic_response -__all__ = ["DeterministicFakeAgent", "DeterministicFakeCandidateProvider"] +__all__ = [ + "DeterministicFakeCandidateProvider", + "DeterministicFakeModel", + "deterministic_response", +] diff --git a/examples/optimization/eval_optimize_loop/fake/agent.py b/examples/optimization/eval_optimize_loop/fake/agent.py deleted file mode 100644 index 2912ab04..00000000 --- a/examples/optimization/eval_optimize_loop/fake/agent.py +++ /dev/null @@ -1,127 +0,0 @@ -# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. -# -# Copyright (C) 2026 Tencent. All rights reserved. -# -# tRPC-Agent-Python is licensed under the Apache License, Version 2.0. -"""Prompt-sensitive deterministic agent used by the offline pipeline mode.""" - -from __future__ import annotations - -import json -import re -from dataclasses import dataclass - -from trpc_agent_sdk.evaluation import TargetPrompt - - -RULE_PREFIX = "deterministic-fake-rule" -_RULE_RE = re.compile( - rf"", - re.IGNORECASE, -) -_ORDER_ID_RE = re.compile(r"\border\s+([A-Za-z0-9][A-Za-z0-9-]*)", re.IGNORECASE) -_CUSTOMER_ID_RE = re.compile(r"\bcustomer\s+([A-Za-z0-9][A-Za-z0-9-]*)", re.IGNORECASE) - - -@dataclass(frozen=True) -class _RoutingPolicy: - account_terms: frozenset[str] = frozenset({"email"}) - order_lookup: bool = False - shipping_policy: bool = False - refund_route: bool = True - - -def _parse_bool(value: str, *, default: bool) -> bool: - normalized = value.strip().lower() - if normalized in {"true", "yes", "1", "enabled"}: - return True - if normalized in {"false", "no", "0", "disabled"}: - return False - return default - - -def _parse_policy(prompt_text: str) -> _RoutingPolicy: - values: dict[str, str] = {} - for key, value in _RULE_RE.findall(prompt_text): - values[key.lower()] = value.strip() - - account_terms = frozenset( - term.strip().lower() - for term in values.get("account_terms", "email").split(",") - if term.strip() - ) - return _RoutingPolicy( - account_terms=account_terms, - order_lookup=_parse_bool(values.get("order_lookup", "false"), default=False), - shipping_policy=_parse_bool(values.get("shipping_policy", "false"), default=False), - refund_route=_parse_bool(values.get("refund_route", "true"), default=True), - ) - - -def _compact_response(route: str, message: str) -> str: - # Keep route before message: the example intentionally uses exact text matching. - return json.dumps({"route": route, "message": message}, ensure_ascii=False, separators=(",", ":")) - - -class DeterministicFakeAgent: - """A black-box ``call_agent`` implementation whose behavior follows prompt rules. - - The agent deliberately receives only the user query. It cannot inspect an - eval id, expected response, scenario name, call count, or external state. - Every call rereads ``target_prompt`` so a candidate written between the - baseline and candidate evaluations takes effect immediately. - """ - - def __init__(self, target_prompt: TargetPrompt) -> None: - self._target_prompt = target_prompt - - async def call_agent(self, query: str) -> str: - if not isinstance(query, str): - raise TypeError("query must be a string") - - prompt_values = await self._target_prompt.read_all() - policy = _parse_policy("\n".join(prompt_values.values())) - normalized = " ".join(query.casefold().split()) - - if policy.refund_route and ( - "charged twice" in normalized - or ("duplicate" in normalized and ("payment" in normalized or "charge" in normalized)) - ): - return _compact_response( - "billing_refund", - "I will route this duplicate charge for refund review.", - ) - - if policy.shipping_policy and "shipping" in normalized and ( - "standard" in normalized or "how long" in normalized - ): - return _compact_response( - "shipping_policy", - "Standard shipping normally takes 3-5 business days.", - ) - - if ( - policy.order_lookup - and "order" in normalized - and (order_match := _ORDER_ID_RE.search(query)) is not None - ): - order_id = order_match.group(1) - customer_match = _CUSTOMER_ID_RE.search(query) - if customer_match is not None: - message = f"Checking order {order_id} for customer {customer_match.group(1)}." - else: - message = f"Checking order {order_id}." - return _compact_response("order_lookup", message) - - account_term = next((term for term in sorted(policy.account_terms) if term in normalized), None) - if account_term and ("update" in normalized or "change" in normalized): - attribute = "email" if "email" in normalized else "address" - return _compact_response( - "account", - f"Open profile settings to update your {attribute}.", - ) - - return _compact_response( - "general_support", - "Please provide more details so I can route your request.", - ) diff --git a/examples/optimization/eval_optimize_loop/fake/candidate_provider.py b/examples/optimization/eval_optimize_loop/fake/candidate_provider.py index 1724e29b..42fd054f 100644 --- a/examples/optimization/eval_optimize_loop/fake/candidate_provider.py +++ b/examples/optimization/eval_optimize_loop/fake/candidate_provider.py @@ -12,11 +12,11 @@ from typing import Mapping from ..schemas import FakeCandidateProposal -from ..schemas import FakeCandidateScenario -from .agent import RULE_PREFIX +from ..schemas import CandidateScenario +from .model import RULE_PREFIX -_SCENARIO_BLOCKS: dict[FakeCandidateScenario, tuple[str, str]] = { +_SCENARIO_BLOCKS: dict[CandidateScenario, tuple[str, str]] = { "improve": ( "Generalize routing across account synonyms, order lookup, shipping policy, and refunds.", "\n".join( @@ -80,7 +80,7 @@ def propose( self, current_prompts: Mapping[str, str], *, - scenario: FakeCandidateScenario, + scenario: CandidateScenario, seed: int, ) -> FakeCandidateProposal: if self._target_field not in current_prompts: diff --git a/examples/optimization/eval_optimize_loop/fake/model.py b/examples/optimization/eval_optimize_loop/fake/model.py new file mode 100644 index 00000000..01460799 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/fake/model.py @@ -0,0 +1,189 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under the Apache License, Version 2.0. +"""由 Prompt 与用户输入驱动的确定性离线模型。""" + +from __future__ import annotations + +import json +import re +from collections.abc import AsyncGenerator +from dataclasses import dataclass + +from trpc_agent_sdk.context import InvocationContext +from trpc_agent_sdk.models import LLMModel +from trpc_agent_sdk.models import LlmRequest +from trpc_agent_sdk.models import LlmResponse +from trpc_agent_sdk.types import Content +from trpc_agent_sdk.types import Part + + +RULE_PREFIX = "deterministic-fake-rule" +_RULE_RE = re.compile( + rf"", + re.IGNORECASE, +) +_ORDER_ID_RE = re.compile( + r"\border\s+([A-Za-z0-9][A-Za-z0-9-]*)", + re.IGNORECASE, +) +_CUSTOMER_ID_RE = re.compile( + r"\bcustomer\s+([A-Za-z0-9][A-Za-z0-9-]*)", + re.IGNORECASE, +) + + +@dataclass(frozen=True) +class _RoutingPolicy: + account_terms: frozenset[str] = frozenset({"email"}) + order_lookup: bool = False + shipping_policy: bool = False + refund_route: bool = True + + +def _parse_bool(value: str, *, default: bool) -> bool: + normalized = value.strip().lower() + if normalized in {"true", "yes", "1", "enabled"}: + return True + if normalized in {"false", "no", "0", "disabled"}: + return False + return default + + +def _parse_policy(prompt_text: str) -> _RoutingPolicy: + values = { + key.lower(): value.strip() + for key, value in _RULE_RE.findall(prompt_text) + } + account_terms = frozenset( + term.strip().lower() + for term in values.get("account_terms", "email").split(",") + if term.strip() + ) + return _RoutingPolicy( + account_terms=account_terms, + order_lookup=_parse_bool( + values.get("order_lookup", "false"), + default=False, + ), + shipping_policy=_parse_bool( + values.get("shipping_policy", "false"), + default=False, + ), + refund_route=_parse_bool( + values.get("refund_route", "true"), + default=True, + ), + ) + + +def _compact_response(route: str, message: str) -> str: + return json.dumps( + {"route": route, "message": message}, + ensure_ascii=False, + separators=(",", ":"), + ) + + +def deterministic_response(instruction: str, user_text: str) -> str: + """仅根据 Prompt instruction 和用户文本生成稳定响应。""" + if not isinstance(instruction, str): + raise TypeError("instruction must be a string") + if not isinstance(user_text, str): + raise TypeError("user text must be a string") + + policy = _parse_policy(instruction) + normalized = " ".join(user_text.casefold().split()) + + if policy.refund_route and ( + "charged twice" in normalized + or ( + "duplicate" in normalized + and ("payment" in normalized or "charge" in normalized) + ) + ): + return _compact_response( + "billing_refund", + "I will route this duplicate charge for refund review.", + ) + + if policy.shipping_policy and "shipping" in normalized and ( + "standard" in normalized or "how long" in normalized + ): + return _compact_response( + "shipping_policy", + "Standard shipping normally takes 3-5 business days.", + ) + + order_match = _ORDER_ID_RE.search(user_text) + if policy.order_lookup and "order" in normalized and order_match is not None: + order_id = order_match.group(1) + customer_match = _CUSTOMER_ID_RE.search(user_text) + message = f"Checking order {order_id}." + if customer_match is not None: + message = ( + f"Checking order {order_id} for customer " + f"{customer_match.group(1)}." + ) + return _compact_response("order_lookup", message) + + account_term = next( + ( + term + for term in sorted(policy.account_terms) + if term in normalized + ), + None, + ) + if account_term and ("update" in normalized or "change" in normalized): + attribute = "email" if "email" in normalized else "address" + return _compact_response( + "account", + f"Open profile settings to update your {attribute}.", + ) + + return _compact_response( + "general_support", + "Please provide more details so I can route your request.", + ) + + +def _last_user_text(request: LlmRequest) -> str: + for content in reversed(request.contents): + if content.role != "user" or not content.parts: + continue + text = "".join(part.text or "" for part in content.parts).strip() + if text: + return text + raise ValueError("LLM request must contain non-empty user text") + + +class DeterministicFakeModel(LLMModel): + """通过 SDK Model 接口提供不访问网络的确定性响应。""" + + def __init__(self) -> None: + super().__init__(model_name="deterministic-fake-model") + + @classmethod + def supported_models(cls) -> list[str]: + return ["deterministic-fake-model"] + + async def _generate_async_impl( + self, + request: LlmRequest, + stream: bool = False, + ctx: InvocationContext | None = None, + ) -> AsyncGenerator[LlmResponse, None]: + del stream, ctx + instruction = "" + if request.config is not None and request.config.system_instruction: + instruction = str(request.config.system_instruction) + response = deterministic_response(instruction, _last_user_text(request)) + yield LlmResponse( + content=Content( + role="model", + parts=[Part.from_text(text=response)], + ) + ) diff --git a/examples/optimization/eval_optimize_loop/pipeline.json b/examples/optimization/eval_optimize_loop/pipeline.json index f3b862e4..1d88ff2a 100644 --- a/examples/optimization/eval_optimize_loop/pipeline.json +++ b/examples/optimization/eval_optimize_loop/pipeline.json @@ -1,9 +1,8 @@ { "config_version": 1, "execution": { - "mode": "fake", - "fake_candidate_scenario": "improve", - "use_fake_judge": false + "mode": "offline", + "candidate_scenario": "improve" }, "inputs": { "train_evalset": "data/train.evalset.json", diff --git a/examples/optimization/eval_optimize_loop/pipeline.py b/examples/optimization/eval_optimize_loop/pipeline.py index bbb26672..2b8a1c89 100644 --- a/examples/optimization/eval_optimize_loop/pipeline.py +++ b/examples/optimization/eval_optimize_loop/pipeline.py @@ -7,6 +7,7 @@ from __future__ import annotations +import json import re import shutil from dataclasses import dataclass @@ -29,6 +30,7 @@ from trpc_agent_sdk.evaluation import load_optimize_config from .analysis import build_evaluation_analysis +from .business_agent import BusinessAgent from .artifact_writer import discover_run_artifacts from .artifact_writer import publish_report_bundle from .artifact_writer import write_failure_report @@ -39,7 +41,7 @@ from .config import PipelineConfig from .config import load_pipeline_config from .evaluation_adapter import EvaluationAnalysisError -from .fake import DeterministicFakeAgent +from .fake import DeterministicFakeModel from .gate import evaluate_gate from .gate import GateEvaluationError from .prompt_workspace import PromptWorkspaceError @@ -49,16 +51,22 @@ from .report_builder import build_failure_report from .report_builder import build_optimization_report from .schemas import InputSnapshot -from .schemas import FakeCandidateScenario -from .schemas import FakeEvaluationSnapshot -from .schemas import FakeStageResult +from .schemas import CandidateScenario +from .schemas import EvaluationSnapshot +from .schemas import OfflineStageResult from .schemas import ObservableValue from .schemas import OptimizerRuntimeParameters from .schemas import ResourceMeasurements from .schemas import RealStageResult from .schemas import ReportPhase from .schemas import ReportProgress +from .schemas import TraceCandidateProposal +from .schemas import TraceInputSnapshot +from .schemas import TracePromptSnapshot +from .schemas import TraceScenarioInputSnapshot +from .schemas import TraceStageResult from .schemas import WorkspaceSnapshot +from .schemas import WritebackResult from .writeback import perform_writeback @@ -71,7 +79,7 @@ class PipelineExecutionError(RuntimeError): # Compatibility for callers and tests written before the real-mode stage. -FakeStageExecutionError = PipelineExecutionError +PipelineStageExecutionError = PipelineExecutionError @dataclass(frozen=True) @@ -142,7 +150,7 @@ async def _record_failure( async def _rollback_written_source( prepared: PreparedRun, - result: FakeStageResult | RealStageResult, + result: OfflineStageResult | RealStageResult | TraceStageResult, ) -> None: """Restore the prepared source Prompt if success reporting cannot publish.""" if result.writeback.status != "written": @@ -172,7 +180,7 @@ async def _handle_stage_failure( prepared: PreparedRun, progress: _MutableReportProgress, error: Exception, - result: FakeStageResult | RealStageResult | None, + result: OfflineStageResult | RealStageResult | TraceStageResult | None, ) -> None: failure_error: Exception = error if progress.current_phase == "reporting" and result is not None: @@ -206,6 +214,18 @@ def _load_evalset(path: Path, label: str) -> EvalSet: raise PipelinePreparationError(f"{label} is not a valid EvalSet: {path}: {exc}") from exc +def _validate_trace_evalset(eval_set: EvalSet, label: str) -> None: + invalid = [ + case.eval_id + for case in eval_set.eval_cases + if case.eval_mode != "trace" or not case.actual_conversation + ] + if invalid: + raise PipelinePreparationError( + f"{label} requires eval_mode='trace' and actual_conversation: {invalid}" + ) + + def _validate_eval_case_ids(train: EvalSet, validation: EvalSet, config: PipelineConfig) -> None: train_ids = [case.eval_id for case in train.eval_cases] validation_ids = [case.eval_id for case in validation.eval_cases] @@ -263,9 +283,9 @@ def _verify_prepared_file(path: Path, *, label: str, expected_sha256: str) -> No try: actual_sha256 = _file_sha256(path) except OSError as exc: - raise FakeStageExecutionError(f"failed to reload prepared {label}: {path}: {exc}") from exc + raise PipelineStageExecutionError(f"failed to reload prepared {label}: {path}: {exc}") from exc if actual_sha256 != expected_sha256: - raise FakeStageExecutionError( + raise PipelineStageExecutionError( f"{label} changed after prepare_run: {path}; " f"expected sha256 {expected_sha256}, got {actual_sha256}" ) @@ -281,11 +301,11 @@ def _reload_prepared_evalset( try: payload = path.read_bytes() except OSError as exc: - raise FakeStageExecutionError(f"failed to reload prepared {label}: {path}: {exc}") from exc + raise PipelineStageExecutionError(f"failed to reload prepared {label}: {path}: {exc}") from exc actual_sha256 = sha256(payload).hexdigest() if actual_sha256 != expected_sha256: - raise FakeStageExecutionError( + raise PipelineStageExecutionError( f"{label} changed after prepare_run: {path}; " f"expected sha256 {expected_sha256}, got {actual_sha256}" ) @@ -293,7 +313,73 @@ def _reload_prepared_evalset( try: return EvalSet.model_validate_json(payload) except Exception as exc: - raise FakeStageExecutionError(f"prepared {label} is no longer a valid EvalSet: {path}: {exc}") from exc + raise PipelineStageExecutionError(f"prepared {label} is no longer a valid EvalSet: {path}: {exc}") from exc + + +def _prepare_trace_inputs( + example_root: Path, + config: PipelineConfig, + baseline_train: EvalSet, + baseline_validation: EvalSet, +) -> TraceInputSnapshot | None: + if config.execution.mode != "trace": + return None + _validate_trace_evalset(baseline_train, "baseline train trace") + _validate_trace_evalset(baseline_validation, "baseline validation trace") + assert config.trace_inputs is not None + train_ids = {case.eval_id for case in baseline_train.eval_cases} + validation_ids = {case.eval_id for case in baseline_validation.eval_cases} + scenarios: dict[str, TraceScenarioInputSnapshot] = {} + for scenario, inputs in config.trace_inputs.candidates.items(): + train_path = resolve_inside_example_root( + example_root, inputs.train_evalset, f"trace {scenario} train" + ) + validation_path = resolve_inside_example_root( + example_root, + inputs.validation_evalset, + f"trace {scenario} validation", + ) + train = _load_evalset(train_path, f"trace {scenario} train") + validation = _load_evalset( + validation_path, f"trace {scenario} validation" + ) + _validate_trace_evalset(train, f"trace {scenario} train") + _validate_trace_evalset(validation, f"trace {scenario} validation") + if {case.eval_id for case in train.eval_cases} != train_ids: + raise PipelinePreparationError( + f"trace {scenario} train eval IDs must match baseline" + ) + if {case.eval_id for case in validation.eval_cases} != validation_ids: + raise PipelinePreparationError( + f"trace {scenario} validation eval IDs must match baseline" + ) + prompt_snapshots: list[TracePromptSnapshot] = [] + for prompt in inputs.prompts: + path = resolve_inside_example_root( + example_root, prompt.path, f"trace {scenario} prompt" + ) + try: + content = path.read_text(encoding="utf-8") + except (OSError, UnicodeError) as exc: + raise PipelinePreparationError( + f"trace {scenario} prompt is invalid: {path}: {exc}" + ) from exc + prompt_snapshots.append( + TracePromptSnapshot( + field_name=prompt.name, + path=str(path), + content=content, + sha256=_file_sha256(path), + ) + ) + scenarios[scenario] = TraceScenarioInputSnapshot( + train_evalset_path=str(train_path), + train_evalset_sha256=_file_sha256(train_path), + validation_evalset_path=str(validation_path), + validation_evalset_sha256=_file_sha256(validation_path), + prompt_snapshots=prompt_snapshots, + ) + return TraceInputSnapshot(scenarios=scenarios) def prepare_run(pipeline_config_path: str | Path, *, run_id: str | None = None) -> PreparedRun: @@ -312,6 +398,9 @@ def prepare_run(pipeline_config_path: str | Path, *, run_id: str | None = None) train_evalset = _load_evalset(train_path, "train_evalset") validation_evalset = _load_evalset(validation_path, "validation_evalset") _validate_eval_case_ids(train_evalset, validation_evalset, config) + trace_inputs = _prepare_trace_inputs( + example_root, config, train_evalset, validation_evalset + ) try: optimizer_config = load_optimize_config(str(optimizer_path)) @@ -366,6 +455,7 @@ def prepare_run(pipeline_config_path: str | Path, *, run_id: str | None = None) validation_evalset_sha256=_file_sha256(validation_path), prompt_snapshots=prompt_snapshots, seed=config.run.seed, + trace_inputs=trace_inputs, ) prepared = PreparedRun( config=config, @@ -410,7 +500,7 @@ def _validate_results( expected_ids = {case.eval_id for case in eval_set.eval_cases} actual_ids = set(eval_results_by_eval_id) if actual_ids != expected_ids: - raise FakeStageExecutionError( + raise PipelineStageExecutionError( f"{phase} {split} evaluation returned case ids {sorted(actual_ids)}; " f"expected {sorted(expected_ids)}" ) @@ -420,7 +510,7 @@ def _validate_results( if len(results) != num_runs } if wrong_run_counts: - raise FakeStageExecutionError( + raise PipelineStageExecutionError( f"{phase} {split} evaluation returned unexpected run counts: {wrong_run_counts}; " f"expected {num_runs}" ) @@ -430,10 +520,10 @@ async def _evaluate_split( *, prepared: PreparedRun, eval_set: EvalSet, - call_agent: CallAgent, + call_agent: CallAgent | None, phase: Literal["baseline", "candidate"], split: Literal["train", "validation"], -) -> FakeEvaluationSnapshot: +) -> EvaluationSnapshot: num_runs = prepared.optimizer_config.evaluate.num_runs try: failed_summary, details_lines, result_lines, eval_results_by_eval_id = ( @@ -448,7 +538,7 @@ async def _evaluate_split( ) ) except Exception as exc: - raise FakeStageExecutionError(f"{phase} {split} evaluation failed: {exc}") from exc + raise PipelineStageExecutionError(f"{phase} {split} evaluation failed: {exc}") from exc _validate_results( eval_set=eval_set, @@ -458,7 +548,7 @@ async def _evaluate_split( split=split, ) passed_cases, total_cases, average_score = _summarize_results(eval_results_by_eval_id) - return FakeEvaluationSnapshot( + return EvaluationSnapshot( phase=phase, split=split, eval_set_id=eval_set.eval_set_id, @@ -484,37 +574,33 @@ async def _restore_working_baseline( await prepared.working_target.write_all(baseline_prompts) restored = await prepared.working_target.read_all() except Exception as exc: - raise FakeStageExecutionError(f"failed to restore optimizer working prompts: {exc}") from exc + raise PipelineStageExecutionError(f"failed to restore optimizer working prompts: {exc}") from exc if restored != baseline_prompts: - raise FakeStageExecutionError("optimizer working prompts did not match baseline after restoration") + raise PipelineStageExecutionError("optimizer working prompts did not match baseline after restoration") return was_modified -async def _execute_fake_stage( +async def _execute_offline_stage( prepared: PreparedRun, *, - scenario: FakeCandidateScenario | None = None, + scenario: CandidateScenario | None = None, progress: _MutableReportProgress, -) -> FakeStageResult: - """Run four deterministic evaluations without a model, judge, or optimizer. +) -> OfflineStageResult: + """Run four evaluations through SDK LlmAgent and a deterministic model. Source prompts are never written. Once generated, the candidate remains in the isolated working target on success or candidate-evaluation failure so the run can be inspected later. """ progress.enter("baseline_train") - if prepared.config.execution.mode != "fake": - raise FakeStageExecutionError( - f"run_fake_stage requires execution.mode='fake', got {prepared.config.execution.mode!r}" - ) - if prepared.config.execution.use_fake_judge: - raise FakeStageExecutionError( - "execution.use_fake_judge=true is not supported in stage two; " - "the example uses deterministic final-response matching" + if prepared.config.execution.mode != "offline": + raise PipelineStageExecutionError( + "run_offline_stage requires execution.mode='offline', got " + f"{prepared.config.execution.mode!r}" ) started_at = perf_counter() - selected_scenario = scenario or prepared.config.execution.fake_candidate_scenario + selected_scenario = scenario or prepared.config.execution.candidate_scenario train_evalset = _reload_prepared_evalset( Path(prepared.input_snapshot.train_evalset_path), label="train_evalset", @@ -529,14 +615,20 @@ async def _execute_fake_stage( try: baseline_prompts = await prepared.working_target.read_all() except Exception as exc: - raise FakeStageExecutionError(f"failed to read prepared working prompts: {exc}") from exc + raise PipelineStageExecutionError(f"failed to read prepared working prompts: {exc}") from exc expected_baseline = { snapshot.field_name: snapshot.content for snapshot in prepared.input_snapshot.prompt_snapshots } if baseline_prompts != expected_baseline: - raise FakeStageExecutionError("working prompts no longer match the prepared baseline snapshot") - - agent = DeterministicFakeAgent(prepared.working_target) + raise PipelineStageExecutionError("working prompts no longer match the prepared baseline snapshot") + + agent = BusinessAgent( + prepared.working_target, + DeterministicFakeModel, + agent_name="eval_optimize_offline_agent", + app_name="eval_optimize_offline", + user_id="offline-evaluation", + ) progress.enter("baseline_train") baseline_train = await _evaluate_split( prepared=prepared, @@ -568,15 +660,15 @@ async def _execute_fake_stage( generated = await FakeCandidateProviderAdapter(selected_scenario).propose(request) candidate = generated.proposal except Exception as exc: - raise FakeStageExecutionError(f"fake candidate generation failed: {exc}") from exc + raise PipelineStageExecutionError(f"fake candidate generation failed: {exc}") from exc try: await prepared.working_target.write_all(candidate.prompts) written_prompts = await prepared.working_target.read_all() except Exception as exc: - raise FakeStageExecutionError(f"candidate prompt write failed: {exc}") from exc + raise PipelineStageExecutionError(f"candidate prompt write failed: {exc}") from exc if written_prompts != candidate.prompts: - raise FakeStageExecutionError("candidate prompt readback did not match the generated proposal") + raise PipelineStageExecutionError("candidate prompt readback did not match the generated proposal") progress.enter("candidate_train") candidate_train = await _evaluate_split( @@ -606,17 +698,17 @@ async def _execute_fake_stage( severe_case_score_drop=prepared.config.gate.severe_case_score_drop, ) except EvaluationAnalysisError as exc: - raise FakeStageExecutionError(f"stage 3a analysis failed: {exc}") from exc + raise PipelineStageExecutionError(f"stage 3a analysis failed: {exc}") from exc measurements = ResourceMeasurements( cost_usd=ObservableValue( status="unavailable", unit="USD", - reason="Fake mode does not report monetary cost.", + reason="Offline deterministic model does not report monetary cost.", ), total_tokens=ObservableValue( status="unavailable", unit="tokens", - reason="Fake mode does not report token usage.", + reason="Offline deterministic model does not report token usage.", ), duration_seconds=ObservableValue( status="available", @@ -633,7 +725,7 @@ async def _execute_fake_stage( measurements, ) except GateEvaluationError as exc: - raise FakeStageExecutionError(f"stage 3b gate failed: {exc}") from exc + raise PipelineStageExecutionError(f"stage 3b gate failed: {exc}") from exc progress.enter("writeback") writeback = await perform_writeback( decision=gate_decision, @@ -642,7 +734,7 @@ async def _execute_fake_stage( source_target=prepared.source_target, candidate=candidate, ) - return FakeStageResult( + return OfflineStageResult( scenario=selected_scenario, candidate=candidate, baseline_train=baseline_train, @@ -666,12 +758,9 @@ async def _execute_real_stage( """Generate a real optimizer candidate and run the full guarded regression.""" progress.enter("baseline_train") if prepared.config.execution.mode != "real": - raise FakeStageExecutionError( + raise PipelineStageExecutionError( f"run_real_stage requires execution.mode='real', got {prepared.config.execution.mode!r}" ) - if prepared.config.execution.use_fake_judge: - raise FakeStageExecutionError("execution.use_fake_judge=true is not supported in real mode") - started_at = perf_counter() _verify_prepared_file( Path(prepared.input_snapshot.optimizer_config_path), @@ -691,12 +780,12 @@ async def _execute_real_stage( try: baseline_prompts = await prepared.working_target.read_all() except Exception as exc: - raise FakeStageExecutionError(f"failed to read prepared working prompts: {exc}") from exc + raise PipelineStageExecutionError(f"failed to read prepared working prompts: {exc}") from exc expected_baseline = { snapshot.field_name: snapshot.content for snapshot in prepared.input_snapshot.prompt_snapshots } if baseline_prompts != expected_baseline: - raise FakeStageExecutionError("working prompts no longer match the prepared baseline snapshot") + raise PipelineStageExecutionError("working prompts no longer match the prepared baseline snapshot") progress.enter("baseline_train") baseline_train = await _evaluate_split( @@ -732,19 +821,19 @@ async def _execute_real_stage( generated = await AgentOptimizerCandidateProvider(call_agent).propose(request) except CandidateProviderError as exc: await _restore_working_baseline(prepared, baseline_prompts) - raise FakeStageExecutionError(f"real candidate generation failed: {exc}") from exc + raise PipelineStageExecutionError(f"real candidate generation failed: {exc}") from exc if await _restore_working_baseline(prepared, baseline_prompts): - raise FakeStageExecutionError("optimizer did not restore working prompts after update_source=False") + raise PipelineStageExecutionError("optimizer did not restore working prompts after update_source=False") candidate = generated.proposal try: await prepared.working_target.write_all(candidate.prompts) written_prompts = await prepared.working_target.read_all() except Exception as exc: - raise FakeStageExecutionError(f"candidate prompt write failed: {exc}") from exc + raise PipelineStageExecutionError(f"candidate prompt write failed: {exc}") from exc if written_prompts != candidate.prompts: - raise FakeStageExecutionError("candidate prompt readback did not match the generated proposal") + raise PipelineStageExecutionError("candidate prompt readback did not match the generated proposal") progress.enter("candidate_train") candidate_train = await _evaluate_split( @@ -774,7 +863,7 @@ async def _execute_real_stage( severe_case_score_drop=prepared.config.gate.severe_case_score_drop, ) except EvaluationAnalysisError as exc: - raise FakeStageExecutionError(f"stage 3a analysis failed: {exc}") from exc + raise PipelineStageExecutionError(f"stage 3a analysis failed: {exc}") from exc measurements = ResourceMeasurements( cost_usd=ObservableValue( status="unavailable", @@ -801,7 +890,7 @@ async def _execute_real_stage( measurements, ) except GateEvaluationError as exc: - raise FakeStageExecutionError(f"stage 3b gate failed: {exc}") from exc + raise PipelineStageExecutionError(f"stage 3b gate failed: {exc}") from exc progress.enter("writeback") writeback = await perform_writeback( @@ -827,16 +916,16 @@ async def _execute_real_stage( ) -async def run_fake_stage( +async def run_offline_stage( prepared: PreparedRun, *, - scenario: FakeCandidateScenario | None = None, -) -> FakeStageResult: - """Run fake regression and atomically publish its audit report.""" + scenario: CandidateScenario | None = None, +) -> OfflineStageResult: + """Run offline SDK-Agent regression and publish its audit report.""" progress = _MutableReportProgress(started_at=datetime.now(timezone.utc)) - result: FakeStageResult | None = None + result: OfflineStageResult | None = None try: - result = await _execute_fake_stage( + result = await _execute_offline_stage( prepared, scenario=scenario, progress=progress, @@ -859,6 +948,162 @@ async def run_fake_stage( raise +async def _execute_trace_stage( + prepared: PreparedRun, + *, + scenario: CandidateScenario | None, + progress: _MutableReportProgress, +) -> TraceStageResult: + if prepared.config.execution.mode != "trace": + raise PipelineExecutionError( + "run_trace_stage requires execution.mode='trace', got " + f"{prepared.config.execution.mode!r}" + ) + trace_inputs = prepared.input_snapshot.trace_inputs + if trace_inputs is None: + raise PipelineExecutionError("prepared trace inputs are missing") + selected = scenario or prepared.config.execution.candidate_scenario + candidate_inputs = trace_inputs.scenarios[selected] + started_at = perf_counter() + + baseline_train_set = _reload_prepared_evalset( + Path(prepared.input_snapshot.train_evalset_path), + label="baseline train trace", + expected_sha256=prepared.input_snapshot.train_evalset_sha256, + ) + baseline_validation_set = _reload_prepared_evalset( + Path(prepared.input_snapshot.validation_evalset_path), + label="baseline validation trace", + expected_sha256=prepared.input_snapshot.validation_evalset_sha256, + ) + candidate_train_set = _reload_prepared_evalset( + Path(candidate_inputs.train_evalset_path), + label=f"candidate {selected} train trace", + expected_sha256=candidate_inputs.train_evalset_sha256, + ) + candidate_validation_set = _reload_prepared_evalset( + Path(candidate_inputs.validation_evalset_path), + label=f"candidate {selected} validation trace", + expected_sha256=candidate_inputs.validation_evalset_sha256, + ) + + progress.enter("baseline_train") + baseline_train = await _evaluate_split( + prepared=prepared, eval_set=baseline_train_set, call_agent=None, + phase="baseline", split="train", + ) + progress.enter("baseline_validation") + baseline_validation = await _evaluate_split( + prepared=prepared, eval_set=baseline_validation_set, call_agent=None, + phase="baseline", split="validation", + ) + progress.enter("candidate_generation") + prompts = { + snapshot.field_name: snapshot.content + for snapshot in candidate_inputs.prompt_snapshots + } + baseline_prompts = { + snapshot.field_name: snapshot.content + for snapshot in prepared.input_snapshot.prompt_snapshots + } + canonical = json.dumps( + prompts, ensure_ascii=False, sort_keys=True, separators=(",", ":") + ) + candidate_hash = sha256(canonical.encode("utf-8")).hexdigest() + parent_canonical = json.dumps( + baseline_prompts, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ) + parent_hash = sha256(parent_canonical.encode("utf-8")).hexdigest() + candidate = TraceCandidateProposal( + scenario=selected, + prompts=prompts, + changed_fields=[ + name for name in baseline_prompts if baseline_prompts[name] != prompts[name] + ], + rationale="Replay the selected pre-recorded candidate trace.", + parent_prompt_sha256=parent_hash, + candidate_prompt_sha256=candidate_hash, + candidate_id=f"trace-{selected}-{candidate_hash[:12]}", + source_trace_sha256={ + "train": candidate_inputs.train_evalset_sha256, + "validation": candidate_inputs.validation_evalset_sha256, + }, + ) + progress.enter("candidate_train") + candidate_train = await _evaluate_split( + prepared=prepared, eval_set=candidate_train_set, call_agent=None, + phase="candidate", split="train", + ) + progress.enter("candidate_validation") + candidate_validation = await _evaluate_split( + prepared=prepared, eval_set=candidate_validation_set, call_agent=None, + phase="candidate", split="validation", + ) + progress.enter("analysis") + analysis = build_evaluation_analysis( + baseline_train=baseline_train, + baseline_validation=baseline_validation, + candidate_train=candidate_train, + candidate_validation=candidate_validation, + hard_case_ids=set(prepared.config.case_labels.hard_case_ids), + critical_case_ids=set(prepared.config.case_labels.critical_case_ids), + severe_case_score_drop=prepared.config.gate.severe_case_score_drop, + ) + measurements = ResourceMeasurements( + cost_usd=ObservableValue(status="unavailable", unit="USD", reason="Trace replay does not call a model."), + total_tokens=ObservableValue(status="unavailable", unit="tokens", reason="Trace replay does not call a model."), + duration_seconds=ObservableValue(status="available", value=perf_counter() - started_at, unit="seconds"), + ) + progress.enter("gate") + gate_decision = evaluate_gate( + analysis, prepared.config.gate, prepared.config.budget, measurements + ) + progress.enter("writeback") + writeback = WritebackResult( + status="skipped", reason="trace_replay", attempted=False + ) + return TraceStageResult( + scenario=selected, candidate=candidate, + baseline_train=baseline_train, + baseline_validation=baseline_validation, + candidate_train=candidate_train, + candidate_validation=candidate_validation, + analysis=analysis, measurements=measurements, + gate_decision=gate_decision, writeback=writeback, + ) + + +async def run_trace_stage( + prepared: PreparedRun, + *, + scenario: CandidateScenario | None = None, +) -> TraceStageResult: + """回放四个 Trace EvalSet，并发布分析与 Gate 报告。""" + progress = _MutableReportProgress(started_at=datetime.now(timezone.utc)) + result: TraceStageResult | None = None + try: + result = await _execute_trace_stage( + prepared, scenario=scenario, progress=progress + ) + progress.enter("reporting") + report = build_optimization_report( + prepared, result, progress=progress.snapshot(), + finished_at=datetime.now(timezone.utc), + ) + publish_report_bundle( + report, + run_dir=Path(prepared.workspace.run_dir), + copy_input_files=prepared.config.artifacts.copy_input_files, + ) + return result + except Exception as exc: + await _handle_stage_failure(prepared, progress, exc, result) + raise + + async def run_real_stage( prepared: PreparedRun, *, diff --git a/examples/optimization/eval_optimize_loop/pipeline.real.json b/examples/optimization/eval_optimize_loop/pipeline.real.json index 1493cec0..1ac989dd 100644 --- a/examples/optimization/eval_optimize_loop/pipeline.real.json +++ b/examples/optimization/eval_optimize_loop/pipeline.real.json @@ -2,8 +2,7 @@ "config_version": 1, "execution": { "mode": "real", - "fake_candidate_scenario": "improve", - "use_fake_judge": false + "candidate_scenario": "improve" }, "inputs": { "train_evalset": "data/train.evalset.json", diff --git a/examples/optimization/eval_optimize_loop/pipeline.trace.json b/examples/optimization/eval_optimize_loop/pipeline.trace.json new file mode 100644 index 00000000..56946e7b --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline.trace.json @@ -0,0 +1,46 @@ +{ + "config_version": 1, + "execution": {"mode": "trace", "candidate_scenario": "improve"}, + "inputs": { + "train_evalset": "data/traces/baseline.train.evalset.json", + "validation_evalset": "data/traces/baseline.validation.evalset.json", + "optimizer_config": "optimizer.json" + }, + "prompts": [{"name": "system_prompt", "path": "prompts/system.md"}], + "trace_inputs": { + "candidates": { + "improve": { + "train_evalset": "data/traces/improve.train.evalset.json", + "validation_evalset": "data/traces/improve.validation.evalset.json", + "prompts": [{"name": "system_prompt", "path": "data/traces/prompts/improve.md"}] + }, + "no_improvement": { + "train_evalset": "data/traces/no_improvement.train.evalset.json", + "validation_evalset": "data/traces/no_improvement.validation.evalset.json", + "prompts": [{"name": "system_prompt", "path": "data/traces/prompts/no_improvement.md"}] + }, + "overfit": { + "train_evalset": "data/traces/overfit.train.evalset.json", + "validation_evalset": "data/traces/overfit.validation.evalset.json", + "prompts": [{"name": "system_prompt", "path": "data/traces/prompts/overfit.md"}] + } + } + }, + "run": {"runs_dir": "runs", "seed": 42}, + "case_labels": { + "hard_case_ids": ["val_knowledge_recall"], + "critical_case_ids": ["val_refund_route"] + }, + "gate": { + "min_validation_score_delta": 0.05, + "reject_on_validation_pass_rate_drop": true, + "reject_new_hard_fail": true, + "reject_critical_regression": true, + "severe_case_score_drop": 0.2, + "required_metrics": ["final_response_avg_score"] + }, + "budget": {"max_duration_seconds": 180, "on_unavailable": "warning"}, + "reporting": {"write_json": true, "write_markdown": true, "include_case_evidence": true}, + "artifacts": {"copy_input_files": true, "retain_optimizer_native_artifacts": true, "audit_all_candidates": false}, + "writeback": {"enabled": false, "require_source_hash_match": true} +} diff --git a/examples/optimization/eval_optimize_loop/real_agent.py b/examples/optimization/eval_optimize_loop/real_agent.py index 013ee11e..1aa0a554 100644 --- a/examples/optimization/eval_optimize_loop/real_agent.py +++ b/examples/optimization/eval_optimize_loop/real_agent.py @@ -8,18 +8,13 @@ from __future__ import annotations import os -import uuid from dataclasses import dataclass from typing import Mapping -from trpc_agent_sdk.agents import LlmAgent from trpc_agent_sdk.evaluation import TargetPrompt from trpc_agent_sdk.models import OpenAIModel -from trpc_agent_sdk.runners import Runner -from trpc_agent_sdk.sessions import InMemorySessionService -from trpc_agent_sdk.types import Content -from trpc_agent_sdk.types import GenerateContentConfig -from trpc_agent_sdk.types import Part + +from .business_agent import BusinessAgent @dataclass(frozen=True) @@ -51,62 +46,22 @@ def load_business_model_config( ) -def _render_instruction(prompts: dict[str, str]) -> str: - """稳定拼接一个或多个工作 Prompt 字段。""" - if len(prompts) == 1: - return next(iter(prompts.values())) - return "\n\n".join(f"## {name}\n{content}" for name, content in prompts.items()) - - class RealBusinessAgent: """以真实模型执行评测，并确保 case 与 Prompt 版本相互隔离。""" def __init__(self, target_prompt: TargetPrompt, config: BusinessModelConfig) -> None: - self._target_prompt = target_prompt - self._config = config - - async def call_agent(self, query: str) -> str: - """重新读取工作 Prompt，运行独立 session，只返回正式最终文本。""" - prompts = await self._target_prompt.read_all() - model = OpenAIModel( - model_name=self._config.model_name, - api_key=self._config.api_key, - base_url=self._config.base_url, - ) - root_agent = LlmAgent( - name="eval_optimize_real_agent", - description="真实模型驱动的评测与 Prompt 优化示例 Agent。", - model=model, - instruction=_render_instruction(prompts), - generate_content_config=GenerateContentConfig( - temperature=0.0, - max_output_tokens=512, + self._delegate = BusinessAgent( + target_prompt, + lambda: OpenAIModel( + model_name=config.model_name, + api_key=config.api_key, + base_url=config.base_url, ), - ) - session_service = InMemorySessionService() - runner = Runner( - app_name="eval_optimize_real_integration", - agent=root_agent, - session_service=session_service, - ) - session_id = str(uuid.uuid4()) - user_id = "real-integration" - await session_service.create_session( + agent_name="eval_optimize_real_agent", app_name="eval_optimize_real_integration", - user_id=user_id, - session_id=session_id, - state={}, + user_id="real-integration", ) - message = Content(role="user", parts=[Part.from_text(text=query)]) - final_text = "" - async for event in runner.run_async( - user_id=user_id, - session_id=session_id, - new_message=message, - ): - if not event.is_final_response() or not event.content or not event.content.parts: - continue - for part in event.content.parts: - if not part.thought and part.text: - final_text += part.text - return final_text.strip() + + async def call_agent(self, query: str) -> str: + """重新读取工作 Prompt，运行独立 session，只返回正式最终文本。""" + return await self._delegate.call_agent(query) diff --git a/examples/optimization/eval_optimize_loop/report_builder.py b/examples/optimization/eval_optimize_loop/report_builder.py index d3190a06..c64bf976 100644 --- a/examples/optimization/eval_optimize_loop/report_builder.py +++ b/examples/optimization/eval_optimize_loop/report_builder.py @@ -9,7 +9,7 @@ from .schemas import FailureReport, OptimizerResourceObservation, OptimizerResourceValue from .schemas import OptimizationReport -from .schemas import PipelineStageResult, RealStageResult, ReportProgress +from .schemas import PipelineStageResult, RealStageResult, ReportProgress, TraceStageResult if TYPE_CHECKING: from .pipeline import PreparedRun @@ -17,7 +17,8 @@ _OPTIMIZER_SCOPE = ( "Optimizer-only observation; excludes complete business Agent evaluation usage." ) -_FAKE_OPTIMIZER_REASON = "Fake mode does not run AgentOptimizer." +_OFFLINE_OPTIMIZER_REASON = "Offline mode uses a deterministic candidate provider." +_TRACE_OPTIMIZER_REASON = "Trace replay does not run a candidate provider or AgentOptimizer." _MISSING_COST_REASON = ( "Reflection LM calls were observed but optimizer cost was not reported." ) @@ -40,12 +41,12 @@ def _not_applicable_optimizer_value( - unit: str, + unit: str, reason: str, ) -> OptimizerResourceValue[object]: return OptimizerResourceValue[object]( status="not_applicable", unit=unit, - reason=_FAKE_OPTIMIZER_REASON, + reason=reason, ) @@ -79,13 +80,18 @@ def _is_complete_token_usage(value: object) -> bool: def _optimizer_resources(result: PipelineStageResult) -> OptimizerResourceObservation: if not isinstance(result, RealStageResult): + reason = ( + _TRACE_OPTIMIZER_REASON + if isinstance(result, TraceStageResult) + else _OFFLINE_OPTIMIZER_REASON + ) return OptimizerResourceObservation( - scope_note=_FAKE_OPTIMIZER_REASON, - total_rounds=_not_applicable_optimizer_value("rounds"), - reflection_lm_calls=_not_applicable_optimizer_value("calls"), - cost_usd=_not_applicable_optimizer_value("USD"), - token_usage=_not_applicable_optimizer_value("tokens"), - duration_seconds=_not_applicable_optimizer_value("seconds"), + scope_note=reason, + total_rounds=_not_applicable_optimizer_value("rounds", reason), + reflection_lm_calls=_not_applicable_optimizer_value("calls", reason), + cost_usd=_not_applicable_optimizer_value("USD", reason), + token_usage=_not_applicable_optimizer_value("tokens", reason), + duration_seconds=_not_applicable_optimizer_value("seconds", reason), ) native = result.optimize_result reflection_calls = native.total_reflection_lm_calls diff --git a/examples/optimization/eval_optimize_loop/run_pipeline.py b/examples/optimization/eval_optimize_loop/run_pipeline.py index b1e3c333..b18801f6 100644 --- a/examples/optimization/eval_optimize_loop/run_pipeline.py +++ b/examples/optimization/eval_optimize_loop/run_pipeline.py @@ -18,12 +18,14 @@ if str(_REPO_ROOT) not in sys.path: sys.path.insert(0, str(_REPO_ROOT)) from examples.optimization.eval_optimize_loop.pipeline import prepare_run - from examples.optimization.eval_optimize_loop.pipeline import run_fake_stage + from examples.optimization.eval_optimize_loop.pipeline import run_offline_stage + from examples.optimization.eval_optimize_loop.pipeline import run_trace_stage from examples.optimization.eval_optimize_loop.config import load_pipeline_config else: from .config import load_pipeline_config from .pipeline import prepare_run - from .pipeline import run_fake_stage + from .pipeline import run_offline_stage + from .pipeline import run_trace_stage def _format_snapshot(label: str, snapshot: object) -> str: @@ -44,7 +46,7 @@ def main() -> None: parser.add_argument( "--scenario", choices=("improve", "no_improvement", "overfit"), - help="Override execution.fake_candidate_scenario for this run.", + help="Override execution.candidate_scenario for this run.", ) args = parser.parse_args() @@ -54,11 +56,14 @@ def main() -> None: "real mode requires an injected business agent; use the Python API " "run_real_stage(prepared, call_agent=...)" ) - if config.execution.mode != "fake": - parser.error("this CLI currently supports execution.mode='fake' only") prepared = prepare_run(args.config, run_id=args.run_id) - result = asyncio.run(run_fake_stage(prepared, scenario=args.scenario)) + if config.execution.mode == "offline": + result = asyncio.run(run_offline_stage(prepared, scenario=args.scenario)) + elif config.execution.mode == "trace": + result = asyncio.run(run_trace_stage(prepared, scenario=args.scenario)) + else: + parser.error(f"unsupported execution mode: {config.execution.mode}") print(f"Completed deterministic pipeline stage: {prepared.workspace.run_dir}") print(f"Candidate: {result.candidate.candidate_id} ({result.scenario})") print(_format_snapshot("Baseline train", result.baseline_train)) diff --git a/examples/optimization/eval_optimize_loop/sample_output/artifact_index.json b/examples/optimization/eval_optimize_loop/sample_output/artifact_index.json new file mode 100644 index 00000000..640ae07a --- /dev/null +++ b/examples/optimization/eval_optimize_loop/sample_output/artifact_index.json @@ -0,0 +1,139 @@ +{ + "schema_version": 1, + "run_id": "stage6_sample", + "generated_at": "2026-07-21T07:36:54.029902Z", + "artifacts": [ + { + "artifact_id": "report.optimization_json", + "artifact_type": "report", + "relative_path": "optimization_report.json", + "required": true, + "produced_by": "reporting", + "status": "available", + "size_bytes": 128820, + "sha256": "451fc8b7a32552f52075dd7fbb6bf4b4014d0477d351c172e2ef21785a323ab0", + "unavailable_reason": null + }, + { + "artifact_id": "report.optimization_markdown", + "artifact_type": "report", + "relative_path": "optimization_report.md", + "required": true, + "produced_by": "reporting", + "status": "available", + "size_bytes": 1335, + "sha256": "ee44b5e042e554bce89662d9d85c64ca861984ce2e69e0e32d5e5ddb7a26570e", + "unavailable_reason": null + }, + { + "artifact_id": "evaluation.baseline_train", + "artifact_type": "evaluation", + "relative_path": "evaluations/baseline_train.json", + "required": true, + "produced_by": "baseline_train", + "status": "available", + "size_bytes": 17082, + "sha256": "e3763b13c8e2909185c91b61b93266757a3fdabe67ee78be134d171fb12d2601", + "unavailable_reason": null + }, + { + "artifact_id": "evaluation.baseline_validation", + "artifact_type": "evaluation", + "relative_path": "evaluations/baseline_validation.json", + "required": true, + "produced_by": "baseline_validation", + "status": "available", + "size_bytes": 17257, + "sha256": "9032bf067f4e0dc129bc63bdd862ab740cb9552960a72145b5b8d24c3b5ca2b3", + "unavailable_reason": null + }, + { + "artifact_id": "evaluation.candidate_train", + "artifact_type": "evaluation", + "relative_path": "evaluations/candidate_train.json", + "required": true, + "produced_by": "candidate_train", + "status": "available", + "size_bytes": 15950, + "sha256": "606d512943a332fcc159fbfa9c35c8cf0d51c4c545ff74067fc1ad7ed3201608", + "unavailable_reason": null + }, + { + "artifact_id": "evaluation.candidate_validation", + "artifact_type": "evaluation", + "relative_path": "evaluations/candidate_validation.json", + "required": true, + "produced_by": "candidate_validation", + "status": "available", + "size_bytes": 16165, + "sha256": "9a8bef5b2d7beedcd9535ab0844af47a91ce9dd3e8031801c375c17308a8bd06", + "unavailable_reason": null + }, + { + "artifact_id": "prompt.baseline.system_prompt", + "artifact_type": "prompt", + "relative_path": "prompts/baseline/000-system_prompt.md", + "required": true, + "produced_by": "baseline_train", + "status": "available", + "size_bytes": 205, + "sha256": "ded1ce1a1b01fa5f42e8a98ff81c8f816452f77b1ec510ce4b951769420e9871", + "unavailable_reason": null + }, + { + "artifact_id": "prompt.candidate.system_prompt", + "artifact_type": "prompt", + "relative_path": "prompts/candidate/000-system_prompt.md", + "required": true, + "produced_by": "candidate_generation", + "status": "available", + "size_bytes": 588, + "sha256": "70613f68877e0d15f123648ee80513dc8ded4ecbc685fad1f040bd901267252c", + "unavailable_reason": null + }, + { + "artifact_id": "input.pipeline_config", + "artifact_type": "input", + "relative_path": "inputs/pipeline_config.json", + "required": true, + "produced_by": "baseline_train", + "status": "available", + "size_bytes": 1210, + "sha256": "6bc16344d7a0691fd2aec05ae29a19fcd1507c81f7eaf5e50535c27a07775fcb", + "unavailable_reason": null + }, + { + "artifact_id": "input.optimizer_config", + "artifact_type": "input", + "relative_path": "inputs/optimizer_config.json", + "required": true, + "produced_by": "candidate_generation", + "status": "available", + "size_bytes": 772, + "sha256": "5f936b9abd6fbd0cfb8e72c5ee8783c3001791434f5881a2196ac3ce65145c49", + "unavailable_reason": null + }, + { + "artifact_id": "input.train_evalset", + "artifact_type": "input", + "relative_path": "inputs/train_evalset.json", + "required": true, + "produced_by": "baseline_train", + "status": "available", + "size_bytes": 1672, + "sha256": "d2849889340afd9b624583d75caccc96c1657281c90a8a3373b60563e332b8a9", + "unavailable_reason": null + }, + { + "artifact_id": "input.validation_evalset", + "artifact_type": "input", + "relative_path": "inputs/validation_evalset.json", + "required": true, + "produced_by": "baseline_validation", + "status": "available", + "size_bytes": 1784, + "sha256": "719cf1c6c95dd2d1e164e3383bc0c28ba9405fc438b6b4f2d34195db51028341", + "unavailable_reason": null + } + ] +} diff --git a/examples/optimization/eval_optimize_loop/sample_output/evaluations/baseline_train.json b/examples/optimization/eval_optimize_loop/sample_output/evaluations/baseline_train.json new file mode 100644 index 00000000..6a031dbd --- /dev/null +++ b/examples/optimization/eval_optimize_loop/sample_output/evaluations/baseline_train.json @@ -0,0 +1,500 @@ +{ + "phase": "baseline", + "split": "train", + "eval_set_id": "eval_optimize_loop_train", + "failed_summary": { + "agentName": "call-agent", + "evalSetId": "eval_optimize_loop_train", + "overallStatus": "failed", + "runs": 1, + "evalCases": [ + { + "evalCaseId": "train_output_format", + "overallStatus": "passed", + "metricResults": [ + { + "metricName": "final_response_avg_score", + "score": 1.0, + "threshold": 1.0, + "evalStatus": "passed" + } + ] + }, + { + "evalCaseId": "train_tool_arguments", + "overallStatus": "failed", + "metricResults": [ + { + "metricName": "final_response_avg_score", + "score": 0.0, + "threshold": 1.0, + "evalStatus": "failed" + } + ] + }, + { + "evalCaseId": "train_tool_choice", + "overallStatus": "failed", + "metricResults": [ + { + "metricName": "final_response_avg_score", + "score": 0.0, + "threshold": 1.0, + "evalStatus": "failed" + } + ] + } + ] + }, + "details_lines": [], + "result_lines": [ + "Eval Set : eval_optimize_loop_train", + "Overall Status: failed", + "Case train_output_format -> passed", + " Metric final_response_avg_score: score 1.0 (threshold 1.0) => passed", + "", + "Case train_tool_arguments -> failed", + " Metric final_response_avg_score: score 0.0 (threshold 1.0) => failed", + "", + "Case train_tool_choice -> failed", + " Metric final_response_avg_score: score 0.0 (threshold 1.0) => failed", + "" + ], + "eval_results_by_eval_id": { + "train_tool_choice": [ + { + "eval_set_id": "eval_optimize_loop_train", + "eval_id": "train_tool_choice", + "run_id": 1, + "final_eval_status": 2, + "error_message": null, + "overall_eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 0.0, + "eval_status": 2, + "details": null + } + ], + "eval_metric_result_per_invocation": [ + { + "actual_invocation": { + "invocation_id": "train-2", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "Check the status of order A100.", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"general_support\",\"message\":\"Please provide more details so I can route your request.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": null + }, + "intermediate_data": null, + "creation_timestamp": 1784619414.0021462 + }, + "expected_invocation": { + "invocation_id": "train-2", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "Check the status of order A100.", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"order_lookup\",\"message\":\"Checking order A100.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "model" + }, + "intermediate_data": null, + "creation_timestamp": 0.0 + }, + "eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 0.0, + "eval_status": 2, + "details": null + } + ] + } + ], + "session_id": "___remote_eval___session___fe46fce6-d440-432c-ac3d-3a1a03db01f1", + "user_id": null, + "session_details": null + } + ], + "train_tool_arguments": [ + { + "eval_set_id": "eval_optimize_loop_train", + "eval_id": "train_tool_arguments", + "run_id": 1, + "final_eval_status": 2, + "error_message": null, + "overall_eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 0.0, + "eval_status": 2, + "details": null + } + ], + "eval_metric_result_per_invocation": [ + { + "actual_invocation": { + "invocation_id": "train-3", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "Look up order B-204 for customer 17.", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"general_support\",\"message\":\"Please provide more details so I can route your request.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": null + }, + "intermediate_data": null, + "creation_timestamp": 1784619413.9986317 + }, + "expected_invocation": { + "invocation_id": "train-3", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "Look up order B-204 for customer 17.", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"order_lookup\",\"message\":\"Checking order B-204 for customer 17.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "model" + }, + "intermediate_data": null, + "creation_timestamp": 0.0 + }, + "eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 0.0, + "eval_status": 2, + "details": null + } + ] + } + ], + "session_id": "___remote_eval___session___a6cd661a-f2ca-4f9b-8c5f-e9d875e61f41", + "user_id": null, + "session_details": null + } + ], + "train_output_format": [ + { + "eval_set_id": "eval_optimize_loop_train", + "eval_id": "train_output_format", + "run_id": 1, + "final_eval_status": 1, + "error_message": null, + "overall_eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 1.0, + "eval_status": 1, + "details": null + } + ], + "eval_metric_result_per_invocation": [ + { + "actual_invocation": { + "invocation_id": "train-1", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "How can I update my email address?", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"account\",\"message\":\"Open profile settings to update your email.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": null + }, + "intermediate_data": null, + "creation_timestamp": 1784619414.0005472 + }, + "expected_invocation": { + "invocation_id": "train-1", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "How can I update my email address?", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"account\",\"message\":\"Open profile settings to update your email.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "model" + }, + "intermediate_data": null, + "creation_timestamp": 0.0 + }, + "eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 1.0, + "eval_status": 1, + "details": null + } + ] + } + ], + "session_id": "___remote_eval___session___168d89d2-fc32-4a89-ae7d-49993d76be8f", + "user_id": null, + "session_details": null + } + ] + }, + "passed_case_count": 1, + "total_case_count": 3, + "average_score": 0.3333333333333333 +} diff --git a/examples/optimization/eval_optimize_loop/sample_output/evaluations/baseline_validation.json b/examples/optimization/eval_optimize_loop/sample_output/evaluations/baseline_validation.json new file mode 100644 index 00000000..e6307421 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/sample_output/evaluations/baseline_validation.json @@ -0,0 +1,500 @@ +{ + "phase": "baseline", + "split": "validation", + "eval_set_id": "eval_optimize_loop_validation", + "failed_summary": { + "agentName": "call-agent", + "evalSetId": "eval_optimize_loop_validation", + "overallStatus": "failed", + "runs": 1, + "evalCases": [ + { + "evalCaseId": "val_knowledge_recall", + "overallStatus": "failed", + "metricResults": [ + { + "metricName": "final_response_avg_score", + "score": 0.0, + "threshold": 1.0, + "evalStatus": "failed" + } + ] + }, + { + "evalCaseId": "val_paraphrase", + "overallStatus": "failed", + "metricResults": [ + { + "metricName": "final_response_avg_score", + "score": 0.0, + "threshold": 1.0, + "evalStatus": "failed" + } + ] + }, + { + "evalCaseId": "val_refund_route", + "overallStatus": "passed", + "metricResults": [ + { + "metricName": "final_response_avg_score", + "score": 1.0, + "threshold": 1.0, + "evalStatus": "passed" + } + ] + } + ] + }, + "details_lines": [], + "result_lines": [ + "Eval Set : eval_optimize_loop_validation", + "Overall Status: failed", + "Case val_knowledge_recall -> failed", + " Metric final_response_avg_score: score 0.0 (threshold 1.0) => failed", + "", + "Case val_paraphrase -> failed", + " Metric final_response_avg_score: score 0.0 (threshold 1.0) => failed", + "", + "Case val_refund_route -> passed", + " Metric final_response_avg_score: score 1.0 (threshold 1.0) => passed", + "" + ], + "eval_results_by_eval_id": { + "val_knowledge_recall": [ + { + "eval_set_id": "eval_optimize_loop_validation", + "eval_id": "val_knowledge_recall", + "run_id": 1, + "final_eval_status": 2, + "error_message": null, + "overall_eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 0.0, + "eval_status": 2, + "details": null + } + ], + "eval_metric_result_per_invocation": [ + { + "actual_invocation": { + "invocation_id": "val-2", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "How long does standard shipping usually take?", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"general_support\",\"message\":\"Please provide more details so I can route your request.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": null + }, + "intermediate_data": null, + "creation_timestamp": 1784619414.0093913 + }, + "expected_invocation": { + "invocation_id": "val-2", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "How long does standard shipping usually take?", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"shipping_policy\",\"message\":\"Standard shipping normally takes 3-5 business days.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "model" + }, + "intermediate_data": null, + "creation_timestamp": 0.0 + }, + "eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 0.0, + "eval_status": 2, + "details": null + } + ] + } + ], + "session_id": "___remote_eval___session___45c563f4-d805-4a5c-82c4-77f2237e4d20", + "user_id": null, + "session_details": null + } + ], + "val_refund_route": [ + { + "eval_set_id": "eval_optimize_loop_validation", + "eval_id": "val_refund_route", + "run_id": 1, + "final_eval_status": 1, + "error_message": null, + "overall_eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 1.0, + "eval_status": 1, + "details": null + } + ], + "eval_metric_result_per_invocation": [ + { + "actual_invocation": { + "invocation_id": "val-3", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "I was charged twice and need the duplicate payment refunded.", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"billing_refund\",\"message\":\"I will route this duplicate charge for refund review.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": null + }, + "intermediate_data": null, + "creation_timestamp": 1784619414.0046532 + }, + "expected_invocation": { + "invocation_id": "val-3", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "I was charged twice and need the duplicate payment refunded.", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"billing_refund\",\"message\":\"I will route this duplicate charge for refund review.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "model" + }, + "intermediate_data": null, + "creation_timestamp": 0.0 + }, + "eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 1.0, + "eval_status": 1, + "details": null + } + ] + } + ], + "session_id": "___remote_eval___session___5a98aee1-19fd-4665-9259-1b1fe3aaf23f", + "user_id": null, + "session_details": null + } + ], + "val_paraphrase": [ + { + "eval_set_id": "eval_optimize_loop_validation", + "eval_id": "val_paraphrase", + "run_id": 1, + "final_eval_status": 2, + "error_message": null, + "overall_eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 0.0, + "eval_status": 2, + "details": null + } + ], + "eval_metric_result_per_invocation": [ + { + "actual_invocation": { + "invocation_id": "val-1", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "Where do I change the address tied to my account?", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"general_support\",\"message\":\"Please provide more details so I can route your request.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": null + }, + "intermediate_data": null, + "creation_timestamp": 1784619414.006062 + }, + "expected_invocation": { + "invocation_id": "val-1", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "Where do I change the address tied to my account?", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"account\",\"message\":\"Open profile settings to update your address.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "model" + }, + "intermediate_data": null, + "creation_timestamp": 0.0 + }, + "eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 0.0, + "eval_status": 2, + "details": null + } + ] + } + ], + "session_id": "___remote_eval___session___3ac5ca32-1910-4b66-b165-9134276d069c", + "user_id": null, + "session_details": null + } + ] + }, + "passed_case_count": 1, + "total_case_count": 3, + "average_score": 0.3333333333333333 +} diff --git a/examples/optimization/eval_optimize_loop/sample_output/evaluations/candidate_train.json b/examples/optimization/eval_optimize_loop/sample_output/evaluations/candidate_train.json new file mode 100644 index 00000000..3a5d2561 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/sample_output/evaluations/candidate_train.json @@ -0,0 +1,457 @@ +{ + "phase": "candidate", + "split": "train", + "eval_set_id": "eval_optimize_loop_train", + "failed_summary": null, + "details_lines": [], + "result_lines": [ + "Eval Set : eval_optimize_loop_train", + "Overall Status: passed", + "Case train_output_format -> passed", + " Metric final_response_avg_score: score 1.0 (threshold 1.0) => passed", + "", + "Case train_tool_arguments -> passed", + " Metric final_response_avg_score: score 1.0 (threshold 1.0) => passed", + "", + "Case train_tool_choice -> passed", + " Metric final_response_avg_score: score 1.0 (threshold 1.0) => passed", + "" + ], + "eval_results_by_eval_id": { + "train_tool_choice": [ + { + "eval_set_id": "eval_optimize_loop_train", + "eval_id": "train_tool_choice", + "run_id": 1, + "final_eval_status": 1, + "error_message": null, + "overall_eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 1.0, + "eval_status": 1, + "details": null + } + ], + "eval_metric_result_per_invocation": [ + { + "actual_invocation": { + "invocation_id": "train-2", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "Check the status of order A100.", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"order_lookup\",\"message\":\"Checking order A100.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": null + }, + "intermediate_data": null, + "creation_timestamp": 1784619414.017015 + }, + "expected_invocation": { + "invocation_id": "train-2", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "Check the status of order A100.", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"order_lookup\",\"message\":\"Checking order A100.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "model" + }, + "intermediate_data": null, + "creation_timestamp": 0.0 + }, + "eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 1.0, + "eval_status": 1, + "details": null + } + ] + } + ], + "session_id": "___remote_eval___session___640410c0-cbbc-4cad-aa4a-d4b169b4fac7", + "user_id": null, + "session_details": null + } + ], + "train_tool_arguments": [ + { + "eval_set_id": "eval_optimize_loop_train", + "eval_id": "train_tool_arguments", + "run_id": 1, + "final_eval_status": 1, + "error_message": null, + "overall_eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 1.0, + "eval_status": 1, + "details": null + } + ], + "eval_metric_result_per_invocation": [ + { + "actual_invocation": { + "invocation_id": "train-3", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "Look up order B-204 for customer 17.", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"order_lookup\",\"message\":\"Checking order B-204 for customer 17.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": null + }, + "intermediate_data": null, + "creation_timestamp": 1784619414.012829 + }, + "expected_invocation": { + "invocation_id": "train-3", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "Look up order B-204 for customer 17.", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"order_lookup\",\"message\":\"Checking order B-204 for customer 17.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "model" + }, + "intermediate_data": null, + "creation_timestamp": 0.0 + }, + "eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 1.0, + "eval_status": 1, + "details": null + } + ] + } + ], + "session_id": "___remote_eval___session___785bd08e-706e-4f83-9c44-ac03997c67c3", + "user_id": null, + "session_details": null + } + ], + "train_output_format": [ + { + "eval_set_id": "eval_optimize_loop_train", + "eval_id": "train_output_format", + "run_id": 1, + "final_eval_status": 1, + "error_message": null, + "overall_eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 1.0, + "eval_status": 1, + "details": null + } + ], + "eval_metric_result_per_invocation": [ + { + "actual_invocation": { + "invocation_id": "train-1", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "How can I update my email address?", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"account\",\"message\":\"Open profile settings to update your email.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": null + }, + "intermediate_data": null, + "creation_timestamp": 1784619414.0143127 + }, + "expected_invocation": { + "invocation_id": "train-1", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "How can I update my email address?", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"account\",\"message\":\"Open profile settings to update your email.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "model" + }, + "intermediate_data": null, + "creation_timestamp": 0.0 + }, + "eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 1.0, + "eval_status": 1, + "details": null + } + ] + } + ], + "session_id": "___remote_eval___session___4c9ebf1d-9e6f-4488-a64c-ca2a0c01f40a", + "user_id": null, + "session_details": null + } + ] + }, + "passed_case_count": 3, + "total_case_count": 3, + "average_score": 1.0 +} diff --git a/examples/optimization/eval_optimize_loop/sample_output/evaluations/candidate_validation.json b/examples/optimization/eval_optimize_loop/sample_output/evaluations/candidate_validation.json new file mode 100644 index 00000000..0a379a3f --- /dev/null +++ b/examples/optimization/eval_optimize_loop/sample_output/evaluations/candidate_validation.json @@ -0,0 +1,457 @@ +{ + "phase": "candidate", + "split": "validation", + "eval_set_id": "eval_optimize_loop_validation", + "failed_summary": null, + "details_lines": [], + "result_lines": [ + "Eval Set : eval_optimize_loop_validation", + "Overall Status: passed", + "Case val_knowledge_recall -> passed", + " Metric final_response_avg_score: score 1.0 (threshold 1.0) => passed", + "", + "Case val_paraphrase -> passed", + " Metric final_response_avg_score: score 1.0 (threshold 1.0) => passed", + "", + "Case val_refund_route -> passed", + " Metric final_response_avg_score: score 1.0 (threshold 1.0) => passed", + "" + ], + "eval_results_by_eval_id": { + "val_knowledge_recall": [ + { + "eval_set_id": "eval_optimize_loop_validation", + "eval_id": "val_knowledge_recall", + "run_id": 1, + "final_eval_status": 1, + "error_message": null, + "overall_eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 1.0, + "eval_status": 1, + "details": null + } + ], + "eval_metric_result_per_invocation": [ + { + "actual_invocation": { + "invocation_id": "val-2", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "How long does standard shipping usually take?", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"shipping_policy\",\"message\":\"Standard shipping normally takes 3-5 business days.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": null + }, + "intermediate_data": null, + "creation_timestamp": 1784619414.024159 + }, + "expected_invocation": { + "invocation_id": "val-2", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "How long does standard shipping usually take?", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"shipping_policy\",\"message\":\"Standard shipping normally takes 3-5 business days.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "model" + }, + "intermediate_data": null, + "creation_timestamp": 0.0 + }, + "eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 1.0, + "eval_status": 1, + "details": null + } + ] + } + ], + "session_id": "___remote_eval___session___426070b6-19e9-4441-941e-321dcbb6b443", + "user_id": null, + "session_details": null + } + ], + "val_refund_route": [ + { + "eval_set_id": "eval_optimize_loop_validation", + "eval_id": "val_refund_route", + "run_id": 1, + "final_eval_status": 1, + "error_message": null, + "overall_eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 1.0, + "eval_status": 1, + "details": null + } + ], + "eval_metric_result_per_invocation": [ + { + "actual_invocation": { + "invocation_id": "val-3", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "I was charged twice and need the duplicate payment refunded.", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"billing_refund\",\"message\":\"I will route this duplicate charge for refund review.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": null + }, + "intermediate_data": null, + "creation_timestamp": 1784619414.0204713 + }, + "expected_invocation": { + "invocation_id": "val-3", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "I was charged twice and need the duplicate payment refunded.", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"billing_refund\",\"message\":\"I will route this duplicate charge for refund review.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "model" + }, + "intermediate_data": null, + "creation_timestamp": 0.0 + }, + "eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 1.0, + "eval_status": 1, + "details": null + } + ] + } + ], + "session_id": "___remote_eval___session___3439f8ac-062e-4b85-b8e3-b13ef260efe3", + "user_id": null, + "session_details": null + } + ], + "val_paraphrase": [ + { + "eval_set_id": "eval_optimize_loop_validation", + "eval_id": "val_paraphrase", + "run_id": 1, + "final_eval_status": 1, + "error_message": null, + "overall_eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 1.0, + "eval_status": 1, + "details": null + } + ], + "eval_metric_result_per_invocation": [ + { + "actual_invocation": { + "invocation_id": "val-1", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "Where do I change the address tied to my account?", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"account\",\"message\":\"Open profile settings to update your address.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": null + }, + "intermediate_data": null, + "creation_timestamp": 1784619414.0225098 + }, + "expected_invocation": { + "invocation_id": "val-1", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "Where do I change the address tied to my account?", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"account\",\"message\":\"Open profile settings to update your address.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "model" + }, + "intermediate_data": null, + "creation_timestamp": 0.0 + }, + "eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 1.0, + "eval_status": 1, + "details": null + } + ] + } + ], + "session_id": "___remote_eval___session___bad8b5f6-d2d4-4512-8536-786f53cd2f5a", + "user_id": null, + "session_details": null + } + ] + }, + "passed_case_count": 3, + "total_case_count": 3, + "average_score": 1.0 +} diff --git a/examples/optimization/eval_optimize_loop/sample_output/inputs/optimizer_config.json b/examples/optimization/eval_optimize_loop/sample_output/inputs/optimizer_config.json new file mode 100644 index 00000000..9efff8be --- /dev/null +++ b/examples/optimization/eval_optimize_loop/sample_output/inputs/optimizer_config.json @@ -0,0 +1,36 @@ +{ + "evaluate": { + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + } + } + ], + "num_runs": 1 + }, + "optimize": { + "eval_case_parallelism": 1, + "stop": { + "required_metrics": "all" + }, + "algorithm": { + "name": "gepa_reflective", + "seed": 42, + "reflection_lm": { + "model_name": "fake-not-used-in-offline-mode", + "api_key": "fake-not-used-in-offline-mode" + }, + "reflection_minibatch_size": 3, + "skip_perfect_score": false, + "max_candidate_proposals": 3 + } + } +} diff --git a/examples/optimization/eval_optimize_loop/sample_output/inputs/pipeline_config.json b/examples/optimization/eval_optimize_loop/sample_output/inputs/pipeline_config.json new file mode 100644 index 00000000..1d88ff2a --- /dev/null +++ b/examples/optimization/eval_optimize_loop/sample_output/inputs/pipeline_config.json @@ -0,0 +1,52 @@ +{ + "config_version": 1, + "execution": { + "mode": "offline", + "candidate_scenario": "improve" + }, + "inputs": { + "train_evalset": "data/train.evalset.json", + "validation_evalset": "data/val.evalset.json", + "optimizer_config": "optimizer.json" + }, + "prompts": [ + { + "name": "system_prompt", + "path": "prompts/system.md" + } + ], + "run": { + "runs_dir": "runs", + "seed": 42 + }, + "case_labels": { + "hard_case_ids": ["val_knowledge_recall"], + "critical_case_ids": ["val_refund_route"] + }, + "gate": { + "min_validation_score_delta": 0.05, + "reject_on_validation_pass_rate_drop": true, + "reject_new_hard_fail": true, + "reject_critical_regression": true, + "severe_case_score_drop": 0.2, + "required_metrics": ["final_response_avg_score"] + }, + "budget": { + "max_duration_seconds": 180, + "on_unavailable": "warning" + }, + "reporting": { + "write_json": true, + "write_markdown": true, + "include_case_evidence": true + }, + "artifacts": { + "copy_input_files": true, + "retain_optimizer_native_artifacts": true, + "audit_all_candidates": false + }, + "writeback": { + "enabled": false, + "require_source_hash_match": true + } +} diff --git a/examples/optimization/eval_optimize_loop/sample_output/inputs/train_evalset.json b/examples/optimization/eval_optimize_loop/sample_output/inputs/train_evalset.json new file mode 100644 index 00000000..6b41ca3a --- /dev/null +++ b/examples/optimization/eval_optimize_loop/sample_output/inputs/train_evalset.json @@ -0,0 +1,40 @@ +{ + "eval_set_id": "eval_optimize_loop_train", + "name": "Evaluation optimization loop - train", + "description": "Three deterministic training cases for format, tool choice, and tool arguments.", + "eval_cases": [ + { + "eval_id": "train_output_format", + "conversation": [ + { + "invocation_id": "train-1", + "user_content": {"parts": [{"text": "How can I update my email address?"}], "role": "user"}, + "final_response": {"parts": [{"text": "{\"route\":\"account\",\"message\":\"Open profile settings to update your email.\"}"}], "role": "model"} + } + ], + "session_input": {"app_name": "eval_optimize_loop", "user_id": "demo", "state": {}} + }, + { + "eval_id": "train_tool_choice", + "conversation": [ + { + "invocation_id": "train-2", + "user_content": {"parts": [{"text": "Check the status of order A100."}], "role": "user"}, + "final_response": {"parts": [{"text": "{\"route\":\"order_lookup\",\"message\":\"Checking order A100.\"}"}], "role": "model"} + } + ], + "session_input": {"app_name": "eval_optimize_loop", "user_id": "demo", "state": {}} + }, + { + "eval_id": "train_tool_arguments", + "conversation": [ + { + "invocation_id": "train-3", + "user_content": {"parts": [{"text": "Look up order B-204 for customer 17."}], "role": "user"}, + "final_response": {"parts": [{"text": "{\"route\":\"order_lookup\",\"message\":\"Checking order B-204 for customer 17.\"}"}], "role": "model"} + } + ], + "session_input": {"app_name": "eval_optimize_loop", "user_id": "demo", "state": {}} + } + ] +} diff --git a/examples/optimization/eval_optimize_loop/sample_output/inputs/validation_evalset.json b/examples/optimization/eval_optimize_loop/sample_output/inputs/validation_evalset.json new file mode 100644 index 00000000..387e6de7 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/sample_output/inputs/validation_evalset.json @@ -0,0 +1,40 @@ +{ + "eval_set_id": "eval_optimize_loop_validation", + "name": "Evaluation optimization loop - validation", + "description": "Three deterministic validation cases for generalization, recall, and critical routing.", + "eval_cases": [ + { + "eval_id": "val_paraphrase", + "conversation": [ + { + "invocation_id": "val-1", + "user_content": {"parts": [{"text": "Where do I change the address tied to my account?"}], "role": "user"}, + "final_response": {"parts": [{"text": "{\"route\":\"account\",\"message\":\"Open profile settings to update your address.\"}"}], "role": "model"} + } + ], + "session_input": {"app_name": "eval_optimize_loop", "user_id": "demo", "state": {}} + }, + { + "eval_id": "val_knowledge_recall", + "conversation": [ + { + "invocation_id": "val-2", + "user_content": {"parts": [{"text": "How long does standard shipping usually take?"}], "role": "user"}, + "final_response": {"parts": [{"text": "{\"route\":\"shipping_policy\",\"message\":\"Standard shipping normally takes 3-5 business days.\"}"}], "role": "model"} + } + ], + "session_input": {"app_name": "eval_optimize_loop", "user_id": "demo", "state": {}} + }, + { + "eval_id": "val_refund_route", + "conversation": [ + { + "invocation_id": "val-3", + "user_content": {"parts": [{"text": "I was charged twice and need the duplicate payment refunded."}], "role": "user"}, + "final_response": {"parts": [{"text": "{\"route\":\"billing_refund\",\"message\":\"I will route this duplicate charge for refund review.\"}"}], "role": "model"} + } + ], + "session_input": {"app_name": "eval_optimize_loop", "user_id": "demo", "state": {}} + } + ] +} diff --git a/examples/optimization/eval_optimize_loop/sample_output/optimization_report.json b/examples/optimization/eval_optimize_loop/sample_output/optimization_report.json new file mode 100644 index 00000000..df7830ef --- /dev/null +++ b/examples/optimization/eval_optimize_loop/sample_output/optimization_report.json @@ -0,0 +1,3666 @@ +{ + "schema_version": 1, + "status": "completed", + "run_id": "stage6_sample", + "execution_mode": "offline", + "seed": 42, + "started_at": "2026-07-21T07:36:53.995239Z", + "finished_at": "2026-07-21T07:36:54.029902Z", + "input_snapshot": { + "pipeline_config_path": "/pipeline.json", + "pipeline_config_sha256": "6bc16344d7a0691fd2aec05ae29a19fcd1507c81f7eaf5e50535c27a07775fcb", + "optimizer_config_path": "/optimizer.json", + "optimizer_config_sha256": "5f936b9abd6fbd0cfb8e72c5ee8783c3001791434f5881a2196ac3ce65145c49", + "train_evalset_path": "/data/train.evalset.json", + "train_evalset_sha256": "d2849889340afd9b624583d75caccc96c1657281c90a8a3373b60563e332b8a9", + "validation_evalset_path": "/data/val.evalset.json", + "validation_evalset_sha256": "719cf1c6c95dd2d1e164e3383bc0c28ba9405fc438b6b4f2d34195db51028341", + "prompt_snapshots": [ + { + "field_name": "system_prompt", + "source_path": "/prompts/system.md", + "working_path": "/runs/stage6_sample/workspace/prompts/01_system_prompt.md", + "content": "You are a customer-support routing assistant.\n\nAnswer with a compact JSON object containing `route` and `message`. Use the\navailable account tool for account-specific requests. Never invent account\nfacts.\n", + "sha256": "ded1ce1a1b01fa5f42e8a98ff81c8f816452f77b1ec510ce4b951769420e9871" + } + ], + "seed": 42, + "trace_inputs": null + }, + "candidate": { + "provider": "fake", + "prompts": { + "system_prompt": "You are a customer-support routing assistant.\n\nAnswer with a compact JSON object containing `route` and `message`. Use the\navailable account tool for account-specific requests. Never invent account\nfacts.\n\n\nApply general customer-support routing rules across equivalent user phrasings.\n\n\n\n\n\n" + }, + "changed_fields": [ + "system_prompt" + ], + "rationale": "Generalize routing across account synonyms, order lookup, shipping policy, and refunds.", + "parent_prompt_sha256": "85420c1a6ffbfc6681d7d4439154e798c02a173b5e336896c4d6046b98c83f3a", + "candidate_prompt_sha256": "60cc05b773e8ba70320c6789ccadcdc0b28da43ed1904743580791601cd6d901", + "candidate_id": "fake-improve-60cc05b773e8", + "scenario": "improve", + "seed": 42 + }, + "baseline_train": { + "phase": "baseline", + "split": "train", + "eval_set_id": "eval_optimize_loop_train", + "failed_summary": { + "agentName": "call-agent", + "evalSetId": "eval_optimize_loop_train", + "overallStatus": "failed", + "runs": 1, + "evalCases": [ + { + "evalCaseId": "train_output_format", + "overallStatus": "passed", + "metricResults": [ + { + "metricName": "final_response_avg_score", + "score": 1.0, + "threshold": 1.0, + "evalStatus": "passed" + } + ] + }, + { + "evalCaseId": "train_tool_arguments", + "overallStatus": "failed", + "metricResults": [ + { + "metricName": "final_response_avg_score", + "score": 0.0, + "threshold": 1.0, + "evalStatus": "failed" + } + ] + }, + { + "evalCaseId": "train_tool_choice", + "overallStatus": "failed", + "metricResults": [ + { + "metricName": "final_response_avg_score", + "score": 0.0, + "threshold": 1.0, + "evalStatus": "failed" + } + ] + } + ] + }, + "details_lines": [], + "result_lines": [ + "Eval Set : eval_optimize_loop_train", + "Overall Status: failed", + "Case train_output_format -> passed", + " Metric final_response_avg_score: score 1.0 (threshold 1.0) => passed", + "", + "Case train_tool_arguments -> failed", + " Metric final_response_avg_score: score 0.0 (threshold 1.0) => failed", + "", + "Case train_tool_choice -> failed", + " Metric final_response_avg_score: score 0.0 (threshold 1.0) => failed", + "" + ], + "eval_results_by_eval_id": { + "train_tool_choice": [ + { + "eval_set_id": "eval_optimize_loop_train", + "eval_id": "train_tool_choice", + "run_id": 1, + "final_eval_status": 2, + "error_message": null, + "overall_eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 0.0, + "eval_status": 2, + "details": null + } + ], + "eval_metric_result_per_invocation": [ + { + "actual_invocation": { + "invocation_id": "train-2", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "Check the status of order A100.", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"general_support\",\"message\":\"Please provide more details so I can route your request.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": null + }, + "intermediate_data": null, + "creation_timestamp": 1784619414.0021462 + }, + "expected_invocation": { + "invocation_id": "train-2", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "Check the status of order A100.", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"order_lookup\",\"message\":\"Checking order A100.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "model" + }, + "intermediate_data": null, + "creation_timestamp": 0.0 + }, + "eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 0.0, + "eval_status": 2, + "details": null + } + ] + } + ], + "session_id": "___remote_eval___session___fe46fce6-d440-432c-ac3d-3a1a03db01f1", + "user_id": null, + "session_details": null + } + ], + "train_tool_arguments": [ + { + "eval_set_id": "eval_optimize_loop_train", + "eval_id": "train_tool_arguments", + "run_id": 1, + "final_eval_status": 2, + "error_message": null, + "overall_eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 0.0, + "eval_status": 2, + "details": null + } + ], + "eval_metric_result_per_invocation": [ + { + "actual_invocation": { + "invocation_id": "train-3", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "Look up order B-204 for customer 17.", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"general_support\",\"message\":\"Please provide more details so I can route your request.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": null + }, + "intermediate_data": null, + "creation_timestamp": 1784619413.9986317 + }, + "expected_invocation": { + "invocation_id": "train-3", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "Look up order B-204 for customer 17.", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"order_lookup\",\"message\":\"Checking order B-204 for customer 17.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "model" + }, + "intermediate_data": null, + "creation_timestamp": 0.0 + }, + "eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 0.0, + "eval_status": 2, + "details": null + } + ] + } + ], + "session_id": "___remote_eval___session___a6cd661a-f2ca-4f9b-8c5f-e9d875e61f41", + "user_id": null, + "session_details": null + } + ], + "train_output_format": [ + { + "eval_set_id": "eval_optimize_loop_train", + "eval_id": "train_output_format", + "run_id": 1, + "final_eval_status": 1, + "error_message": null, + "overall_eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 1.0, + "eval_status": 1, + "details": null + } + ], + "eval_metric_result_per_invocation": [ + { + "actual_invocation": { + "invocation_id": "train-1", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "How can I update my email address?", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"account\",\"message\":\"Open profile settings to update your email.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": null + }, + "intermediate_data": null, + "creation_timestamp": 1784619414.0005472 + }, + "expected_invocation": { + "invocation_id": "train-1", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "How can I update my email address?", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"account\",\"message\":\"Open profile settings to update your email.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "model" + }, + "intermediate_data": null, + "creation_timestamp": 0.0 + }, + "eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 1.0, + "eval_status": 1, + "details": null + } + ] + } + ], + "session_id": "___remote_eval___session___168d89d2-fc32-4a89-ae7d-49993d76be8f", + "user_id": null, + "session_details": null + } + ] + }, + "passed_case_count": 1, + "total_case_count": 3, + "average_score": 0.3333333333333333 + }, + "baseline_validation": { + "phase": "baseline", + "split": "validation", + "eval_set_id": "eval_optimize_loop_validation", + "failed_summary": { + "agentName": "call-agent", + "evalSetId": "eval_optimize_loop_validation", + "overallStatus": "failed", + "runs": 1, + "evalCases": [ + { + "evalCaseId": "val_knowledge_recall", + "overallStatus": "failed", + "metricResults": [ + { + "metricName": "final_response_avg_score", + "score": 0.0, + "threshold": 1.0, + "evalStatus": "failed" + } + ] + }, + { + "evalCaseId": "val_paraphrase", + "overallStatus": "failed", + "metricResults": [ + { + "metricName": "final_response_avg_score", + "score": 0.0, + "threshold": 1.0, + "evalStatus": "failed" + } + ] + }, + { + "evalCaseId": "val_refund_route", + "overallStatus": "passed", + "metricResults": [ + { + "metricName": "final_response_avg_score", + "score": 1.0, + "threshold": 1.0, + "evalStatus": "passed" + } + ] + } + ] + }, + "details_lines": [], + "result_lines": [ + "Eval Set : eval_optimize_loop_validation", + "Overall Status: failed", + "Case val_knowledge_recall -> failed", + " Metric final_response_avg_score: score 0.0 (threshold 1.0) => failed", + "", + "Case val_paraphrase -> failed", + " Metric final_response_avg_score: score 0.0 (threshold 1.0) => failed", + "", + "Case val_refund_route -> passed", + " Metric final_response_avg_score: score 1.0 (threshold 1.0) => passed", + "" + ], + "eval_results_by_eval_id": { + "val_knowledge_recall": [ + { + "eval_set_id": "eval_optimize_loop_validation", + "eval_id": "val_knowledge_recall", + "run_id": 1, + "final_eval_status": 2, + "error_message": null, + "overall_eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 0.0, + "eval_status": 2, + "details": null + } + ], + "eval_metric_result_per_invocation": [ + { + "actual_invocation": { + "invocation_id": "val-2", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "How long does standard shipping usually take?", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"general_support\",\"message\":\"Please provide more details so I can route your request.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": null + }, + "intermediate_data": null, + "creation_timestamp": 1784619414.0093913 + }, + "expected_invocation": { + "invocation_id": "val-2", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "How long does standard shipping usually take?", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"shipping_policy\",\"message\":\"Standard shipping normally takes 3-5 business days.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "model" + }, + "intermediate_data": null, + "creation_timestamp": 0.0 + }, + "eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 0.0, + "eval_status": 2, + "details": null + } + ] + } + ], + "session_id": "___remote_eval___session___45c563f4-d805-4a5c-82c4-77f2237e4d20", + "user_id": null, + "session_details": null + } + ], + "val_refund_route": [ + { + "eval_set_id": "eval_optimize_loop_validation", + "eval_id": "val_refund_route", + "run_id": 1, + "final_eval_status": 1, + "error_message": null, + "overall_eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 1.0, + "eval_status": 1, + "details": null + } + ], + "eval_metric_result_per_invocation": [ + { + "actual_invocation": { + "invocation_id": "val-3", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "I was charged twice and need the duplicate payment refunded.", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"billing_refund\",\"message\":\"I will route this duplicate charge for refund review.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": null + }, + "intermediate_data": null, + "creation_timestamp": 1784619414.0046532 + }, + "expected_invocation": { + "invocation_id": "val-3", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "I was charged twice and need the duplicate payment refunded.", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"billing_refund\",\"message\":\"I will route this duplicate charge for refund review.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "model" + }, + "intermediate_data": null, + "creation_timestamp": 0.0 + }, + "eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 1.0, + "eval_status": 1, + "details": null + } + ] + } + ], + "session_id": "___remote_eval___session___5a98aee1-19fd-4665-9259-1b1fe3aaf23f", + "user_id": null, + "session_details": null + } + ], + "val_paraphrase": [ + { + "eval_set_id": "eval_optimize_loop_validation", + "eval_id": "val_paraphrase", + "run_id": 1, + "final_eval_status": 2, + "error_message": null, + "overall_eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 0.0, + "eval_status": 2, + "details": null + } + ], + "eval_metric_result_per_invocation": [ + { + "actual_invocation": { + "invocation_id": "val-1", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "Where do I change the address tied to my account?", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"general_support\",\"message\":\"Please provide more details so I can route your request.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": null + }, + "intermediate_data": null, + "creation_timestamp": 1784619414.006062 + }, + "expected_invocation": { + "invocation_id": "val-1", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "Where do I change the address tied to my account?", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"account\",\"message\":\"Open profile settings to update your address.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "model" + }, + "intermediate_data": null, + "creation_timestamp": 0.0 + }, + "eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 0.0, + "eval_status": 2, + "details": null + } + ] + } + ], + "session_id": "___remote_eval___session___3ac5ca32-1910-4b66-b165-9134276d069c", + "user_id": null, + "session_details": null + } + ] + }, + "passed_case_count": 1, + "total_case_count": 3, + "average_score": 0.3333333333333333 + }, + "candidate_train": { + "phase": "candidate", + "split": "train", + "eval_set_id": "eval_optimize_loop_train", + "failed_summary": null, + "details_lines": [], + "result_lines": [ + "Eval Set : eval_optimize_loop_train", + "Overall Status: passed", + "Case train_output_format -> passed", + " Metric final_response_avg_score: score 1.0 (threshold 1.0) => passed", + "", + "Case train_tool_arguments -> passed", + " Metric final_response_avg_score: score 1.0 (threshold 1.0) => passed", + "", + "Case train_tool_choice -> passed", + " Metric final_response_avg_score: score 1.0 (threshold 1.0) => passed", + "" + ], + "eval_results_by_eval_id": { + "train_tool_choice": [ + { + "eval_set_id": "eval_optimize_loop_train", + "eval_id": "train_tool_choice", + "run_id": 1, + "final_eval_status": 1, + "error_message": null, + "overall_eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 1.0, + "eval_status": 1, + "details": null + } + ], + "eval_metric_result_per_invocation": [ + { + "actual_invocation": { + "invocation_id": "train-2", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "Check the status of order A100.", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"order_lookup\",\"message\":\"Checking order A100.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": null + }, + "intermediate_data": null, + "creation_timestamp": 1784619414.017015 + }, + "expected_invocation": { + "invocation_id": "train-2", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "Check the status of order A100.", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"order_lookup\",\"message\":\"Checking order A100.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "model" + }, + "intermediate_data": null, + "creation_timestamp": 0.0 + }, + "eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 1.0, + "eval_status": 1, + "details": null + } + ] + } + ], + "session_id": "___remote_eval___session___640410c0-cbbc-4cad-aa4a-d4b169b4fac7", + "user_id": null, + "session_details": null + } + ], + "train_tool_arguments": [ + { + "eval_set_id": "eval_optimize_loop_train", + "eval_id": "train_tool_arguments", + "run_id": 1, + "final_eval_status": 1, + "error_message": null, + "overall_eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 1.0, + "eval_status": 1, + "details": null + } + ], + "eval_metric_result_per_invocation": [ + { + "actual_invocation": { + "invocation_id": "train-3", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "Look up order B-204 for customer 17.", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"order_lookup\",\"message\":\"Checking order B-204 for customer 17.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": null + }, + "intermediate_data": null, + "creation_timestamp": 1784619414.012829 + }, + "expected_invocation": { + "invocation_id": "train-3", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "Look up order B-204 for customer 17.", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"order_lookup\",\"message\":\"Checking order B-204 for customer 17.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "model" + }, + "intermediate_data": null, + "creation_timestamp": 0.0 + }, + "eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 1.0, + "eval_status": 1, + "details": null + } + ] + } + ], + "session_id": "___remote_eval___session___785bd08e-706e-4f83-9c44-ac03997c67c3", + "user_id": null, + "session_details": null + } + ], + "train_output_format": [ + { + "eval_set_id": "eval_optimize_loop_train", + "eval_id": "train_output_format", + "run_id": 1, + "final_eval_status": 1, + "error_message": null, + "overall_eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 1.0, + "eval_status": 1, + "details": null + } + ], + "eval_metric_result_per_invocation": [ + { + "actual_invocation": { + "invocation_id": "train-1", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "How can I update my email address?", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"account\",\"message\":\"Open profile settings to update your email.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": null + }, + "intermediate_data": null, + "creation_timestamp": 1784619414.0143127 + }, + "expected_invocation": { + "invocation_id": "train-1", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "How can I update my email address?", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"account\",\"message\":\"Open profile settings to update your email.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "model" + }, + "intermediate_data": null, + "creation_timestamp": 0.0 + }, + "eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 1.0, + "eval_status": 1, + "details": null + } + ] + } + ], + "session_id": "___remote_eval___session___4c9ebf1d-9e6f-4488-a64c-ca2a0c01f40a", + "user_id": null, + "session_details": null + } + ] + }, + "passed_case_count": 3, + "total_case_count": 3, + "average_score": 1.0 + }, + "candidate_validation": { + "phase": "candidate", + "split": "validation", + "eval_set_id": "eval_optimize_loop_validation", + "failed_summary": null, + "details_lines": [], + "result_lines": [ + "Eval Set : eval_optimize_loop_validation", + "Overall Status: passed", + "Case val_knowledge_recall -> passed", + " Metric final_response_avg_score: score 1.0 (threshold 1.0) => passed", + "", + "Case val_paraphrase -> passed", + " Metric final_response_avg_score: score 1.0 (threshold 1.0) => passed", + "", + "Case val_refund_route -> passed", + " Metric final_response_avg_score: score 1.0 (threshold 1.0) => passed", + "" + ], + "eval_results_by_eval_id": { + "val_knowledge_recall": [ + { + "eval_set_id": "eval_optimize_loop_validation", + "eval_id": "val_knowledge_recall", + "run_id": 1, + "final_eval_status": 1, + "error_message": null, + "overall_eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 1.0, + "eval_status": 1, + "details": null + } + ], + "eval_metric_result_per_invocation": [ + { + "actual_invocation": { + "invocation_id": "val-2", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "How long does standard shipping usually take?", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"shipping_policy\",\"message\":\"Standard shipping normally takes 3-5 business days.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": null + }, + "intermediate_data": null, + "creation_timestamp": 1784619414.024159 + }, + "expected_invocation": { + "invocation_id": "val-2", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "How long does standard shipping usually take?", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"shipping_policy\",\"message\":\"Standard shipping normally takes 3-5 business days.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "model" + }, + "intermediate_data": null, + "creation_timestamp": 0.0 + }, + "eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 1.0, + "eval_status": 1, + "details": null + } + ] + } + ], + "session_id": "___remote_eval___session___426070b6-19e9-4441-941e-321dcbb6b443", + "user_id": null, + "session_details": null + } + ], + "val_refund_route": [ + { + "eval_set_id": "eval_optimize_loop_validation", + "eval_id": "val_refund_route", + "run_id": 1, + "final_eval_status": 1, + "error_message": null, + "overall_eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 1.0, + "eval_status": 1, + "details": null + } + ], + "eval_metric_result_per_invocation": [ + { + "actual_invocation": { + "invocation_id": "val-3", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "I was charged twice and need the duplicate payment refunded.", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"billing_refund\",\"message\":\"I will route this duplicate charge for refund review.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": null + }, + "intermediate_data": null, + "creation_timestamp": 1784619414.0204713 + }, + "expected_invocation": { + "invocation_id": "val-3", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "I was charged twice and need the duplicate payment refunded.", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"billing_refund\",\"message\":\"I will route this duplicate charge for refund review.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "model" + }, + "intermediate_data": null, + "creation_timestamp": 0.0 + }, + "eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 1.0, + "eval_status": 1, + "details": null + } + ] + } + ], + "session_id": "___remote_eval___session___3439f8ac-062e-4b85-b8e3-b13ef260efe3", + "user_id": null, + "session_details": null + } + ], + "val_paraphrase": [ + { + "eval_set_id": "eval_optimize_loop_validation", + "eval_id": "val_paraphrase", + "run_id": 1, + "final_eval_status": 1, + "error_message": null, + "overall_eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 1.0, + "eval_status": 1, + "details": null + } + ], + "eval_metric_result_per_invocation": [ + { + "actual_invocation": { + "invocation_id": "val-1", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "Where do I change the address tied to my account?", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"account\",\"message\":\"Open profile settings to update your address.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": null + }, + "intermediate_data": null, + "creation_timestamp": 1784619414.0225098 + }, + "expected_invocation": { + "invocation_id": "val-1", + "user_content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "Where do I change the address tied to my account?", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "{\"route\":\"account\",\"message\":\"Open profile settings to update your address.\"}", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "model" + }, + "intermediate_data": null, + "creation_timestamp": 0.0 + }, + "eval_metric_results": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + }, + "score": 1.0, + "eval_status": 1, + "details": null + } + ] + } + ], + "session_id": "___remote_eval___session___bad8b5f6-d2d4-4512-8536-786f53cd2f5a", + "user_id": null, + "session_details": null + } + ] + }, + "passed_case_count": 3, + "total_case_count": 3, + "average_score": 1.0 + }, + "analysis": { + "baseline_train": { + "phase": "baseline", + "split": "train", + "eval_set_id": "eval_optimize_loop_train", + "cases": [ + { + "eval_id": "train_output_format", + "status": "passed", + "average_score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "status": "passed", + "score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "reason": null + } + ], + "runs": [ + { + "run_id": 1, + "status": "passed", + "error_message": null, + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "status": "passed", + "score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "reason": null + } + ], + "invocations": [ + { + "invocation_id": "train-1", + "user_text": "How can I update my email address?", + "expected_response": "{\"route\":\"account\",\"message\":\"Open profile settings to update your email.\"}", + "actual_response": "{\"route\":\"account\",\"message\":\"Open profile settings to update your email.\"}", + "expected_tools": [], + "actual_tools": [], + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "status": "passed", + "score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "reason": null + } + ] + } + ] + } + ], + "attribution": null + }, + { + "eval_id": "train_tool_arguments", + "status": "failed", + "average_score": { + "status": "available", + "value": 0.0, + "unit": null, + "reason": null + }, + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "status": "failed", + "score": { + "status": "available", + "value": 0.0, + "unit": null, + "reason": null + }, + "reason": null + } + ], + "runs": [ + { + "run_id": 1, + "status": "failed", + "error_message": null, + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "status": "failed", + "score": { + "status": "available", + "value": 0.0, + "unit": null, + "reason": null + }, + "reason": null + } + ], + "invocations": [ + { + "invocation_id": "train-3", + "user_text": "Look up order B-204 for customer 17.", + "expected_response": "{\"route\":\"order_lookup\",\"message\":\"Checking order B-204 for customer 17.\"}", + "actual_response": "{\"route\":\"general_support\",\"message\":\"Please provide more details so I can route your request.\"}", + "expected_tools": [], + "actual_tools": [], + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "status": "failed", + "score": { + "status": "available", + "value": 0.0, + "unit": null, + "reason": null + }, + "reason": null + } + ] + } + ] + } + ], + "attribution": { + "primary_category": "routing_error", + "secondary_categories": [ + "final_response_mismatch" + ], + "summary": "Expected route 'order_lookup', got 'general_support'.", + "evidence": [ + { + "evidence_type": "response", + "message": "Expected route 'order_lookup', got 'general_support'.", + "run_id": 1, + "invocation_id": "train-3", + "metric_name": null, + "expected": "order_lookup", + "actual": "general_support" + }, + { + "evidence_type": "response", + "message": "Final response did not satisfy metric 'final_response_avg_score'.", + "run_id": 1, + "invocation_id": "train-3", + "metric_name": "final_response_avg_score", + "expected": "{\"route\":\"order_lookup\",\"message\":\"Checking order B-204 for customer 17.\"}", + "actual": "{\"route\":\"general_support\",\"message\":\"Please provide more details so I can route your request.\"}" + } + ] + } + }, + { + "eval_id": "train_tool_choice", + "status": "failed", + "average_score": { + "status": "available", + "value": 0.0, + "unit": null, + "reason": null + }, + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "status": "failed", + "score": { + "status": "available", + "value": 0.0, + "unit": null, + "reason": null + }, + "reason": null + } + ], + "runs": [ + { + "run_id": 1, + "status": "failed", + "error_message": null, + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "status": "failed", + "score": { + "status": "available", + "value": 0.0, + "unit": null, + "reason": null + }, + "reason": null + } + ], + "invocations": [ + { + "invocation_id": "train-2", + "user_text": "Check the status of order A100.", + "expected_response": "{\"route\":\"order_lookup\",\"message\":\"Checking order A100.\"}", + "actual_response": "{\"route\":\"general_support\",\"message\":\"Please provide more details so I can route your request.\"}", + "expected_tools": [], + "actual_tools": [], + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "status": "failed", + "score": { + "status": "available", + "value": 0.0, + "unit": null, + "reason": null + }, + "reason": null + } + ] + } + ] + } + ], + "attribution": { + "primary_category": "routing_error", + "secondary_categories": [ + "final_response_mismatch" + ], + "summary": "Expected route 'order_lookup', got 'general_support'.", + "evidence": [ + { + "evidence_type": "response", + "message": "Expected route 'order_lookup', got 'general_support'.", + "run_id": 1, + "invocation_id": "train-2", + "metric_name": null, + "expected": "order_lookup", + "actual": "general_support" + }, + { + "evidence_type": "response", + "message": "Final response did not satisfy metric 'final_response_avg_score'.", + "run_id": 1, + "invocation_id": "train-2", + "metric_name": "final_response_avg_score", + "expected": "{\"route\":\"order_lookup\",\"message\":\"Checking order A100.\"}", + "actual": "{\"route\":\"general_support\",\"message\":\"Please provide more details so I can route your request.\"}" + } + ] + } + } + ], + "passed_case_count": 1, + "failed_case_count": 2, + "not_evaluated_case_count": 0, + "average_score": { + "status": "available", + "value": 0.3333333333333333, + "unit": null, + "reason": null + } + }, + "baseline_validation": { + "phase": "baseline", + "split": "validation", + "eval_set_id": "eval_optimize_loop_validation", + "cases": [ + { + "eval_id": "val_knowledge_recall", + "status": "failed", + "average_score": { + "status": "available", + "value": 0.0, + "unit": null, + "reason": null + }, + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "status": "failed", + "score": { + "status": "available", + "value": 0.0, + "unit": null, + "reason": null + }, + "reason": null + } + ], + "runs": [ + { + "run_id": 1, + "status": "failed", + "error_message": null, + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "status": "failed", + "score": { + "status": "available", + "value": 0.0, + "unit": null, + "reason": null + }, + "reason": null + } + ], + "invocations": [ + { + "invocation_id": "val-2", + "user_text": "How long does standard shipping usually take?", + "expected_response": "{\"route\":\"shipping_policy\",\"message\":\"Standard shipping normally takes 3-5 business days.\"}", + "actual_response": "{\"route\":\"general_support\",\"message\":\"Please provide more details so I can route your request.\"}", + "expected_tools": [], + "actual_tools": [], + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "status": "failed", + "score": { + "status": "available", + "value": 0.0, + "unit": null, + "reason": null + }, + "reason": null + } + ] + } + ] + } + ], + "attribution": { + "primary_category": "routing_error", + "secondary_categories": [ + "final_response_mismatch" + ], + "summary": "Expected route 'shipping_policy', got 'general_support'.", + "evidence": [ + { + "evidence_type": "response", + "message": "Expected route 'shipping_policy', got 'general_support'.", + "run_id": 1, + "invocation_id": "val-2", + "metric_name": null, + "expected": "shipping_policy", + "actual": "general_support" + }, + { + "evidence_type": "response", + "message": "Final response did not satisfy metric 'final_response_avg_score'.", + "run_id": 1, + "invocation_id": "val-2", + "metric_name": "final_response_avg_score", + "expected": "{\"route\":\"shipping_policy\",\"message\":\"Standard shipping normally takes 3-5 business days.\"}", + "actual": "{\"route\":\"general_support\",\"message\":\"Please provide more details so I can route your request.\"}" + } + ] + } + }, + { + "eval_id": "val_paraphrase", + "status": "failed", + "average_score": { + "status": "available", + "value": 0.0, + "unit": null, + "reason": null + }, + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "status": "failed", + "score": { + "status": "available", + "value": 0.0, + "unit": null, + "reason": null + }, + "reason": null + } + ], + "runs": [ + { + "run_id": 1, + "status": "failed", + "error_message": null, + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "status": "failed", + "score": { + "status": "available", + "value": 0.0, + "unit": null, + "reason": null + }, + "reason": null + } + ], + "invocations": [ + { + "invocation_id": "val-1", + "user_text": "Where do I change the address tied to my account?", + "expected_response": "{\"route\":\"account\",\"message\":\"Open profile settings to update your address.\"}", + "actual_response": "{\"route\":\"general_support\",\"message\":\"Please provide more details so I can route your request.\"}", + "expected_tools": [], + "actual_tools": [], + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "status": "failed", + "score": { + "status": "available", + "value": 0.0, + "unit": null, + "reason": null + }, + "reason": null + } + ] + } + ] + } + ], + "attribution": { + "primary_category": "routing_error", + "secondary_categories": [ + "final_response_mismatch" + ], + "summary": "Expected route 'account', got 'general_support'.", + "evidence": [ + { + "evidence_type": "response", + "message": "Expected route 'account', got 'general_support'.", + "run_id": 1, + "invocation_id": "val-1", + "metric_name": null, + "expected": "account", + "actual": "general_support" + }, + { + "evidence_type": "response", + "message": "Final response did not satisfy metric 'final_response_avg_score'.", + "run_id": 1, + "invocation_id": "val-1", + "metric_name": "final_response_avg_score", + "expected": "{\"route\":\"account\",\"message\":\"Open profile settings to update your address.\"}", + "actual": "{\"route\":\"general_support\",\"message\":\"Please provide more details so I can route your request.\"}" + } + ] + } + }, + { + "eval_id": "val_refund_route", + "status": "passed", + "average_score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "status": "passed", + "score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "reason": null + } + ], + "runs": [ + { + "run_id": 1, + "status": "passed", + "error_message": null, + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "status": "passed", + "score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "reason": null + } + ], + "invocations": [ + { + "invocation_id": "val-3", + "user_text": "I was charged twice and need the duplicate payment refunded.", + "expected_response": "{\"route\":\"billing_refund\",\"message\":\"I will route this duplicate charge for refund review.\"}", + "actual_response": "{\"route\":\"billing_refund\",\"message\":\"I will route this duplicate charge for refund review.\"}", + "expected_tools": [], + "actual_tools": [], + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "status": "passed", + "score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "reason": null + } + ] + } + ] + } + ], + "attribution": null + } + ], + "passed_case_count": 1, + "failed_case_count": 2, + "not_evaluated_case_count": 0, + "average_score": { + "status": "available", + "value": 0.3333333333333333, + "unit": null, + "reason": null + } + }, + "candidate_train": { + "phase": "candidate", + "split": "train", + "eval_set_id": "eval_optimize_loop_train", + "cases": [ + { + "eval_id": "train_output_format", + "status": "passed", + "average_score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "status": "passed", + "score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "reason": null + } + ], + "runs": [ + { + "run_id": 1, + "status": "passed", + "error_message": null, + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "status": "passed", + "score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "reason": null + } + ], + "invocations": [ + { + "invocation_id": "train-1", + "user_text": "How can I update my email address?", + "expected_response": "{\"route\":\"account\",\"message\":\"Open profile settings to update your email.\"}", + "actual_response": "{\"route\":\"account\",\"message\":\"Open profile settings to update your email.\"}", + "expected_tools": [], + "actual_tools": [], + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "status": "passed", + "score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "reason": null + } + ] + } + ] + } + ], + "attribution": null + }, + { + "eval_id": "train_tool_arguments", + "status": "passed", + "average_score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "status": "passed", + "score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "reason": null + } + ], + "runs": [ + { + "run_id": 1, + "status": "passed", + "error_message": null, + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "status": "passed", + "score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "reason": null + } + ], + "invocations": [ + { + "invocation_id": "train-3", + "user_text": "Look up order B-204 for customer 17.", + "expected_response": "{\"route\":\"order_lookup\",\"message\":\"Checking order B-204 for customer 17.\"}", + "actual_response": "{\"route\":\"order_lookup\",\"message\":\"Checking order B-204 for customer 17.\"}", + "expected_tools": [], + "actual_tools": [], + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "status": "passed", + "score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "reason": null + } + ] + } + ] + } + ], + "attribution": null + }, + { + "eval_id": "train_tool_choice", + "status": "passed", + "average_score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "status": "passed", + "score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "reason": null + } + ], + "runs": [ + { + "run_id": 1, + "status": "passed", + "error_message": null, + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "status": "passed", + "score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "reason": null + } + ], + "invocations": [ + { + "invocation_id": "train-2", + "user_text": "Check the status of order A100.", + "expected_response": "{\"route\":\"order_lookup\",\"message\":\"Checking order A100.\"}", + "actual_response": "{\"route\":\"order_lookup\",\"message\":\"Checking order A100.\"}", + "expected_tools": [], + "actual_tools": [], + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "status": "passed", + "score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "reason": null + } + ] + } + ] + } + ], + "attribution": null + } + ], + "passed_case_count": 3, + "failed_case_count": 0, + "not_evaluated_case_count": 0, + "average_score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + } + }, + "candidate_validation": { + "phase": "candidate", + "split": "validation", + "eval_set_id": "eval_optimize_loop_validation", + "cases": [ + { + "eval_id": "val_knowledge_recall", + "status": "passed", + "average_score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "status": "passed", + "score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "reason": null + } + ], + "runs": [ + { + "run_id": 1, + "status": "passed", + "error_message": null, + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "status": "passed", + "score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "reason": null + } + ], + "invocations": [ + { + "invocation_id": "val-2", + "user_text": "How long does standard shipping usually take?", + "expected_response": "{\"route\":\"shipping_policy\",\"message\":\"Standard shipping normally takes 3-5 business days.\"}", + "actual_response": "{\"route\":\"shipping_policy\",\"message\":\"Standard shipping normally takes 3-5 business days.\"}", + "expected_tools": [], + "actual_tools": [], + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "status": "passed", + "score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "reason": null + } + ] + } + ] + } + ], + "attribution": null + }, + { + "eval_id": "val_paraphrase", + "status": "passed", + "average_score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "status": "passed", + "score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "reason": null + } + ], + "runs": [ + { + "run_id": 1, + "status": "passed", + "error_message": null, + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "status": "passed", + "score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "reason": null + } + ], + "invocations": [ + { + "invocation_id": "val-1", + "user_text": "Where do I change the address tied to my account?", + "expected_response": "{\"route\":\"account\",\"message\":\"Open profile settings to update your address.\"}", + "actual_response": "{\"route\":\"account\",\"message\":\"Open profile settings to update your address.\"}", + "expected_tools": [], + "actual_tools": [], + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "status": "passed", + "score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "reason": null + } + ] + } + ] + } + ], + "attribution": null + }, + { + "eval_id": "val_refund_route", + "status": "passed", + "average_score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "status": "passed", + "score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "reason": null + } + ], + "runs": [ + { + "run_id": 1, + "status": "passed", + "error_message": null, + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "status": "passed", + "score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "reason": null + } + ], + "invocations": [ + { + "invocation_id": "val-3", + "user_text": "I was charged twice and need the duplicate payment refunded.", + "expected_response": "{\"route\":\"billing_refund\",\"message\":\"I will route this duplicate charge for refund review.\"}", + "actual_response": "{\"route\":\"billing_refund\",\"message\":\"I will route this duplicate charge for refund review.\"}", + "expected_tools": [], + "actual_tools": [], + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "status": "passed", + "score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "reason": null + } + ] + } + ] + } + ], + "attribution": null + } + ], + "passed_case_count": 3, + "failed_case_count": 0, + "not_evaluated_case_count": 0, + "average_score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + } + }, + "train_diff": { + "split": "train", + "eval_set_id": "eval_optimize_loop_train", + "cases": [ + { + "eval_id": "train_output_format", + "split": "train", + "baseline_status": "passed", + "candidate_status": "passed", + "baseline_score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "candidate_score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "score_delta": { + "status": "available", + "value": 0.0, + "unit": null, + "reason": null + }, + "change": "unchanged", + "metrics": [ + { + "metric_name": "final_response_avg_score", + "baseline_status": "passed", + "candidate_status": "passed", + "baseline_score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "candidate_score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "score_delta": { + "status": "available", + "value": 0.0, + "unit": null, + "reason": null + }, + "change": "unchanged" + } + ], + "baseline_attribution": null, + "candidate_attribution": null, + "is_hard": false, + "is_critical": false, + "severe_regression": false + }, + { + "eval_id": "train_tool_arguments", + "split": "train", + "baseline_status": "failed", + "candidate_status": "passed", + "baseline_score": { + "status": "available", + "value": 0.0, + "unit": null, + "reason": null + }, + "candidate_score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "score_delta": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "change": "newly_passed", + "metrics": [ + { + "metric_name": "final_response_avg_score", + "baseline_status": "failed", + "candidate_status": "passed", + "baseline_score": { + "status": "available", + "value": 0.0, + "unit": null, + "reason": null + }, + "candidate_score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "score_delta": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "change": "newly_passed" + } + ], + "baseline_attribution": { + "primary_category": "routing_error", + "secondary_categories": [ + "final_response_mismatch" + ], + "summary": "Expected route 'order_lookup', got 'general_support'.", + "evidence": [ + { + "evidence_type": "response", + "message": "Expected route 'order_lookup', got 'general_support'.", + "run_id": 1, + "invocation_id": "train-3", + "metric_name": null, + "expected": "order_lookup", + "actual": "general_support" + }, + { + "evidence_type": "response", + "message": "Final response did not satisfy metric 'final_response_avg_score'.", + "run_id": 1, + "invocation_id": "train-3", + "metric_name": "final_response_avg_score", + "expected": "{\"route\":\"order_lookup\",\"message\":\"Checking order B-204 for customer 17.\"}", + "actual": "{\"route\":\"general_support\",\"message\":\"Please provide more details so I can route your request.\"}" + } + ] + }, + "candidate_attribution": null, + "is_hard": false, + "is_critical": false, + "severe_regression": false + }, + { + "eval_id": "train_tool_choice", + "split": "train", + "baseline_status": "failed", + "candidate_status": "passed", + "baseline_score": { + "status": "available", + "value": 0.0, + "unit": null, + "reason": null + }, + "candidate_score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "score_delta": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "change": "newly_passed", + "metrics": [ + { + "metric_name": "final_response_avg_score", + "baseline_status": "failed", + "candidate_status": "passed", + "baseline_score": { + "status": "available", + "value": 0.0, + "unit": null, + "reason": null + }, + "candidate_score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "score_delta": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "change": "newly_passed" + } + ], + "baseline_attribution": { + "primary_category": "routing_error", + "secondary_categories": [ + "final_response_mismatch" + ], + "summary": "Expected route 'order_lookup', got 'general_support'.", + "evidence": [ + { + "evidence_type": "response", + "message": "Expected route 'order_lookup', got 'general_support'.", + "run_id": 1, + "invocation_id": "train-2", + "metric_name": null, + "expected": "order_lookup", + "actual": "general_support" + }, + { + "evidence_type": "response", + "message": "Final response did not satisfy metric 'final_response_avg_score'.", + "run_id": 1, + "invocation_id": "train-2", + "metric_name": "final_response_avg_score", + "expected": "{\"route\":\"order_lookup\",\"message\":\"Checking order A100.\"}", + "actual": "{\"route\":\"general_support\",\"message\":\"Please provide more details so I can route your request.\"}" + } + ] + }, + "candidate_attribution": null, + "is_hard": false, + "is_critical": false, + "severe_regression": false + } + ], + "baseline_average_score": { + "status": "available", + "value": 0.3333333333333333, + "unit": null, + "reason": null + }, + "candidate_average_score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "score_delta": { + "status": "available", + "value": 0.6666666666666667, + "unit": null, + "reason": null + }, + "newly_passed_count": 2, + "newly_failed_count": 0, + "improved_count": 0, + "regressed_count": 0, + "unchanged_count": 1, + "incomparable_count": 0 + }, + "validation_diff": { + "split": "validation", + "eval_set_id": "eval_optimize_loop_validation", + "cases": [ + { + "eval_id": "val_knowledge_recall", + "split": "validation", + "baseline_status": "failed", + "candidate_status": "passed", + "baseline_score": { + "status": "available", + "value": 0.0, + "unit": null, + "reason": null + }, + "candidate_score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "score_delta": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "change": "newly_passed", + "metrics": [ + { + "metric_name": "final_response_avg_score", + "baseline_status": "failed", + "candidate_status": "passed", + "baseline_score": { + "status": "available", + "value": 0.0, + "unit": null, + "reason": null + }, + "candidate_score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "score_delta": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "change": "newly_passed" + } + ], + "baseline_attribution": { + "primary_category": "routing_error", + "secondary_categories": [ + "final_response_mismatch" + ], + "summary": "Expected route 'shipping_policy', got 'general_support'.", + "evidence": [ + { + "evidence_type": "response", + "message": "Expected route 'shipping_policy', got 'general_support'.", + "run_id": 1, + "invocation_id": "val-2", + "metric_name": null, + "expected": "shipping_policy", + "actual": "general_support" + }, + { + "evidence_type": "response", + "message": "Final response did not satisfy metric 'final_response_avg_score'.", + "run_id": 1, + "invocation_id": "val-2", + "metric_name": "final_response_avg_score", + "expected": "{\"route\":\"shipping_policy\",\"message\":\"Standard shipping normally takes 3-5 business days.\"}", + "actual": "{\"route\":\"general_support\",\"message\":\"Please provide more details so I can route your request.\"}" + } + ] + }, + "candidate_attribution": null, + "is_hard": true, + "is_critical": false, + "severe_regression": false + }, + { + "eval_id": "val_paraphrase", + "split": "validation", + "baseline_status": "failed", + "candidate_status": "passed", + "baseline_score": { + "status": "available", + "value": 0.0, + "unit": null, + "reason": null + }, + "candidate_score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "score_delta": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "change": "newly_passed", + "metrics": [ + { + "metric_name": "final_response_avg_score", + "baseline_status": "failed", + "candidate_status": "passed", + "baseline_score": { + "status": "available", + "value": 0.0, + "unit": null, + "reason": null + }, + "candidate_score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "score_delta": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "change": "newly_passed" + } + ], + "baseline_attribution": { + "primary_category": "routing_error", + "secondary_categories": [ + "final_response_mismatch" + ], + "summary": "Expected route 'account', got 'general_support'.", + "evidence": [ + { + "evidence_type": "response", + "message": "Expected route 'account', got 'general_support'.", + "run_id": 1, + "invocation_id": "val-1", + "metric_name": null, + "expected": "account", + "actual": "general_support" + }, + { + "evidence_type": "response", + "message": "Final response did not satisfy metric 'final_response_avg_score'.", + "run_id": 1, + "invocation_id": "val-1", + "metric_name": "final_response_avg_score", + "expected": "{\"route\":\"account\",\"message\":\"Open profile settings to update your address.\"}", + "actual": "{\"route\":\"general_support\",\"message\":\"Please provide more details so I can route your request.\"}" + } + ] + }, + "candidate_attribution": null, + "is_hard": false, + "is_critical": false, + "severe_regression": false + }, + { + "eval_id": "val_refund_route", + "split": "validation", + "baseline_status": "passed", + "candidate_status": "passed", + "baseline_score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "candidate_score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "score_delta": { + "status": "available", + "value": 0.0, + "unit": null, + "reason": null + }, + "change": "unchanged", + "metrics": [ + { + "metric_name": "final_response_avg_score", + "baseline_status": "passed", + "candidate_status": "passed", + "baseline_score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "candidate_score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "score_delta": { + "status": "available", + "value": 0.0, + "unit": null, + "reason": null + }, + "change": "unchanged" + } + ], + "baseline_attribution": null, + "candidate_attribution": null, + "is_hard": false, + "is_critical": true, + "severe_regression": false + } + ], + "baseline_average_score": { + "status": "available", + "value": 0.3333333333333333, + "unit": null, + "reason": null + }, + "candidate_average_score": { + "status": "available", + "value": 1.0, + "unit": null, + "reason": null + }, + "score_delta": { + "status": "available", + "value": 0.6666666666666667, + "unit": null, + "reason": null + }, + "newly_passed_count": 2, + "newly_failed_count": 0, + "improved_count": 0, + "regressed_count": 0, + "unchanged_count": 1, + "incomparable_count": 0 + }, + "overfit_status": "not_detected", + "overfit_reason": "Train score delta is 0.666667; validation score delta is 0.666667." + }, + "pipeline_resources": { + "cost_usd": { + "status": "unavailable", + "value": null, + "unit": "USD", + "reason": "Offline deterministic model does not report monetary cost." + }, + "total_tokens": { + "status": "unavailable", + "value": null, + "unit": "tokens", + "reason": "Offline deterministic model does not report token usage." + }, + "duration_seconds": { + "status": "available", + "value": 0.03433158100233413, + "unit": "seconds", + "reason": null + } + }, + "optimizer_resources": { + "scope_note": "Offline mode uses a deterministic candidate provider.", + "total_rounds": { + "status": "not_applicable", + "value": null, + "unit": "rounds", + "reason": "Offline mode uses a deterministic candidate provider." + }, + "reflection_lm_calls": { + "status": "not_applicable", + "value": null, + "unit": "calls", + "reason": "Offline mode uses a deterministic candidate provider." + }, + "cost_usd": { + "status": "not_applicable", + "value": null, + "unit": "USD", + "reason": "Offline mode uses a deterministic candidate provider." + }, + "token_usage": { + "status": "not_applicable", + "value": null, + "unit": "tokens", + "reason": "Offline mode uses a deterministic candidate provider." + }, + "duration_seconds": { + "status": "not_applicable", + "value": null, + "unit": "seconds", + "reason": "Offline mode uses a deterministic candidate provider." + } + }, + "gate_decision": { + "decision": "accept", + "rule_results": [ + { + "rule_id": "evaluation_completeness", + "outcome": "pass", + "message": "All four evaluations contain complete case and metric results.", + "case_ids": [], + "metric_names": [], + "observed": {}, + "threshold": null + }, + { + "rule_id": "minimum_validation_score_delta", + "outcome": "pass", + "message": "Validation score improvement meets the configured minimum.", + "case_ids": [], + "metric_names": [], + "observed": { + "validation_score_delta": { + "status": "available", + "value": 0.6666666666666667, + "unit": null, + "reason": null + } + }, + "threshold": 0.05 + }, + { + "rule_id": "validation_pass_rate_non_decrease", + "outcome": "pass", + "message": "Validation pass rate did not decrease.", + "case_ids": [], + "metric_names": [], + "observed": { + "baseline_validation_pass_rate": { + "status": "available", + "value": 0.3333333333333333, + "unit": "ratio", + "reason": null + }, + "candidate_validation_pass_rate": { + "status": "available", + "value": 1.0, + "unit": "ratio", + "reason": null + } + }, + "threshold": null + }, + { + "rule_id": "no_new_hard_fail", + "outcome": "pass", + "message": "No new hard failures were found.", + "case_ids": [], + "metric_names": [], + "observed": {}, + "threshold": null + }, + { + "rule_id": "no_critical_regression", + "outcome": "pass", + "message": "No critical-case regressions were found.", + "case_ids": [], + "metric_names": [], + "observed": {}, + "threshold": null + }, + { + "rule_id": "no_severe_regression", + "outcome": "pass", + "message": "No severe case regressions were found.", + "case_ids": [], + "metric_names": [], + "observed": {}, + "threshold": null + }, + { + "rule_id": "required_metrics", + "outcome": "pass", + "message": "All required candidate metrics are available and passed.", + "case_ids": [], + "metric_names": [], + "observed": {}, + "threshold": null + }, + { + "rule_id": "no_overfitting", + "outcome": "pass", + "message": "No train-improvement/validation-regression pattern was detected.", + "case_ids": [], + "metric_names": [], + "observed": {}, + "threshold": null + }, + { + "rule_id": "cost_budget", + "outcome": "skipped", + "message": "cost_usd budget is not configured.", + "case_ids": [], + "metric_names": [], + "observed": {}, + "threshold": null + }, + { + "rule_id": "token_budget", + "outcome": "skipped", + "message": "total_tokens budget is not configured.", + "case_ids": [], + "metric_names": [], + "observed": {}, + "threshold": null + }, + { + "rule_id": "duration_budget", + "outcome": "pass", + "message": "duration_seconds is within the configured budget.", + "case_ids": [], + "metric_names": [], + "observed": { + "duration_seconds": { + "status": "available", + "value": 0.03433158100233413, + "unit": "seconds", + "reason": null + } + }, + "threshold": 180.0 + } + ], + "rejection_reasons": [], + "warnings": [] + }, + "writeback": { + "status": "skipped", + "reason": "disabled", + "attempted": false, + "changed_fields": [], + "source_hashes_before": {}, + "source_hashes_after": {}, + "error_message": null + } +} diff --git a/examples/optimization/eval_optimize_loop/sample_output/optimization_report.md b/examples/optimization/eval_optimize_loop/sample_output/optimization_report.md new file mode 100644 index 00000000..f5100362 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/sample_output/optimization_report.md @@ -0,0 +1,44 @@ +# Optimization Report + +- Run: `stage6_sample` +- Mode: `offline` +- Gate decision: ACCEPT +- Candidate: `fake-improve-60cc05b773e8` + +## Full Evaluations + +- Baseline train: 1/3 passed; average score=0.3333333333333333 +- Baseline validation: 1/3 passed; average score=0.3333333333333333 +- Candidate train: 3/3 passed; average score=1.0 +- Candidate validation: 3/3 passed; average score=1.0 + +## Gate + +- No rejection reasons or warnings. + +## Candidate Changes + +- system_prompt + +## Overfit +- Status: not_detected +- Reason: Train score delta is 0.666667; validation score delta is 0.666667. + +## Writeback +- Status: skipped +- Reason: disabled + +## Pipeline Observations +- Cost: unavailable +- Tokens: unavailable +- Duration: available + +## Optimizer Resources +- Rounds: not_applicable; unit=rounds; reason=Offline mode uses a deterministic candidate provider. +- Reflection calls: not_applicable; unit=calls; reason=Offline mode uses a deterministic candidate provider. +- Cost: not_applicable; unit=USD; reason=Offline mode uses a deterministic candidate provider. +- Token usage: not_applicable; unit=tokens; reason=Offline mode uses a deterministic candidate provider. +- Duration: not_applicable; unit=seconds; reason=Offline mode uses a deterministic candidate provider. + +## Optimizer Scope +- Offline mode uses a deterministic candidate provider. diff --git a/examples/optimization/eval_optimize_loop/sample_output/prompts/baseline/000-system_prompt.md b/examples/optimization/eval_optimize_loop/sample_output/prompts/baseline/000-system_prompt.md new file mode 100644 index 00000000..345c342d --- /dev/null +++ b/examples/optimization/eval_optimize_loop/sample_output/prompts/baseline/000-system_prompt.md @@ -0,0 +1,5 @@ +You are a customer-support routing assistant. + +Answer with a compact JSON object containing `route` and `message`. Use the +available account tool for account-specific requests. Never invent account +facts. diff --git a/examples/optimization/eval_optimize_loop/sample_output/prompts/candidate/000-system_prompt.md b/examples/optimization/eval_optimize_loop/sample_output/prompts/candidate/000-system_prompt.md new file mode 100644 index 00000000..42ad8d8f --- /dev/null +++ b/examples/optimization/eval_optimize_loop/sample_output/prompts/candidate/000-system_prompt.md @@ -0,0 +1,13 @@ +You are a customer-support routing assistant. + +Answer with a compact JSON object containing `route` and `message`. Use the +available account tool for account-specific requests. Never invent account +facts. + + +Apply general customer-support routing rules across equivalent user phrasings. + + + + + diff --git a/examples/optimization/eval_optimize_loop/schemas.py b/examples/optimization/eval_optimize_loop/schemas.py index cb1d955c..5698fe5b 100644 --- a/examples/optimization/eval_optimize_loop/schemas.py +++ b/examples/optimization/eval_optimize_loop/schemas.py @@ -29,7 +29,7 @@ from trpc_agent_sdk.evaluation import OptimizeResult -FakeCandidateScenario = Literal["improve", "no_improvement", "overfit"] +CandidateScenario = Literal["improve", "no_improvement", "overfit"] EvaluationStatus = Literal["passed", "failed", "not_evaluated"] FailureCategory = Literal[ "evaluation_error", @@ -74,6 +74,7 @@ "write_error", "readback_mismatch", "written", + "trace_replay", ] @@ -123,6 +124,27 @@ class PromptSnapshot(EvalBaseModel): sha256: str = Field(pattern=r"^[0-9a-f]{64}$") +class TracePromptSnapshot(EvalBaseModel): + """Trace 候选随附的只读 Prompt 快照。""" + + field_name: str + path: str + content: str + sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + + +class TraceScenarioInputSnapshot(EvalBaseModel): + train_evalset_path: str + train_evalset_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + validation_evalset_path: str + validation_evalset_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + prompt_snapshots: list[TracePromptSnapshot] + + +class TraceInputSnapshot(EvalBaseModel): + scenarios: dict[CandidateScenario, TraceScenarioInputSnapshot] + + class InputSnapshot(EvalBaseModel): """Immutable file identities captured before a pipeline run starts.""" @@ -136,6 +158,7 @@ class InputSnapshot(EvalBaseModel): validation_evalset_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") prompt_snapshots: list[PromptSnapshot] seed: int + trace_inputs: Optional[TraceInputSnapshot] = None class WorkspaceSnapshot(EvalBaseModel): @@ -150,7 +173,7 @@ class WorkspaceSnapshot(EvalBaseModel): class CandidateProposal(EvalBaseModel): """Common, serializable identity and prompt payload for any provider.""" - provider: Literal["fake", "agent_optimizer"] + provider: Literal["fake", "agent_optimizer", "trace"] prompts: dict[str, str] changed_fields: list[str] rationale: str @@ -163,7 +186,7 @@ class FakeCandidateProposal(CandidateProposal): """One deterministic prompt proposal produced without a real optimizer.""" provider: Literal["fake"] = "fake" - scenario: FakeCandidateScenario + scenario: CandidateScenario seed: int candidate_id: str = Field(pattern=r"^fake-(improve|no_improvement|overfit)-[0-9a-f]{12}$") @@ -181,8 +204,17 @@ class OptimizerCandidateProposal(CandidateProposal): candidate_id: str = Field(pattern=r"^optimizer-[0-9a-f]{12}$") -class FakeEvaluationSnapshot(EvalBaseModel): - """Complete SDK outputs from one fake-agent dataset evaluation.""" +class TraceCandidateProposal(CandidateProposal): + """由预录制轨迹和 Prompt 快照标识的候选版本。""" + + provider: Literal["trace"] = "trace" + scenario: CandidateScenario + source_trace_sha256: dict[Literal["train", "validation"], str] + candidate_id: str = Field(pattern=r"^trace-(improve|no_improvement|overfit)-[0-9a-f]{12}$") + + +class EvaluationSnapshot(EvalBaseModel): + """Complete SDK outputs from one evaluation split.""" phase: Literal["baseline", "candidate"] split: Literal["train", "validation"] @@ -398,20 +430,20 @@ class WritebackResult(EvalBaseModel): class PipelineStageResult(EvalBaseModel): """Evaluation, analysis, Gate, and writeback fields shared by all modes.""" - baseline_train: FakeEvaluationSnapshot - baseline_validation: FakeEvaluationSnapshot - candidate_train: FakeEvaluationSnapshot - candidate_validation: FakeEvaluationSnapshot + baseline_train: EvaluationSnapshot + baseline_validation: EvaluationSnapshot + candidate_train: EvaluationSnapshot + candidate_validation: EvaluationSnapshot analysis: EvaluationAnalysis measurements: ResourceMeasurements gate_decision: GateDecision writeback: WritebackResult -class FakeStageResult(PipelineStageResult): - """Full deterministic fake-mode pipeline result.""" +class OfflineStageResult(PipelineStageResult): + """Full deterministic offline-mode pipeline result.""" - scenario: FakeCandidateScenario + scenario: CandidateScenario candidate: FakeCandidateProposal @@ -421,6 +453,13 @@ class RealStageResult(PipelineStageResult): candidate: OptimizerCandidateProposal optimize_result: OptimizeResult + +class TraceStageResult(PipelineStageResult): + """完整的 Trace 回放、分析与 Gate 结果。""" + + scenario: CandidateScenario + candidate: TraceCandidateProposal + ReportPhase = Literal[ "baseline_train", "baseline_validation", "candidate_generation", "candidate_train", "candidate_validation", "analysis", "gate", "writeback", "reporting", @@ -512,16 +551,16 @@ class OptimizationReport(EvalBaseModel): schema_version: Literal[1] = 1 status: Literal["completed"] = "completed" run_id: str - execution_mode: Literal["fake", "real"] + execution_mode: Literal["offline", "real", "trace"] seed: int started_at: datetime finished_at: datetime input_snapshot: InputSnapshot - candidate: Union[FakeCandidateProposal, OptimizerCandidateProposal] - baseline_train: FakeEvaluationSnapshot - baseline_validation: FakeEvaluationSnapshot - candidate_train: FakeEvaluationSnapshot - candidate_validation: FakeEvaluationSnapshot + candidate: Union[FakeCandidateProposal, OptimizerCandidateProposal, TraceCandidateProposal] + baseline_train: EvaluationSnapshot + baseline_validation: EvaluationSnapshot + candidate_train: EvaluationSnapshot + candidate_validation: EvaluationSnapshot analysis: EvaluationAnalysis pipeline_resources: ResourceMeasurements optimizer_resources: OptimizerResourceObservation @@ -532,7 +571,7 @@ class FailureReport(EvalBaseModel): schema_version: Literal[1] = 1 status: Literal["failed"] = "failed" run_id: str - execution_mode: Literal["fake", "real"] + execution_mode: Literal["offline", "real", "trace"] failed_phase: ReportPhase exception_type: str error_message: str diff --git a/tests/evaluation/test_eval_optimize_loop_real_integration.py b/tests/evaluation/test_eval_optimize_loop_real_integration.py index fe258b0d..7f4c2662 100644 --- a/tests/evaluation/test_eval_optimize_loop_real_integration.py +++ b/tests/evaluation/test_eval_optimize_loop_real_integration.py @@ -147,12 +147,18 @@ async def run_async(self, **kwargs): yield FakeEvent(True, [FakePart("thinking", thought=True), FakePart("answer")]) monkeypatch.setattr("examples.optimization.eval_optimize_loop.real_agent.OpenAIModel", lambda **kwargs: object()) - monkeypatch.setattr("examples.optimization.eval_optimize_loop.real_agent.LlmAgent", FakeLlmAgent) monkeypatch.setattr( - "examples.optimization.eval_optimize_loop.real_agent.InMemorySessionService", + "examples.optimization.eval_optimize_loop.business_agent.LlmAgent", + FakeLlmAgent, + ) + monkeypatch.setattr( + "examples.optimization.eval_optimize_loop.business_agent.InMemorySessionService", FakeSessionService, ) - monkeypatch.setattr("examples.optimization.eval_optimize_loop.real_agent.Runner", FakeRunner) + monkeypatch.setattr( + "examples.optimization.eval_optimize_loop.business_agent.Runner", + FakeRunner, + ) agent = RealBusinessAgent( FakeTarget(), BusinessModelConfig(api_key="secret", base_url="https://example.test", model_name="model"), diff --git a/tests/evaluation/test_eval_optimize_loop_stage1.py b/tests/evaluation/test_eval_optimize_loop_stage1.py index 58adc76f..bd6f8df0 100644 --- a/tests/evaluation/test_eval_optimize_loop_stage1.py +++ b/tests/evaluation/test_eval_optimize_loop_stage1.py @@ -67,7 +67,7 @@ def test_pipeline_config_loads_complete_example_and_camel_case(tmp_path: Path): root = _copy_example(tmp_path) config = load_pipeline_config(root / "pipeline.json") assert config.config_version == 1 - assert config.execution.mode == "fake" + assert config.execution.mode == "offline" assert config.prompts[0].name == "system_prompt" assert config.gate.required_metrics == ["final_response_avg_score"] @@ -81,6 +81,26 @@ def test_pipeline_config_loads_complete_example_and_camel_case(tmp_path: Path): assert camel_config.case_labels.hard_case_ids == ["val_knowledge_recall"] +def test_pipeline_config_rejects_legacy_fake_mode_with_migration_message( + tmp_path: Path, +): + root = _copy_example(tmp_path) + payload = _read_config(root) + payload["execution"]["mode"] = "fake" + + with pytest.raises(ValidationError, match="renamed to 'offline'"): + PipelineConfig.model_validate(payload) + + +def test_pipeline_config_rejects_removed_fake_judge_option(tmp_path: Path): + root = _copy_example(tmp_path) + payload = _read_config(root) + payload["execution"]["use_fake_judge"] = True + + with pytest.raises(ValidationError, match="evaluation metrics or rubric"): + PipelineConfig.model_validate(payload) + + @pytest.mark.parametrize( "mutate", [ diff --git a/tests/evaluation/test_eval_optimize_loop_stage2.py b/tests/evaluation/test_eval_optimize_loop_stage2.py index 96d7719b..383f7a87 100644 --- a/tests/evaluation/test_eval_optimize_loop_stage2.py +++ b/tests/evaluation/test_eval_optimize_loop_stage2.py @@ -16,12 +16,13 @@ import pytest -from examples.optimization.eval_optimize_loop.fake import DeterministicFakeAgent +from examples.optimization.eval_optimize_loop.business_agent import BusinessAgent +from examples.optimization.eval_optimize_loop.fake import DeterministicFakeModel from examples.optimization.eval_optimize_loop.fake import DeterministicFakeCandidateProvider -from examples.optimization.eval_optimize_loop.pipeline import FakeStageExecutionError +from examples.optimization.eval_optimize_loop.pipeline import PipelineStageExecutionError from examples.optimization.eval_optimize_loop.pipeline import prepare_run -from examples.optimization.eval_optimize_loop.pipeline import run_fake_stage -from examples.optimization.eval_optimize_loop.schemas import FakeStageResult +from examples.optimization.eval_optimize_loop.pipeline import run_offline_stage +from examples.optimization.eval_optimize_loop.schemas import OfflineStageResult from trpc_agent_sdk.evaluation import EvalStatus from trpc_agent_sdk.evaluation import TargetPrompt @@ -30,6 +31,16 @@ _EXAMPLE = _REPO_ROOT / "examples" / "optimization" / "eval_optimize_loop" +def _offline_agent(target: TargetPrompt) -> BusinessAgent: + return BusinessAgent( + target, + DeterministicFakeModel, + agent_name="stage2_offline_agent", + app_name="stage2_offline_test", + user_id="stage2-test", + ) + + def _copy_example(tmp_path: Path, name: str = "eval_optimize_loop") -> Path: target = tmp_path / name shutil.copytree(_EXAMPLE, target, ignore=shutil.ignore_patterns("runs", "__pycache__")) @@ -101,7 +112,7 @@ def test_overfit_candidate_contains_rules_not_eval_ids_or_full_examples(): @pytest.mark.asyncio async def test_fake_agent_rereads_prompt_and_is_deterministic(tmp_path: Path): target = _target_for_prompt(tmp_path / "system.md", "baseline") - agent = DeterministicFakeAgent(target) + agent = _offline_agent(target) email_query = "How can I update my email address?" order_query = "Check the status of order A100." @@ -125,7 +136,7 @@ async def test_fake_agent_rereads_prompt_and_is_deterministic(tmp_path: Path): @pytest.mark.asyncio async def test_fake_agent_unknown_query_has_stable_fallback(tmp_path: Path): target = _target_for_prompt(tmp_path / "system.md", "baseline") - agent = DeterministicFakeAgent(target) + agent = _offline_agent(target) expected = ( '{"route":"general_support","message":' '"Please provide more details so I can route your request."}' @@ -156,7 +167,7 @@ async def test_fake_stage_scenario_matrix( baseline_source = source.read_text(encoding="utf-8") prepared = prepare_run(root / "pipeline.json", run_id=f"stage2_{scenario}") - result = await run_fake_stage(prepared, scenario=scenario) # type: ignore[arg-type] + result = await run_offline_stage(prepared, scenario=scenario) # type: ignore[arg-type] assert result.baseline_train.passed_case_count == 1 assert result.baseline_validation.passed_case_count == 1 @@ -174,7 +185,7 @@ async def test_fake_stage_scenario_matrix( if scenario == "improve": assert result.candidate_train.failed_summary is None assert result.candidate_validation.failed_summary is None - restored = FakeStageResult.model_validate_json(result.model_dump_json()) + restored = OfflineStageResult.model_validate_json(result.model_dump_json()) assert restored.candidate.candidate_id == result.candidate.candidate_id elif scenario == "overfit": baseline_refund = result.baseline_validation.eval_results_by_eval_id["val_refund_route"][0] @@ -192,7 +203,7 @@ async def test_fake_stage_keeps_all_configured_runs(tmp_path: Path): _write_json(optimizer_path, optimizer) prepared = prepare_run(root / "pipeline.json", run_id="two_runs") - result = await run_fake_stage(prepared, scenario="improve") + result = await run_offline_stage(prepared, scenario="improve") for snapshot in ( result.baseline_train, @@ -204,21 +215,8 @@ async def test_fake_stage_keeps_all_configured_runs(tmp_path: Path): @pytest.mark.asyncio -async def test_fake_stage_rejects_unimplemented_fake_judge(tmp_path: Path): - root = _copy_example(tmp_path) - config_path = root / "pipeline.json" - config = _read_json(config_path) - config["execution"]["use_fake_judge"] = True - _write_json(config_path, config) - prepared = prepare_run(config_path, run_id="fake_judge") - - with pytest.raises(FakeStageExecutionError, match="use_fake_judge=true"): - await run_fake_stage(prepared) - - -@pytest.mark.parametrize("mode", ["real", "trace"]) -@pytest.mark.asyncio -async def test_fake_stage_rejects_non_fake_execution_modes(tmp_path: Path, mode: str): +async def test_offline_stage_rejects_real_execution_mode(tmp_path: Path): + mode = "real" root = _copy_example(tmp_path, mode) config_path = root / "pipeline.json" config = _read_json(config_path) @@ -227,10 +225,10 @@ async def test_fake_stage_rejects_non_fake_execution_modes(tmp_path: Path, mode: prepared = prepare_run(config_path, run_id=f"mode_{mode}") with pytest.raises( - FakeStageExecutionError, - match=rf"requires execution.mode='fake', got '{mode}'", + PipelineStageExecutionError, + match=rf"requires execution.mode='offline', got '{mode}'", ): - await run_fake_stage(prepared) + await run_offline_stage(prepared) @pytest.mark.parametrize( @@ -254,10 +252,10 @@ async def test_fake_stage_rejects_evalset_changes_after_prepare_run( _write_json(evalset_path, evalset) with pytest.raises( - FakeStageExecutionError, + PipelineStageExecutionError, match=rf"{expected_label} changed after prepare_run", ): - await run_fake_stage(prepared) + await run_offline_stage(prepared) @pytest.mark.asyncio @@ -272,8 +270,8 @@ async def fail_evaluation(*_args, **_kwargs): "examples.optimization.eval_optimize_loop.pipeline.AgentEvaluator.evaluate_eval_set", fail_evaluation, ) - with pytest.raises(FakeStageExecutionError, match="baseline train evaluation failed"): - await run_fake_stage(prepared) + with pytest.raises(PipelineStageExecutionError, match="baseline train evaluation failed"): + await run_offline_stage(prepared) assert await prepared.working_target.read_all() == { snapshot.field_name: snapshot.content for snapshot in prepared.input_snapshot.prompt_snapshots } @@ -299,8 +297,8 @@ async def fail_candidate_train(*args, **kwargs): return await original(*args, **kwargs) monkeypatch.setattr(pipeline_module.AgentEvaluator, "evaluate_eval_set", fail_candidate_train) - with pytest.raises(FakeStageExecutionError, match="candidate train evaluation failed"): - await run_fake_stage(prepared, scenario="improve") + with pytest.raises(PipelineStageExecutionError, match="candidate train evaluation failed"): + await run_offline_stage(prepared, scenario="improve") working = await prepared.working_target.read_all() assert "deterministic-fake-candidate:start" in working["system_prompt"] diff --git a/tests/evaluation/test_eval_optimize_loop_stage3a.py b/tests/evaluation/test_eval_optimize_loop_stage3a.py index 22370bda..bad25f3b 100644 --- a/tests/evaluation/test_eval_optimize_loop_stage3a.py +++ b/tests/evaluation/test_eval_optimize_loop_stage3a.py @@ -18,9 +18,9 @@ from examples.optimization.eval_optimize_loop.evaluation_adapter import EvaluationAnalysisError from examples.optimization.eval_optimize_loop.evaluation_adapter import standardize_snapshot from examples.optimization.eval_optimize_loop.pipeline import prepare_run -from examples.optimization.eval_optimize_loop.pipeline import run_fake_stage -from examples.optimization.eval_optimize_loop.schemas import FakeEvaluationSnapshot -from examples.optimization.eval_optimize_loop.schemas import FakeStageResult +from examples.optimization.eval_optimize_loop.pipeline import run_offline_stage +from examples.optimization.eval_optimize_loop.schemas import EvaluationSnapshot +from examples.optimization.eval_optimize_loop.schemas import OfflineStageResult from trpc_agent_sdk.evaluation import EvalCaseResult from trpc_agent_sdk.evaluation import EvalMetricResult from trpc_agent_sdk.evaluation import EvalMetricResultDetails @@ -107,8 +107,8 @@ def _case_run( ) -def _snapshot(*runs: EvalCaseResult) -> FakeEvaluationSnapshot: - return FakeEvaluationSnapshot( +def _snapshot(*runs: EvalCaseResult) -> EvaluationSnapshot: + return EvaluationSnapshot( phase="baseline", split="train", eval_set_id="train-set", @@ -510,7 +510,7 @@ async def test_stage3a_scenario_analysis_matrix( root = _copy_example(tmp_path, scenario) prepared = prepare_run(root / "pipeline.json", run_id=f"stage3a_{scenario}") - result = await run_fake_stage(prepared, scenario=scenario) # type: ignore[arg-type] + result = await run_offline_stage(prepared, scenario=scenario) # type: ignore[arg-type] analysis = result.analysis assert analysis.train_diff.newly_passed_count == train_newly_passed @@ -529,7 +529,7 @@ async def test_stage3a_scenario_analysis_matrix( assert refund.candidate_attribution is not None assert refund.candidate_attribution.primary_category == "routing_error" - restored = FakeStageResult.model_validate_json(result.model_dump_json()) + restored = OfflineStageResult.model_validate_json(result.model_dump_json()) assert restored.analysis == analysis @@ -544,4 +544,4 @@ def fail_analysis(**_kwargs): monkeypatch.setattr(pipeline_module, "build_evaluation_analysis", fail_analysis) with pytest.raises(ValueError, match="injected programming error"): - await run_fake_stage(prepared, scenario="improve") + await run_offline_stage(prepared, scenario="improve") diff --git a/tests/evaluation/test_eval_optimize_loop_stage3b.py b/tests/evaluation/test_eval_optimize_loop_stage3b.py index c1516ab5..4a9a7223 100644 --- a/tests/evaluation/test_eval_optimize_loop_stage3b.py +++ b/tests/evaluation/test_eval_optimize_loop_stage3b.py @@ -20,11 +20,11 @@ from examples.optimization.eval_optimize_loop.config import BudgetConfig from examples.optimization.eval_optimize_loop.config import GateConfig from examples.optimization.eval_optimize_loop.pipeline import prepare_run -from examples.optimization.eval_optimize_loop.pipeline import FakeStageExecutionError -from examples.optimization.eval_optimize_loop.pipeline import run_fake_stage +from examples.optimization.eval_optimize_loop.pipeline import PipelineStageExecutionError +from examples.optimization.eval_optimize_loop.pipeline import run_offline_stage from examples.optimization.eval_optimize_loop.schemas import CaseEvaluation from examples.optimization.eval_optimize_loop.schemas import EvaluationAnalysis -from examples.optimization.eval_optimize_loop.schemas import FakeStageResult +from examples.optimization.eval_optimize_loop.schemas import OfflineStageResult from examples.optimization.eval_optimize_loop.schemas import MetricOutcome from examples.optimization.eval_optimize_loop.schemas import ObservableValue from examples.optimization.eval_optimize_loop.schemas import ResourceMeasurements @@ -506,7 +506,7 @@ async def test_fake_stage_returns_measurements_and_accepts_improving_candidate(t baseline_source = source.read_text(encoding="utf-8") prepared = prepare_run(root / "pipeline.json", run_id="stage3b_improve") - result = await run_fake_stage(prepared, scenario="improve") + result = await run_offline_stage(prepared, scenario="improve") assert result.gate_decision.decision == "accept" assert result.gate_decision.rejection_reasons == [] @@ -555,7 +555,7 @@ async def test_stage3b_scenario_gate_matrix( baseline_source = source.read_text(encoding="utf-8") prepared = prepare_run(root / "pipeline.json", run_id=f"stage3b_{scenario}") - result = await run_fake_stage(prepared, scenario=scenario) # type: ignore[arg-type] + result = await run_offline_stage(prepared, scenario=scenario) # type: ignore[arg-type] assert result.gate_decision.decision == expected_decision assert [ @@ -564,7 +564,7 @@ async def test_stage3b_scenario_gate_matrix( if rule.outcome == "reject" ] == expected_reject_rules assert source.read_text(encoding="utf-8") == baseline_source - assert FakeStageResult.model_validate_json(result.model_dump_json()) == result + assert OfflineStageResult.model_validate_json(result.model_dump_json()) == result @pytest.mark.asyncio @@ -579,8 +579,8 @@ def fail_gate(*_args, **_kwargs): monkeypatch.setattr(pipeline_module, "evaluate_gate", fail_gate) - with pytest.raises(FakeStageExecutionError, match="stage 3b gate failed"): - await run_fake_stage(prepared, scenario="improve") + with pytest.raises(PipelineStageExecutionError, match="stage 3b gate failed"): + await run_offline_stage(prepared, scenario="improve") @pytest.mark.asyncio @@ -594,7 +594,7 @@ def fail_gate(*_args, **_kwargs): monkeypatch.setattr(pipeline_module, "evaluate_gate", fail_gate) with pytest.raises(ValueError, match="injected programming error"): - await run_fake_stage(prepared, scenario="improve") + await run_offline_stage(prepared, scenario="improve") def test_stage3b_cli_prints_gate_decision_and_rejection_reasons(tmp_path: Path): diff --git a/tests/evaluation/test_eval_optimize_loop_stage4.py b/tests/evaluation/test_eval_optimize_loop_stage4.py index b746096f..281a4c03 100644 --- a/tests/evaluation/test_eval_optimize_loop_stage4.py +++ b/tests/evaluation/test_eval_optimize_loop_stage4.py @@ -21,22 +21,34 @@ from examples.optimization.eval_optimize_loop.candidate_provider import CandidateProviderError from examples.optimization.eval_optimize_loop.candidate_provider import CandidateRequest from examples.optimization.eval_optimize_loop.config import WritebackConfig -from examples.optimization.eval_optimize_loop.fake import DeterministicFakeAgent +from examples.optimization.eval_optimize_loop.business_agent import BusinessAgent +from examples.optimization.eval_optimize_loop.fake import DeterministicFakeModel from examples.optimization.eval_optimize_loop.fake import DeterministicFakeCandidateProvider from examples.optimization.eval_optimize_loop.pipeline import prepare_run -from examples.optimization.eval_optimize_loop.pipeline import FakeStageExecutionError -from examples.optimization.eval_optimize_loop.pipeline import run_fake_stage +from examples.optimization.eval_optimize_loop.pipeline import PipelineStageExecutionError +from examples.optimization.eval_optimize_loop.pipeline import run_offline_stage from examples.optimization.eval_optimize_loop.pipeline import run_real_stage from examples.optimization.eval_optimize_loop.schemas import GateDecision from examples.optimization.eval_optimize_loop.schemas import RealStageResult from examples.optimization.eval_optimize_loop.writeback import perform_writeback from trpc_agent_sdk.evaluation import OptimizeResult +from trpc_agent_sdk.evaluation import TargetPrompt _REPO_ROOT = Path(__file__).resolve().parents[2] _EXAMPLE = _REPO_ROOT / "examples" / "optimization" / "eval_optimize_loop" +def _offline_agent(target: TargetPrompt) -> BusinessAgent: + return BusinessAgent( + target, + DeterministicFakeModel, + agent_name="stage4_offline_agent", + app_name="stage4_offline_test", + user_id="stage4-test", + ) + + def _copy_example(tmp_path: Path, name: str = "eval_optimize_loop") -> Path: target = tmp_path / name shutil.copytree(_EXAMPLE, target, ignore=shutil.ignore_patterns("runs", "__pycache__")) @@ -174,7 +186,7 @@ async def fake_optimize(**kwargs): "examples.optimization.eval_optimize_loop.candidate_provider.AgentOptimizer.optimize", fake_optimize, ) - agent = DeterministicFakeAgent(prepared.working_target) + agent = _offline_agent(prepared.working_target) result = await run_real_stage(prepared, call_agent=agent.call_agent) @@ -210,7 +222,7 @@ async def test_real_stage_writes_accepted_candidate_after_hash_check( result = await run_real_stage( prepared, - call_agent=DeterministicFakeAgent(prepared.working_target).call_agent, + call_agent=_offline_agent(prepared.working_target).call_agent, ) assert result.gate_decision.decision == "accept" @@ -242,7 +254,7 @@ async def test_real_stage_blocks_writeback_when_source_drifted( result = await run_real_stage( prepared, - call_agent=DeterministicFakeAgent(prepared.working_target).call_agent, + call_agent=_offline_agent(prepared.working_target).call_agent, ) assert result.gate_decision.decision == "accept" @@ -275,7 +287,7 @@ async def fail_write(prompts: dict[str, str]) -> None: result = await run_real_stage( prepared, - call_agent=DeterministicFakeAgent(prepared.working_target).call_agent, + call_agent=_offline_agent(prepared.working_target).call_agent, ) assert result.writeback.status == "failed" @@ -294,7 +306,7 @@ async def test_fake_stage_uses_the_same_guarded_writeback_path(tmp_path: Path): path.write_text(json.dumps(payload), encoding="utf-8") prepared = prepare_run(root / "pipeline.json", run_id="stage4_fake_write") - result = await run_fake_stage(prepared, scenario="improve") + result = await run_offline_stage(prepared, scenario="improve") assert result.gate_decision.decision == "accept" assert result.writeback.status == "written" @@ -311,7 +323,7 @@ async def test_fake_stage_never_writes_a_gate_rejected_candidate(tmp_path: Path) prepared = prepare_run(root / "pipeline.json", run_id="stage4_fake_reject") source_before = await prepared.source_target.read_all() - result = await run_fake_stage(prepared, scenario="no_improvement") + result = await run_offline_stage(prepared, scenario="no_improvement") assert result.gate_decision.decision == "reject" assert result.writeback.status == "skipped" @@ -346,7 +358,7 @@ async def failed_optimize(**kwargs): ) with pytest.raises(CandidateProviderError, match="returned FAILED"): - await AgentOptimizerCandidateProvider(DeterministicFakeAgent(prepared.working_target).call_agent).propose( + await AgentOptimizerCandidateProvider(_offline_agent(prepared.working_target).call_agent).propose( request ) @@ -376,10 +388,10 @@ async def should_not_optimize(**kwargs): should_not_optimize, ) - with pytest.raises(FakeStageExecutionError, match="optimizer_config changed"): + with pytest.raises(PipelineStageExecutionError, match="optimizer_config changed"): await run_real_stage( prepared, - call_agent=DeterministicFakeAgent(prepared.working_target).call_agent, + call_agent=_offline_agent(prepared.working_target).call_agent, ) assert called is False @@ -449,10 +461,10 @@ async def mutating_failure(**kwargs): mutating_failure, ) - with pytest.raises(FakeStageExecutionError, match="real candidate generation failed"): + with pytest.raises(PipelineStageExecutionError, match="real candidate generation failed"): await run_real_stage( prepared, - call_agent=DeterministicFakeAgent(prepared.working_target).call_agent, + call_agent=_offline_agent(prepared.working_target).call_agent, ) assert await prepared.working_target.read_all() == baseline @@ -482,7 +494,7 @@ async def test_real_candidate_provider_can_discard_native_artifacts_after_extrac ) generated = await AgentOptimizerCandidateProvider( - DeterministicFakeAgent(prepared.working_target).call_agent + _offline_agent(prepared.working_target).call_agent ).propose(request) assert generated.optimize_result is not None @@ -582,5 +594,5 @@ def fail_cleanup(path: Path, **kwargs) -> None: with pytest.raises(CandidateProviderError, match="discard optimizer artifacts"): await AgentOptimizerCandidateProvider( - DeterministicFakeAgent(prepared.working_target).call_agent + _offline_agent(prepared.working_target).call_agent ).propose(request) diff --git a/tests/evaluation/test_eval_optimize_loop_stage5_artifacts.py b/tests/evaluation/test_eval_optimize_loop_stage5_artifacts.py index 2cff48b1..beb26990 100644 --- a/tests/evaluation/test_eval_optimize_loop_stage5_artifacts.py +++ b/tests/evaluation/test_eval_optimize_loop_stage5_artifacts.py @@ -39,7 +39,7 @@ async def _build_fake_report_for_root(root: Path, run_id: str): execution_progress = pipeline_module._MutableReportProgress( started_at=datetime(2026, 7, 18, tzinfo=timezone.utc) ) - result = await pipeline_module._execute_fake_stage( + result = await pipeline_module._execute_offline_stage( prepared, scenario="improve", progress=execution_progress, diff --git a/tests/evaluation/test_eval_optimize_loop_stage5_pipeline.py b/tests/evaluation/test_eval_optimize_loop_stage5_pipeline.py index f913cc3a..121c41df 100644 --- a/tests/evaluation/test_eval_optimize_loop_stage5_pipeline.py +++ b/tests/evaluation/test_eval_optimize_loop_stage5_pipeline.py @@ -19,23 +19,35 @@ from examples.optimization.eval_optimize_loop import pipeline as pipeline_module from examples.optimization.eval_optimize_loop import run_real_integration -from examples.optimization.eval_optimize_loop.fake import DeterministicFakeAgent +from examples.optimization.eval_optimize_loop.business_agent import BusinessAgent +from examples.optimization.eval_optimize_loop.fake import DeterministicFakeModel from examples.optimization.eval_optimize_loop.fake import DeterministicFakeCandidateProvider from examples.optimization.eval_optimize_loop.pipeline import PipelineExecutionError from examples.optimization.eval_optimize_loop.pipeline import prepare_run -from examples.optimization.eval_optimize_loop.pipeline import run_fake_stage +from examples.optimization.eval_optimize_loop.pipeline import run_offline_stage from examples.optimization.eval_optimize_loop.pipeline import run_real_stage from examples.optimization.eval_optimize_loop.schemas import ArtifactIndex from examples.optimization.eval_optimize_loop.schemas import FailureReport from examples.optimization.eval_optimize_loop.schemas import OptimizationReport from examples.optimization.eval_optimize_loop.schemas import OptimizerRuntimeParameters from trpc_agent_sdk.evaluation import OptimizeResult +from trpc_agent_sdk.evaluation import TargetPrompt _REPO_ROOT = Path(__file__).resolve().parents[2] _EXAMPLE = _REPO_ROOT / "examples" / "optimization" / "eval_optimize_loop" +def _offline_agent(target: TargetPrompt) -> BusinessAgent: + return BusinessAgent( + target, + DeterministicFakeModel, + agent_name="stage5_offline_agent", + app_name="stage5_offline_test", + user_id="stage5-test", + ) + + def _copy_example(tmp_path: Path, name: str = "eval_optimize_loop") -> Path: target = tmp_path / name shutil.copytree( @@ -87,7 +99,7 @@ async def test_fake_stage_publishes_complete_report_bundle( root = _copy_example(tmp_path) prepared = prepare_run(root / "pipeline.json", run_id=f"stage5_{scenario}") - result = await run_fake_stage(prepared, scenario=scenario) + result = await run_offline_stage(prepared, scenario=scenario) run_dir = Path(prepared.workspace.run_dir) report_dir = run_dir / "report" @@ -131,7 +143,7 @@ async def fake_optimize(**kwargs: object) -> OptimizeResult: result = await run_real_stage( prepared, - call_agent=DeterministicFakeAgent(prepared.working_target).call_agent, + call_agent=_offline_agent(prepared.working_target).call_agent, optimizer_parameters=OptimizerRuntimeParameters( model_name="stage5-reflection-model", max_candidate_proposals=1, @@ -209,7 +221,7 @@ async def test_fake_stage_writes_phase_specific_failure_report( _install_stage_failure(monkeypatch, target) with pytest.raises(Exception, match="simulated stage failure"): - await run_fake_stage(prepared, scenario="improve") + await run_offline_stage(prepared, scenario="improve") run_dir = Path(prepared.workspace.run_dir) failure = FailureReport.model_validate_json( @@ -248,7 +260,7 @@ def fail_failure_report(*args: object, **kwargs: object) -> None: ) with pytest.raises(PipelineExecutionError) as raised: - await run_fake_stage(prepared, scenario="improve") + await run_offline_stage(prepared, scenario="improve") assert "simulated stage failure" in str(raised.value) assert "failed to write failure report" in str(raised.value) @@ -275,7 +287,7 @@ def fail_report(*args: object, **kwargs: object) -> None: monkeypatch.setattr(pipeline_module, "publish_report_bundle", fail_report) with pytest.raises(RuntimeError, match="simulated reporting failure"): - await run_fake_stage(prepared, scenario="improve") + await run_offline_stage(prepared, scenario="improve") assert await prepared.source_target.read_all() == source_before failure = FailureReport.model_validate_json( @@ -308,7 +320,7 @@ async def fail_rollback(*args: object, **kwargs: object) -> None: monkeypatch.setattr(pipeline_module, "_rollback_written_source", fail_rollback) with pytest.raises(PipelineExecutionError) as raised: - await run_fake_stage(prepared, scenario="improve") + await run_offline_stage(prepared, scenario="improve") assert "simulated reporting failure" in str(raised.value) assert "simulated source rollback failure" in str(raised.value) @@ -380,7 +392,7 @@ async def test_all_fake_scenarios_duration_is_under_three_minutes_with_reports( root = _copy_example(tmp_path, name=f"eval_optimize_loop_{scenario}") prepared = prepare_run(root / "pipeline.json", run_id=f"stage5_{scenario}") - await run_fake_stage(prepared, scenario=scenario) + await run_offline_stage(prepared, scenario=scenario) report_dir = Path(prepared.workspace.run_dir) / "report" assert (report_dir / "optimization_report.json").is_file() diff --git a/tests/evaluation/test_eval_optimize_loop_stage5_report_builder.py b/tests/evaluation/test_eval_optimize_loop_stage5_report_builder.py index 7ff42c82..4bc4badd 100644 --- a/tests/evaluation/test_eval_optimize_loop_stage5_report_builder.py +++ b/tests/evaluation/test_eval_optimize_loop_stage5_report_builder.py @@ -6,7 +6,7 @@ import pytest -from examples.optimization.eval_optimize_loop.pipeline import prepare_run, run_fake_stage +from examples.optimization.eval_optimize_loop.pipeline import prepare_run, run_offline_stage from examples.optimization.eval_optimize_loop.report_builder import ( build_failure_report, build_optimization_report, render_optimization_markdown, ) @@ -49,11 +49,11 @@ async def test_fake_report_is_serializable_and_marks_optimizer_resources_not_app root = _copy_example(tmp_path) prepared = prepare_run(root / "pipeline.json", run_id="report_fake") report = build_optimization_report( - prepared, await run_fake_stage(prepared, scenario="improve"), progress=_progress(), + prepared, await run_offline_stage(prepared, scenario="improve"), progress=_progress(), finished_at=datetime(2026, 7, 18, 0, 1, tzinfo=timezone.utc), ) assert report.status == "completed" - assert report.execution_mode == "fake" + assert report.execution_mode == "offline" optimizer_resources = report.optimizer_resources for observation in ( optimizer_resources.total_rounds, @@ -64,7 +64,9 @@ async def test_fake_report_is_serializable_and_marks_optimizer_resources_not_app ): assert observation.status == "not_applicable" assert observation.value is None - assert observation.reason == "Fake mode does not run AgentOptimizer." + assert observation.reason == ( + "Offline mode uses a deterministic candidate provider." + ) assert report.pipeline_resources.total_tokens.status == "unavailable" assert OptimizationReport.model_validate_json(report.model_dump_json()) == report @@ -73,7 +75,7 @@ async def test_fake_report_is_serializable_and_marks_optimizer_resources_not_app async def test_real_report_includes_optimizer_resource_observations(tmp_path): root = _copy_example(tmp_path) prepared = prepare_run(root / "pipeline.json", run_id="report_real") - fake_result = await run_fake_stage(prepared, scenario="improve") + fake_result = await run_offline_stage(prepared, scenario="improve") candidate = OptimizerCandidateProposal.model_validate({ **fake_result.candidate.model_dump(exclude={"scenario", "seed"}), "provider": "agent_optimizer", "optimizer_status": "SUCCEEDED", "finish_reason": "completed", @@ -105,7 +107,7 @@ async def test_real_report_includes_optimizer_resource_observations(tmp_path): async def test_real_report_marks_incomplete_cost_and_token_telemetry_unavailable(tmp_path): root = _copy_example(tmp_path) prepared = prepare_run(root / "pipeline.json", run_id="report_real_incomplete") - fake_result = await run_fake_stage(prepared, scenario="improve") + fake_result = await run_offline_stage(prepared, scenario="improve") candidate = OptimizerCandidateProposal.model_validate({ **fake_result.candidate.model_dump(exclude={"scenario", "seed"}), "provider": "agent_optimizer", "optimizer_status": "SUCCEEDED", "finish_reason": "completed", @@ -154,13 +156,13 @@ async def test_markdown_includes_accept_and_reject_report_evidence(tmp_path): root = _copy_example(tmp_path) prepared = prepare_run(root / "pipeline.json", run_id="report_markdown") accepted = build_optimization_report( - prepared, await run_fake_stage(prepared, scenario="improve"), progress=_progress(), + prepared, await run_offline_stage(prepared, scenario="improve"), progress=_progress(), finished_at=datetime(2026, 7, 18, 0, 1, tzinfo=timezone.utc), ) rejected_root = _copy_example(tmp_path, "eval_optimize_loop_reject") rejected_prepared = prepare_run(rejected_root / "pipeline.json", run_id="report_markdown_reject") rejected = build_optimization_report( - rejected_prepared, await run_fake_stage(rejected_prepared, scenario="no_improvement"), progress=_progress(), + rejected_prepared, await run_offline_stage(rejected_prepared, scenario="no_improvement"), progress=_progress(), finished_at=datetime(2026, 7, 18, 0, 1, tzinfo=timezone.utc), ) markdown = render_optimization_markdown(accepted) @@ -264,7 +266,7 @@ def test_failure_report_redacts_hyphenated_keys_and_bare_urls(tmp_path): async def test_real_report_marks_malformed_token_usage_unavailable(tmp_path, token_usage): root = _copy_example(tmp_path) prepared = prepare_run(root / "pipeline.json", run_id="report_real_bad_tokens") - fake_result = await run_fake_stage(prepared, scenario="improve") + fake_result = await run_offline_stage(prepared, scenario="improve") candidate = OptimizerCandidateProposal.model_validate({ **fake_result.candidate.model_dump(exclude={"scenario", "seed"}), "provider": "agent_optimizer", "optimizer_status": "SUCCEEDED", "finish_reason": "completed", @@ -292,7 +294,7 @@ async def test_real_report_marks_malformed_token_usage_unavailable(tmp_path, tok async def test_real_report_allows_consistent_zero_token_usage_without_reflection_calls(tmp_path): root = _copy_example(tmp_path) prepared = prepare_run(root / "pipeline.json", run_id="report_real_zero_tokens") - fake_result = await run_fake_stage(prepared, scenario="improve") + fake_result = await run_offline_stage(prepared, scenario="improve") candidate = OptimizerCandidateProposal.model_validate({ **fake_result.candidate.model_dump(exclude={"scenario", "seed"}), "provider": "agent_optimizer", "optimizer_status": "SUCCEEDED", "finish_reason": "completed", diff --git a/tests/evaluation/test_eval_optimize_loop_stage6_acceptance.py b/tests/evaluation/test_eval_optimize_loop_stage6_acceptance.py new file mode 100644 index 00000000..c7e4f272 --- /dev/null +++ b/tests/evaluation/test_eval_optimize_loop_stage6_acceptance.py @@ -0,0 +1,60 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under the Apache License, Version 2.0. +"""Repository-level Stage 6 sample-output acceptance tests.""" + +from __future__ import annotations + +from hashlib import sha256 +from pathlib import Path + +from examples.optimization.eval_optimize_loop.schemas import ArtifactIndex +from examples.optimization.eval_optimize_loop.schemas import OptimizationReport + + +_REPO_ROOT = Path(__file__).resolve().parents[2] +_SAMPLE = ( + _REPO_ROOT + / "examples" + / "optimization" + / "eval_optimize_loop" + / "sample_output" +) + + +def test_sample_output_is_schema_valid_and_self_consistent(): + report_path = _SAMPLE / "optimization_report.json" + markdown_path = _SAMPLE / "optimization_report.md" + index_path = _SAMPLE / "artifact_index.json" + report = OptimizationReport.model_validate_json( + report_path.read_text(encoding="utf-8") + ) + index = ArtifactIndex.model_validate_json(index_path.read_text(encoding="utf-8")) + + assert report.execution_mode == "offline" + assert report.gate_decision.decision == "accept" + assert index.run_id == report.run_id + assert report.run_id in markdown_path.read_text(encoding="utf-8") + for reference in index.artifacts: + if reference.status != "available": + continue + path = _SAMPLE / reference.relative_path # type: ignore[arg-type] + payload = path.read_bytes() + assert len(payload) == reference.size_bytes + assert sha256(payload).hexdigest() == reference.sha256 + + +def test_sample_output_does_not_contain_credentials_or_machine_paths(): + combined = "\n".join( + path.read_text(encoding="utf-8") + for path in ( + _SAMPLE / "optimization_report.json", + _SAMPLE / "optimization_report.md", + _SAMPLE / "artifact_index.json", + ) + ) + assert "/home/" not in combined + assert "TRPC_AGENT_API_KEY" not in combined + assert "TRPC_AGENT_BASE_URL" not in combined diff --git a/tests/evaluation/test_eval_optimize_loop_stage6_offline.py b/tests/evaluation/test_eval_optimize_loop_stage6_offline.py new file mode 100644 index 00000000..2a4f4c30 --- /dev/null +++ b/tests/evaluation/test_eval_optimize_loop_stage6_offline.py @@ -0,0 +1,143 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under the Apache License, Version 2.0. +"""Stage 6 offline-mode tests for the SDK-backed deterministic model.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from examples.optimization.eval_optimize_loop.business_agent import BusinessAgent +from examples.optimization.eval_optimize_loop.fake.model import ( + DeterministicFakeModel, +) +from trpc_agent_sdk.evaluation import TargetPrompt +from trpc_agent_sdk.models import LlmRequest +from trpc_agent_sdk.types import Content +from trpc_agent_sdk.types import GenerateContentConfig +from trpc_agent_sdk.types import Part + + +def _request(instruction: str, user_text: str) -> LlmRequest: + return LlmRequest( + contents=[ + Content(role="user", parts=[Part.from_text(text=user_text)]), + ], + config=GenerateContentConfig(system_instruction=instruction), + ) + + +async def _text(model: DeterministicFakeModel, request: LlmRequest) -> str: + responses = [response async for response in model.generate_async(request)] + assert len(responses) == 1 + assert responses[0].content is not None + return "".join(part.text or "" for part in responses[0].content.parts or []) + + +def test_deterministic_fake_model_registers_a_stable_model_pattern(): + assert DeterministicFakeModel.supported_models() == [ + "deterministic-fake-model" + ] + + +@pytest.mark.asyncio +async def test_deterministic_fake_model_uses_system_instruction(): + model = DeterministicFakeModel() + query = "Check the status of order A100." + + baseline = await _text(model, _request("baseline", query)) + improved = await _text( + model, + _request( + "", + query, + ), + ) + + assert '"route":"general_support"' in baseline + assert improved == ( + '{"route":"order_lookup","message":"Checking order A100."}' + ) + assert await _text(model, _request("baseline", query)) == baseline + + +@pytest.mark.asyncio +async def test_deterministic_fake_model_uses_last_user_message(): + request = _request( + "", + "ignored", + ) + request.contents.extend( + [ + Content(role="model", parts=[Part.from_text(text="intermediate")]), + Content( + role="user", + parts=[Part.from_text(text="How long is standard shipping?")], + ), + ] + ) + + assert await _text(DeterministicFakeModel(), request) == ( + '{"route":"shipping_policy","message":' + '"Standard shipping normally takes 3-5 business days."}' + ) + + +@pytest.mark.asyncio +async def test_deterministic_fake_model_rejects_missing_user_text(): + request = LlmRequest( + contents=[Content(role="model", parts=[Part.from_text(text="only model")])], + config=GenerateContentConfig(system_instruction="baseline"), + ) + + responses = [ + response + async for response in DeterministicFakeModel().generate_async(request) + ] + + assert len(responses) == 1 + assert responses[0].content is None + assert responses[0].error_code == "API_ERROR" + assert "user text" in (responses[0].error_message or "") + + +@pytest.mark.asyncio +async def test_business_agent_rereads_prompt_through_llm_agent(tmp_path: Path): + prompt_path = tmp_path / "system.md" + prompt_path.write_text("baseline", encoding="utf-8") + target = TargetPrompt().add_path("system_prompt", str(prompt_path)) + created_models: list[DeterministicFakeModel] = [] + + def model_factory() -> DeterministicFakeModel: + model = DeterministicFakeModel() + created_models.append(model) + return model + + agent = BusinessAgent( + target, + model_factory, + agent_name="stage6_offline_agent", + app_name="stage6_offline_test", + user_id="stage6-test", + ) + query = "Check the status of order A100." + + baseline = await agent.call_agent(query) + await target.write_all( + { + "system_prompt": ( + "" + ) + } + ) + candidate = await agent.call_agent(query) + + assert '"route":"general_support"' in baseline + assert candidate == ( + '{"route":"order_lookup","message":"Checking order A100."}' + ) + assert len(created_models) == 2 diff --git a/tests/evaluation/test_eval_optimize_loop_stage6_trace.py b/tests/evaluation/test_eval_optimize_loop_stage6_trace.py new file mode 100644 index 00000000..c8b2a9a8 --- /dev/null +++ b/tests/evaluation/test_eval_optimize_loop_stage6_trace.py @@ -0,0 +1,99 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under the Apache License, Version 2.0. +"""Stage 6 tests for deterministic SDK Trace replay.""" + +from __future__ import annotations + +import shutil +from pathlib import Path + +import pytest + +from examples.optimization.eval_optimize_loop import pipeline as pipeline_module +from examples.optimization.eval_optimize_loop.pipeline import prepare_run +from examples.optimization.eval_optimize_loop.pipeline import run_trace_stage +from examples.optimization.eval_optimize_loop.schemas import ArtifactIndex +from examples.optimization.eval_optimize_loop.schemas import OptimizationReport + + +_REPO_ROOT = Path(__file__).resolve().parents[2] +_EXAMPLE = _REPO_ROOT / "examples" / "optimization" / "eval_optimize_loop" + + +def _copy_example(tmp_path: Path, name: str) -> Path: + target = tmp_path / name + shutil.copytree( + _EXAMPLE, + target, + ignore=shutil.ignore_patterns("runs", "__pycache__"), + ) + return target + + +@pytest.mark.parametrize( + ("scenario", "train_passed", "validation_passed", "decision"), + [ + ("improve", 3, 3, "accept"), + ("no_improvement", 1, 1, "reject"), + ("overfit", 3, 0, "reject"), + ], +) +@pytest.mark.asyncio +async def test_trace_stage_replays_four_evalsets_without_writeback( + tmp_path: Path, + scenario: str, + train_passed: int, + validation_passed: int, + decision: str, + monkeypatch: pytest.MonkeyPatch, +): + class _ForbiddenComponent: + def __init__(self, *_args, **_kwargs): + raise AssertionError("trace mode must not construct runtime components") + + monkeypatch.setattr(pipeline_module, "BusinessAgent", _ForbiddenComponent) + monkeypatch.setattr( + pipeline_module, + "FakeCandidateProviderAdapter", + _ForbiddenComponent, + ) + root = _copy_example(tmp_path, f"trace_{scenario}") + source = root / "prompts" / "system.md" + baseline_source = source.read_text(encoding="utf-8") + prepared = prepare_run( + root / "pipeline.trace.json", + run_id=f"trace_{scenario}", + ) + baseline_working = await prepared.working_target.read_all() + + result = await run_trace_stage( + prepared, + scenario=scenario, # type: ignore[arg-type] + ) + + assert result.baseline_train.passed_case_count == 1 + assert result.baseline_validation.passed_case_count == 1 + assert result.candidate_train.passed_case_count == train_passed + assert result.candidate_validation.passed_case_count == validation_passed + assert result.gate_decision.decision == decision + assert result.writeback.status == "skipped" + assert result.writeback.reason == "trace_replay" + assert result.writeback.attempted is False + assert source.read_text(encoding="utf-8") == baseline_source + assert await prepared.working_target.read_all() == baseline_working + assert len(result.analysis.train_diff.cases) == 3 + assert len(result.analysis.validation_diff.cases) == 3 + report_dir = Path(prepared.workspace.run_dir) / "report" + report = OptimizationReport.model_validate_json( + (report_dir / "optimization_report.json").read_text(encoding="utf-8") + ) + index = ArtifactIndex.model_validate_json( + (report_dir / "artifact_index.json").read_text(encoding="utf-8") + ) + assert report.execution_mode == "trace" + artifact_ids = {artifact.artifact_id for artifact in index.artifacts} + assert "input.trace.candidate_train" in artifact_ids + assert "input.trace.candidate_validation" in artifact_ids From 7c2aaf1b03d2e3bf5e151d94f51babb081a1e72f Mon Sep 17 00:00:00 2001 From: guocfu Date: Tue, 21 Jul 2026 18:57:13 +0800 Subject: [PATCH 17/18] =?UTF-8?q?fix(evaluation):=20=E8=A1=A5=E9=BD=90?= =?UTF-8?q?=E5=BC=82=E5=B8=B8=E8=84=B1=E6=95=8F=E4=B8=8E=20Trace=20?= =?UTF-8?q?=E9=94=99=E8=AF=AF=E4=B8=8A=E4=B8=8B=E6=96=87?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../eval_optimize_loop/pipeline.py | 37 ++++++++++++------ .../eval_optimize_loop/report_builder.py | 5 ++- .../run_real_integration.py | 7 +++- ...est_eval_optimize_loop_real_integration.py | 34 ++++++++++++++++ .../test_eval_optimize_loop_stage6_trace.py | 39 +++++++++++++++++++ 5 files changed, 107 insertions(+), 15 deletions(-) diff --git a/examples/optimization/eval_optimize_loop/pipeline.py b/examples/optimization/eval_optimize_loop/pipeline.py index 2b8a1c89..a67828e9 100644 --- a/examples/optimization/eval_optimize_loop/pipeline.py +++ b/examples/optimization/eval_optimize_loop/pipeline.py @@ -1043,24 +1043,37 @@ async def _execute_trace_stage( phase="candidate", split="validation", ) progress.enter("analysis") - analysis = build_evaluation_analysis( - baseline_train=baseline_train, - baseline_validation=baseline_validation, - candidate_train=candidate_train, - candidate_validation=candidate_validation, - hard_case_ids=set(prepared.config.case_labels.hard_case_ids), - critical_case_ids=set(prepared.config.case_labels.critical_case_ids), - severe_case_score_drop=prepared.config.gate.severe_case_score_drop, - ) + try: + analysis = build_evaluation_analysis( + baseline_train=baseline_train, + baseline_validation=baseline_validation, + candidate_train=candidate_train, + candidate_validation=candidate_validation, + hard_case_ids=set(prepared.config.case_labels.hard_case_ids), + critical_case_ids=set(prepared.config.case_labels.critical_case_ids), + severe_case_score_drop=prepared.config.gate.severe_case_score_drop, + ) + except EvaluationAnalysisError as exc: + raise PipelineStageExecutionError( + f"stage 3a analysis failed: {exc}" + ) from exc measurements = ResourceMeasurements( cost_usd=ObservableValue(status="unavailable", unit="USD", reason="Trace replay does not call a model."), total_tokens=ObservableValue(status="unavailable", unit="tokens", reason="Trace replay does not call a model."), duration_seconds=ObservableValue(status="available", value=perf_counter() - started_at, unit="seconds"), ) progress.enter("gate") - gate_decision = evaluate_gate( - analysis, prepared.config.gate, prepared.config.budget, measurements - ) + try: + gate_decision = evaluate_gate( + analysis, + prepared.config.gate, + prepared.config.budget, + measurements, + ) + except GateEvaluationError as exc: + raise PipelineStageExecutionError( + f"stage 3b gate failed: {exc}" + ) from exc progress.enter("writeback") writeback = WritebackResult( status="skipped", reason="trace_replay", attempted=False diff --git a/examples/optimization/eval_optimize_loop/report_builder.py b/examples/optimization/eval_optimize_loop/report_builder.py index c64bf976..d5cc3182 100644 --- a/examples/optimization/eval_optimize_loop/report_builder.py +++ b/examples/optimization/eval_optimize_loop/report_builder.py @@ -50,7 +50,8 @@ def _not_applicable_optimizer_value( ) -def _redact_error_message(error: Exception) -> str: +def redact_error_message(error: Exception) -> str: + """移除异常文本中的环境凭据、认证字段和连接地址。""" message = str(error) environment_values = { os.environ.get(name, "") @@ -152,7 +153,7 @@ def build_failure_report( return FailureReport( run_id=prepared.workspace.run_id, execution_mode=prepared.config.execution.mode, failed_phase=progress.current_phase, exception_type=type(error).__name__, - error_message=_redact_error_message(error), generated_at=generated_at, + error_message=redact_error_message(error), generated_at=generated_at, input_snapshot=prepared.input_snapshot, source_prompt_hashes=dict(sorted(source_prompt_hashes.items())), completed_phases=progress.completed_phases, existing_artifacts=sorted(existing_artifacts), diff --git a/examples/optimization/eval_optimize_loop/run_real_integration.py b/examples/optimization/eval_optimize_loop/run_real_integration.py index 03d0d5e3..c326e5ea 100644 --- a/examples/optimization/eval_optimize_loop/run_real_integration.py +++ b/examples/optimization/eval_optimize_loop/run_real_integration.py @@ -25,6 +25,7 @@ from examples.optimization.eval_optimize_loop.real_agent import BusinessModelConfig from examples.optimization.eval_optimize_loop.real_agent import RealBusinessAgent from examples.optimization.eval_optimize_loop.real_agent import load_business_model_config + from examples.optimization.eval_optimize_loop.report_builder import redact_error_message from examples.optimization.eval_optimize_loop.schemas import OptimizerRuntimeParameters else: from .config import load_pipeline_config @@ -33,6 +34,7 @@ from .real_agent import BusinessModelConfig from .real_agent import RealBusinessAgent from .real_agent import load_business_model_config + from .report_builder import redact_error_message from .schemas import OptimizerRuntimeParameters @@ -116,7 +118,10 @@ def main() -> int: try: prepared, result = asyncio.run(_run(args, business_config, parameters)) except Exception as exc: - print(f"Real integration failed: {exc}", file=sys.stderr) + print( + f"Real integration failed: {redact_error_message(exc)}", + file=sys.stderr, + ) return 1 print(f"Completed real-model pipeline: {prepared.workspace.run_dir}") diff --git a/tests/evaluation/test_eval_optimize_loop_real_integration.py b/tests/evaluation/test_eval_optimize_loop_real_integration.py index 7f4c2662..3920ecf9 100644 --- a/tests/evaluation/test_eval_optimize_loop_real_integration.py +++ b/tests/evaluation/test_eval_optimize_loop_real_integration.py @@ -335,6 +335,40 @@ async def fail(*args, **kwargs): assert "offline failure" in capsys.readouterr().err +def test_real_cli_redacts_environment_secrets_from_pipeline_error( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +): + api_key = "integration-secret-key" + base_url = "https://private-gateway.example.test/v1" + + async def fail(*args, **kwargs): + raise RuntimeError( + f"request failed api_key={api_key} base_url={base_url}" + ) + + monkeypatch.setenv("TRPC_AGENT_API_KEY", api_key) + monkeypatch.setenv("TRPC_AGENT_BASE_URL", base_url) + monkeypatch.setenv("TRPC_AGENT_MODEL_NAME", "business-model") + monkeypatch.setattr(run_real_integration, "_run", fail) + monkeypatch.setattr( + sys, + "argv", + [ + "run_real_integration.py", + "--run-real", + "--optimizer-model-name", + "optimizer-model", + ], + ) + + assert run_real_integration.main() == 1 + error = capsys.readouterr().err + assert api_key not in error + assert base_url not in error + assert "[REDACTED]" in error + + def test_real_cli_requires_explicit_confirmation_before_creating_workspace(tmp_path: Path): root = _copy_example(tmp_path) completed = subprocess.run( diff --git a/tests/evaluation/test_eval_optimize_loop_stage6_trace.py b/tests/evaluation/test_eval_optimize_loop_stage6_trace.py index c8b2a9a8..1fb682de 100644 --- a/tests/evaluation/test_eval_optimize_loop_stage6_trace.py +++ b/tests/evaluation/test_eval_optimize_loop_stage6_trace.py @@ -14,6 +14,7 @@ from examples.optimization.eval_optimize_loop import pipeline as pipeline_module from examples.optimization.eval_optimize_loop.pipeline import prepare_run +from examples.optimization.eval_optimize_loop.pipeline import PipelineStageExecutionError from examples.optimization.eval_optimize_loop.pipeline import run_trace_stage from examples.optimization.eval_optimize_loop.schemas import ArtifactIndex from examples.optimization.eval_optimize_loop.schemas import OptimizationReport @@ -97,3 +98,41 @@ def __init__(self, *_args, **_kwargs): artifact_ids = {artifact.artifact_id for artifact in index.artifacts} assert "input.trace.candidate_train" in artifact_ids assert "input.trace.candidate_validation" in artifact_ids + + +@pytest.mark.parametrize( + ("target", "error_type", "expected"), + [ + ( + "build_evaluation_analysis", + pipeline_module.EvaluationAnalysisError, + "stage 3a analysis failed", + ), + ( + "evaluate_gate", + pipeline_module.GateEvaluationError, + "stage 3b gate failed", + ), + ], +) +@pytest.mark.asyncio +async def test_trace_stage_wraps_analysis_and_gate_errors_with_phase_context( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + target: str, + error_type: type[Exception], + expected: str, +): + root = _copy_example(tmp_path, f"trace_{target}") + prepared = prepare_run( + root / "pipeline.trace.json", + run_id=f"trace_{target}", + ) + + def fail(*_args, **_kwargs): + raise error_type("injected failure") + + monkeypatch.setattr(pipeline_module, target, fail) + + with pytest.raises(PipelineStageExecutionError, match=expected): + await run_trace_stage(prepared, scenario="improve") From 68c318e1dfe7e569e79474ade8eb6169bd6d9056 Mon Sep 17 00:00:00 2001 From: guocfu Date: Tue, 21 Jul 2026 21:57:59 +0800 Subject: [PATCH 18/18] =?UTF-8?q?fix(evaluation):=20=E7=BB=9F=E4=B8=80?= =?UTF-8?q?=E5=AE=A1=E8=AE=A1=E5=8F=A3=E5=BE=84=E5=B9=B6=E8=A1=A5=E5=BC=BA?= =?UTF-8?q?=E6=8F=90=E7=A4=BA=E8=AF=8D=E6=81=A2=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../eval_optimize_loop/artifact_writer.py | 87 ++---------- .../eval_optimize_loop/candidate_provider.py | 22 +--- .../eval_optimize_loop/pipeline.py | 70 ++++++---- .../eval_optimize_loop/sensitive_config.py | 124 ++++++++++++++++++ ...est_eval_optimize_loop_real_integration.py | 27 +++- .../test_eval_optimize_loop_stage3a.py | 46 +++++++ .../test_eval_optimize_loop_stage4.py | 36 +++++ ...est_eval_optimize_loop_stage5_artifacts.py | 18 +++ ...test_eval_optimize_loop_stage5_pipeline.py | 21 +++ 9 files changed, 322 insertions(+), 129 deletions(-) create mode 100644 examples/optimization/eval_optimize_loop/sensitive_config.py diff --git a/examples/optimization/eval_optimize_loop/artifact_writer.py b/examples/optimization/eval_optimize_loop/artifact_writer.py index 2353177c..41e82169 100644 --- a/examples/optimization/eval_optimize_loop/artifact_writer.py +++ b/examples/optimization/eval_optimize_loop/artifact_writer.py @@ -24,6 +24,8 @@ from .schemas import ArtifactIndex, ArtifactReference, FailureReport from .schemas import OptimizationReport, ReportPhase from .schemas import TraceCandidateProposal +from .sensitive_config import SensitiveConfigError +from .sensitive_config import validate_persisted_sensitive_values ArtifactType: TypeAlias = Literal[ @@ -36,46 +38,6 @@ ] _INPUT_COPY_DISABLED = "artifacts.copy_input_files=false" -_SENSITIVE_CONFIG_KEYS = { - "accesstoken", - "apikey", - "auth", - "authorization", - "authtoken", - "baseurl", - "bearertoken", - "clientsecret", - "credential", - "credentials", - "password", - "passwd", - "privatekey", - "secret", - "secretkey", - "token", - "xapikey", -} -_SENSITIVE_CONFIG_KEY_SUFFIXES = { - "accesstoken", - "apikey", - "authtoken", - "baseurl", - "bearertoken", - "clientsecret", - "credential", - "credentials", - "endpointurl", - "password", - "passwd", - "privatekey", - "secretkey", -} -_APPROVED_SENSITIVE_VALUES = { - "", - "${TRPC_AGENT_API_KEY}", - "${TRPC_AGENT_BASE_URL}", - "fake-not-used-in-offline-mode", -} _AT_FDCWD = -100 _RENAME_NOREPLACE = 1 _RENAME_EXCL = 0x4 @@ -179,44 +141,6 @@ def _json_text(model: BaseModel) -> str: return model.model_dump_json(by_alias=False, indent=2) + "\n" -def _normalized_sensitive_key(key: str) -> str: - return key.replace("_", "").replace("-", "").casefold() - - -def _is_sensitive_config_key(key: str) -> bool: - normalized = _normalized_sensitive_key(key) - return normalized in _SENSITIVE_CONFIG_KEYS or any( - normalized.endswith(suffix) - for suffix in _SENSITIVE_CONFIG_KEY_SUFFIXES - ) - - -def _validate_sensitive_config_values(value: object, *, path: str = "$") -> None: - if isinstance(value, str): - if value.strip().casefold().startswith(("http://", "https://")): - raise ArtifactWriteError( - "sensitive optimizer config value is not an approved " - f"placeholder: {path}" - ) - return - if isinstance(value, list): - for index, item in enumerate(value): - _validate_sensitive_config_values(item, path=f"{path}[{index}]") - return - if not isinstance(value, dict): - return - for key, item in value.items(): - item_path = f"{path}.{key}" - if _is_sensitive_config_key(key): - if not isinstance(item, str) or item not in _APPROVED_SENSITIVE_VALUES: - raise ArtifactWriteError( - "sensitive optimizer config value is not an approved " - f"placeholder: {item_path}" - ) - else: - _validate_sensitive_config_values(item, path=item_path) - - def _validate_optimizer_config_for_copy(path: Path) -> None: try: payload = json.loads(path.read_text(encoding="utf-8")) @@ -224,7 +148,10 @@ def _validate_optimizer_config_for_copy(path: Path) -> None: raise ArtifactWriteError( f"failed to parse optimizer config snapshot: {path}: {exc}" ) from exc - _validate_sensitive_config_values(payload) + try: + validate_persisted_sensitive_values(payload) + except SensitiveConfigError as exc: + raise ArtifactWriteError(str(exc)) from exc def _rename_directory_no_replace(source: Path, target: Path) -> None: @@ -621,6 +548,8 @@ def publish_report_bundle( for relative in native_paths: native_path = root / relative + if native_path.name == "optimizer.runtime.json": + _validate_optimizer_config_for_copy(native_path) references.append( _available_reference( root, diff --git a/examples/optimization/eval_optimize_loop/candidate_provider.py b/examples/optimization/eval_optimize_loop/candidate_provider.py index 69b65822..3b3882dc 100644 --- a/examples/optimization/eval_optimize_loop/candidate_provider.py +++ b/examples/optimization/eval_optimize_loop/candidate_provider.py @@ -24,6 +24,7 @@ from .schemas import CandidateScenario from .schemas import OptimizerCandidateProposal from .schemas import OptimizerRuntimeParameters +from .sensitive_config import replace_persisted_sensitive_values class CandidateProviderError(RuntimeError): @@ -96,26 +97,7 @@ def __init__(self, call_agent: CallAgent) -> None: @staticmethod def _replace_persisted_connection_values(value: object) -> object: """递归将可能被 SDK 复制到产物的连接值替换为环境占位符。""" - if isinstance(value, list): - return [ - AgentOptimizerCandidateProvider._replace_persisted_connection_values(item) - for item in value - ] - if not isinstance(value, dict): - return value - placeholders = { - "api_key": "${TRPC_AGENT_API_KEY}", - "apiKey": "${TRPC_AGENT_API_KEY}", - "base_url": "${TRPC_AGENT_BASE_URL}", - "baseUrl": "${TRPC_AGENT_BASE_URL}", - } - return { - key: placeholders.get( - key, - AgentOptimizerCandidateProvider._replace_persisted_connection_values(item), - ) - for key, item in value.items() - } + return replace_persisted_sensitive_values(value) @staticmethod def _prepare_runtime_config(request: CandidateRequest) -> Path: diff --git a/examples/optimization/eval_optimize_loop/pipeline.py b/examples/optimization/eval_optimize_loop/pipeline.py index a67828e9..9657f3de 100644 --- a/examples/optimization/eval_optimize_loop/pipeline.py +++ b/examples/optimization/eval_optimize_loop/pipeline.py @@ -24,7 +24,6 @@ from trpc_agent_sdk.evaluation import CallAgent from trpc_agent_sdk.evaluation import EvalCaseResult from trpc_agent_sdk.evaluation import EvalSet -from trpc_agent_sdk.evaluation import EvalStatus from trpc_agent_sdk.evaluation import OptimizeConfigFile from trpc_agent_sdk.evaluation import TargetPrompt from trpc_agent_sdk.evaluation import load_optimize_config @@ -41,6 +40,7 @@ from .config import PipelineConfig from .config import load_pipeline_config from .evaluation_adapter import EvaluationAnalysisError +from .evaluation_adapter import standardize_snapshot from .fake import DeterministicFakeModel from .gate import evaluate_gate from .gate import GateEvaluationError @@ -473,22 +473,6 @@ def prepare_run(pipeline_config_path: str | Path, *, run_id: str | None = None) raise -def _summarize_results( - eval_results_by_eval_id: dict[str, list[EvalCaseResult]], -) -> tuple[int, int, float | None]: - passed_cases = 0 - scores: list[float] = [] - for runs in eval_results_by_eval_id.values(): - if runs and all(getattr(run, "final_eval_status", None) == EvalStatus.PASSED for run in runs): - passed_cases += 1 - for run in runs: - for metric in getattr(run, "overall_eval_metric_results", []): - if metric.score is not None: - scores.append(float(metric.score)) - average_score = sum(scores) / len(scores) if scores else None - return passed_cases, len(eval_results_by_eval_id), average_score - - def _validate_results( *, eval_set: EvalSet, @@ -547,8 +531,7 @@ async def _evaluate_split( phase=phase, split=split, ) - passed_cases, total_cases, average_score = _summarize_results(eval_results_by_eval_id) - return EvaluationSnapshot( + snapshot = EvaluationSnapshot( phase=phase, split=split, eval_set_id=eval_set.eval_set_id, @@ -556,9 +539,26 @@ async def _evaluate_split( details_lines=details_lines, result_lines=result_lines, eval_results_by_eval_id=eval_results_by_eval_id, - passed_case_count=passed_cases, - total_case_count=total_cases, - average_score=average_score, + passed_case_count=0, + total_case_count=len(eval_results_by_eval_id), + average_score=None, + ) + try: + standardized = standardize_snapshot(snapshot) + except EvaluationAnalysisError as exc: + raise PipelineStageExecutionError( + f"{phase} {split} evaluation result standardization failed: {exc}" + ) from exc + return snapshot.model_copy( + update={ + "passed_case_count": standardized.passed_case_count, + "total_case_count": len(standardized.cases), + "average_score": ( + standardized.average_score.value + if standardized.average_score.status == "available" + else None + ), + } ) @@ -567,14 +567,34 @@ async def _restore_working_baseline( baseline_prompts: dict[str, str], ) -> bool: """Restore optimizer leftovers and prove the isolated baseline is present.""" + initial_read_error: Exception | None = None + was_modified = True try: current = await prepared.working_target.read_all() + except Exception as exc: + initial_read_error = exc + else: was_modified = current != baseline_prompts - if was_modified: + + if was_modified: + try: await prepared.working_target.write_all(baseline_prompts) + except Exception as exc: + if initial_read_error is not None: + raise PipelineStageExecutionError( + "failed to restore optimizer working prompts after initial " + f"read failed ({initial_read_error}): {exc}" + ) from exc + raise PipelineStageExecutionError( + f"failed to restore optimizer working prompts: {exc}" + ) from exc + + try: restored = await prepared.working_target.read_all() except Exception as exc: - raise PipelineStageExecutionError(f"failed to restore optimizer working prompts: {exc}") from exc + raise PipelineStageExecutionError( + f"failed to verify restored optimizer working prompts: {exc}" + ) from exc if restored != baseline_prompts: raise PipelineStageExecutionError("optimizer working prompts did not match baseline after restoration") return was_modified @@ -592,7 +612,6 @@ async def _execute_offline_stage( the isolated working target on success or candidate-evaluation failure so the run can be inspected later. """ - progress.enter("baseline_train") if prepared.config.execution.mode != "offline": raise PipelineStageExecutionError( "run_offline_stage requires execution.mode='offline', got " @@ -756,7 +775,6 @@ async def _execute_real_stage( progress: _MutableReportProgress, ) -> RealStageResult: """Generate a real optimizer candidate and run the full guarded regression.""" - progress.enter("baseline_train") if prepared.config.execution.mode != "real": raise PipelineStageExecutionError( f"run_real_stage requires execution.mode='real', got {prepared.config.execution.mode!r}" diff --git a/examples/optimization/eval_optimize_loop/sensitive_config.py b/examples/optimization/eval_optimize_loop/sensitive_config.py new file mode 100644 index 00000000..d909f23f --- /dev/null +++ b/examples/optimization/eval_optimize_loop/sensitive_config.py @@ -0,0 +1,124 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under the Apache License, Version 2.0. +"""共享的敏感配置识别、占位符替换与持久化校验规则。""" + +from __future__ import annotations + + +API_KEY_PLACEHOLDER = "${TRPC_AGENT_API_KEY}" +BASE_URL_PLACEHOLDER = "${TRPC_AGENT_BASE_URL}" + +_SENSITIVE_CONFIG_KEYS = { + "accesstoken", + "apikey", + "auth", + "authorization", + "authtoken", + "baseurl", + "bearertoken", + "clientsecret", + "credential", + "credentials", + "password", + "passwd", + "privatekey", + "secret", + "secretkey", + "token", + "xapikey", +} +_SENSITIVE_CONFIG_KEY_SUFFIXES = { + "accesstoken", + "apikey", + "authtoken", + "baseurl", + "bearertoken", + "clientsecret", + "credential", + "credentials", + "endpointurl", + "password", + "passwd", + "privatekey", + "secretkey", +} +_URL_CONFIG_KEY_SUFFIXES = {"baseurl", "endpointurl"} +_APPROVED_SENSITIVE_VALUES = { + "", + API_KEY_PLACEHOLDER, + BASE_URL_PLACEHOLDER, + "fake-not-used-in-offline-mode", +} + + +class SensitiveConfigError(ValueError): + """配置中存在不允许持久化的连接信息或凭据。""" + + +def _normalized_key(key: str) -> str: + return key.replace("_", "").replace("-", "").casefold() + + +def _is_sensitive_key(key: str) -> bool: + normalized = _normalized_key(key) + return normalized in _SENSITIVE_CONFIG_KEYS or any( + normalized.endswith(suffix) + for suffix in _SENSITIVE_CONFIG_KEY_SUFFIXES + ) + + +def _placeholder_for_key(key: str) -> str: + normalized = _normalized_key(key) + if any(normalized.endswith(suffix) for suffix in _URL_CONFIG_KEY_SUFFIXES): + return BASE_URL_PLACEHOLDER + return API_KEY_PLACEHOLDER + + +def replace_persisted_sensitive_values(value: object) -> object: + """递归替换任何可能进入运行产物的连接地址和凭据。""" + if isinstance(value, str): + if value.strip().casefold().startswith(("http://", "https://")): + return BASE_URL_PLACEHOLDER + return value + if isinstance(value, list): + return [replace_persisted_sensitive_values(item) for item in value] + if not isinstance(value, dict): + return value + return { + key: ( + _placeholder_for_key(key) + if _is_sensitive_key(key) + else replace_persisted_sensitive_values(item) + ) + for key, item in value.items() + } + + +def validate_persisted_sensitive_values(value: object, *, path: str = "$") -> None: + """拒绝不符合共享占位符策略的持久化配置。""" + if isinstance(value, str): + if value.strip().casefold().startswith(("http://", "https://")): + raise SensitiveConfigError( + "sensitive optimizer config value is not an approved " + f"placeholder: {path}" + ) + return + if isinstance(value, list): + for index, item in enumerate(value): + validate_persisted_sensitive_values(item, path=f"{path}[{index}]") + return + if not isinstance(value, dict): + return + for key, item in value.items(): + item_path = f"{path}.{key}" + if _is_sensitive_key(key): + if not isinstance(item, str) or item not in _APPROVED_SENSITIVE_VALUES: + raise SensitiveConfigError( + "sensitive optimizer config value is not an approved " + f"placeholder: {item_path}" + ) + else: + validate_persisted_sensitive_values(item, path=item_path) diff --git a/tests/evaluation/test_eval_optimize_loop_real_integration.py b/tests/evaluation/test_eval_optimize_loop_real_integration.py index 3920ecf9..7c626066 100644 --- a/tests/evaluation/test_eval_optimize_loop_real_integration.py +++ b/tests/evaluation/test_eval_optimize_loop_real_integration.py @@ -214,6 +214,11 @@ async def fake_optimize(**kwargs): template["evaluate"]["credential_probe"] = { "api_key": "judge-secret", "base_url": "https://judge.example.test", + "authorization": "Bearer judge-authorization-secret", + "access_token": "judge-access-token-secret", + "client_secret": "judge-client-secret", + "password": "judge-password-secret", + "provider_endpoint_url": "https://judge-alt.example.test/v1", } request.optimizer_config_path.write_text(json.dumps(template), encoding="utf-8") @@ -236,11 +241,25 @@ async def fake_optimize(**kwargs): assert runtime["evaluate"]["credential_probe"] == { "api_key": "${TRPC_AGENT_API_KEY}", "base_url": "${TRPC_AGENT_BASE_URL}", + "authorization": "${TRPC_AGENT_API_KEY}", + "access_token": "${TRPC_AGENT_API_KEY}", + "client_secret": "${TRPC_AGENT_API_KEY}", + "password": "${TRPC_AGENT_API_KEY}", + "provider_endpoint_url": "${TRPC_AGENT_BASE_URL}", } - runtime_text = runtime_path.read_text(encoding="utf-8") - snapshot_text = (request.output_dir / "config.snapshot.json").read_text(encoding="utf-8") - assert "judge-secret" not in runtime_text - assert "judge-secret" not in snapshot_text + persisted_texts = [ + runtime_path.read_text(encoding="utf-8"), + (request.output_dir / "config.snapshot.json").read_text(encoding="utf-8"), + ] + for secret in ( + "judge-secret", + "judge-authorization-secret", + "judge-access-token-secret", + "judge-client-secret", + "judge-password-secret", + "https://judge-alt.example.test/v1", + ): + assert all(secret not in text for text in persisted_texts) assert original_config != request.optimizer_config_path.read_text(encoding="utf-8") diff --git a/tests/evaluation/test_eval_optimize_loop_stage3a.py b/tests/evaluation/test_eval_optimize_loop_stage3a.py index bad25f3b..a9680c75 100644 --- a/tests/evaluation/test_eval_optimize_loop_stage3a.py +++ b/tests/evaluation/test_eval_optimize_loop_stage3a.py @@ -199,6 +199,52 @@ def test_standardize_snapshot_treats_failed_metric_without_score_as_not_evaluate assert case.runs[0].metrics[0].status == "not_evaluated" +@pytest.mark.asyncio +async def test_evaluation_snapshot_uses_standardized_pass_count( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + root = _copy_example(tmp_path, "standardized_snapshot_summary") + prepared = prepare_run(root / "pipeline.json", run_id="standardized_snapshot_summary") + eval_set = pipeline_module._reload_prepared_evalset( + Path(prepared.input_snapshot.train_evalset_path), + label="train_evalset", + expected_sha256=prepared.input_snapshot.train_evalset_sha256, + ) + results: dict[str, list[EvalCaseResult]] = {} + for index, case in enumerate(eval_set.eval_cases): + score = None if index == 0 else 1.0 + run = _case_run( + run_id=1, + status=EvalStatus.PASSED, + score=score, + ).model_copy( + update={"eval_set_id": eval_set.eval_set_id, "eval_id": case.eval_id} + ) + results[case.eval_id] = [run] + + async def fake_evaluate_eval_set(*args, **kwargs): + return None, [], [], results + + monkeypatch.setattr( + pipeline_module.AgentEvaluator, + "evaluate_eval_set", + fake_evaluate_eval_set, + ) + + snapshot = await pipeline_module._evaluate_split( + prepared=prepared, + eval_set=eval_set, + call_agent=None, + phase="baseline", + split="train", + ) + standardized = standardize_snapshot(snapshot) + + assert snapshot.passed_case_count == len(eval_set.eval_cases) - 1 + assert snapshot.passed_case_count == standardized.passed_case_count + + def test_standardize_snapshot_rejects_duplicate_run_ids_and_metric_threshold_drift(): duplicate_runs = _snapshot( _case_run(run_id=1, status=EvalStatus.PASSED, score=1.0), diff --git a/tests/evaluation/test_eval_optimize_loop_stage4.py b/tests/evaluation/test_eval_optimize_loop_stage4.py index 281a4c03..8b2c84ab 100644 --- a/tests/evaluation/test_eval_optimize_loop_stage4.py +++ b/tests/evaluation/test_eval_optimize_loop_stage4.py @@ -16,6 +16,7 @@ import pytest from examples.optimization.eval_optimize_loop import candidate_provider as candidate_provider_module +from examples.optimization.eval_optimize_loop import pipeline as pipeline_module from examples.optimization.eval_optimize_loop import writeback as writeback_module from examples.optimization.eval_optimize_loop.candidate_provider import AgentOptimizerCandidateProvider from examples.optimization.eval_optimize_loop.candidate_provider import CandidateProviderError @@ -471,6 +472,41 @@ async def mutating_failure(**kwargs): assert await prepared.source_target.read_all() == source_before +@pytest.mark.asyncio +async def test_restore_working_baseline_falls_back_when_initial_read_fails( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + root = _copy_example(tmp_path, "restore_after_read_failure") + prepared = prepare_run(root / "pipeline.json", run_id="stage4_restore_read_failure") + baseline = await prepared.working_target.read_all() + changed = { + name: f"{content}\nleft by optimizer" + for name, content in baseline.items() + } + await prepared.working_target.write_all(changed) + original_read_all = prepared.working_target.read_all + read_count = 0 + + async def fail_initial_read(): + nonlocal read_count + read_count += 1 + if read_count == 1: + raise OSError("simulated transient read failure") + return await original_read_all() + + monkeypatch.setattr(prepared.working_target, "read_all", fail_initial_read) + + was_modified = await pipeline_module._restore_working_baseline( + prepared, + baseline, + ) + + assert was_modified is True + assert read_count == 2 + assert await original_read_all() == baseline + + @pytest.mark.asyncio async def test_real_candidate_provider_can_discard_native_artifacts_after_extraction( tmp_path: Path, diff --git a/tests/evaluation/test_eval_optimize_loop_stage5_artifacts.py b/tests/evaluation/test_eval_optimize_loop_stage5_artifacts.py index beb26990..d6bf7137 100644 --- a/tests/evaluation/test_eval_optimize_loop_stage5_artifacts.py +++ b/tests/evaluation/test_eval_optimize_loop_stage5_artifacts.py @@ -243,6 +243,24 @@ async def test_publish_rejects_literal_url_hidden_under_provider_specific_key( ) +@pytest.mark.asyncio +async def test_publish_rejects_plaintext_secret_in_optimizer_runtime_artifact(tmp_path): + prepared, report = await _build_fake_report(tmp_path, "artifact_runtime_secret") + run_dir = Path(prepared.workspace.run_dir) + runtime_path = run_dir / "optimizer.runtime.json" + secret = "runtime-authorization-secret" + runtime_path.write_text( + json.dumps({"reflection_lm": {"authorization": secret}}), + encoding="utf-8", + ) + + with pytest.raises(ArtifactWriteError, match="sensitive optimizer config"): + publish_report_bundle(report, run_dir=run_dir, copy_input_files=True) + + assert not (run_dir / "report").exists() + assert list(run_dir.glob(".report.tmp-*")) == [] + + @pytest.mark.asyncio async def test_publish_rejects_existing_report_directory(tmp_path): prepared, report = await _build_fake_report(tmp_path, "artifact_existing") diff --git a/tests/evaluation/test_eval_optimize_loop_stage5_pipeline.py b/tests/evaluation/test_eval_optimize_loop_stage5_pipeline.py index 121c41df..808efbbd 100644 --- a/tests/evaluation/test_eval_optimize_loop_stage5_pipeline.py +++ b/tests/evaluation/test_eval_optimize_loop_stage5_pipeline.py @@ -119,6 +119,18 @@ async def test_real_stage_reports_optimizer_native_artifacts( ) -> None: root = _copy_example(tmp_path) _set_real_mode(root) + pipeline_path = root / "pipeline.json" + pipeline_payload = json.loads(pipeline_path.read_text(encoding="utf-8")) + pipeline_payload["artifacts"]["copy_input_files"] = False + pipeline_path.write_text(json.dumps(pipeline_payload), encoding="utf-8") + optimizer_path = root / "optimizer.json" + optimizer_payload = json.loads(optimizer_path.read_text(encoding="utf-8")) + optimizer_payload["optimize"]["algorithm"]["reflection_lm"]["extra_fields"] = { + "authorization": "Bearer stage5-authorization-secret", + "token": "stage5-provider-token", + "client_secret": "stage5-client-secret", + } + optimizer_path.write_text(json.dumps(optimizer_payload), encoding="utf-8") prepared = prepare_run(root / "pipeline.json", run_id="stage5_real") baseline = await prepared.working_target.read_all() best = DeterministicFakeCandidateProvider().propose( @@ -167,6 +179,15 @@ async def fake_optimize(**kwargs: object) -> OptimizeResult: "optimizer/result.json", "optimizer/rounds/round_001.json", } + runtime_text = ( + Path(prepared.workspace.run_dir) / "optimizer.runtime.json" + ).read_text(encoding="utf-8") + for secret in ( + "stage5-authorization-secret", + "stage5-provider-token", + "stage5-client-secret", + ): + assert secret not in runtime_text assert report.candidate == result.candidate assert report.optimizer_resources.total_rounds.status == "available" assert report.optimizer_resources.reflection_lm_calls.status == "available"