diff --git a/.claude/commands/agent-runtime-kit/upgrade.md b/.claude/commands/agent-runtime-kit/upgrade.md index d19d38a..890c9a5 100644 --- a/.claude/commands/agent-runtime-kit/upgrade.md +++ b/.claude/commands/agent-runtime-kit/upgrade.md @@ -67,11 +67,15 @@ If unspecified, inspect all packages: ```bash gh auth status env -u UV_EXCLUDE_NEWER -u UV_EXCLUDE_NEWER_PACKAGE \ - uv run python -m examples.sdk_evolution_agent --help + uv run --locked python -m examples.sdk_evolution_agent --help ``` 5. Resolve the runtime that will run the AI-backed stages. Use `claude-agent-sdk` unless the user explicitly selected another runtime. + Change the runtime and uv extra together: + - `claude-agent-sdk` -> `--extra claude` + - `codex-agent-sdk` -> `--extra codex` + - `antigravity-agent-sdk` -> `--extra antigravity` 6. Verify provider auth through supported mechanisms only: - Claude: Anthropic API key, Claude Code auth, or Claude Code provider @@ -89,7 +93,7 @@ If unspecified, inspect all packages: ```bash env -u UV_EXCLUDE_NEWER -u UV_EXCLUDE_NEWER_PACKAGE \ - uv run --extra codex python -m examples.sdk_evolution_agent.auth ensure-codex + uv run --locked --extra codex python -m examples.sdk_evolution_agent.auth ensure-codex ``` The helper creates `~/.codex_agent_runtime_sdk`, removes uv freshness cutoff @@ -99,9 +103,9 @@ If unspecified, inspect all packages: and refresh the normal Codex login cache: ```bash - uv run --extra codex codex login --device-auth + uv run --locked --extra codex codex login --device-auth env -u UV_EXCLUDE_NEWER -u UV_EXCLUDE_NEWER_PACKAGE \ - uv run --extra codex python -m examples.sdk_evolution_agent.auth ensure-codex + uv run --locked --extra codex python -m examples.sdk_evolution_agent.auth ensure-codex ``` ## Report-Only Evidence Pass @@ -111,17 +115,22 @@ upstream SDK releases are the point of this workflow: ```bash env -u UV_EXCLUDE_NEWER -u UV_EXCLUDE_NEWER_PACKAGE \ - uv run python -m examples.sdk_evolution_agent \ + uv run --locked --extra claude python -m examples.sdk_evolution_agent \ --runtime claude-agent-sdk \ --refresh-preview \ + --inspect-candidates \ --package claude-agent-sdk \ --package openai-codex \ --package openai-codex-cli-bin \ --package google-antigravity ``` -If the user chose another runtime, replace only the `--runtime` value. Do not -add direct model calls. +`--inspect-candidates` is explicit consent to install and import a missing or +drifted locked baseline and resolver-selected candidates in credential-scrubbed +temporary environments. + +If the user chose another runtime, replace both the `--runtime` value and the +matching uv extra using the mapping above. Do not add direct model calls. Inspect the newest `reports/sdk-evolution//` directory and summarize: @@ -130,6 +139,7 @@ Inspect the newest `reports/sdk-evolution//` directory and summarize: - `api_diffs.json` - `behavior_probes.json` - `behavior_diffs.json` +- `behavior_summary.json` - `current_state.json` - `direction_analysis.json` - `architecture_decision.json` @@ -140,7 +150,8 @@ Stop before implementation if any of these are true: - required candidate API diffs are missing, - required release-note evidence could not be collected, -- `behavior_diffs.json` contains breaking adapter-contract drift, +- `behavior_summary.json` is missing, malformed, has an unknown status, or + reports `fail` / `incomplete`, - `architecture_decision.json` has `manual_design_required: true`, - the reviewer rejects the evidence or design, - recursive self-adaptation is required and the report does not include a safe @@ -148,6 +159,11 @@ Stop before implementation if any of these are true: adapters, output schemas, event sinks, permission profiles, or `AgentResult`. +`pass` means complete unchanged evidence; `changed` means complete +non-breaking evidence; `incomplete` means required observations could not be +proved; and `fail` means a required contract failed or a breaking diff was +observed. + ## Implementation Pass Only run implementation when the report-only pass supports it and the user wants @@ -157,9 +173,10 @@ an upgrade branch or PR: BRANCH="sdk-evolution-upgrade-$(date +%Y%m%d-%H%M%S)" env -u UV_EXCLUDE_NEWER -u UV_EXCLUDE_NEWER_PACKAGE \ - uv run python -m examples.sdk_evolution_agent \ + uv run --locked --extra claude python -m examples.sdk_evolution_agent \ --runtime claude-agent-sdk \ --refresh-preview \ + --inspect-candidates \ --implementation-enabled \ --create-branch \ --branch-name "$BRANCH" \ @@ -173,7 +190,8 @@ env -u UV_EXCLUDE_NEWER -u UV_EXCLUDE_NEWER_PACKAGE \ --package google-antigravity ``` -If the user chose another runtime, replace only the `--runtime` value. +If the user chose another runtime, replace both the `--runtime` value and the +matching uv extra using the mapping above. ## Verification @@ -181,12 +199,13 @@ After implementation, run or verify: ```bash env -u UV_EXCLUDE_NEWER -u UV_EXCLUDE_NEWER_PACKAGE uv lock --check -env -u UV_EXCLUDE_NEWER -u UV_EXCLUDE_NEWER_PACKAGE uv run ruff check . -env -u UV_EXCLUDE_NEWER -u UV_EXCLUDE_NEWER_PACKAGE uv run mypy -env -u UV_EXCLUDE_NEWER -u UV_EXCLUDE_NEWER_PACKAGE uv run pytest +env -u UV_EXCLUDE_NEWER -u UV_EXCLUDE_NEWER_PACKAGE uv run --locked ruff check . +env -u UV_EXCLUDE_NEWER -u UV_EXCLUDE_NEWER_PACKAGE uv run --locked mypy +env -u UV_EXCLUDE_NEWER -u UV_EXCLUDE_NEWER_PACKAGE uv run --locked pytest ``` If a draft PR was created, watch CI until it finishes or clearly report that it is still running. Include the PR URL, report path, changed SDK versions, -architecture decision, reviewer result, test results, uncertainty, and manual -review checklist in the final response. +`behavior_summary.json` status and reasons, architecture decision, reviewer +result, test results, uncertainty, and manual review checklist in the final +response. diff --git a/.codex/skills/agent-runtime-kit-upgrade/SKILL.md b/.codex/skills/agent-runtime-kit-upgrade/SKILL.md index fc52cc8..ad6f43b 100644 --- a/.codex/skills/agent-runtime-kit-upgrade/SKILL.md +++ b/.codex/skills/agent-runtime-kit-upgrade/SKILL.md @@ -39,11 +39,15 @@ Default runtime for this Codex skill: `codex-agent-sdk`. ```bash gh auth status env -u UV_EXCLUDE_NEWER -u UV_EXCLUDE_NEWER_PACKAGE \ - uv run python -m examples.sdk_evolution_agent --help + uv run --locked python -m examples.sdk_evolution_agent --help ``` 4. Resolve the runtime that will run the AI-backed stages. Use `codex-agent-sdk` unless the user explicitly selected another runtime. + Change the runtime and uv extra together: + - `claude-agent-sdk` -> `--extra claude` + - `codex-agent-sdk` -> `--extra codex` + - `antigravity-agent-sdk` -> `--extra antigravity` 5. Use supported provider auth only: - Claude: Anthropic API key, Claude Code auth, or Claude Code provider @@ -61,7 +65,7 @@ Default runtime for this Codex skill: `codex-agent-sdk`. ```bash env -u UV_EXCLUDE_NEWER -u UV_EXCLUDE_NEWER_PACKAGE \ - uv run --extra codex python -m examples.sdk_evolution_agent.auth ensure-codex + uv run --locked --extra codex python -m examples.sdk_evolution_agent.auth ensure-codex ``` The helper creates `~/.codex_agent_runtime_sdk`, removes uv freshness cutoff @@ -71,9 +75,9 @@ Default runtime for this Codex skill: `codex-agent-sdk`. and refresh the normal Codex login cache: ```bash - uv run --extra codex codex login --device-auth + uv run --locked --extra codex codex login --device-auth env -u UV_EXCLUDE_NEWER -u UV_EXCLUDE_NEWER_PACKAGE \ - uv run --extra codex python -m examples.sdk_evolution_agent.auth ensure-codex + uv run --locked --extra codex python -m examples.sdk_evolution_agent.auth ensure-codex ``` ## Report-Only First @@ -83,17 +87,23 @@ freshness cutoffs: ```bash env -u UV_EXCLUDE_NEWER -u UV_EXCLUDE_NEWER_PACKAGE \ - uv run python -m examples.sdk_evolution_agent \ + uv run --locked --extra codex python -m examples.sdk_evolution_agent \ --runtime codex-agent-sdk \ --refresh-preview \ + --inspect-candidates \ --package claude-agent-sdk \ --package openai-codex \ --package openai-codex-cli-bin \ --package google-antigravity ``` -If the user explicitly chooses another runtime, replace only the `--runtime` -value. Codex-backed runs should use the runner's built-in `gpt-5.5` and +`--inspect-candidates` is explicit consent to install and import a missing or +drifted locked baseline and resolver-selected candidates in credential-scrubbed +temporary environments. + +If the user explicitly chooses another runtime, replace both the `--runtime` +value and the matching uv extra using the mapping above. Codex-backed runs +should use the runner's built-in `gpt-5.5` and `reasoning_effort=xhigh` policy; do not implement model selection outside the runner. @@ -104,6 +114,7 @@ Inspect the newest `reports/sdk-evolution//` directory: - `api_diffs.json` - `behavior_probes.json` - `behavior_diffs.json` +- `behavior_summary.json` - `current_state.json` - `direction_analysis.json` - `architecture_decision.json` @@ -111,9 +122,13 @@ Inspect the newest `reports/sdk-evolution//` directory: - `report.md` Stop before implementation when candidate API diffs are missing, required -release-note evidence is missing, behavior probes show breaking adapter-contract -drift, `manual_design_required` is true, the reviewer rejects the evidence or -design, or recursive self-adaptation lacks a safe migration plan. +release-note evidence is missing, `behavior_summary.json` is missing, malformed, +has an unknown status, or reports `fail` / `incomplete`, +`manual_design_required` is true, the reviewer rejects the evidence or design, +or recursive self-adaptation lacks a safe migration plan. `pass` means complete +unchanged evidence; `changed` means complete non-breaking evidence; +`incomplete` means required observations could not be proved; and `fail` means +a required contract failed or a breaking diff was observed. Recursive self-adaptation means the upgrade affects the runner's own use of `AgentTask`, `RuntimeRegistry`, adapters, output schemas, event sinks, @@ -130,9 +145,10 @@ an upgrade branch or PR: BRANCH="sdk-evolution-upgrade-$(date +%Y%m%d-%H%M%S)" env -u UV_EXCLUDE_NEWER -u UV_EXCLUDE_NEWER_PACKAGE \ - uv run python -m examples.sdk_evolution_agent \ + uv run --locked --extra codex python -m examples.sdk_evolution_agent \ --runtime codex-agent-sdk \ --refresh-preview \ + --inspect-candidates \ --implementation-enabled \ --create-branch \ --branch-name "$BRANCH" \ @@ -152,12 +168,12 @@ Run or verify: ```bash env -u UV_EXCLUDE_NEWER -u UV_EXCLUDE_NEWER_PACKAGE uv lock --check -env -u UV_EXCLUDE_NEWER -u UV_EXCLUDE_NEWER_PACKAGE uv run ruff check . -env -u UV_EXCLUDE_NEWER -u UV_EXCLUDE_NEWER_PACKAGE uv run mypy -env -u UV_EXCLUDE_NEWER -u UV_EXCLUDE_NEWER_PACKAGE uv run pytest +env -u UV_EXCLUDE_NEWER -u UV_EXCLUDE_NEWER_PACKAGE uv run --locked ruff check . +env -u UV_EXCLUDE_NEWER -u UV_EXCLUDE_NEWER_PACKAGE uv run --locked mypy +env -u UV_EXCLUDE_NEWER -u UV_EXCLUDE_NEWER_PACKAGE uv run --locked pytest ``` If a draft PR was created, watch CI until it finishes or clearly report that it is still running. Final output should include the PR URL, report path, changed -SDK versions, architecture decision, reviewer result, test results, uncertainty, -and manual review checklist. +SDK versions, `behavior_summary.json` status and reasons, architecture decision, +reviewer result, test results, uncertainty, and manual review checklist. diff --git a/docs/sdk-evolution-agent-design.md b/docs/sdk-evolution-agent-design.md index 6ad1166..b5f20d9 100644 --- a/docs/sdk-evolution-agent-design.md +++ b/docs/sdk-evolution-agent-design.md @@ -150,27 +150,34 @@ evidence, but they should not invent evidence that was not collected. The default command should be report-only: ```bash -python -m examples.sdk_evolution_agent --runtime fake --refresh-preview +uv run --locked python -m examples.sdk_evolution_agent \ + --runtime fake \ + --refresh-preview ``` This mode collects evidence, writes artifacts, runs the analysis stages through -the selected runtime, and stops before editing the workspace. The fake runtime is -allowed only as a deterministic development harness. It proves the pipeline and -schemas, not the quality of AI reasoning. +the selected runtime, and stops before editing the workspace. The fake runtime +is allowed only as a deterministic pipeline-shape harness. It proves the +pipeline and schemas, not upgrade safety or the quality of AI reasoning. When +updates exist, the default no-inspection run records incomplete candidate +evidence. A real analysis run should select one configured runtime: ```bash -python -m examples.sdk_evolution_agent --runtime claude-agent-sdk --refresh-preview -python -m examples.sdk_evolution_agent --runtime codex-agent-sdk --refresh-preview -python -m examples.sdk_evolution_agent --runtime antigravity-agent-sdk --refresh-preview +uv run --locked --extra claude python -m examples.sdk_evolution_agent \ + --runtime claude-agent-sdk --refresh-preview --inspect-candidates +uv run --locked --extra codex python -m examples.sdk_evolution_agent \ + --runtime codex-agent-sdk --refresh-preview --inspect-candidates +uv run --locked --extra antigravity python -m examples.sdk_evolution_agent \ + --runtime antigravity-agent-sdk --refresh-preview --inspect-candidates ``` Before a Codex-backed run, prepare the dedicated SDK evolution auth home: ```bash env -u UV_EXCLUDE_NEWER -u UV_EXCLUDE_NEWER_PACKAGE \ - uv run --extra codex python -m examples.sdk_evolution_agent.auth ensure-codex + uv run --locked --extra codex python -m examples.sdk_evolution_agent.auth ensure-codex ``` This helper owns the operator-readiness boundary for Codex-backed runs. It @@ -188,9 +195,10 @@ Package filters narrow evidence collection for debugging, but normal evolution runs should inspect all tracked packages: ```bash -python -m examples.sdk_evolution_agent \ +uv run --locked --extra antigravity python -m examples.sdk_evolution_agent \ --runtime antigravity-agent-sdk \ --refresh-preview \ + --inspect-candidates \ --package claude-agent-sdk \ --package openai-codex \ --package openai-codex-cli-bin \ @@ -201,15 +209,18 @@ python -m examples.sdk_evolution_agent \ since update candidates without candidate API snapshots are not actionable. The shipped behavior deliberately inverts that default: candidate inspection installs and imports freshly downloaded upstream code, so it is opt-in and -runs in a credential-scrubbed environment, with explicit `skip` records when -it is off. +runs in a credential-scrubbed environment. The same explicit flag is required +to install a locked baseline that is missing or drifted in the active +environment. When inspection is off, those gaps become explicit `skip` records +rather than claimed observations. Implementation mode should remain explicitly gated: ```bash -python -m examples.sdk_evolution_agent \ +uv run --locked --extra antigravity python -m examples.sdk_evolution_agent \ --runtime antigravity-agent-sdk \ --refresh-preview \ + --inspect-candidates \ --implementation-enabled ``` @@ -347,6 +358,7 @@ Behavior probe output should be a first-class report artifact, for example: ```text behavior_probes.json behavior_diffs.json +behavior_summary.json ``` Each probe result should include: @@ -358,15 +370,21 @@ Each probe result should include: - stdout/stderr summary, - skipped reason when optional credentials are missing. -`behavior_diffs.json` compares current-environment probes against -candidate-version probes for resolver-selected updates. Breaking candidate probe -changes block implementation deterministically before any local lock update. +`behavior_diffs.json` compares observed locked-baseline probes against +candidate-version probes for resolver-selected updates. Breaking candidate +probe changes block implementation deterministically before any local lock +update. `behavior_probes.json` may include observed SDK fields or parameters that are not part of the adapter contract. `behavior_diffs.json` compares the required adapter contract, not every optional field. Public API and signature churn remains visible in `api_diffs.json` and probe details, but it should only block implementation when the required behavior contract fails or becomes ambiguous. +`behavior_summary.json` is the deterministic hand-off: `pass` means complete +unchanged evidence, `changed` means complete non-breaking evidence, +`incomplete` means required observations could not be proved, and `fail` means +a required contract failed or a breaking diff was observed. Missing, malformed, +or unknown summary states also block implementation. ### 5. Runtime-Generated Analysis @@ -518,6 +536,7 @@ api_snapshots/ api_diffs.json behavior_probes.json behavior_diffs.json +behavior_summary.json current_state.json direction_analysis.json architecture_decision.json @@ -552,6 +571,7 @@ artifact-aware. It should record: - paths or content hashes for current API snapshots, - paths or content hashes for release-note evidence, - paths or content hashes for behavior probe results, +- a path or content hash for the deterministic behavior summary, - whether the baseline was promoted, refreshed, skipped, or blocked. Promotion rules should be conservative: @@ -631,13 +651,16 @@ The example implements the deterministic evidence artifacts described above: - `behavior_probes.json` records current and candidate adapter-contract probes. - `behavior_diffs.json` records behavior differences between current and candidate probes. +- `behavior_summary.json` records the recomputed deterministic status, counts, + and reasons handed to implementation guards and operators. - `current_state.json` records the run baseline, lockfile hash, accepted package versions, artifact hashes, and promotion status. The implementation path is gated by deterministic checks before the local lockfile update runs. Missing candidate API diffs, unavailable required -release-note evidence, breaking behavior diffs, reviewer rejection, -`manual_design_required`, and unresolved recursive self-adaptation all block -implementation. When implementation is allowed, the example applies the -resolver-selected SDK lock update locally, runs verification, writes the report -artifacts, commits them, pushes the branch, and opens a draft PR when configured. +release-note evidence, behavior summaries that are failed, incomplete, missing, +malformed, or unknown, reviewer rejection, `manual_design_required`, and +unresolved recursive self-adaptation all block implementation. When +implementation is allowed, the example applies the resolver-selected SDK lock +update locally, runs verification, writes the report artifacts, commits them, +pushes the branch, and opens a draft PR when configured. diff --git a/docs/sdk-evolution-agent.md b/docs/sdk-evolution-agent.md index 03b43c1..8d2c8bf 100644 --- a/docs/sdk-evolution-agent.md +++ b/docs/sdk-evolution-agent.md @@ -11,16 +11,22 @@ changelog strategy, caveats, and alternatives, see Run it from the repository: ```bash -python -m examples.sdk_evolution_agent --runtime fake +uv run --locked python -m examples.sdk_evolution_agent --runtime fake ``` The `fake` runtime is deterministic and useful for checking the local pipeline -without credentials. For real AI reasoning, select a configured runtime: +without credentials. It proves pipeline shape only, never upgrade safety. When +updates exist, its default no-inspection run records incomplete candidate +evidence. For real AI reasoning, select a configured runtime and its matching +uv extra: ```bash -python -m examples.sdk_evolution_agent --runtime claude-agent-sdk -python -m examples.sdk_evolution_agent --runtime codex-agent-sdk -python -m examples.sdk_evolution_agent --runtime antigravity-agent-sdk +uv run --locked --extra claude python -m examples.sdk_evolution_agent \ + --runtime claude-agent-sdk --refresh-preview --inspect-candidates +uv run --locked --extra codex python -m examples.sdk_evolution_agent \ + --runtime codex-agent-sdk --refresh-preview --inspect-candidates +uv run --locked --extra antigravity python -m examples.sdk_evolution_agent \ + --runtime antigravity-agent-sdk --refresh-preview --inspect-candidates ``` Every AI-backed stage is dispatched as an `AgentTask` through a runtime resolved @@ -37,7 +43,7 @@ Run the auth preflight before real Codex-backed SDK evolution runs: ```bash env -u UV_EXCLUDE_NEWER -u UV_EXCLUDE_NEWER_PACKAGE \ - uv run --extra codex python -m examples.sdk_evolution_agent.auth ensure-codex + uv run --locked --extra codex python -m examples.sdk_evolution_agent.auth ensure-codex ``` The helper checks `codex login status` against the dedicated home and can refresh @@ -46,9 +52,9 @@ the helper reports that the isolated home is not authenticated, refresh the normal Codex login cache and rerun the helper: ```bash -uv run --extra codex codex login --device-auth +uv run --locked --extra codex codex login --device-auth env -u UV_EXCLUDE_NEWER -u UV_EXCLUDE_NEWER_PACKAGE \ - uv run --extra codex python -m examples.sdk_evolution_agent.auth ensure-codex + uv run --locked --extra codex python -m examples.sdk_evolution_agent.auth ensure-codex ``` Codex-backed SDK evolution runs explicitly choose `gpt-5.5` with @@ -113,16 +119,18 @@ cutoff variables must not hide candidate releases. ## Candidate API Inspection -The command treats `uv.lock` as the current baseline. If the active `.venv` -contains a different installed version, the agent inspects the locked baseline -in a temporary isolated virtualenv instead of trusting the drifted environment. +The command treats `uv.lock` as the current baseline. If the locked SDK is +missing from the active `.venv` or the installed version differs, the agent can +inspect the locked baseline in a temporary isolated virtualenv instead of +trusting the missing or drifted environment. When a refresh preview is available, package update candidates come from the resolver's `uv lock --dry-run -P ...` output, not only from PyPI's `latest` -metadata. With `--inspect-candidates`, the agent installs each resolver update -candidate in a temporary isolated virtualenv — with a credential-scrubbed -environment (throwaway `HOME`, `PATH` only) — and writes an API snapshot plus -`api_diffs.json` entry, and runs the behavior probes against the candidate the -same way. This avoids false downgrade diffs for packages whose locked +metadata. With `--inspect-candidates`, the agent installs each missing or +drifted locked baseline and each resolver update candidate in a temporary +isolated virtualenv — with a credential-scrubbed environment (throwaway `HOME`, +`PATH` only) — and writes API snapshots plus an `api_diffs.json` entry, and runs +the behavior probes against the candidate the same way. This avoids false +downgrade diffs for packages whose locked prerelease is newer than PyPI's stable latest field. Candidate inspection is opt-in because it executes freshly downloaded upstream code; without the flag, candidates are recorded as explicit `skip` entries rather than silently @@ -163,7 +171,11 @@ a stale or contradictory cached summary is never presented as the run result. Report-only mode is the default. To allow the implementation stage, pass: ```bash -python -m examples.sdk_evolution_agent --runtime claude-agent-sdk --implementation-enabled +uv run --locked --extra claude python -m examples.sdk_evolution_agent \ + --runtime claude-agent-sdk \ + --refresh-preview \ + --inspect-candidates \ + --implementation-enabled ``` Implementation is still blocked when: @@ -172,8 +184,9 @@ Implementation is still blocked when: - the reviewer rejects the evidence or design, - a resolver-selected update lacks a candidate API diff, - required release-note evidence could not be collected, -- behavior evidence is failed, incomplete, malformed, internally inconsistent, - or uses the wrong package/version transition, +- `behavior_summary.json` is missing, malformed, has an unknown status, reports + `fail` / `incomplete`, is internally inconsistent, or uses the wrong + package/version transition, - required structured output or permission behavior is unsupported by the selected runtime, - recursive self-adaptation is required but no safe migration plan exists. @@ -190,8 +203,10 @@ design review. Draft PR creation is opt-in: ```bash -python -m examples.sdk_evolution_agent \ +uv run --locked --extra claude python -m examples.sdk_evolution_agent \ --runtime claude-agent-sdk \ + --refresh-preview \ + --inspect-candidates \ --implementation-enabled \ --create-branch \ --branch-name sdk-evolution-update \ diff --git a/tests/test_sdk_evolution_docs.py b/tests/test_sdk_evolution_docs.py new file mode 100644 index 0000000..e3c7ab8 --- /dev/null +++ b/tests/test_sdk_evolution_docs.py @@ -0,0 +1,353 @@ +from __future__ import annotations + +import shlex +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +CODEX_RUNBOOK = Path(".codex/skills/agent-runtime-kit-upgrade/SKILL.md") +CLAUDE_RUNBOOK = Path(".claude/commands/agent-runtime-kit/upgrade.md") +PUBLIC_GUIDE = Path("docs/sdk-evolution-agent.md") +DESIGN_GUIDE = Path("docs/sdk-evolution-agent-design.md") +AUTHORITATIVE_DOCS = (CODEX_RUNBOOK, CLAUDE_RUNBOOK, PUBLIC_GUIDE, DESIGN_GUIDE) +RUNBOOKS = (CODEX_RUNBOOK, CLAUDE_RUNBOOK) +PUBLIC_DOCS = (PUBLIC_GUIDE, DESIGN_GUIDE) +RUNTIME_EXTRAS = { + "claude-agent-sdk": "claude", + "codex-agent-sdk": "codex", + "antigravity-agent-sdk": "antigravity", +} +AGENT_MODULE = ("python", "-m", "examples.sdk_evolution_agent") +AUTH_MODULE = ("python", "-m", "examples.sdk_evolution_agent.auth") + + +def test_sdk_evolution_bash_commands_use_locked_uv() -> None: + for path in AUTHORITATIVE_DOCS: + for command in _bash_commands(path): + _assert_locked_command(path, command) + + +def test_real_sdk_evolution_commands_match_runtime_extras() -> None: + for path in AUTHORITATIVE_DOCS: + for command in _agent_commands(path): + _assert_real_runtime_command(path, command) + + +def test_public_guides_cover_every_runtime_extra_mapping() -> None: + for path in PUBLIC_DOCS: + commands = _agent_commands(path) + observed = { + runtime + for command in commands + if (runtime := _option_value(command, "--runtime")) in RUNTIME_EXTRAS + } + assert observed == set(RUNTIME_EXTRAS), f"incomplete runtime mapping in {path}" + + for path in RUNBOOKS: + text = _read(path) + for runtime, extra in RUNTIME_EXTRAS.items(): + assert f"`{runtime}` -> `--extra {extra}`" in text + + +def test_runbooks_include_report_and_implementation_commands() -> None: + defaults = { + CODEX_RUNBOOK: "codex-agent-sdk", + CLAUDE_RUNBOOK: "claude-agent-sdk", + } + for path, runtime in defaults.items(): + commands = [ + command + for command in _agent_commands(path) + if _option_value(command, "--runtime") == runtime + ] + assert any("--implementation-enabled" not in command for command in commands) + assert any("--implementation-enabled" in command for command in commands) + + +def test_fake_commands_are_pipeline_shape_checks_only() -> None: + for path in PUBLIC_DOCS: + commands = [ + command + for command in _agent_commands(path) + if _option_value(command, "--runtime") == "fake" + ] + assert commands, f"missing fake pipeline-shape command in {path}" + for command in commands: + assert "--inspect-candidates" not in command + assert "--implementation-enabled" not in command + assert "--draft-pr" not in command + prose = _read(path).lower().replace("-", " ") + assert "pipeline shape" in prose + assert "never upgrade safety" in prose or "not upgrade safety" in prose + + +def test_candidate_inspection_remains_explicit_opt_in() -> None: + for path in AUTHORITATIVE_DOCS: + prose = " ".join(_read(path).lower().split()) + assert "--inspect-candidates" in prose + assert _has_candidate_opt_in(prose) + + +def test_behavior_summary_is_in_artifacts_and_runbook_handoffs() -> None: + for path in AUTHORITATIVE_DOCS: + text = _read(path) + assert "behavior_summary.json" in text + for status in ("pass", "changed", "incomplete", "fail"): + assert f"`{status}`" in text + + for path in RUNBOOKS: + text = _read(path) + normalized = " ".join(text.split()) + assert "`behavior_summary.json` status and reasons" in text + assert "a missing or drifted locked baseline" in normalized + assert "credential-scrubbed" in text + for blocker in ("missing", "malformed", "unknown status", "`fail`", "`incomplete`"): + assert blocker in text + + +def test_codex_auth_commands_use_the_locked_codex_extra() -> None: + for path in AUTHORITATIVE_DOCS: + for command in _bash_commands(path): + payload = _command_payload(command) + if _starts_with(payload, AUTH_MODULE) or _starts_with(payload, ("codex", "login")): + assert _starts_with( + _command_argv(command), + ("uv", "run", "--locked", "--extra", "codex"), + ) + + +@pytest.mark.parametrize( + "body", + ( + "uv run --locked python -m examples.sdk_evolution_agent " + "--runtime=codex-agent-sdk", + "COMPLIANT='uv run --locked --extra codex python -m " + "examples.sdk_evolution_agent' python -m examples.sdk_evolution_agent " + "--runtime codex-agent-sdk --refresh-preview --inspect-candidates", + "uv run --locked python -m examples.sdk_evolution_agent --help && " + "python -m examples.sdk_evolution_agent --runtime codex-agent-sdk " + "--refresh-preview --inspect-candidates", + ), +) +def test_noncompliant_shell_forms_cannot_evade_locked_command_checks( + tmp_path: Path, body: str +) -> None: + path = tmp_path / "commands.md" + path.write_text(f"```bash\n{body}\n```\n", encoding="utf-8") + commands = _bash_commands(path) + with pytest.raises(AssertionError): + for command in commands: + _assert_locked_command(path, command) + _assert_real_runtime_command(path, command) + + +def test_runtime_equals_form_cannot_evade_runtime_extra_checks(tmp_path: Path) -> None: + path = tmp_path / "commands.md" + path.write_text( + "```bash\n" + "uv run --locked python -m examples.sdk_evolution_agent " + "--runtime=codex-agent-sdk\n" + "```\n", + encoding="utf-8", + ) + command = _agent_commands(path)[0] + assert _option_value(command, "--runtime") == "codex-agent-sdk" + with pytest.raises(AssertionError, match="runtime/extra mismatch"): + _assert_real_runtime_command(path, command) + + +def test_echoes_and_comments_do_not_count_as_executable_commands(tmp_path: Path) -> None: + path = tmp_path / "commands.md" + path.write_text( + "```bash\n" + "echo uv run --locked --extra codex python -m " + "examples.sdk_evolution_agent --runtime codex-agent-sdk " + "--refresh-preview --inspect-candidates\n" + "# uv run --locked --extra codex python -m " + "examples.sdk_evolution_agent --runtime codex-agent-sdk " + "--refresh-preview --inspect-candidates\n" + "```\n", + encoding="utf-8", + ) + assert _agent_commands(path) == [] + + +def test_unrelated_opt_in_prose_does_not_satisfy_candidate_consent() -> None: + prose = "--inspect-candidates is automatic. draft pr creation is opt-in." + assert not _has_candidate_opt_in(prose) + + +def _read(path: Path) -> str: + return (ROOT / path).read_text(encoding="utf-8") + + +def _agent_commands(path: Path) -> list[list[str]]: + return [ + command + for command in _bash_commands(path) + if _starts_with(_command_payload(command), AGENT_MODULE) + ] + + +def _bash_commands(path: Path) -> list[list[str]]: + commands: list[list[str]] = [] + pending: list[str] = [] + in_bash = False + for raw_line in _read(path).splitlines(): + stripped = raw_line.strip() + if stripped == "```bash": + in_bash = True + pending = [] + continue + if in_bash and stripped == "```": + if pending: + commands.extend(_split_shell_commands(" ".join(pending))) + in_bash = False + pending = [] + continue + if not in_bash: + continue + if not stripped: + if pending: + commands.extend(_split_shell_commands(" ".join(pending))) + pending = [] + continue + continued = stripped.endswith("\\") + pending.append(stripped[:-1].rstrip() if continued else stripped) + if not continued: + commands.extend(_split_shell_commands(" ".join(pending))) + pending = [] + return commands + + +def _split_shell_commands(command: str) -> list[list[str]]: + lexer = shlex.shlex(command, posix=True, punctuation_chars=";&|") + lexer.whitespace_split = True + lexer.commenters = "#" + commands: list[list[str]] = [] + pending: list[str] = [] + for token in lexer: + if token and set(token) <= {";", "&", "|"}: + if pending: + commands.append(pending) + pending = [] + continue + pending.append(token) + if pending: + commands.append(pending) + return commands + + +def _option_value(command: list[str], option: str) -> str | None: + for index, part in enumerate(command): + if part == option: + return command[index + 1] if index + 1 < len(command) else None + prefix = f"{option}=" + if part.startswith(prefix): + return part.removeprefix(prefix) or None + return None + + +def _assert_locked_command(path: Path, command: list[str]) -> None: + argv = _command_argv(command) + if _starts_with(argv, ("uv", "run")): + assert _starts_with(argv, ("uv", "run", "--locked")), ( + f"unlocked uv command in {path}: {_display(command)}" + ) + payload = _command_payload(command) + if _starts_with(payload, AGENT_MODULE): + assert _starts_with(argv, ("uv", "run", "--locked")), ( + f"bare actionable SDK evolution command in {path}: {_display(command)}" + ) + + +def _assert_real_runtime_command(path: Path, command: list[str]) -> None: + runtime = _option_value(command, "--runtime") + if runtime is None or runtime == "fake" or "--help" in command: + return + assert runtime in RUNTIME_EXTRAS, f"unknown runtime in {path}: {_display(command)}" + extra = RUNTIME_EXTRAS[runtime] + argv = _command_argv(command) + assert _starts_with(argv, ("uv", "run", "--locked", "--extra", extra)), ( + f"runtime/extra mismatch in {path}: {_display(command)}" + ) + assert _starts_with(_command_payload(command), AGENT_MODULE), ( + f"runtime command is not the SDK evolution entrypoint in {path}: {_display(command)}" + ) + assert "--refresh-preview" in command, ( + f"missing refresh in {path}: {_display(command)}" + ) + assert "--inspect-candidates" in command, ( + f"missing candidate inspection in {path}: {_display(command)}" + ) + + +def _command_argv(command: list[str]) -> list[str]: + index = 0 + while index < len(command) and _is_assignment(command[index]): + index += 1 + if index >= len(command) or command[index] != "env": + return command[index:] + + index += 1 + while index < len(command): + token = command[index] + if _is_assignment(token): + index += 1 + elif token in {"-u", "--unset"}: + index += 2 + elif token.startswith("--unset="): + index += 1 + elif token == "--": + index += 1 + break + else: + break + return command[index:] + + +def _command_payload(command: list[str]) -> list[str]: + argv = _command_argv(command) + if not _starts_with(argv, ("uv", "run")): + return argv + + index = 2 + while index < len(argv): + token = argv[index] + if token == "--": + return argv[index + 1 :] + if token == "--extra": + index += 2 + continue + if token == "--locked" or token.startswith("--extra="): + index += 1 + continue + break + return argv[index:] + + +def _is_assignment(token: str) -> bool: + name, separator, _value = token.partition("=") + return bool(separator and name and name.replace("_", "a").isalnum() and not name[0].isdigit()) + + +def _has_candidate_opt_in(prose: str) -> bool: + normalized = prose.replace("opt in", "opt-in") + sentences = normalized.split(".") + return ( + "`--inspect-candidates` is explicit consent" in normalized + or "candidate inspection is opt-in" in normalized + or any( + "candidate inspection" in sentence and "so it is opt-in" in sentence + for sentence in sentences + ) + ) + + +def _starts_with(command: list[str], sequence: tuple[str, ...]) -> bool: + return tuple(command[: len(sequence)]) == sequence + + +def _display(command: list[str]) -> str: + return shlex.join(command)