From f6a93cc1aef85d89eeb1f8f6d942d8f09a6d4e32 Mon Sep 17 00:00:00 2001 From: Chris Huber Date: Thu, 16 Jul 2026 01:02:04 -0400 Subject: [PATCH] Use WP Codebox v0.12.21 reviewer projection --- .github/workflows/README.md | 4 ++-- .github/workflows/maintain-docs.yml | 4 ++-- .github/workflows/validate.yml | 2 +- README.md | 4 ++-- tests/validate-docs-agent-packages.php | 7 ++++--- ...alidate-wp-codebox-run-agent-task-contract.php | 15 +++++++++++++-- tests/wp-codebox-release.fixture.json | 7 ++++--- 7 files changed, 28 insertions(+), 15 deletions(-) diff --git a/.github/workflows/README.md b/.github/workflows/README.md index ba8e6a5..3e264a3 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -14,9 +14,9 @@ The reusable workflow declares the expected typed review artifacts for Docs Agen The target repository grants `contents: write`, `pull-requests: write`, and `issues: write`. Docs Agent forwards the caller-scoped `${{ github.token }}` to WP Codebox for same-repository publication, so consumers do not configure `ACCESS_TOKEN`. `OPENAI_API_KEY` is an optional workflow secret and is required only for a live OpenAI run; skipped and dry-run calls do not require it. `EXTERNAL_PACKAGE_SOURCE_POLICY` remains a separate required v1 JSON secret. Migrate its value to the exact one-line JSON in the root README: it authorizes the selected Docs Agent package, the pinned Agents API component, the pinned PHP AI Client overlay, and the checksum-pinned OpenAI provider artifact. Both secrets are forwarded to WP Codebox without serialization into the task descriptor. -Docs Agent consumes the released [WP Codebox v0.12.20](https://github.com/Automattic/wp-codebox/releases/tag/v0.12.20) workflow and passes the matching `wp_codebox_release_ref: v0.12.20` input. WP Codebox validates the paired tags, materializes the declared native runtime closure, executes the selected package, applies bounded workspace changes, runs verification, and publishes approved host changes. +Docs Agent consumes the released [WP Codebox v0.12.21](https://github.com/Automattic/wp-codebox/releases/tag/v0.12.21) workflow and passes the matching `wp_codebox_release_ref: v0.12.21` input. WP Codebox validates the paired tags, materializes the declared native runtime closure, executes the selected package, applies bounded workspace changes, runs verification, and publishes approved host changes. -Provider artifacts retain only allowlisted review artifacts, controlled workflow envelopes, and canonical runtime-source and seed provenance. WP Codebox captures one trusted canonical `codebox-transcript` through a pre-sanitization reviewer-evidence descriptor, validates its artifact-root containment, schema, digest, and size, then emits the compact reviewer transcript as a `wp-codebox/reviewer-agent-transcript/v1` projection. Before persistence or upload, it sanitizes private runtime paths from nested values, object keys, diagnostics, and command arguments. v0.12.20 adds sanitized canonical Agents API tool-lifecycle evidence while excluding raw stdout, tool arguments, result bodies, raw errors, source trees, secrets, package payloads, and private paths. Lifecycle and downstream failures retain runtime evidence in their normalized failed result even when artifact preparation cannot complete. Diagnostic messages that name runtime classes remain reviewable, while PHP-shaped runtime source remains blocked. See [WP Codebox #1767](https://github.com/Automattic/wp-codebox/issues/1767), [WP Codebox #1786](https://github.com/Automattic/wp-codebox/issues/1786), [WP Codebox #1788](https://github.com/Automattic/wp-codebox/issues/1788), [WP Codebox #1790](https://github.com/Automattic/wp-codebox/pull/1790), [WP Codebox #1791](https://github.com/Automattic/wp-codebox/issues/1791), [WP Codebox #1795](https://github.com/Automattic/wp-codebox/issues/1795), [WP Codebox #1809](https://github.com/Automattic/wp-codebox/pull/1809), and [WP Codebox #1811](https://github.com/Automattic/wp-codebox/pull/1811). Regression reference: [hosted canonical reviewer-evidence run `29350690551`](https://github.com/Automattic/build-with-wordpress/actions/runs/29350690551). +Provider artifacts retain only allowlisted review artifacts, controlled workflow envelopes, and canonical runtime-source and seed provenance. WP Codebox v0.12.21 uploads a reviewer-safe workflow-result projection: public control, verification, publication, output-projection, access, failure, artifact-declaration, and canonical transcript-provenance fields remain, while raw `runtime_result`, `outputs.engine_data`, model/provider/tool payloads, source content, private paths, and secrets are excluded. Before persistence or upload, it sanitizes private runtime paths from nested values, object keys, diagnostics, and command arguments. WP Codebox captures one trusted canonical `codebox-transcript` through a pre-sanitization reviewer-evidence descriptor, validates its artifact-root containment, schema, digest, and size, then emits the compact reviewer transcript as a `wp-codebox/reviewer-agent-transcript/v1` projection. The canonical transcript remains the bounded tool-observability surface. Lifecycle and downstream failures retain runtime evidence in their internal normalized failed result even when artifact preparation cannot complete. Diagnostic messages that name runtime classes remain reviewable, while PHP-shaped runtime source remains blocked. See [WP Codebox #1767](https://github.com/Automattic/wp-codebox/issues/1767), [WP Codebox #1814](https://github.com/Automattic/wp-codebox/issues/1814), and [WP Codebox #1815](https://github.com/Automattic/wp-codebox/pull/1815). Regression reference: [hosted reviewer-safe workflow-result run `29468956857`](https://github.com/Automattic/agents-api/actions/runs/29468956857). ## Docs Agent Runner Recipe diff --git a/.github/workflows/maintain-docs.yml b/.github/workflows/maintain-docs.yml index 1415268..cbb52b5 100644 --- a/.github/workflows/maintain-docs.yml +++ b/.github/workflows/maintain-docs.yml @@ -249,9 +249,9 @@ jobs: contents: write pull-requests: write issues: write - uses: Automattic/wp-codebox/.github/workflows/run-agent-task.yml@v0.12.20 + uses: Automattic/wp-codebox/.github/workflows/run-agent-task.yml@v0.12.21 with: - wp_codebox_release_ref: v0.12.20 + wp_codebox_release_ref: v0.12.21 external_package_source: ${{ needs.prepare.outputs.external_package_source }} runtime_sources: ${{ needs.prepare.outputs.runtime_sources }} workload_id: docs-agent-${{ inputs.audience }}-${{ inputs.run_kind }} diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index feb07cf..67347cd 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -17,7 +17,7 @@ jobs: uses: actions/checkout@v4 with: repository: Automattic/wp-codebox - ref: v0.12.20 + ref: v0.12.21 path: .wp-codebox - name: Validate Docs Agent env: diff --git a/README.md b/README.md index 4278beb..6b7a70d 100644 --- a/README.md +++ b/README.md @@ -111,7 +111,7 @@ Docs Agent declares the review artifacts it expects the runner to materialize as `maintain-docs.yml` writes `expected_artifacts` and `artifact_declarations` into a portable Docs Agent recipe and exposes the same declaration objects as `declared_artifacts_json`. -WP Codebox v0.12.20 separately uploads one compact reviewer transcript through a pre-sanitization canonical reviewer-evidence descriptor. It resolves exactly one trusted `codebox-transcript` under the artifact root and records its artifact-relative path, schema, verified source digest, and size before runtime-result sanitization. The uploader revalidates that descriptor before producing the `wp-codebox/reviewer-agent-transcript/v1` projection, which preserves bounded command, status, diagnostic, workspace-tool, and canonical Agents API tool-lifecycle evidence with verified source and projection digests. It excludes raw stdout, tool payloads, result bodies, raw errors, source trees, secrets, setup stacks, package data, and private paths. +WP Codebox v0.12.21 uploads a reviewer-safe workflow-result projection with public control and publication fields plus canonical transcript provenance. It excludes raw `runtime_result`, `outputs.engine_data`, model/provider/tool payloads, source content, private paths, and secrets. The canonical `codebox-transcript` remains the bounded tool-observability surface: its pre-sanitization reviewer-evidence descriptor records the trusted artifact-relative path, schema, verified source digest, and size, which the uploader revalidates before producing the `wp-codebox/reviewer-agent-transcript/v1` projection. Docs Agent owns native package selection, lane, artifact, prompt, and workspace mapping. Execution, credentials, AI provider selection, sandboxing, and publication are runner-owned concerns outside this repository. @@ -125,7 +125,7 @@ Docs Agent opens or updates one canonical PR for the configured branch. - If maintenance is needed, changes are written only under `writable_paths`. - If the canonical PR is already open, later runs reuse the same `docs_branch` and PR instead of creating duplicates. - `validation_dependencies` is an optional caller-owned reusable-workflow input. It is passed through the portable recipe and runs before verification commands when a live runner execution needs setup. -- `job_status`, `transcript_summary`, `credential_mode`, `success_requires_pr`, `validation_dependencies`, and bounded `projected_outputs_json` are exposed as reusable workflow outputs. A `run_agent: false` call returns `job_status: skipped`; a `dry_run: true` call validates without starting a model run. `OPENAI_API_KEY` is only required for a live OpenAI run and is never included in recipes, workflow outputs, or artifacts. Bootstrap lanes require a published pull request and its projected URL for success; maintenance lanes allow a no-change result with no publication projection. Typed artifact declarations remain optional and are exposed as `declared_artifacts_json` for review/debugging. Raw engine data is retained in runner artifacts rather than exposed as a workflow output. +- `job_status`, `transcript_summary`, `credential_mode`, `success_requires_pr`, `validation_dependencies`, and bounded `projected_outputs_json` are exposed as reusable workflow outputs. A `run_agent: false` call returns `job_status: skipped`; a `dry_run: true` call validates without starting a model run. `OPENAI_API_KEY` is only required for a live OpenAI run and is never included in recipes, workflow outputs, or artifacts. Bootstrap lanes require a published pull request and its projected URL for success; maintenance lanes allow a no-change result with no publication projection. Typed artifact declarations remain optional and are exposed as `declared_artifacts_json` for review/debugging. Raw engine data is not exposed as a workflow output or uploaded reviewer artifact. ## Quality Bar diff --git a/tests/validate-docs-agent-packages.php b/tests/validate-docs-agent-packages.php index 8be7229..3f6bda3 100644 --- a/tests/validate-docs-agent-packages.php +++ b/tests/validate-docs-agent-packages.php @@ -91,11 +91,11 @@ $transitional_homeboy_extensions_workflow = 'uses: Extra-Chill/homeboy-extensions/.github/workflows/runtime-agent-full-run.yml@main'; $forbidden_docs_agent_codebox_workflow = 'uses: Automattic/wp-codebox/.github/workflows/docs-agent-runner.yml@main'; -$generic_codebox_agent_task_workflow = 'uses: Automattic/wp-codebox/.github/workflows/run-agent-task.yml@v0.12.20'; +$generic_codebox_agent_task_workflow = 'uses: Automattic/wp-codebox/.github/workflows/run-agent-task.yml@v0.12.21'; $assert( ! str_contains( $maintain_docs_workflow, $transitional_homeboy_extensions_workflow ), 'maintain-docs.yml must not call Homeboy Extensions directly.' ); $assert( ! str_contains( $maintain_docs_workflow, $forbidden_docs_agent_codebox_workflow ), 'maintain-docs.yml must not call a Codebox-owned Docs Agent wrapper.' ); $assert( str_contains( $maintain_docs_workflow, $generic_codebox_agent_task_workflow ), 'maintain-docs.yml must call the generic Codebox agent-task workflow.' ); -$assert( str_contains( $maintain_docs_workflow, 'wp_codebox_release_ref: v0.12.20' ), 'maintain-docs.yml must pass the matching WP Codebox release tag.' ); +$assert( str_contains( $maintain_docs_workflow, 'wp_codebox_release_ref: v0.12.21' ), 'maintain-docs.yml must pass the matching WP Codebox release tag.' ); $workflow_blocked_runtime_fragments = array_values( array_diff( $blocked_runtime_fragments, array( 'wp-codebox', 'Automattic/wp-codebox', 'OPENAI_API_KEY' ) ) ); $workflow_internal_fragments = array_merge( $workflow_blocked_runtime_fragments, array( 'homeboy_extensions_ref:', 'runtime_ref:', 'runtime_ref }}', 'runtime_provider:', 'runtime_provider }}', 'runtime_profile:', 'runtime_profile }}', 'runtime_profiles:', 'runtime_profiles }}', 'runtime_execution:', 'runtime_execution }}', 'runtime_config:', 'runtime_config }}', 'component_contracts:', 'component_contracts }}', 'ability_requirements:', 'ability_requirements }}', 'runtime_components:', 'runtime_components }}', 'runtime_mounts:', 'runtime_mounts }}', 'required_abilities:', 'required_abilities }}', 'extra_wp_config_defines:' ) ); @@ -132,7 +132,7 @@ foreach ( array( 'Docs Agent Runner Recipe', 'portable recipe', 'Docs Agent owns the native package' ) as $migration_note_text ) { $assert( str_contains( $workflow_readme, $migration_note_text ), "Workflow README missing agent runtime note: {$migration_note_text}" ); } -$assert( str_contains( $workflow_readme, 'v0.12.20' ), 'Workflow README must record the WP Codebox release tag.' ); +$assert( str_contains( $workflow_readme, 'v0.12.21' ), 'Workflow README must record the WP Codebox release tag.' ); $assert( str_contains( $workflow_readme, 'Diagnostic messages that name runtime classes remain reviewable' ), 'Workflow README must document diagnostic-versus-source detection.' ); $assert( str_contains( $workflow_readme, 'allowlisted review artifacts' ), 'Workflow README must document the WP Codebox upload allowlist.' ); $assert( str_contains( $workflow_readme, 'normalized failed result' ), 'Workflow README must document normalized WP Codebox failures.' ); @@ -143,6 +143,7 @@ $assert( str_contains( $workflow_readme, 'compact reviewer transcript' ), 'Workflow README must document the compact reviewer transcript upload.' ); $assert( str_contains( $workflow_readme, 'private paths' ), 'Workflow README must document the reviewer transcript privacy boundary.' ); $assert( str_contains( $workflow_readme, 'pre-sanitization reviewer-evidence descriptor' ), 'Workflow README must document the pre-sanitization reviewer-evidence descriptor.' ); +$assert( str_contains( $workflow_readme, 'reviewer-safe workflow-result projection' ), 'Workflow README must document the reviewer-safe workflow-result projection.' ); $public_docs = strtolower( (string) file_get_contents( $root . '/README.md' ) . "\n" . (string) file_get_contents( $root . '/.github/workflows/README.md' ) ); foreach ( array( 'hidden internals', 'implementation details', 'compatibility plumbing', 'consumers should not know', 'should not know', 'implementation internals', 'implementation evidence internal', 'plumbing to consumer workflows' ) as $old_boundary_phrase ) { diff --git a/tests/validate-wp-codebox-run-agent-task-contract.php b/tests/validate-wp-codebox-run-agent-task-contract.php index d643f1a..bd18c96 100644 --- a/tests/validate-wp-codebox-run-agent-task-contract.php +++ b/tests/validate-wp-codebox-run-agent-task-contract.php @@ -77,6 +77,7 @@ $assert( true === ( $release['canonical_compact_reviewer_transcript_upload'] ?? null ), 'WP Codebox release fixture must upload the canonical compact reviewer transcript.' ); $assert( true === ( $release['canonical_transcript_privacy_boundary'] ?? null ), 'WP Codebox release fixture must preserve the canonical transcript privacy boundary.' ); $assert( true === ( $release['pre_sanitization_canonical_reviewer_evidence_descriptor'] ?? null ), 'WP Codebox release fixture must preserve the pre-sanitization canonical reviewer-evidence descriptor.' ); +$assert( true === ( $release['reviewer_safe_workflow_result_projection'] ?? null ), 'WP Codebox release fixture must require the reviewer-safe workflow-result projection.' ); $native_result_path = $release['native_result_path'] ?? null; $workflow_result_path = $release['workflow_result_path'] ?? null; $assert( '.codebox/native-agent-task-result.json' === $native_result_path, 'WP Codebox release fixture must declare the controlled native result path.' ); @@ -266,6 +267,16 @@ $assert( str_contains( $producer_upload, 'schema: "wp-codebox/reviewer-agent-transcript/v1"' ) && str_contains( $producer_upload, '".codebox", "agent-task-artifacts", "transcript.json"' ), 'WP Codebox must upload the compact reviewer transcript at its controlled path.' ); $assert( str_contains( $producer_upload, 'source_sha256: actualDigest, projection_sha256: projectionDigest' ) && str_contains( $producer_upload, 'canonical_transcripts: [transcript]' ), 'WP Codebox must retain verified canonical transcript provenance.' ); $assert( str_contains( $producer_upload, 'omitted_payloads' ) && str_contains( $producer_upload, '[host-path]' ) && str_contains( $producer_upload, '[redacted-source-content]' ), 'WP Codebox reviewer transcripts must omit payloads and redact private paths or source content.' ); +$projection_start = strpos( $producer_upload, 'function projectWorkflowResult(value)' ); +$projection_end = strpos( $producer_upload, 'function safeTargetPath(value)', $projection_start ); +$assert( false !== $projection_start && false !== $projection_end, 'WP Codebox must define the reviewer-safe workflow-result projection.' ); +$workflow_result_projection = substr( $producer_upload, $projection_start, $projection_end - $projection_start ); +foreach ( array( 'schema: result.schema', 'run_id: result.run_id', 'status: result.status', 'success: result.success', 'request_path: result.request_path', 'reviewer_evidence:', 'verification,', 'publication: result.publication', 'transcript: result.transcript', 'artifacts: result.artifacts', 'outputs: outputs.projections', 'access: result.access', 'publication_verification: result.publication_verification', 'publication_error: result.publication_error', 'failure: result.failure', 'projection_error: result.projection_error' ) as $public_projection_field ) { + $assert( str_contains( $workflow_result_projection, $public_projection_field ), "WP Codebox reviewer-safe workflow-result projection must retain {$public_projection_field}." ); +} +foreach ( array( 'runtime_result', 'engine_data', 'model', 'provider', 'tool_calls', 'tool_results', 'source_package_root' ) as $private_projection_field ) { + $assert( ! str_contains( $workflow_result_projection, $private_projection_field ), "WP Codebox reviewer-safe workflow-result projection must exclude {$private_projection_field}." ); +} $assert( (string) $upload_regression_run === ( $producer_upload_regression['run_id'] ?? null ), 'WP Codebox upload regression fixture must match the recorded run.' ); $assert( (string) $diagnostic_regression_run === ( $producer_diagnostic_regression['run_id'] ?? null ), 'WP Codebox diagnostic regression fixture must match the recorded run.' ); $diagnostic = $producer_diagnostic_regression['result']['diagnostics'][0] ?? null; @@ -274,7 +285,7 @@ $assert( in_array( '.codebox/agent-task-request.json', $producer_upload_regression['observed']['uploaded'] ?? array(), true ), 'WP Codebox upload regression fixture must retain the controlled request upload.' ); $assert( ! array_intersect( array( 'MODEL_PROVIDER_SECRET_1', 'MODEL_PROVIDER_SECRET_2', 'MODEL_PROVIDER_SECRET_3', 'MODEL_PROVIDER_SECRET_4', 'MODEL_PROVIDER_SECRET_5' ), array_keys( $caller_secrets ) ), 'Docs Agent must forward only the OPENAI_API_KEY provider secret name.' ); -$assert( str_contains( $workflow, 'output_projections="$(jq -cn --arg path \'metadata.runner_workspace_publication.url\' --argjson required "$success_requires_pr" \'{docs_agent_publication:{path:$path,required:$required}}\')"' ), 'Docs Agent must define the v0.12.20 publication projection descriptor.' ); +$assert( str_contains( $workflow, 'output_projections="$(jq -cn --arg path \'metadata.runner_workspace_publication.url\' --argjson required "$success_requires_pr" \'{docs_agent_publication:{path:$path,required:$required}}\')"' ), 'Docs Agent must define the v0.12.21 publication projection descriptor.' ); $docs_projections = array( 'docs_agent_publication' => array( 'path' => 'metadata.runner_workspace_publication.url', @@ -284,7 +295,7 @@ $publication_descriptor = $docs_projections['docs_agent_publication'] ?? null; $assert( is_array( $publication_descriptor ), 'Docs Agent must define the docs_agent_publication projection descriptor.' ); $publication_path = $publication_descriptor['path'] ?? null; -$assert( 'metadata.runner_workspace_publication.url' === $publication_path, 'Docs Agent publication projection must use the v0.12.20 runner workspace publication URL path.' ); +$assert( 'metadata.runner_workspace_publication.url' === $publication_path, 'Docs Agent publication projection must use the v0.12.21 runner workspace publication URL path.' ); $producer_request_fixture = $read_json( rtrim( $wp_codebox_dir, '/' ) . '/contracts/agent-task-workflow-request.fixture.json' ); $producer_projection_paths = array_values( $producer_request_fixture['outputs']['projections'] ?? array() ); diff --git a/tests/wp-codebox-release.fixture.json b/tests/wp-codebox-release.fixture.json index 6152f03..106f523 100644 --- a/tests/wp-codebox-release.fixture.json +++ b/tests/wp-codebox-release.fixture.json @@ -1,7 +1,7 @@ { - "tag": "v0.12.20", - "package_version": "0.12.20", - "run": "29350690551", + "tag": "v0.12.21", + "package_version": "0.12.21", + "run": "29468956857", "diagnostic_regression_run": "29307978522", "upload_regression_run": "29306539573", "successful_noop_optional_artifacts": true, @@ -24,6 +24,7 @@ "canonical_compact_reviewer_transcript_upload": true, "canonical_transcript_privacy_boundary": true, "pre_sanitization_canonical_reviewer_evidence_descriptor": true, + "reviewer_safe_workflow_result_projection": true, "native_result_path": ".codebox/native-agent-task-result.json", "workflow_result_path": ".codebox/agent-task-workflow-result.json" }