diff --git a/.github/workflows/verify.yaml b/.github/workflows/verify.yaml index 34c8e66..921df69 100644 --- a/.github/workflows/verify.yaml +++ b/.github/workflows/verify.yaml @@ -40,6 +40,24 @@ jobs: - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version: stable # go.mod stays the minimum; CI runs latest stable (govulncheck needs it) + - name: AUD2 exit gate (AUD2-S05 — REQ-AUD2-S05-01..05) + # The four 2026-08-18 remediations (REL-01/02/07, REL-03, SEC-03, + # TEST-02) are still present. It runs HERE, in the job that fires on + # pull_request — NOT only in release-exitgate, whose + # `if: github.event_name != 'pull_request'` is RELSE-08, the blind spot + # that let AUD-S18's stale CHECK_STAGES pin merge green four times. + # POSITION IS LOAD-BEARING: this gate runs `go test` against reverted + # copies of the tree, so it must come AFTER actions/setup-go. It sat + # above it until now — with `toolchain go1.26.6` in go.mod the first + # `go test` would have had to download a toolchain with no Go cache + # present, and this branch has never had a CI run to reveal it. + # Keep it undisarmed and argument-free: the gate asserts its own step + # carries no `if:`/`continue-on-error:` and no arguments (`--text-only` + # would skip the behavioural runs), so weakening this reddens it. + # These comment lines live INSIDE the step on purpose — hack/lint/ + # workflow_pins_test.sh isolates the step ABOVE up to the next `- ` + # marker and caps it at six lines. + run: bash hack/audit/aud2_exitgate_test.sh - run: go vet ./... - run: go vet -tags e2e ./... # compile+vet e2e-tagged wiring (test/e2e, P4-E1-S09) - name: golangci-lint (incl. gosec) @@ -140,7 +158,8 @@ jobs: - name: E9 release exit gate (RELSE-03 — REQ-E9-S13) run: bash hack/release/exitgate_test.sh # AUD-S18 (REQ-AUD-S18-01/02) — the P5-AUD exit gate: the 2026-08-06 audit's - # conditions closed, all 14 `task check` stages green, coverage at the raised + # conditions closed, all 19 `task check` stages green (the count is pinned in + # that script's CHECK_STAGES array — it read "14" until AUD2-S05), coverage at the raised # bar, the frozen JSON schemas unchanged against the v0.1.0 tag, and every # audit finding dispositioned. Invoked with NO arguments on purpose: # `--text-only` skips the toolchain layer, and the gate's own CI-wiring check diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c15d88..a2c4e1a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -85,9 +85,21 @@ repository still gets a decision, never by following the link; no release carrie - :memo: docs(spec): decompose the 2026-08-18 audit Next wave as epic P5-AUD2 (D-152) - :memo: docs(spec): AUD2-S02 names resolveRunFacts, not the nonexistent loadProviderHosts - :memo: docs(openspec): amend AUD2-S03 to the shipped pin and log D-153 (SEC-03) +- :memo: docs(audit): document the AUD2 gate and correct stale line counts (F4, F6) +- :memo: docs(backlog): mark P5-AUD2 complete (S01-S05 landed) + +### Features +- :sparkles: feat(audit): measure REL-03/REL-07 by real mutation run, not by name ### Fixes - :bug: docs(decisions): reference the AUD2 spec as a path, not an mkdocs link +- :bug: fix(audit): scope the REL-03 branch assertion to the guard block (F1) +- :bug: fix(audit): measure the REL-03 guard's terminal path, not "a return appears" +- :bug: fix(audit): assert no fall-through, not where the return sits +- :bug: fix(audit): require an ALL-of FAIL quorum and refuse bash < 4.3 + +### Refactoring +- :recycle: refactor(audit): pin REL-03 and REL-07 by test name, demote source shape ### Security - :lock: fix(cmd): discriminate absent provider declaration from forge failure (REL-03) @@ -98,6 +110,8 @@ repository still gets a decision, never by following the link; no release carrie ### Testing - :white_check_mark: test(cmd): assert the REL-03 error wrap as one contiguous substring - :test: test(compare): kill the surviving EffectChallenge intervention mutant (TEST-02) +- :white_check_mark: test(audit): pin the AUD2 remediations behind a PR-visible exit gate +- :white_check_mark: test(audit): assert verify.yaml still triggers on pull_request ## [0.3.0] - 2026-08-18 ### Chores diff --git a/Taskfile.yml b/Taskfile.yml index 03dd465..6c3ac41 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -127,6 +127,13 @@ tasks: # Operator 2026-08-13: Dependabot is the updater; a reintroduced # renovate.json would otherwise merge green. - task: ci-audit-test + # AUD2-S05 (REQ-AUD2-S05-03): the P5-AUD2 exit gate. NOT free — it runs + # `go test` against reverted copies of the tree (~40-50s), which is the only + # way it can tell a live test from a hollowed-out one. Placed last, after + # `test` has already built the packages. It also runs in the + # pull-request-visible `verify` job, which is the point: the four + # 2026-08-18 remediations must not be revertible behind a green PR. + - task: audit-aud2-exitgate-test ci-audit-test: desc: "E9-S04 + Dependabot-only: single CodeQL workflow; no Renovate config" @@ -267,6 +274,11 @@ tasks: cmds: - bash hack/release/install_cosign_pin_test.sh + audit-aud2-exitgate-test: + desc: "AUD2-S05 exit gate: the four 2026-08-18 remediations (REL-01/02/07, REL-03, SEC-03, TEST-02) are still present (REQ-AUD2-S05-01..05)" + cmds: + - bash hack/audit/aud2_exitgate_test.sh + audit-exitgate-test: desc: "AUD-S18 exit gate: 2026-08-06 audit conditions closed + gates green at the new bar (REQ-AUD-S18-01/02). NOT in `check` — it runs `task check` itself." cmds: diff --git a/hack/audit/README.md b/hack/audit/README.md index 5da4561..c2252fe 100644 --- a/hack/audit/README.md +++ b/hack/audit/README.md @@ -1,4 +1,7 @@ -# `hack/audit` — the P5-AUD exit gate +# `hack/audit` — the audit exit gates + +Two gates live here, one per audit wave: `exitgate_test.sh` (2026-08-06, P5-AUD) and +`aud2_exitgate_test.sh` (2026-08-18, P5-AUD2). ## `exitgate_test.sh` @@ -9,7 +12,7 @@ epic's bar still holds. Every failure names the audit finding ID it reopens. | # | Asserts | Names on failure | | --- | --- | --- | | 1 | The AUD-S01 fail-closed cassettes PASS **by name** (not "`go test` exited 0", which a `-run` regex matching nothing also does) | `REL-07` | -| 2 | `task check` green with all **14** pinned stages actually executed — including `changelog-verify` — and measured `./internal/...` coverage at or above the 91.0% bar | `RELSE-01`, `TEST-03` | +| 2 | `task check` green with all **19** pinned stages actually executed — including `changelog-verify` — and measured `./internal/...` coverage at or above the 91.0% bar | `RELSE-01`, `TEST-03` | | 3 | The release job still runs the verify-green-on-tag-SHA gate, before anything is built/signed/published, and undisarmed | `RELSE-05` | | 4 | The AUD-S05/S06 truth pins pass and no retired pre-release phrase has resurfaced on a front-of-house surface | `DOC-05`…`DOC-11` | | 5 | The determinism double-run gate is green **and** every test its `-run` patterns name really ran, twice | `AUD-S18 determinism` | @@ -117,3 +120,202 @@ derived from, an evidence line that interpolates `${pct}` rather than a literal, comparison that still reads `{{.COVERAGE_MIN}}`. `lint-workflow-pins-test:` is pinned to its script the same way. Same mechanism as `hack/release/changelog_gate_test.sh`, which already pins four other stage bodies — `coverage:` was not one of them. + +--- + +## `aud2_exitgate_test.sh` + +`bash hack/audit/aud2_exitgate_test.sh` is the **AUD2-S05 exit gate** (REQ-AUD2-S05-01..05): one +invocation that proves the four remediations of the 2026-08-18 audit's *risk-reduction* wave are +still present in the tree. Every failure names the audit finding ID it reopens. + +| # | Asserts | Names on failure | +| --- | --- | --- | +| 1 | `CallExec` bounds child stdout at `MaxResponseBytes` and sets a non-zero `cmd.WaitDelay` (source pins); its stderr fold is held by a **behavioural mutation run** — drop the fold in a copy of the tree, `TestExecStderrFoldedIntoError` must fail | `REL-01`, `REL-02`, `REL-07` | +| 2 | `resolveRunFacts` discriminates `forge.ErrNotFound`, held by a **behavioural mutation run** — narrow the guard's condition in a copy of the tree, the two named tests must fail | `REL-03` | +| 3 | `hack/install.sh` pins the cosign signer identity and OIDC issuer, byte-identically with `SECURITY.md` | `SEC-03` | +| 4 | `isStricterInterventionEffect` still classifies `EffectChallenge`, and the named unit case that kills the auditor's demonstrated mutant still exists | `TEST-02` | +| 5 | The gate is a `check:` stage, is pinned in `CHECK_STAGES`, and runs in the **pull-request-visible** `verify` job, undisarmed | `REQ-AUD2-S05-03/04/05` | + +Unlike `exitgate_test.sh`, this one **is** a `task check` stage (`audit-aud2-exitgate-test`, the +19th) and is also a step of `verify.yaml`'s `verify` job. That placement is the point, and the +reason is *not* that the gate is cheap — it runs `go test` against reverted copies of the tree +and takes ~40-50s. It is that the `verify` job is the one that fires on **pull requests**, while +`release-exitgate`, where the AUD-S18 gate lives, carries +`if: github.event_name != 'pull_request'` — the RELSE-08 blind spot that let AUD-S18's own stale +`CHECK_STAGES` pin merge green four times. Because it needs a Go toolchain, the step sits **after** +`actions/setup-go` in that job; only `--text-only` is toolchain-free, and neither wiring site is +allowed to pass it. + +### What actually holds each remediation closed + +Not the same mechanism for all four, and the difference matters: + +| Remediation | Held closed by | Secondary | +| --- | --- | --- | +| `SEC-03` | source pins over `hack/install.sh` + `SECURITY.md`, plus the behavioural stub-`cosign` gate in `hack/release/install_cosign_pin_test.sh` | — | +| `REL-01`, `REL-02` | source pins over `CallExec` (bounded sink by constructor, non-zero `WaitDelay`, single assignment) | — | +| `TEST-02` | the named unit case, pinned by name | the `EffectChallenge` term in the classifier | +| `REL-03`, `REL-07` | **a behavioural mutation run** — the revert applied to a throwaway copy of the tree, the pinned tests required to FAIL against it | name/case pins (fast, precise pre-check) and source-shape heuristics | + +**Correction, recorded because this README asserted it twice and it was false.** An earlier +version claimed "a revert that *keeps* the tests is caught by `go test`; one that *deletes or +renames* them is caught by the name pins", and presented that as a partition. It is not one. +Keeping the names and **emptying the bodies** is a third case, and it defeated both halves at +once: + +```go +func TestProviderDeclarationForgeErrorAbortsResolveRunFacts(t *testing.T) { + t.Skip("temporarily disabled") +} +``` + +With that plus the one-line condition narrowing: `gofmt` silent, `go build` OK, `go vet` OK, +`go test ./cmd/assent/` **ok**, and the gate **green** — REL-03 fully reverted. `t.Skip("flaky")` +is a routine event, not a contrivance, and there is no third backstop here: `cmd/` sits outside +`./internal/...`, so the coverage floor does not move when those tests are gutted. + +**What replaced it.** The gate now *measures* the property instead of reading text or names. For +each of `REL-03` and `REL-07` it copies the working tree's tracked files to `$WORK`, applies the +revert **to the copy**, and runs the pinned tests against it — requiring them to **fail**, as +tests, naming a pinned test in a `--- FAIL:` line. Three ways that can go wrong are refused +explicitly: + +* the pinned tests must first **`--- PASS:` by name** on an unmutated copy — `go test` exits 0 + both when a `-run` pattern matches nothing and when a test calls `t.Skip`, so an exit code + would treat a disabled test as a passing one; +* a `[build failed]` transcript on the reverted copy is **refused** — "the mutant does not + compile" is not evidence about behaviour; +* the reverted run must fail *and* name a pinned test, not some neighbour. + +This terminates the escape class: it is indifferent to how the production revert is spelled +(condition narrowing, polarity swap, string-literal return) *and* to how a test might be hollowed +out (`t.Skip`, a no-op body, a deleted assertion), because it reports what the tests **do**. +Nothing is written to the real tree. Measured cost: the tracked-file copy is ~1,200 files / 8.6 MB +and is effectively instant; the two `go test -run ` runs take a few seconds each. The +whole gate runs in ~40-50s, at `check:` stage 19, after `task test` has already built the packages. + +The name and case pins are kept beside it — they say *which* test or case was lost, which a +go-test transcript does not — but they are a pre-check, not the property. Case pins now read a +**comment-stripped** view: `[]int{401, 403}` was previously satisfied out of a Go comment left +behind by a gutted body. + +### `--text-only` + +`bash hack/audit/aud2_exitgate_test.sh --text-only` runs the pattern layer plus its **full** +mutation harness, skipping only the behavioural runs (which need Go and git). It exists so the +patterns can be exercised under Linux/GNU grep + mawk: + +```sh +docker run --rm -v "$PWD:$PWD" -w "$PWD" debian:stable-slim bash hack/audit/aud2_exitgate_test.sh --text-only +``` + +Its PASS banner says plainly that it certifies **nothing** about whether `REL-03` and `REL-07` are +still held closed. The control floor is mode-aware (38 full, 32 text-only) so neither mode can +quietly shrink, and both the `task check` stage and the `verify.yaml` step are asserted to invoke +the script with **no arguments**, so the flag cannot be used to disarm the gate where it counts. + +### Scoping is the whole gate + +Three of the four remediations have a **decoy occurrence in the same file**: `MaxResponseBytes` +also appears in `CallHTTP`, `errors.Is(err, forge.ErrNotFound)` also appears in +`loadResourceOwnerRegistry` (the D-130 guard), and `EffectChallenge` is all over the compare +tests. A file-level `grep` for any of them stays **green on a reverted tree**. Every assertion +therefore reads a function body extracted by name, with a positive control on a known-present +anchor, a `^func ` count that refuses an over-extracted body, and a next-function-below guard. +The `REL-03` and `REL-07` branch assertions go further still, because between them they are the +ones an independent reviewer got past four times — every mutant `gofmt`-clean, `go vet`-clean, +`golangci-lint`-clean and compiling, and each one a real revert that `go test` catches: + +1. **Block, not body.** `resolveRunFacts` contains three `return nil, nil, fmt.Errorf(` lines and + `CallExec` returns in several places, so a body-wide grep for "a return that mentions X" is + satisfied on every possible tree — including the half-revert where the guard is kept and its + body downgraded to `continue`. Each assertion isolates the block it is about (the + `forge.ErrNotFound` guard; the `cmd.Run()` failure branch), brace-matched on indentation. +2. **The terminal path, as a necessary condition.** Inside that block the required return must + sit at exactly the block's body indent, so a return reachable only under a further condition + does not count. This is necessary but **not sufficient**, and the README used to claim + otherwise: it measures indent *position*, and the polarity-swapped revert + + ```go + if !errors.Is(err, context.Canceled) { continue } + return nil, nil, fmt.Errorf(...) + ``` + + puts a real return at exactly that indent while skipping every 503, 401, 403 and throttle. It + is also the *more* idiomatic of the two spellings. +3. **No fall-through / no sink-less return** — the strongest of the shape rules, and still only + a heuristic. For `REL-03`: no `continue`, `break` or `goto` anywhere inside the guard, at any + depth. For `REL-07`: no `return` inside the `cmd.Run()` failure branch that omits the stderr + sink. Between them these kill six revert shapes. They do **not** decide whether the function + still fails closed, and two mutants prove it, both recorded as executable controls in the + script rather than left as prose: + + * a one-line **condition narrowing** in the guard's own `if` — + `if !errors.Is(err, forge.ErrNotFound) && errors.Is(err, context.Canceled) {` — has no + fall-through at all and returns at the correct indent; + * a **bare return whose message text contains the identifier** — + `return nil, fmt.Errorf("provider exec failed (stderr suppressed): %w", err)` — satisfies + both halves of the `REL-07` rule, because both halves are substring tests, not data flow. + + The second one is why no further filter work can close this: it is an ordinary double-quoted + string literal, not a comment. Recognising "does this function still fail closed" from text + means re-implementing Go's type checker and data flow, and four rounds of shape-by-shape + patching were each broken by a fresh reviewer, twice with idiomatic one-liners. +4. **Two comment filters, because absence and presence fail in opposite directions.** Presence + assertions read `code_only`, which strips line *and* block comments aggressively: quoting the + required return inside a `/* … */` span otherwise satisfies them, and over-stripping there can + only cause a false RED. Absence assertions (item 3) invert that — over-stripping would hide + the very token being hunted and go falsely **green** — so they read `code_conservative`, which + strips only a leading `//` line or a leading `/*` span and never a trailing comment or a + mid-line marker, and therefore can only ever keep too much. Two residuals, neither assumed + away: a multi-line raw string literal is refused outright by a backtick check; and the + fail-closed argument itself **does not hold for `REL-07`'s absence rule**, because that rule + is a *negated* match — extra kept text is fail-**open** there. That is one more reason the + `REL-07` property is pinned by test name and not by this filter. +5. **Single assignment.** `rhs_of` reads the *first* assignment to `cmd.Stdout` / `cmd.Stderr` / + `cmd.WaitDelay`; Go runs the *last*. Appending a second assignment reverts the remediation + while every other check still reads the good statement, so each of those lvalues — and each + bounded-capture sink variable — must be assigned exactly once. + +### Anti-vacuity discipline + +Held to the same bar as `exitgate_test.sh` and `hack/release/install_cosign_pin_test.sh`: every +check is a **function** over file arguments, run against the real tree (must be green) *and* +against a mutant carrying the very defect it exists to catch (must be red, and red **for its own +stated reason** — `expect_red` pins a message fragment). 38 mutation controls run per invocation (32 under `--text-only`) +and the count is asserted against a floor, so a skipped section fails the gate instead of +quietly shrinking it. **What the PASS banner claims, precisely**: that those controls ran, that +each went red for its own pinned message, and that every REQ-bearing assertion carries one — *not* +that every finding the script can emit is controlled. The script emits roughly forty distinct +findings; thirty-eight carry a control, and the rest are extraction and positive-control failures +whose own reason to exist is that they fire when a pattern stops matching. An exit gate whose +banner overstates its coverage is the D-124 failure mode this epic exists to close, so the banner +is worded to what is actually proved. Mutants are **file copies** under `mktemp -d`; `git checkout --` is never +used, because it reverts to `HEAD` — which can be behind the working tree, making a "mutation" a +no-op or a real edit. Mutations are **surgical**: the `REL-01` mutant leaves `CallHTTP`'s +`MaxResponseBytes` intact and the `REL-03` mutants leave the D-130 guard intact, and the gate +asserts those survivals, so a red can only come from reading the right function. + +Portability: the gate runs on `ubuntu-latest` for every PR, so it is exercised under GNU +coreutils as well as BSD: + +```sh +docker run --rm -v "$PWD:$PWD" -w "$PWD" debian:stable-slim bash hack/audit/aud2_exitgate_test.sh +``` + +Note for anyone extending it: `awk -v` strips one level of escaping, so a pattern passed that way +must not contain escaped parentheses — `\(` arrives as an unbalanced group and mawk/gawk reject +it outright, while `\.` silently becomes "any character". + +### What it does not certify + +Release readiness, and not even the whole 2026-08-18 audit — only the four *risk-reduction* +findings. The **Later (hygiene)** wave (TEST-01/03/04/05/06, SEC-04..07, RELSE-03/04, ARCH-02/03/05, +REL-04/05/06, DOC-02..06) and **WG-S01 / D-145** stay open in the audit report and the backlog. +It also does not make itself a *required* check: whether the `verify` workflow blocks a merge is a +live branch-protection setting no in-tree change can reach — the operator half of **RELSE-08** — +and the `task check` **stage** still does not run on PRs, because the only job that runs +`task check` is `release-exitgate`. The gate's PR coverage is the standalone script invocation, +not the stage. `check_pr_wiring` asserts exactly what it can: the step exists in the `verify` job, +that job carries no job-level `if:`, and the workflow's `on:` block still contains `pull_request`. diff --git a/hack/audit/aud2_exitgate_test.sh b/hack/audit/aud2_exitgate_test.sh new file mode 100755 index 0000000..968ac77 --- /dev/null +++ b/hack/audit/aud2_exitgate_test.sh @@ -0,0 +1,2121 @@ +#!/usr/bin/env bash +# REQ-AUD2-S05-01..05 — the P5-AUD2 exit gate for the 2026-08-18 project audit's +# "Next (risk reduction)" wave. +# +# ONE invocation asserts that all four AUD2 remediations are still present, and +# every failure names the audit finding ID it reopens: +# +# REL-01/02/07 (AUD2-S01) — internal/provider/transport.go's CallExec bounds +# child stdout at MaxResponseBytes, sets cmd.WaitDelay, and +# captures stderr into the returned error. +# REL-03 (AUD2-S02) — cmd/assent/provider_host.go's resolveRunFacts +# discriminates forge.ErrNotFound from every other FileAtRef +# error on the provider-declaration fetch. +# SEC-03 (AUD2-S03) — hack/install.sh pins the cosign signer identity and +# OIDC issuer, byte-identically with SECURITY.md. +# TEST-02 (AUD2-S04) — internal/compare's isStricterInterventionEffect +# still carries EffectChallenge, and the named unit case that +# kills the auditor's demonstrated mutant still exists. +# +# WHAT THIS GATE COSTS, corrected. It was a pure-text gate for five rounds and +# that sentence outlived the fact: it now RUNS `go test` twice, against reverted +# copies of the tree, and takes ~50s (measured 52s here, 38.6s by the reviewer) with the Go toolchain and +# git on PATH. `--text-only` is the mode that is still toolchain-free, and it is +# explicitly NOT what `task check` or CI invoke — both are asserted to pass no +# arguments, because that mode certifies nothing about REL-03/REL-07. +# +# WHY IT RUNS IN THE PR-VISIBLE JOB. Not because it is cheap — because the +# remediations must not be revertible behind a green PR. `task test` already +# runs these tests, but what tests do NOT do is survive their own deletion or +# hollowing: TEST-02 is precisely the finding that a fix can be reverted with +# every wired gate still green. So the gate lives in the `verify` job +# (REQ-AUD2-S05-05) rather than only in the push-only release-exitgate job +# (RELSE-08, the blind spot that hid AUD-S18's stale pin for four merges), and +# it is placed AFTER actions/setup-go there, since it needs a Go toolchain. +# +# SCOPING IS THE WHOLE GAME HERE. Three of the four assertions have a decoy in +# the very same file: +# * MaxResponseBytes also appears in CallHTTP, twenty lines above CallExec; +# * errors.Is(err, forge.ErrNotFound) also appears in +# loadResourceOwnerRegistry, the D-130 guard further down provider_host.go; +# * EffectChallenge also appears throughout the compare test files. +# A file-level grep for any of those stays GREEN with the remediation reverted. +# Every assertion below therefore reads a FUNCTION BODY extracted by name, with +# a positive control (the body is non-empty and contains a known-present anchor) +# and a scoping control (the body does NOT contain the neighbouring decoy +# function's header) so a broken extraction fails loudly instead of vacuously. +# +# ANTI-VACUITY DISCIPLINE (D-124, AUD-S18: this repo has a documented history of +# gates that cannot fail). Every check is a FUNCTION over file arguments, run +# twice: once against the real tree (must be GREEN) and once against a mutant +# copy carrying the very defect it exists to catch (must be RED, and red for its +# STATED reason — expect_red pins the message fragment). The mutations are +# SURGICAL: the REL-03 mutant deletes only resolveRunFacts' guard and the gate +# then asserts the decoy at loadResourceOwnerRegistry is still there, which is +# what distinguishes a scoped assertion from one that merely happens to pass. +# +# Mutants are file COPIES under mktemp -d. `git checkout --` is never used: it +# reverts to HEAD, which can be behind the working tree, so a "mutation" would +# silently be a no-op or, worse, a real edit to the tree. +set -euo pipefail + +# BASH >= 4.3 REQUIRED, ASSERTED FIRST — before anything else can partially run. +# This script uses namerefs (`local -n`) to pass array names into check functions. +# Stock macOS /bin/bash is 3.2, which has no namerefs: it aborted at the first +# nameref, after one section header had already printed, and EXITED 0. A tree +# with every AUD2 remediation reverted would then pass `task check` stage 19 for +# anyone whose `bash` resolves to 3.2 — and AGENTS.md rule 4 makes a green local +# `task check` a per-commit precondition, so that is not a theoretical path. CI +# is ubuntu/bash 5, so nothing merges through it; the anti-vacuity gate silently +# certifying nothing on a maintainer's machine is the defect regardless. +if ((BASH_VERSINFO[0] < 4 || (BASH_VERSINFO[0] == 4 && BASH_VERSINFO[1] < 3))); then + echo "FAIL: hack/audit/aud2_exitgate_test.sh requires bash >= 4.3 (namerefs); this is bash ${BASH_VERSION}." >&2 + echo " Refusing to run rather than aborting mid-way and exiting 0 — an exit gate that certifies nothing must say so." >&2 + echo " On macOS: run it under a modern bash (e.g. \`brew install bash\`), which is what 'task check' and CI use." >&2 + exit 1 +fi + +# MODES. Default: everything, including the behavioural mutation runs, which +# need the Go toolchain and git. `--text-only`: the pure-text layer plus its full +# mutation harness, for exercising the patterns under Linux/GNU grep + mawk in a +# container that has neither. The wiring checks assert that `task check` and +# verify.yaml invoke this script with NO arguments, so the flag cannot be used to +# disarm the gate where it counts (the AUD-S18 idiom, same reason). +TEXT_ONLY=0 +while (($# > 0)); do + case "$1" in + --text-only) TEXT_ONLY=1 ;; + *) + echo "usage: $0 [--text-only]" >&2 + exit 2 + ;; + esac + shift +done + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cd "$ROOT" + +TRANSPORT="$ROOT/internal/provider/transport.go" +PROVIDER_HOST="$ROOT/cmd/assent/provider_host.go" +INSTALL="$ROOT/hack/install.sh" +SECURITY="$ROOT/SECURITY.md" +CLASSIFY="$ROOT/internal/compare/classify_intervention.go" +CLASSIFY_TEST="$ROOT/internal/compare/classify_intervention_test.go" +TASKFILE="$ROOT/Taskfile.yml" +AUD_GATE="$ROOT/hack/audit/exitgate_test.sh" +WORKFLOW="$ROOT/.github/workflows/verify.yaml" + +# This gate's own `task check` stage, and the script the stage must invoke. +STAGE="audit-aud2-exitgate-test" +SELF="hack/audit/aud2_exitgate_test.sh" +# The named unit case AUD2-S04 added; TEST-02 is closed by its existence. +CHALLENGE_TEST="TestClassifyStricterInterventionAddedChallengeEffect" + +# THE PRIMARY PIN for REL-03 and REL-07, and the reason this gate stopped trying +# to re-derive those two remediations from unparsed Go source text. +# +# Four rounds of source-shape assertions were each defeated by a fresh reviewer +# within the hour, twice by idiomatic one-liners that need no camouflage at all: +# a condition NARROWING in the guard's own `if` +# if !errors.Is(err, forge.ErrNotFound) && errors.Is(err, context.Canceled) { +# (no fall-through, return at the right indent), and a bare return whose text +# merely CONTAINS the sink identifier +# return nil, fmt.Errorf("provider exec failed (stderr suppressed): %w", err) +# The second one is decisive: it is a plain double-quoted string literal, so no +# comment filter and no filter polarity can fix it. Recognising "does this +# function still fail closed" from text means re-implementing Go's type checker +# and data flow; the space of spellings does not shrink as patches accumulate. +# +# So the property is pinned the way TEST-02 already pins its own — BY TEST NAME — +# and the source-shape checks are demoted to a secondary heuristic that names the +# defect precisely on the shapes it does recognise. +# +# WHAT THESE NAME PINS ARE, PRECISELY — corrected after round five claimed more. +# "A revert that keeps the tests is caught by `go test`; one that deletes them is +# caught by the name pin" was presented as a partition and is not one: KEEPING +# the names while emptying the bodies is a third case, and it defeated both +# halves. These pins are therefore a fast, precisely-worded pre-check — they say +# WHICH test or case was lost, which a go-test transcript does not — and the +# behavioural mutation run below is the actual property. +REL03_TEST_FILE="cmd/assent/provider_host_test.go" +REL03_TESTS=( + TestProviderDeclarationForgeErrorAbortsResolveRunFacts + TestProviderDeclarationUnauthorizedAbortsResolveRunFacts +) +# Case-level pins, matched VERBATIM (grep -F) rather than auto-quoted, because +# the two files spell their cases differently: the REL-03 subtests are driven by +# `strconv.Itoa(status)` over an int table, the REL-07 ones are literal t.Run +# names. Each entry below is exactly the text that must survive. +REL03_SUBTESTS=('[]int{401, 403}') + +REL07_TEST_FILE="internal/provider/transport_test.go" +REL07_TESTS=(TestExecStderrFoldedIntoError) +REL07_SUBTESTS=( + 't.Run("nonzero_exit_reports_stderr"' + 't.Run("stderr_never_merged_into_facts"' + 't.Run("runaway_stderr_is_bounded_and_truncated"' +) + +# The reverts the behavioural run applies to a COPY of the tree. Each must +# COMPILE — a mutant that does not build proves nothing about behaviour, and +# check_behavioural_revert refuses a `[build failed]` transcript explicitly. +# +# REL-03: the condition narrowing that beat every source-shape heuristic. One +# line, gofmt/vet/build clean, `context` is already imported by that file. +REL03_PKG="./cmd/assent/" +REL03_SRC="cmd/assent/provider_host.go" +# INDENT-ANCHORED so the revert lands on the provider-declaration guard ONLY. +# The identical guard in loadResourceOwnerRegistry (D-130) sits at one tab; this +# one is inside `for` + `if err != nil` at three. An unanchored program reverts +# both, and the run would then be red partly for a finding this control is not +# about — check_behavioural_revert's "a PINNED test must be named in a --- FAIL: +# line" catches that, but a mutant should mean what it says. +REL03_REVERT=$'s|^\t\t\tif !errors.Is(err, forge.ErrNotFound) {|\t\t\tif !errors.Is(err, forge.ErrNotFound) \&\& errors.Is(err, context.Canceled) {|' +REL03_REVERT_WITNESS='errors.Is(err, context.Canceled)' +REL03_RUN='TestProviderDeclarationForgeErrorAbortsResolveRunFacts|TestProviderDeclarationUnauthorizedAbortsResolveRunFacts' +# THE QUORUM. Every one of these must appear in a `--- FAIL:` line when the +# revert is applied. "At least one pinned test failed" was the previous rule and +# it is weaker than the claim built on it: gutting ONLY +# TestProviderDeclarationUnauthorizedAbortsResolveRunFacts (a no-op body, the +# case table kept as live code so the text pins stay green) left the other test +# failing, satisfied the quorum, and passed the gate at exit 0 with production +# source untouched. Subtests are pinned too, so dropping just the 401 case is +# caught at case granularity rather than only at function granularity. +REL03_MUSTFAIL=( + TestProviderDeclarationForgeErrorAbortsResolveRunFacts + TestProviderDeclarationUnauthorizedAbortsResolveRunFacts/401 + TestProviderDeclarationUnauthorizedAbortsResolveRunFacts/403 +) + +# REL-07: drop the fold. `err` is in scope, so this compiles as written and is +# exactly the pre-fix behaviour — a bare "exit status 1" with no diagnostic. +REL07_PKG="./internal/provider/" +REL07_SRC="internal/provider/transport.go" +REL07_REVERT='s|return nil, execFailure(err, stderr)|return nil, err|' +REL07_REVERT_WITNESS='return nil, err' +REL07_RUN='TestExecStderrFoldedIntoError' +# ASYMMETRY, and it is deliberate: do NOT require all three REL-07 subtests. +# Under the fold-dropping revert, `stderr_never_merged_into_facts` LEGITIMATELY +# PASSES — it asserts stderr does not leak into the fact map, which stays true +# when the fold is removed. Demanding it would make the gate red for a property +# the revert does not break, i.e. an assertion that cannot be satisfied by any +# correct tree. Only the two canaries that DO detect the revert are pinned; the +# third is still held by the case pin in check_named_tests, which requires it to +# exist. Measured, not assumed: see the transcript names in the commit body. +REL07_MUSTFAIL=( + TestExecStderrFoldedIntoError/nonzero_exit_reports_stderr + TestExecStderrFoldedIntoError/runaway_stderr_is_bounded_and_truncated +) + + +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT + +fail() { + echo "FAIL: $*" >&2 + exit 1 +} + +# --------------------------------------------------------------- helpers -- + +# extract_func — the body of a top-level Go func (or method), +# from its `func` line to the first column-0 `}`. Column-0 anchoring is what +# keeps a neighbouring function out of the result. +extract_func() { + awk -v name="$2" ' + !inf && $0 ~ "^func (\\([^)]*\\) )?"name"\\(" { inf = 1; print; next } + inf && /^}/ { print; exit } + inf { print } + ' "$1" +} + +# code_only — filter dropping whole-line Go comments from stdin, so an assertion +# can never be satisfied by prose that merely mentions the construct it wants. +# +# Strips BOTH comment forms. The first version stripped `//` lines only, and an +# independent reviewer immediately built a gofmt-clean, compiling half-revert +# that hid the required return inside a `/* ... */` span — so the claim "a +# return quoted in a comment cannot satisfy the assertion" was false as written. +# Block spans carry across lines, hence the state machine rather than a grep. +# +# It does not model Go string literals: a `//` or `/*` inside a quoted string +# would over-strip. That direction is fail-CLOSED (the assertion sees less code, +# never more), and no such literal exists in the region this filter is applied +# to, so the simpler scanner is the safer one. +code_only() { + awk ' + { + line = $0 + out = "" + while (length(line) > 0) { + if (inblock) { + p = index(line, "*/") + if (p == 0) { line = ""; break } + line = substr(line, p + 2) + inblock = 0 + continue + } + pb = index(line, "/*") + pl = index(line, "//") + if (pl > 0 && (pb == 0 || pl < pb)) { + out = out substr(line, 1, pl - 1) + line = "" + break + } + if (pb > 0) { + out = out substr(line, 1, pb - 1) + line = substr(line, pb + 2) + inblock = 1 + continue + } + out = out line + line = "" + } + print out + } + ' +} + +# code_conservative — the OTHER filter, for ABSENCE assertions. +# +# THE INVERSION, stated plainly because getting it wrong trades one unfailable +# assertion for another. code_only above is tuned for PRESENCE assertions ("a +# return must appear"): over-stripping there hides a return and causes a false +# RED, which is safe. An ABSENCE assertion ("no `continue` may appear") has the +# opposite polarity: over-stripping hides the very token being hunted and causes +# a false GREEN. code_only's own documented weakness — it does not model Go +# string literals, so `u := "http://x"` truncates the line — would therefore +# become a way to smuggle a fall-through past the gate. +# +# So absence assertions use this filter instead, which can only ever keep TOO +# MUCH. It removes a construct only when the construct is unambiguous without +# parsing Go: +# * a line whose FIRST non-blank characters are `//` — everything after a +# leading `//` is comment, whatever quotes appear later on that line; +# * a span whose opening line's FIRST non-blank characters are `/*`, up to and +# including the line carrying `*/`. +# A TRAILING `// ...` comment is deliberately NOT stripped, and a block comment +# opened mid-line is deliberately NOT recognised: both would require deciding +# whether the marker sits inside a string literal, and guessing wrong in the +# stripping direction is the false-green this filter exists to rule out. The +# cost is false REDs on prose that names a fall-through in a trailing comment — +# fail-closed, and loud. +# +# Residual, named rather than assumed away: a multi-line RAW string literal +# (backticks) whose interior line begins with `//` would still be stripped. +# assert_no_raw_string below refuses any region containing a backtick, so that +# case fails closed instead of silently weakening an absence assertion. +code_conservative() { + awk ' + inblock { if (index($0, "*/") > 0) { inblock = 0 }; next } + { + t = $0 + sub(/^[ \t]+/, "", t) + if (substr(t, 1, 2) == "//") { next } + if (substr(t, 1, 2) == "/*") { + if (index(t, "*/") == 0) { inblock = 1 } + next + } + print + } + ' +} + +# assert_no_raw_string — code_conservative's one +# remaining ambiguity is a multi-line raw string literal. No such literal exists +# in any region this gate reads; if one ever appears, fail closed and say why +# rather than let an absence assertion quietly weaken. +assert_no_raw_string() { + if grep -q '`' "$1"; then + echo " $2: the isolated region contains a backtick — a multi-line raw string literal would let code_conservative strip a line that is not a comment, which is the one way an ABSENCE assertion here can go falsely green. Extend the filter before allowing this shape (AUD2-S05)" >&2 + return 1 + fi + return 0 +} + +# block_from — the brace-matched block a line +# opens: from the first line matching the pattern to the `}` at the SAME +# indentation. gofmt (check stage 1) is what makes indentation load-bearing +# rather than cosmetic, so no brace counting is needed. +block_from() { + awk -v re="$2" ' + !ing && $0 ~ re { + match($0, /^[ \t]*/) + indent = substr($0, 1, RLENGTH) + ing = 1 + print + next + } + ing { + print + if ($0 == indent "}") exit + } + ' "$1" +} + +# guard_block — the `if !errors.Is(err, forge.ErrNotFound) { … }` +# block, from its `if` line to the `}` at the SAME indentation. Brace-matched on +# indentation rather than counting braces: gofmt guarantees the closing brace of +# a block sits at the block's own indent, and the whole tree is gofmt-clean +# (`task fmt` is check stage 1). +guard_block() { + awk ' + !ing && /if !errors\.Is\(err, forge\.ErrNotFound\)/ { + match($0, /^[ \t]*/) + indent = substr($0, 1, RLENGTH) + ing = 1 + print + next + } + ing { + print + if ($0 == indent "}") exit + } + ' "$1" +} + +# body_of — extract +# with all three controls. Returns non-zero (naming ) if the extraction +# produced nothing, lost its anchor, or ran past the end of the function. +# +# On the argument: extract_func only ever reads DOWNWARD from +# its target's `func` line, so a guard naming a function that sits ABOVE the +# target can never fire — a control that cannot fail, which is the same species +# of defect this gate exists to catch. The argument must therefore name the +# function immediately BELOW the target (or be empty when the target is the last +# function in the file, as isStricterInterventionEffect is). The `^func ` count +# below is the direction-independent backstop and is live in every case. +body_of() { + local file="$1" fn="$2" out="$3" finding="$4" anchor="$5" next_fn="$6" + if [[ ! -f "$file" ]]; then + echo " ${finding}: missing ${file#"$ROOT"/}" >&2 + return 1 + fi + extract_func "$file" "$fn" >"$out" + if [[ ! -s "$out" ]]; then + echo " ${finding}: func ${fn} was not found in ${file#"$ROOT"/} — it was renamed or deleted, and every assertion over its body would be vacuous" >&2 + return 1 + fi + if ! grep -Fq -- "$anchor" "$out"; then + echo " ${finding}: the extracted ${fn} body does not contain the known-present anchor '${anchor}' — the extraction is wrong, so the assertions below would be vacuous" >&2 + return 1 + fi + # Direction-independent over-extraction control: a body that swallowed a + # neighbour carries two `func` headers, whichever side the neighbour is on. + local n_func + n_func="$(grep -cE '^func ' "$out" || true)" + if [[ "$n_func" -ne 1 ]]; then + echo " ${finding}: the extracted ${fn} body carries ${n_func} top-level 'func' header(s), not 1 — the extraction ran past the end of the function, so a same-file grep is what is really being graded, not this function" >&2 + return 1 + fi + if [[ -n "$next_fn" ]] && grep -Fq -- "$next_fn" "$out"; then + echo " ${finding}: the extracted ${fn} body ALSO contains '${next_fn}', the function immediately below it — the extraction ran past the closing brace (AUD2-S05: this guard names the function BELOW the target on purpose; extract_func only reads downward, so naming one above it would be a control that cannot fire)" >&2 + return 1 + fi + return 0 +} + +# rhs_of — the right-hand side of the first assignment to +# an lvalue, with any leading `&` stripped. Empty when there is no assignment. +rhs_of() { + sed -nE "s/^[[:space:]]*$2[[:space:]]*=[[:space:]]*&?(.+)$/\1/p" "$1" | head -1 +} + +# delete_first — delete only the FIRST line matching the pattern, +# in place. Surgical by construction: a later decoy occurrence survives, which +# is what makes the mutation a real test of the assertion's scoping. +delete_first() { + local file="$1" re="$2" + awk -v re="$re" 'done != 1 && $0 ~ re { done = 1; next } { print }' "$file" >"$file.mut" + mv "$file.mut" "$file" +} + +# downgrade_first_return — replace the FIRST line matching the +# pattern with a `continue` at the same indentation. This is the HALF-REVERT +# shape: the guard stays, its body stops stopping the run. It compiles and it is +# gofmt-clean, which is exactly why a body-wide grep could not see it. +downgrade_first_return() { + local file="$1" re="$2" + awk -v re="$re" ' + done != 1 && $0 ~ re { + match($0, /^[ \t]*/) + print substr($0, 1, RLENGTH) "continue" + done = 1 + next + } + { print } + ' "$file" >"$file.mut" + mv "$file.mut" "$file" +} + +# splice_first_return — replace the FIRST +# `return nil, nil, fmt…` line with the replacement file's lines, each prefixed +# with the replaced line's own indentation (so the replacement is written +# RELATIVE, with its own leading tabs for nesting). Used to build the two +# camouflaged half-reverts: a return hidden in a block comment, and a return +# reachable only under a nested condition. Both compile and are gofmt-clean. +splice_first_return() { + local file="$1" rep="$2" + awk ' + FNR == NR { line[++n] = $0; next } + done != 1 && $0 ~ /^[[:space:]]*return nil, nil, fmt/ { + match($0, /^[ \t]*/) + ind = substr($0, 1, RLENGTH) + for (i = 1; i <= n; i++) print ind line[i] + done = 1 + next + } + { print } + ' "$rep" "$file" >"$file.mut" + mv "$file.mut" "$file" +} + +# mutate — sed in place, then prove the mutation +# actually landed. A mutant identical to the clean input proves nothing. +mutate() { + local file="$1" program="$2" witness="$3" before after + before="$(cat "$file")" + sed "$program" "$file" >"$file.mut" + mv "$file.mut" "$file" + after="$(cat "$file")" + [[ "$before" != "$after" ]] || + fail "mutation harness: sed '$program' did not change ${file#"$WORK"/} — the mutant equals the clean input, so any 'the check fires' conclusion would be false" + grep -Fq -- "$witness" "$file" || + fail "mutation harness: expected '$witness' in ${file#"$WORK"/} after mutation, but it is absent" +} + +# ============================================================================ +# BEHAVIOURAL MUTATION RUN — the primary evidence for REL-03 and REL-07 +# ============================================================================ +# +# WHY THIS EXISTS. Round five pinned these two remediations by TEST NAME, on the +# reasoning that `go test` catches a revert that keeps the tests and the name pin +# catches one that deletes them. A reviewer then kept the names and emptied the +# bodies — +# func TestProviderDeclarationForgeErrorAbortsResolveRunFacts(t *testing.T) { +# t.Skip("temporarily disabled") +# } +# — applied the one-line condition narrowing, and got `gofmt` silent, `go build` +# OK, `go vet` OK, `go test ./cmd/assent/` OK and this gate GREEN, with REL-03 +# fully reverted. A NAME IS NOT A BEHAVIOUR, and "keeps the tests" vs "deletes +# the tests" was never a partition: gutting them is a third case that both halves +# missed. `t.Skip("flaky")` is a routine event, not a contrivance. There is also +# no third backstop for REL-03: cmd/ sits outside ./internal/..., so the coverage +# floor moves not at all when those tests are gutted. +# +# So the property is measured, not read: copy the tree, apply the revert to the +# COPY, run the pinned tests against the copy, and require them to FAIL. That +# terminates the class — it tests behaviour rather than text or names, and it is +# indifferent to how either the production revert or the test gutting is spelled. +# +# Cost, measured: the tree copy is ~1,200 tracked files / 8.6 MB and takes well +# under a second; each `go test -run ` over one package takes ~1-2s. Both +# findings together add ~4s to a stage that already runs after `task test`. +# +# NOTHING IS WRITTEN TO THE REAL TREE. Every mutation lands in $WORK. + +# mutation_tree — a throwaway copy of the WORKING TREE's tracked files. +# Tracked-only (git ls-files) to skip .git and build junk, but the CONTENT comes +# from the working tree, so the copy grades what is about to be committed rather +# than what HEAD already holds. +mutation_tree() { + local dir="$WORK/tree-$1" + if [[ -d "$dir" ]]; then + printf '%s\n' "$dir" + return 0 + fi + mkdir -p "$dir" + if [[ ! -s "$WORK/tracked.tar" ]]; then + (cd "$ROOT" && git ls-files -z | tar -cf - --null -T -) >"$WORK/tracked.tar" + fi + tar -xf "$WORK/tracked.tar" -C "$dir" + printf '%s\n' "$dir" +} + +# drop_unused_imports — emptying a test body orphans the +# imports it used, and Go rejects an unused import. The reviewer's own mutant +# "dropped the now-unused imports"; this does the same thing deterministically +# instead of hardcoding a list: compile, read back the `"x" imported and not +# used` diagnostics, delete exactly those import lines, repeat. Bounded, and it +# returns quietly on any other error so the caller still grades the real +# transcript rather than this loop's opinion of it. +drop_unused_imports() { + local tree="$1" pkg="$2" rel="$3" i out unused imp + for ((i = 0; i < 12; i++)); do + out="$WORK/imports.$$.$i" + (cd "$tree" && go build "$pkg" && go vet "$pkg") >"$out" 2>&1 && return 0 + unused="$(sed -nE 's/.*"([^"]+)" imported and not used.*/\1/p' "$out" | sort -u)" + [[ -n "$unused" ]] || return 0 + while IFS= read -r imp; do + [[ -n "$imp" ]] || continue + awk -v q="\"${imp}\"" ' + { + t = $0 + sub(/^[ \t]+/, "", t) + if (t == q) { next } + if (t ~ ("^[A-Za-z0-9_.]+ " q "$")) { next } + print + } + ' "$tree/$rel" >"$tree/$rel.imp" + mv "$tree/$rel.imp" "$tree/$rel" + done <<<"$unused" + done + return 0 +} + +# go_test_named — `go test -v -run` in a copy. +# Returns go test's own exit code; the caller grades the transcript, never the +# exit code alone (a build failure also exits non-zero and proves nothing about +# behaviour). +go_test_named() { + local tree="$1" run="$2" pkg="$3" out="$4" rc=0 + (cd "$tree" && go test -count=1 -timeout 120s -v -run "$run" "$pkg") >"$out" 2>&1 || rc=$? + return "$rc" +} + +# check_behavioural_revert +# +# The pinned tests must actually DETECT the revert. Three graded runs: +# (1) positive control — the pinned tests PASS by name on an unmutated copy. +# This is also what catches a gutted body that still returns success only +# because it does nothing: see the emptiness probe below. +# (2) the revert applied to a second copy — the pinned tests must FAIL, and +# fail as TESTS: a `[build failed]` transcript is refused explicitly, +# because "the mutant does not compile" is not evidence about behaviour. +# (3) at least one FAIL line must name a PINNED test, not some neighbour. +# Optional 10th argument OVERLAY, "=", +# copied into BOTH the control copy and the reverted copy before either runs. +# That is how the gutted-test controls work: they do not touch production source +# at all, they replace the test file and let the run report what that costs. +check_behavioural_revert() { + local finding="$1" tag="$2" rel="$3" prog="$4" witness="$5" run="$6" pkg="$7" fn_list="$8" remediation="$9" + local overlay="${10:-}" + local mustfail_list="${11:?check_behavioural_revert: a must-fail quorum list is required}" + local -n fns="$fn_list" + local -n mustfail="$mustfail_list" + local clean mut out_clean out_mut fn rc=0 + + clean="$(mutation_tree "${tag}-control")" + if [[ -n "$overlay" ]]; then + cp "${overlay#*=}" "$clean/${overlay%%=*}" + drop_unused_imports "$clean" "$pkg" "${overlay%%=*}" + fi + out_clean="$WORK/gotest.${tag}.clean" + if go_test_named "$clean" "$run" "$pkg" "$out_clean"; then :; else + if grep -Fq 'build failed' "$out_clean"; then + echo " ${finding}: the pinned tests do not BUILD on an unmutated copy of the tree — this gate cannot measure anything until that is fixed (AUD2-S05 behavioural pin)" >&2 + else + echo " ${finding}: the pinned tests are RED on an unmutated copy of the tree — ${remediation} is already broken, or the tests are (AUD2-S05 behavioural pin)" >&2 + fi + sed 's/^/ /' <(grep -E "^(---|\s+---|FAIL|ok)" "$out_clean" | head -8) >&2 + return 1 + fi + for fn in "${fns[@]}"; do + if ! grep -Fq -- "--- PASS: ${fn} " "$out_clean"; then + echo " ${finding}: '${fn}' did not report '--- PASS:' on an unmutated copy — it was renamed, deleted or skipped. \`go test\` exits 0 both when a -run pattern matches nothing and when a test calls t.Skip, so grading its exit code alone would treat a disabled test as a passing one (AUD2-S05 behavioural pin)" >&2 + rc=1 + fi + done + ((rc == 0)) || return "$rc" + + mut="$(mutation_tree "${tag}-reverted")" + if [[ -n "$overlay" ]]; then + cp "${overlay#*=}" "$mut/${overlay%%=*}" + drop_unused_imports "$mut" "$pkg" "${overlay%%=*}" + fi + local target="$mut/$rel" + sed "$prog" "$target" >"$target.mut" && mv "$target.mut" "$target" + if ! grep -Fq -- "$witness" "$target"; then + fail "mutation harness (${finding}): the behavioural revert did not land in ${rel} — expected '${witness}' after applying the sed program, so the run below would grade an unmutated copy and pass for the wrong reason" + fi + + out_mut="$WORK/gotest.${tag}.mut" + if go_test_named "$mut" "$run" "$pkg" "$out_mut"; then + echo " ${finding}: the pinned tests still PASS with ${remediation} REVERTED in a copy of the tree — they no longer detect the defect they exist to detect. A name that survives with an emptied body (t.Skip, a no-op loop, a deleted assertion) is not a behaviour, and neither \`go test\` nor a name pin can tell the difference. This is the ONLY assertion here that measures the remediation rather than reading it (AUD2-S05 behavioural pin). Revert applied: ${witness}" >&2 + rc=1 + else + if grep -Fq 'build failed' "$out_mut"; then + echo " ${finding}: the reverted copy failed to BUILD rather than failing its tests — that is not evidence about behaviour, and this control must not be allowed to pass on it. Fix the revert program so it produces compiling code (AUD2-S05 behavioural pin)" >&2 + rc=1 + else + # QUORUM: EVERY entry of the finding's must-fail list, not "at least one". + # The trailing space matters — `--- FAIL: X ` must not be satisfied by + # `--- FAIL: X/401`, or a function-level pin would be met by one of its + # own subtests. + local missing="$WORK/quorum.${tag}.missing" + : >"$missing" + for fn in "${mustfail[@]}"; do + grep -Fq -- "--- FAIL: ${fn} " "$out_mut" || printf '%s\n' "$fn" >>"$missing" + done + if [[ -s "$missing" ]]; then + echo " ${finding}: the reverted copy is red, but these pinned tests did NOT report '--- FAIL:' — they no longer detect ${remediation}, and a sibling failing in their place does not close that. Emptying ONE test's body while its siblings still fail is exactly the escape this quorum exists to refuse (AUD2-S05 behavioural pin):" >&2 + sed 's/^/ missing: /' "$missing" >&2 + echo " what the reverted copy actually reported:" >&2 + sed 's/^/ /' <(grep -E '^[[:space:]]*--- (FAIL|PASS|SKIP):' "$out_mut" | head -8) >&2 + rc=1 + fi + fi + fi + return "$rc" +} + +# check_named_tests +# +# The primary pin: the named behavioural tests still exist, by name, in the named +# file — the same idiom TEST-02 is pinned by, and the half of the composition +# that a revert-plus-delete-the-tests cannot walk around. It deliberately does +# NOT run them: `go test` is check stage 4 and a step of the same PR-visible +# `verify` job, so a revert that keeps the tests is already red before this gate +# runs. What is unique here is catching their DELETION or RENAME, which `go test` +# reports as success. +check_named_tests() { + local finding="$1" rel="$2" fn_list="$3" sub_list="$4" remediation="$5" rc=0 + # An absolute path is a mutant copy under $WORK; a relative one is the tree. + local file="$ROOT/$rel" + [[ "$rel" == /* ]] && file="$rel" + if [[ ! -f "$file" ]]; then + echo " ${finding}: ${rel} does not exist — the behavioural tests that hold ${remediation} closed are gone, and 'go test' reports a deleted test as success (AUD2-S05 primary pin)" >&2 + return 1 + fi + # There is deliberately NO `grep -cE "^func Test" >= 1` corpus control here. + # It cannot independently fail: any file satisfying a `func (` pin + # necessarily contains a `^func Test` line, so it would be an assertion that + # is true whenever the assertions after it are — the D-124 species this epic + # exists to close. The real defence against an emptied corpus is the + # behavioural run, which reports what the tests DO. + local -n fns="$fn_list" + local -n subs="$sub_list" + local fn sub + for fn in "${fns[@]}"; do + if ! grep -Fq "func ${fn}(" "$file"; then + echo " ${finding}: the named behavioural test ${fn} is absent from ${rel} — deleting or renaming it is invisible to 'go test' (a test that does not exist cannot fail), so ${remediation} would then be held closed by nothing but this gate's source-shape heuristics, which are documented as insufficient on their own (AUD2-S05 primary pin)" >&2 + rc=1 + fi + done + # F4: the case pins read a COMMENT-STRIPPED view. `[]int{401, 403}` was + # satisfied out of a Go comment by a gutted test whose body had been replaced + # with `// coverage note: this used to iterate []int{401, 403}` — every + # production-source assertion in this script already filters comments, and + # this one did not. code_conservative (not code_only) because these are + # PRESENCE assertions over a test file: over-stripping causes a false RED. + local code_view="$WORK/testfile.code.$(basename "$rel")" + code_conservative <"$file" >"$code_view" + if [[ ! -s "$code_view" ]]; then + echo " ${finding}: the comment-stripped view of ${rel} is empty — the filter is over-stripping, so the case pins below would be graded against nothing" >&2 + return 1 + fi + for sub in "${subs[@]}"; do + if ! grep -Fq -- "$sub" "$code_view"; then + echo " ${finding}: the case pin '${sub}' is absent from the CODE of ${rel} — a case can be dropped without touching any test function name, which narrows the coverage that holds ${remediation} closed while every function-level pin stays green. Comments do not count: a gutted body that leaves the table behind in prose satisfies nothing here (AUD2-S05 primary pin)" >&2 + rc=1 + fi + done + return "$rc" +} + +# assign_count — how many times an lvalue is +# assigned in a region. rhs_of reads the FIRST assignment; at runtime the LAST +# one wins, so "assigned exactly once" is what makes reading the first sound. +assign_count() { + grep -Ec "^[[:space:]]*$2[[:space:]]*:?=" "$1" || true +} + +# assert_assignment_gone — the mutation really +# removed the ASSIGNMENT, not some prose mention of it. Deleting a doc comment +# also "changes the file", which is why assert_changed alone is not enough. +assert_assignment_gone() { + ! grep -Eq "^[[:space:]]*$2[[:space:]]*=" "$1" || + fail "mutation harness ($3): ${1#"$WORK"/} still assigns $2 — the deletion hit a different line (a comment, most likely) and the control would grade the wrong thing" +} + +# assert_changed — the mutant really differs from its source. +assert_changed() { + ! diff -q "$1" "$2" >/dev/null 2>&1 || + fail "mutation harness: ${2#"$WORK"/} is byte-identical to its source — the mutation did not land" +} + +expect_green() { #