From 0750fa8d6f1f26b86592ecd13462b0ccf0ab7fc0 Mon Sep 17 00:00:00 2001
From: Aryam Goyal <aryamgoyal18@gmail.com>
Date: Thu, 20 Aug 2026 17:58:59 +0530
Subject: [PATCH] fix: harden controlled agent study evaluation

---
 benchmarks/agent-study/README.md              |  38 +++
 benchmarks/agent-study/protocol.json          |  26 +-
 benchmarks/agent-study/tasks.json             | 122 +++++++++
 docs/AGENT_STUDY.md                           | 119 +++++---
 docs/assets/fixmap-cli-demo.svg               |   2 +-
 .../reports/declines-fabricated-identifier.md |   2 +-
 examples/reports/declines-unmatched-terms.md  |   2 +-
 examples/reports/declines-vague-task.md       |   2 +-
 examples/reports/workspace-api-discount.md    |   2 +-
 examples/reports/workspace-utils-rounding.md  |   2 +-
 package.json                                  |   5 +-
 scripts/evaluate-agent-study.mjs              | 177 +++++++++---
 scripts/evaluate-agent-study.test.mjs         | 255 ++++++++++++++++++
 13 files changed, 677 insertions(+), 77 deletions(-)
 create mode 100644 benchmarks/agent-study/README.md
 create mode 100644 benchmarks/agent-study/tasks.json
 create mode 100644 scripts/evaluate-agent-study.test.mjs

diff --git a/benchmarks/agent-study/README.md b/benchmarks/agent-study/README.md
new file mode 100644
index 0000000..e273e82
--- /dev/null
+++ b/benchmarks/agent-study/README.md
@@ -0,0 +1,38 @@
+# Controlled agent-study artifacts
+
+This directory defines study structure, not study results.
+
+- `protocol.json` freezes the four arms, enforced requirements, metrics, aggregation policy, and
+  the path to the task manifest.
+- `tasks.json` freezes the complete task universe for `fixmap-navigation-heldout-v1`.
+
+## Task-manifest schema
+
+Top-level fields:
+
+| Field | Contract |
+| --- | --- |
+| `manifestVersion` | `1` |
+| `protocolVersion` | Must equal `protocol.json` (`3`) |
+| `studyId` | Non-empty stable study identity required in every run row |
+| `status` | Must be `frozen` |
+| `frozenAt` | Date on which task membership and text stopped changing |
+| `taskSelection` | Source, deterministic inclusion rule, and pre-outcome rationale |
+| `tasks` | Non-empty array with unique `taskId` values |
+
+Every task requires `taskId`, exact `taskText`, its lowercase hexadecimal `taskTextSha256`, public
+`repository`, exact 40-character `revision`, `sourceIssue`, and `selectionRationale`. The evaluator
+recalculates every manifest task hash before it accepts the protocol.
+
+## Publication boundary
+
+The evaluator has no partial-results mode. With no `--input`, it validates only the protocol and
+manifest and explicitly claims no result. With `--input`, it requires one global `--model`,
+`--model-version`, and `--fixmap-revision`, verifies the exact manifest task × protocol arm
+cross-product and transcript bytes, and emits an aggregate only when the whole group passes.
+Every row also needs a unique declared `contextId`; this prevents accidental context-ID reuse but
+does not prove process isolation.
+
+Separate models require separate complete run files and evaluator invocations. The evaluator does
+not pool model groups. Failed and timed-out runs remain in their arm totals and all-run outcome
+rate denominators.
diff --git a/benchmarks/agent-study/protocol.json b/benchmarks/agent-study/protocol.json
index 8616463..dbf2e59 100644
--- a/benchmarks/agent-study/protocol.json
+++ b/benchmarks/agent-study/protocol.json
@@ -1,25 +1,35 @@
 {
-  "protocolVersion": 2,
-  "status": "protocol-only",
+  "protocolVersion": 3,
+  "status": "frozen-no-runs",
   "suite": "navigation-focused pinned tasks",
+  "taskManifest": "benchmarks/agent-study/tasks.json",
   "arms": [
     "baseline",
     "fixmap-available",
     "fixmap-instructed",
     "fixmap-impact"
   ],
+  "aggregationPolicy": {
+    "publishableOnlyWhenComplete": true,
+    "modelGrouping": "single-global-model-version",
+    "failedAndTimedOutRuns": "retained"
+  },
   "requirements": {
-    "sameModelVersion": true,
+    "sameGlobalModelVersion": true,
     "sameTaskText": true,
     "sameRepositoryRevision": true,
-    "freshContextPerRun": true,
-    "randomizedArmOrder": true,
+    "uniqueContextIdPerRun": true,
+    "uniqueArmOrderPerTask": true,
     "fixedTimeoutAndBudget": true,
     "noFixMapChangesMidStudy": true,
     "rawTranscriptsRequired": true,
-    "taskSuccessRubricFrozen": true,
-    "tokenAccountingSourceRecorded": true,
-    "costRateCardRecorded": true,
+    "transcriptContentHashVerified": true,
+    "taskManifestFrozen": true,
+    "completeTaskArmCrossProduct": true,
+    "taskTextMatchesManifest": true,
+    "taskSuccessRubricIdRecorded": true,
+    "providerReportedTokenCountersOnly": true,
+    "priceSheetIdRecorded": true,
     "failuresRetained": true
   },
   "metrics": [
diff --git a/benchmarks/agent-study/tasks.json b/benchmarks/agent-study/tasks.json
new file mode 100644
index 0000000..df95899
--- /dev/null
+++ b/benchmarks/agent-study/tasks.json
@@ -0,0 +1,122 @@
+{
+  "manifestVersion": 1,
+  "protocolVersion": 3,
+  "studyId": "fixmap-navigation-heldout-v1",
+  "status": "frozen",
+  "frozenAt": "2026-08-20",
+  "taskSelection": {
+    "source": "All cases in benchmarks/heldout/dataset.json at FixMap commit bbc8469c937397fecac787c71c04521a6eb1a87d.",
+    "rule": "Use the complete pre-existing 12-case held-out suite in its frozen order; no task may be added, removed, or rewritten after controlled runs begin.",
+    "rationale": "These public MIT-licensed issue tasks and pinned pre-fix revisions were selected mechanically before any controlled agent-study outcome existed."
+  },
+  "tasks": [
+    {
+      "taskId": "heldout-automattic-mongoose-16379",
+      "taskText": "Document.isModified(path) rebuilds the full modifiedPaths() set on every call — O(N^2) when checked per-path in loops (including mongoose's own validation)\n\n## Document.isModified(path) rebuilds the full modifiedPaths() set on every call — O(N^2) when checked per-path in loops (including mongoose's own validation)\n\n**Repo:** `Automattic/mongoose`\n**Location:** lib/document.js:2339 (isModified -> modifiedPaths rebuild) called per-path from lib/document.js:2965 (_getPathsToValidate) and user hooks\n**Severity:** medium · **Confidence:** 0.8\n**Type:** complexity-at-a-distance\n\n### Description\n\nDocument.prototype.isModified in lib/document.js (line ~2321): when `path` is not directly modified, it falls through to `this[documentModifiedPaths]()` (line ~",
+      "taskTextSha256": "abab98df3c8461269aa9250a9cd3938adcd80765679b54afc3c9e9384740dfc9",
+      "repository": "https://github.com/Automattic/mongoose.git",
+      "revision": "0c3034d2886b5ff11eac8f3749cb33a896f53a57",
+      "sourceIssue": "https://github.com/Automattic/mongoose/issues/16379",
+      "selectionRationale": "Included by the pre-existing mechanical held-out selection rule before any controlled agent-study run; public issue 16379 in MIT-licensed Automattic/mongoose at the pinned pre-fix revision."
+    },
+    {
+      "taskId": "heldout-immerjs-immer-1045",
+      "taskText": "`produceWithPatches` wrong return type\n\n## 🐛 Bug Report\r\n\r\nThe curried form of `produceWithPatches` returns the wrong type, while `produce` does.\r\n\r\n## Link to repro\r\n\r\n[https://codesandbox.io/s/producewithpatches-wrong-return-type-mr5hdp?file=/src/index.ts](https://codesandbox.io/s/producewithpatches-wrong-return-type-mr5hdp?file=/src/index.ts)\r\n\r\n## Observed behavior\r\n\r\nThe state returned by `produceWithPatches((draft: Draft<N>, m: number) => { ... })` is of type `WritableDraft<N>` but should be of type `N`.\r\n\r\nThe type of `produceWithPatches<N, [number]>((draft, m) => { ... })` is `never` while the type of `produce<N, [number]>(",
+      "taskTextSha256": "0cff254c49c2db9178c3251b4ffb920258042dfaec273c2302c8a639762d53ba",
+      "repository": "https://github.com/immerjs/immer.git",
+      "revision": "bf2d15439259887f98f2737cf7ebde4234d5adea",
+      "sourceIssue": "https://github.com/immerjs/immer/issues/1045",
+      "selectionRationale": "Included by the pre-existing mechanical held-out selection rule before any controlled agent-study run; public issue 1045 in MIT-licensed immerjs/immer at the pinned pre-fix revision."
+    },
+    {
+      "taskId": "heldout-jestjs-jest-16174",
+      "taskText": "[Bug]: Spurious type error when calling `mockRejectedValue` on mocks of certain overloaded functions\n\n### Version\n\n30.4\n\n### Steps to reproduce\n\nConsider a method like [`pg.Client.end`](https://github.com/DefinitelyTyped/DefinitelyTyped/blob/5b8b3cdc7c82f3b424e9cbb18c229f40b6497cec/types/pg/index.d.ts#L307-L308). It has two overloads:\n\n```\n    end(): Promise<void>;\n    end(callback: (err: Error) => void): void;\n```\n\nWhen using a mocked instance, one might want to mock resolved or rejected values:\n\n```\ntest('mock pg', async () => {\n  const client: jest.Mocked<pg.Client> = setUpMockClient();\n  client.end.mockRejectedValueOnce(new Error('test error when closing client')) // Type error\n  await exp",
+      "taskTextSha256": "e2d59db074a767feac25c1c2cbf3800f6f661ed4fea2fe951b07c10b369a9106",
+      "repository": "https://github.com/jestjs/jest.git",
+      "revision": "4c3091b4204d703f4ebe343b8ac9d8a28ac4388e",
+      "sourceIssue": "https://github.com/jestjs/jest/issues/16174",
+      "selectionRationale": "Included by the pre-existing mechanical held-out selection rule before any controlled agent-study run; public issue 16174 in MIT-licensed jestjs/jest at the pinned pre-fix revision."
+    },
+    {
+      "taskId": "heldout-knex-knex-5053",
+      "taskText": "BUG - FOR UPDATE must specify unqualified relation names\n\n# Environment\r\n\r\nKnex version: 1.0.3\r\nDatabase + version: PostgreSQL 13\r\nOS: Node 14\r\n\r\n# Bug\r\n\r\nThe error comes up when you build a query like knex('users').withSchema('myschema').where('id','xyz').forUpdate('users')\r\nIt seems incompatible to use withSchema and forUpdate in the same query because withSchema add the schema also in front of the forUpdate table name, but should not.\r\n\r\nError message: \"FOR UPDATE must specify unqualified relation names\"\r\n\r\nPs.: the above query works if I do not specify table name in forUpdate, but in query with joins you must specify the table name otherwise you ",
+      "taskTextSha256": "c17185088d4cc6080774013036641447ee39ecee9f807ccf687e24f2b0e9e754",
+      "repository": "https://github.com/knex/knex.git",
+      "revision": "6f42421e595e47653b131a621c2bd20fcb3337a4",
+      "sourceIssue": "https://github.com/knex/knex/issues/5053",
+      "selectionRationale": "Included by the pre-existing mechanical held-out selection rule before any controlled agent-study run; public issue 5053 in MIT-licensed knex/knex at the pinned pre-fix revision."
+    },
+    {
+      "taskId": "heldout-mochajs-mocha-4526",
+      "taskText": "🐛 Bug: xunit reporter does not strip ansi escape sequences, leading to invalid XML\n\n<!--\r\nHave you read Mocha's Code of Conduct? By filing an Issue, you are expected to comply with it, including treating everyone with respect: https://github.com/mochajs/mocha/blob/master/.github/CODE_OF_CONDUCT.md\r\nFor more, check out the Mocha Gitter chat room: https://gitter.im/mochajs/mocha\r\n\r\nDetail the steps necessary to reproduce the problem. To get the fastest support, create an MCVE and upload it to GitHub.\r\ncreate an [MCVE](https://stackoverflow.com/help/mcve) and upload it to GitHub.\r\n-->\r\n\r\n### Prerequisites\r\n\r\n<!--\r\nPlace an `x` between the square brackets on the lines below for e",
+      "taskTextSha256": "a8366216fe4f1ca018817c98282b318d0d758c8868735f34ad8352c60ab03163",
+      "repository": "https://github.com/mochajs/mocha.git",
+      "revision": "d602cc7ab9b93169916d267e19071fbeb6cd0468",
+      "sourceIssue": "https://github.com/mochajs/mocha/issues/4526",
+      "selectionRationale": "Included by the pre-existing mechanical held-out selection rule before any controlled agent-study run; public issue 4526 in MIT-licensed mochajs/mocha at the pinned pre-fix revision."
+    },
+    {
+      "taskId": "heldout-react-hook-form-react-hook-form-13608",
+      "taskText": "[7.76.0 regression] setValue rebuilds the entire dirtyFields object every call, breaking referential stability (incl. proposed fix)\n\n### Version Number\n\n7.76.0 – 7.81.0 (regression introduced in 7.76.0; still present on `main`)\n\n### Codesandbox/Expo snack\n\nRunnable with no UI — the regression is in `createFormControl`'s dirty-tracking. Node snippet below (also reproducible in any sandbox).\n\nChange the version between 7.81.0 and 7.75.0 to see the differences:\nhttps://codesandbox.io/p/devbox/suspicious-joliot-flkj2p \n\n### Steps to reproduce\n\n`setValue(field, value, { shouldDirty: true })` on an already-dirty field, repeatedly, and observe the **object identity** of `formState.dirtyFields`:\n\n```js\nconst { createFormControl } =",
+      "taskTextSha256": "1983080a5b6f7ff579b1ae4525f845005a85c05021a0b1e68269982faea39c42",
+      "repository": "https://github.com/react-hook-form/react-hook-form.git",
+      "revision": "ee7593ec554c7f2cb74a8ace5984bcb8cb136633",
+      "sourceIssue": "https://github.com/react-hook-form/react-hook-form/issues/13608",
+      "selectionRationale": "Included by the pre-existing mechanical held-out selection rule before any controlled agent-study run; public issue 13608 in MIT-licensed react-hook-form/react-hook-form at the pinned pre-fix revision."
+    },
+    {
+      "taskId": "heldout-socketio-socket-io-5462",
+      "taskText": "socket.io-client - transports options array modified from [string] to [object] by io constructor\n\n**Describe the bug**\n\n```\nconst options = {transports: [\"websocket\",\"polling\"]};\nconst s = io(<URL>, options);\n```\noptions object is modified, and transports is set to  [ class WS, class XHR ]\nNot really a big issue, but quite unexpected and an unusual behaviour. Especially when you need to serialize the options object after connection.\n\n**Expected behavior**\noptions object should not be mutated.",
+      "taskTextSha256": "c3029cb8a565a9d8c3bc807a5493a9f683882d69126e5fc758aa56a13209d0b2",
+      "repository": "https://github.com/socketio/socket.io.git",
+      "revision": "0e591b92d727b1ca84e530d4ab65fa21cfdc4ae6",
+      "sourceIssue": "https://github.com/socketio/socket.io/issues/5462",
+      "selectionRationale": "Included by the pre-existing mechanical held-out selection rule before any controlled agent-study run; public issue 5462 in MIT-licensed socketio/socket.io at the pinned pre-fix revision."
+    },
+    {
+      "taskId": "heldout-sveltejs-svelte-18555",
+      "taskText": "A `<svelte:boundary>` hydrated in the failed state can never be reset\n\n### Describe the bug\n\nA boundary that fails during SSR renders its `failed` snippet and hydrates back into the failed state from the serialized marker. During that hydration `onerror` is never called, and the `reset` handed to the `failed` snippet is a no-op ([boundary.js#L200-L210](https://github.com/sveltejs/svelte/blob/4a6a85b5f149cc96514ed3bf5e59083b9246d394/packages/svelte/src/internal/client/dom/blocks/boundary.js#L200-L210)). Once a boundary hydrates as failed there is no way to leave that state short of destroying the component, even when its children would render fine on the client.\n\n",
+      "taskTextSha256": "8cce4f73d0cc0378d6921361207d65a94c1827b64a29e5934a38cf0b31d48bf8",
+      "repository": "https://github.com/sveltejs/svelte.git",
+      "revision": "2bace308e37ac1def958be750bd699ed302bb715",
+      "sourceIssue": "https://github.com/sveltejs/svelte/issues/18555",
+      "selectionRationale": "Included by the pre-existing mechanical held-out selection rule before any controlled agent-study run; public issue 18555 in MIT-licensed sveltejs/svelte at the pinned pre-fix revision."
+    },
+    {
+      "taskId": "heldout-vitejs-vite-10136",
+      "taskText": "[Panic]: thread 'rolldown-worker' (11046093) panicked at crates/rolldown_common/src/types/hybrid_index_vec.rs:76:28\n\n### Panic message\n\n```Shell\nRolldown panicked. This is a bug in Rolldown, not your code.\n\nthread 'rolldown-worker' (11046093) panicked at crates/rolldown_common/src/types/hybrid_index_vec.rs:76:28:\nHybridIndexVec::Map missing idx 2 (len=1)\nnote: run with `RUST_BACKTRACE=1` environment variable to display a backtrace\n\nPlease report this issue at: https://github.com/rolldown/rolldown/issues/new?template=panic_report.yml\n```\n\n### Reproduction\n\nhttps://github.com/btea/rolldown-panic\n\n1. Running `pnpm dev` works perfectly.\n2. The following changes were made to line 2 of the `src/App.vue` file, and ",
+      "taskTextSha256": "27a0f764d5375ce57c1f5ca128701520c5078fdbfae97c477d6c88572b9dcdec",
+      "repository": "https://github.com/vitejs/vite.git",
+      "revision": "791843e1591ec2d65a401560bc1fc0a85b32ee32",
+      "sourceIssue": "https://github.com/vitejs/vite/issues/10136",
+      "selectionRationale": "Included by the pre-existing mechanical held-out selection rule before any controlled agent-study run; public issue 10136 in MIT-licensed vitejs/vite at the pinned pre-fix revision."
+    },
+    {
+      "taskId": "heldout-vuejs-core-11564",
+      "taskText": "Missing support TS 5.5, without `DOM` lib in tsconfig\n\n### Vue version\n\nsince 3.0.0, to 3.5 alpha\n\n### Link to minimal reproduction\n\nhttps://github.com/sxzz/vue-ref-issue\n\n### Steps to reproduce\n\n- Install TS 5.5.4\r\n- Ensure there is no `DOM` lib in the `tsconfig.json`\r\n- Define `ref(100)` with its type as `Ref<any>`\n\n### What is expected?\n\n`Ref<number>`\n\n### What is actually happening?\n\n`Ref<any>`\n\n### System Info\n\n_No response_\n\n### Any additional comments?\n\nWorks on TS 5.4",
+      "taskTextSha256": "0c7bc999814525f02df677118328d63f32e5d43ab2185811fe8434943d21ee9f",
+      "repository": "https://github.com/vuejs/core.git",
+      "revision": "b555f02eede06f500deb25d3ba5ee44217ce1265",
+      "sourceIssue": "https://github.com/vuejs/core/issues/11564",
+      "selectionRationale": "Included by the pre-existing mechanical held-out selection rule before any controlled agent-study run; public issue 11564 in MIT-licensed vuejs/core at the pinned pre-fix revision."
+    },
+    {
+      "taskId": "heldout-winstonjs-winston-2610",
+      "taskText": "[Bug]: DeprecationWarning: fs.F_OK is deprecated, use fs.constants.F_OK instead (DEP0176)\n\n### 🔎 Search Terms\n\nfs.F_OK\n\n### The problem\n\nRunning winston on Node ≥ 22 emits:\n\n(node:55907) [DEP0176] DeprecationWarning: fs.F_OK is deprecated, use fs.constants.F_OK instead\n\nThe warning comes from FileTransport._tail, which still reads F_OK directly off the fs module.\n\nNode 22 deprecated fs.F_OK / fs.R_OK / fs.W_OK / fs.X_OK as direct properties on fs — see DEP0176. Replacement is fs.constants.F_OK.\n\nHappy to send a PR if it helps.\n\n### What version of Winston presents the issue?\n\nv3.13.0\n\n### What version of Node are you using?\n\nv24.8.0\n\n### If this worked in a previous version of Wins",
+      "taskTextSha256": "1da88d4efb3fa63a9f524e4e8dff58d9e799f9ca43d8072ff5e67237e93c9fdf",
+      "repository": "https://github.com/winstonjs/winston.git",
+      "revision": "1f89222a796225923b87f0462497b92e274915ce",
+      "sourceIssue": "https://github.com/winstonjs/winston/issues/2610",
+      "selectionRationale": "Included by the pre-existing mechanical held-out selection rule before any controlled agent-study run; public issue 2610 in MIT-licensed winstonjs/winston at the pinned pre-fix revision."
+    },
+    {
+      "taskId": "heldout-yargs-yargs-2497",
+      "taskText": "Vulnerability Report: Prototype Pollution in `apply-extends`\n\n\n## Summary\nA prototype pollution vulnerability was identified in the `apply-extends` helper function within `lib/utils/apply-extends.ts`. This function is responsible for merging configuration objects, including those loaded via the `extends` keyword. The vulnerability allowed an attacker to inject properties into `Object.prototype` by including a `__proto__` key in a configuration file.\n\n## Technical Details\n\n### Vulnerability Location\nThe vulnerability existed in the `mergeDeep` function within `lib/utils/apply-extends.ts`. This function recursively merged two objects without checking if th",
+      "taskTextSha256": "fef7a6f78bb4b43585ba1dc390979b6d0413bee60e69527cf3299524b38d6216",
+      "repository": "https://github.com/yargs/yargs.git",
+      "revision": "971e351705f0fbc5566c6ed1dfd707fa65e11c0d",
+      "sourceIssue": "https://github.com/yargs/yargs/issues/2497",
+      "selectionRationale": "Included by the pre-existing mechanical held-out selection rule before any controlled agent-study run; public issue 2497 in MIT-licensed yargs/yargs at the pinned pre-fix revision."
+    }
+  ]
+}
diff --git a/docs/AGENT_STUDY.md b/docs/AGENT_STUDY.md
index 2a86ef5..533e570 100644
--- a/docs/AGENT_STUDY.md
+++ b/docs/AGENT_STUDY.md
@@ -1,50 +1,109 @@
 # FixMap differential agent study
 
-FixMap includes a frozen, machine-checked four-arm protocol. It does **not** publish an
-effectiveness percentage until complete raw runs exist for every task and arm.
+FixMap includes a frozen, machine-checked four-arm protocol and a frozen task manifest. It
+does **not** include controlled agent runs or publish an effectiveness, token, cost, or time
+result.
 
-The arms are:
+The four arms are:
 
 1. Baseline agent with ordinary repository tools.
 2. FixMap available, with no instruction requiring its use.
 3. FixMap explicitly instructed before exploration and Verify after editing.
 4. The same instructed workflow with the 0.9 Impact Graph and compact agent output.
 
-Every paired task must use the same model and model version, task text, repository revision,
-timeout, and budget. Each run starts with fresh context, arm order is randomized, FixMap is
-frozen during the study, the task-success rubric and price sheet are recorded before execution,
-and the raw transcript is retained. Failed and timed-out runs stay in the dataset. The primary
-navigation metric is tool calls to the first relevant file, not turns to the first edit.
+## Frozen task set
 
-Each JSONL row records provider-reported input, cached-input, output, reasoning, and total tokens
-when the provider exposes them; model cost in USD under the frozen price sheet; turns; total tool
-calls; repository-search calls; files and source bytes read; wall-clock time; correctness; test
-selection; and failure status. Unsupported provider counters are `null`, never estimated from
-characters or repository size.
+[`benchmarks/agent-study/tasks.json`](../benchmarks/agent-study/tasks.json) is the sole task
+manifest for `fixmap-navigation-heldout-v1`. It freezes all 12 public, MIT-licensed issue tasks
+from the pre-existing held-out suite at their exact pre-fix repository revisions. The tasks were
+selected mechanically before any controlled agent-study outcome existed. A run file cannot add,
+remove, rename, or rewrite a task.
 
-The row also pins `taskId`, `taskTextSha256`, `arm`, randomized `runOrder`, model and version,
-repository and revision, `environmentId`, `timeoutMs`, `budgetId`, `priceSheetId`, and transcript
-reference plus SHA-256. Those controls let the evaluator reject a comparison where more than the
-experimental arm changed.
+Each manifest entry contains:
 
-Validate the protocol without claiming a result:
+- `taskId`: stable identity used in run rows;
+- `taskText`: exact prompt text;
+- `taskTextSha256`: SHA-256 of the UTF-8 task text;
+- `repository`: public Git repository URL;
+- `revision`: exact 40-character pre-fix Git revision;
+- `sourceIssue`: public issue URL;
+- `selectionRationale`: why the task belongs in the frozen set.
 
-```bash
-npm run study:agent:check
-```
+`npm run study:agent:check` verifies protocol version 3, the manifest schema, task uniqueness,
+exact task-text hashes, and pinned revision syntax without evaluating or claiming a result.
+
+## Run contract
+
+Every study group must contain exactly one run for every manifest task and every arm: 12 tasks ×
+4 arms = 48 rows. Each row records `studyId`, the exact `taskText` and its hash, arm and randomized
+order, repository and revision, environment, unique declared context ID, timeout, budget, task-specific rubric ID, price
+sheet, one globally pinned model and model version, one globally pinned FixMap revision,
+transcript reference and SHA-256, run status, failure reason, and every declared quality or
+efficiency metric.
+
+The transcript reference must be a unique path relative to the JSONL file. It cannot be absolute
+or escape the run-data directory. The evaluator reads the referenced bytes and compares their
+real SHA-256 with `transcriptSha256`; a syntactically valid but incorrect hash is rejected.
+
+Token counters must use `tokenAccountingSource: "provider-reported"`. Provider-reported input,
+cached-input, output, reasoning, and total counters are recorded when exposed. An unsupported
+counter is `null`; the evaluator never estimates it from characters, source bytes, other token
+counters, or repository size. Model cost may be recorded under the frozen `priceSheetId`.
+
+Failed and timed-out runs remain required rows. They need a non-empty `failureReason`, stay in
+each arm's `runs` denominator, and are reported separately as `failedRuns` and `timedOutRuns`.
+They are never filtered out to make an arm look better.
+
+## Global model pin
 
-When paid or externally metered runs have been authorized and collected, store one JSON object
-per line outside the repository and evaluate it explicitly:
+One evaluator invocation is one model study group. Evaluation requires explicit global pins:
 
 ```bash
-node scripts/evaluate-agent-study.mjs --input path/to/runs.jsonl
+node scripts/evaluate-agent-study.mjs \
+  --input path/to/study/runs.jsonl \
+  --model exact-model-name \
+  --model-version exact-provider-version \
+  --fixmap-revision exact-40-character-git-revision
 ```
 
-The evaluator rejects incomplete task/arm sets, duplicate runs, invalid metric types, mismatched
-models or repository revisions, missing transcript hashes, inconsistent task controls, and
-unaccounted failures. Its summary reports quality and efficiency together; a cheaper run is not a
-win if task resolution or patch acceptance falls.
+Every row must match both model pins. Mixed models or versions are rejected before aggregation. A
+multi-model investigation must keep one complete JSONL file and one evaluator output per exact
+model/version; those aggregates must be reported separately and cannot be pooled by this tool.
+Every row must also match the global FixMap revision, including baseline rows where the tool is
+not exposed, so the experimental implementation cannot drift mid-study.
+
+## Publishable aggregate gate
+
+The evaluator emits `publishableAggregate: true` only after all of these checks pass:
+
+- the checked-in protocol and task manifest are valid and frozen;
+- the run set contains exactly the 48 required task/arm pairs;
+- there are no missing, extra, unknown, or duplicate task/arm runs;
+- every row matches the frozen task text, task hash, repository, revision, and study ID;
+- every row has a unique declared context ID;
+- every task holds environment, timeout, budget, rubric ID, and price sheet constant and assigns each arm a unique order from 1 through 4;
+- every row matches the globally pinned model and model version;
+- every row matches the globally pinned FixMap revision;
+- every transcript exists inside the run-data directory and its bytes match its SHA-256;
+- token counters are declared provider-reported and every metric has the required type;
+- completed runs have no failure reason, while failed and timed-out runs retain one.
+
+The aggregate reports outcome rates over **all** required runs plus status counts and medians for
+available numeric values. `publishableAggregate` means the artifact is structurally complete and
+auditable under this protocol. It does not by itself establish causal validity, generalization,
+adequate statistical power, unbiased human scoring, or a marketing claim.
+
+## Data handling and authorization
 
 Run data is deliberately not checked in by default: transcripts can contain source, prompts,
-and account metadata. Any public study must use consented, reviewed, redacted artifacts and link
-the exact model/version and frozen task-selection record.
+tool output, and account metadata. Any public study must use consented, reviewed, redacted
+artifacts and publish the exact manifest hash, model/version, FixMap revision, rubric, price
+sheet, environment, context IDs, and evaluator output.
+
+Controlled model runs may incur cost. Do not run them without explicit authorization. The
+checked-in validation and rejection tests use synthetic local files only and make no model or API
+calls.
+
+The evaluator verifies that context IDs are unique and that each task records a complete arm-order
+permutation; it cannot prove that the underlying agent processes were actually fresh or that order
+assignment was random. Those execution controls still require independent collection records.
diff --git a/docs/assets/fixmap-cli-demo.svg b/docs/assets/fixmap-cli-demo.svg
index 140863b..99aeda3 100644
--- a/docs/assets/fixmap-cli-demo.svg
+++ b/docs/assets/fixmap-cli-demo.svg
@@ -26,7 +26,7 @@
   <text class="l" style="animation-delay:5.75s" x="22" y="352" fill="#e6edf3"><tspan>- </tspan><tspan fill="#79c0ff">test/auth/reset-password.test.ts</tspan><tspan> (high confidence, impact 13): this file imports</tspan></text>
   <text class="l" style="animation-delay:6.00s" x="22" y="373" fill="#e6edf3"><tspan>    src/auth/reset-password.ts; routed test for src/auth/reset-password.ts via npm run test</tspan></text>
   <text class="l" style="animation-delay:6.25s" x="22" y="415" fill="#e6edf3"><tspan>Inspection order: </tspan><tspan fill="#79c0ff">src/auth/reset-password.ts</tspan><tspan> → </tspan><tspan fill="#79c0ff">test/auth/reset-password.test.ts</tspan><tspan>.</tspan></text>
-  <text class="l" style="animation-delay:6.50s" x="22" y="436" fill="#e6edf3"><tspan>History evidence: 29 eligible commits.</tspan></text>
+  <text class="l" style="animation-delay:6.50s" x="22" y="436" fill="#e6edf3"><tspan>History evidence: 30 eligible commits.</tspan></text>
   <text class="l" style="animation-delay:7.65s" x="22" y="478" fill="#e6edf3"><tspan fill="#74f0ba" font-weight="bold">## Test Routes</tspan></text>
   <text class="l" style="animation-delay:7.90s" x="22" y="520" fill="#e6edf3"><tspan>- </tspan><tspan fill="#79c0ff">npm run test</tspan><tspan>: repository root script named test. Related:</tspan></text>
   <text class="l" style="animation-delay:8.15s" x="22" y="541" fill="#e6edf3"><tspan>    </tspan><tspan fill="#79c0ff">test/auth/reset-password.test.ts</tspan><tspan>.</tspan></text>
diff --git a/examples/reports/declines-fabricated-identifier.md b/examples/reports/declines-fabricated-identifier.md
index 05f4cda..a0a3017 100644
--- a/examples/reports/declines-fabricated-identifier.md
+++ b/examples/reports/declines-fabricated-identifier.md
@@ -19,7 +19,7 @@ FixMap found 0 context files, 0 impact files, and generated 0 test routes.
 - None found
 
 Inspection order: None.
-History evidence: 29 eligible commits.
+History evidence: 30 eligible commits.
 
 ## Test Routes
 
diff --git a/examples/reports/declines-unmatched-terms.md b/examples/reports/declines-unmatched-terms.md
index 6062385..13b9fdd 100644
--- a/examples/reports/declines-unmatched-terms.md
+++ b/examples/reports/declines-unmatched-terms.md
@@ -19,7 +19,7 @@ FixMap found 0 context files, 0 impact files, and generated 0 test routes.
 - None found
 
 Inspection order: None.
-History evidence: 29 eligible commits.
+History evidence: 30 eligible commits.
 
 ## Test Routes
 
diff --git a/examples/reports/declines-vague-task.md b/examples/reports/declines-vague-task.md
index d98ca20..b17a081 100644
--- a/examples/reports/declines-vague-task.md
+++ b/examples/reports/declines-vague-task.md
@@ -19,7 +19,7 @@ FixMap found 0 context files, 0 impact files, and generated 0 test routes.
 - None found
 
 Inspection order: None.
-History evidence: 29 eligible commits.
+History evidence: 30 eligible commits.
 
 ## Test Routes
 
diff --git a/examples/reports/workspace-api-discount.md b/examples/reports/workspace-api-discount.md
index b9d97a6..ad87740 100644
--- a/examples/reports/workspace-api-discount.md
+++ b/examples/reports/workspace-api-discount.md
@@ -13,7 +13,7 @@ FixMap found 2 context files, 2 impact files, and generated 3 test routes.
 - `packages/utils/test/currency.test.ts` (high confidence, impact 13): this file imports packages/utils/src/currency.ts; routed test for packages/utils/src/currency.ts via pnpm --dir packages/utils run test
 
 Inspection order: `apps/api/src/orders.ts` → `packages/utils/src/currency.ts` → `apps/api/test/orders.test.ts` → `packages/utils/test/currency.test.ts`.
-History evidence: 64 eligible commits.
+History evidence: 65 eligible commits.
 
 ## Test Routes
 
diff --git a/examples/reports/workspace-utils-rounding.md b/examples/reports/workspace-utils-rounding.md
index b0d44b9..93453e4 100644
--- a/examples/reports/workspace-utils-rounding.md
+++ b/examples/reports/workspace-utils-rounding.md
@@ -11,7 +11,7 @@ FixMap found 1 context file, 1 impact file, and generated 2 test routes.
 - `packages/utils/test/currency.test.ts` (high confidence, impact 13): this file imports packages/utils/src/currency.ts; routed test for packages/utils/src/currency.ts via pnpm --dir packages/utils run test
 
 Inspection order: `packages/utils/src/currency.ts` → `packages/utils/test/currency.test.ts`.
-History evidence: 64 eligible commits.
+History evidence: 65 eligible commits.
 
 ## Test Routes
 
diff --git a/package.json b/package.json
index 0f02336..0563146 100644
--- a/package.json
+++ b/package.json
@@ -51,7 +51,7 @@
     "benchmark:check": "npm run build:core && node scripts/benchmark-scan.mjs --tier 1000 --check",
     "benchmark:savings": "npm run build:core && node scripts/benchmark-savings.mjs",
     "benchmark:savings:record": "npm run build:core && node scripts/benchmark-savings.mjs --record",
-    "ci": "npm run typecheck && npm test && npm run audit:all && npm run lint && npm run build:ci && npm run check:action-metadata && npm run check:server-manifest && npm run check:action-bundle && npm run check:rendered && npm run smoke && npm run study:agent:check && npm run evaluate && npm run evaluate:adversarial:gate && npm run benchmark:check",
+    "ci": "npm run typecheck && npm test && npm run audit:all && npm run lint && npm run build:ci && npm run check:action-metadata && npm run check:server-manifest && npm run check:action-bundle && npm run check:rendered && npm run smoke && npm run study:agent:check && npm run study:agent:test && npm run evaluate && npm run evaluate:adversarial:gate && npm run benchmark:check",
     "evaluate": "npm run build:core && node scripts/evaluate.mjs",
     "evaluate:external": "npm run build:core && node scripts/evaluate-external.mjs",
     "evaluate:external:record": "npm run build:core && node scripts/evaluate-external.mjs --record",
@@ -68,7 +68,8 @@
     "evaluate:adversarial:record": "npm run build:core && node scripts/evaluate-adversarial.mjs --record",
     "render:examples": "npm run build:core && node scripts/render-honest-examples.mjs",
     "check:rendered": "node scripts/check-generated.mjs rendered",
-    "study:agent:check": "node scripts/evaluate-agent-study.mjs"
+    "study:agent:check": "node scripts/evaluate-agent-study.mjs",
+    "study:agent:test": "node --test scripts/evaluate-agent-study.test.mjs"
   },
   "engines": {
     "node": ">=20.11"
diff --git a/scripts/evaluate-agent-study.mjs b/scripts/evaluate-agent-study.mjs
index 518d0ad..5a84e55 100644
--- a/scripts/evaluate-agent-study.mjs
+++ b/scripts/evaluate-agent-study.mjs
@@ -1,10 +1,15 @@
+import { createHash } from "node:crypto";
 import { readFile } from "node:fs/promises";
 import { fileURLToPath } from "node:url";
-import { dirname, join, resolve } from "node:path";
+import { dirname, isAbsolute, join, relative, resolve } from "node:path";
 
 const root = resolve(dirname(fileURLToPath(import.meta.url)), "..");
 const protocolPath = join(root, "benchmarks", "agent-study", "protocol.json");
-const protocol = JSON.parse(await readFile(protocolPath, "utf8"));
+const manifestPath = join(root, "benchmarks", "agent-study", "tasks.json");
+const protocolBytes = await readFile(protocolPath);
+const manifestBytes = await readFile(manifestPath);
+const protocol = JSON.parse(protocolBytes.toString("utf8"));
+const manifest = JSON.parse(manifestBytes.toString("utf8"));
 const requiredArms = ["baseline", "fixmap-available", "fixmap-instructed", "fixmap-impact"];
 const requiredMetrics = [
   "runStatus", "failureReason",
@@ -14,10 +19,17 @@ const requiredMetrics = [
   "turns", "repositorySearchCalls", "filesRead", "sourceBytesRead", "wallClockMs",
   "testsSelectedCorrectly", "finalPatchAccepted", "fixmapPlanUsed", "verifyUsefulWarnings"
 ];
+const requiredRequirements = [
+  "sameGlobalModelVersion", "sameTaskText", "sameRepositoryRevision", "uniqueContextIdPerRun",
+  "uniqueArmOrderPerTask", "fixedTimeoutAndBudget", "noFixMapChangesMidStudy",
+  "rawTranscriptsRequired", "transcriptContentHashVerified", "taskManifestFrozen",
+  "completeTaskArmCrossProduct", "taskTextMatchesManifest", "taskSuccessRubricIdRecorded",
+  "providerReportedTokenCountersOnly", "priceSheetIdRecorded", "failuresRetained"
+];
 const identityFields = [
-  "taskId", "taskTextSha256", "arm", "runOrder", "model", "modelVersion", "repository",
-  "revision", "environmentId", "timeoutMs", "budgetId", "priceSheetId", "transcript",
-  "transcriptSha256"
+  "studyId", "taskId", "taskText", "taskTextSha256", "arm", "runOrder", "model",
+  "modelVersion", "fixmapRevision", "repository", "revision", "environmentId", "contextId", "timeoutMs",
+  "budgetId", "rubricId", "priceSheetId", "tokenAccountingSource", "transcript", "transcriptSha256"
 ];
 const booleanMetrics = [
   "taskResolved", "correctFileInFirstThreeOpened", "testsSelectedCorrectly",
@@ -31,35 +43,119 @@ const numericMetrics = [
 ];
 const runStatuses = ["completed", "failed", "timed-out"];
 
-if (protocol.protocolVersion !== 2 || protocol.status !== "protocol-only" ||
+const sha256 = (value) => createHash("sha256").update(value).digest("hex");
+const option = (name) => {
+  const index = process.argv.indexOf(name);
+  if (index === -1) return null;
+  const value = process.argv[index + 1];
+  if (!value || value.startsWith("--")) throw new Error(`${name} requires a value.`);
+  return value;
+};
+
+if (protocol.protocolVersion !== 3 || protocol.status !== "frozen-no-runs" ||
+  protocol.taskManifest !== "benchmarks/agent-study/tasks.json" ||
   JSON.stringify(protocol.arms) !== JSON.stringify(requiredArms) ||
   requiredMetrics.some((metric) => !protocol.metrics.includes(metric)) ||
-  Object.values(protocol.requirements).some((value) => value !== true)) {
-  throw new Error("Agent-study protocol is incomplete or has drifted from its frozen four-arm contract.");
+  requiredRequirements.some((requirement) => protocol.requirements[requirement] !== true) ||
+  protocol.aggregationPolicy?.publishableOnlyWhenComplete !== true ||
+  protocol.aggregationPolicy?.modelGrouping !== "single-global-model-version" ||
+  protocol.aggregationPolicy?.failedAndTimedOutRuns !== "retained") {
+  throw new Error("Agent-study protocol is incomplete or has drifted from its frozen contract.");
+}
+
+if (manifest.manifestVersion !== 1 || manifest.protocolVersion !== protocol.protocolVersion ||
+  manifest.status !== "frozen" || typeof manifest.studyId !== "string" || !manifest.studyId.trim() ||
+  !/^\d{4}-\d{2}-\d{2}$/.test(manifest.frozenAt) ||
+  ["source", "rule", "rationale"].some((field) => typeof manifest.taskSelection?.[field] !== "string" || !manifest.taskSelection[field].trim()) ||
+  !Array.isArray(manifest.tasks) || manifest.tasks.length === 0) {
+  throw new Error("Agent-study task manifest is incomplete or is not frozen.");
 }
 
-const inputIndex = process.argv.indexOf("--input");
-if (inputIndex === -1) {
-  process.stdout.write("Agent-study protocol valid. No run data supplied; no effectiveness result is claimed.\n");
+const manifestTasks = new Map();
+for (const [index, task] of manifest.tasks.entries()) {
+  for (const field of ["taskId", "taskText", "taskTextSha256", "repository", "revision", "sourceIssue", "selectionRationale"]) {
+    if (typeof task[field] !== "string" || !task[field].trim()) throw new Error(`Manifest task ${index + 1} is missing ${field}.`);
+  }
+  if (manifestTasks.has(task.taskId)) throw new Error(`Duplicate manifest taskId ${task.taskId}.`);
+  if (!/^[a-z0-9][a-z0-9-]+$/.test(task.taskId)) throw new Error(`Manifest task ${index + 1} has an invalid taskId.`);
+  if (!/^[a-f0-9]{64}$/.test(task.taskTextSha256) || sha256(task.taskText) !== task.taskTextSha256) {
+    throw new Error(`Manifest task ${task.taskId} has a task-text hash mismatch.`);
+  }
+  for (const [field, url] of [["repository", task.repository], ["sourceIssue", task.sourceIssue]]) {
+    try {
+      if (new URL(url).protocol !== "https:") throw new Error();
+    } catch {
+      throw new Error(`Manifest task ${task.taskId} has an invalid ${field} URL.`);
+    }
+  }
+  if (!/^[a-f0-9]{40}$/i.test(task.revision)) throw new Error(`Manifest task ${task.taskId} has an invalid pinned revision.`);
+  manifestTasks.set(task.taskId, task);
+}
+
+const inputPathOption = option("--input");
+if (!inputPathOption) {
+  process.stdout.write(`Agent-study protocol and ${manifest.tasks.length}-task frozen manifest valid. No run data supplied; no effectiveness result is claimed.\n`);
   process.exit(0);
 }
-const inputPath = process.argv[inputIndex + 1];
-if (!inputPath) throw new Error("--input requires a JSONL run file.");
-const rows = (await readFile(resolve(inputPath), "utf8"))
+
+const expectedModel = option("--model");
+const expectedModelVersion = option("--model-version");
+const expectedFixMapRevision = option("--fixmap-revision");
+if (!expectedModel || !expectedModelVersion || !expectedFixMapRevision) {
+  throw new Error("Run evaluation requires globally pinned --model, --model-version, and --fixmap-revision values.");
+}
+if (!/^[a-f0-9]{40}$/i.test(expectedFixMapRevision)) throw new Error("--fixmap-revision must be an exact 40-character Git revision.");
+
+const inputPath = resolve(inputPathOption);
+const inputDirectory = dirname(inputPath);
+const rows = (await readFile(inputPath, "utf8"))
   .split(/\r?\n/).filter(Boolean).map((line, index) => {
     try { return JSON.parse(line); } catch (error) { throw new Error(`Invalid JSONL at line ${index + 1}: ${error.message}`); }
   });
 if (rows.length === 0) throw new Error("Agent-study input contains no runs.");
 
 const keys = new Set();
+const transcripts = new Set();
+const contexts = new Set();
 for (const [index, row] of rows.entries()) {
   for (const field of [...identityFields, ...requiredMetrics]) {
     if (!(field in row)) throw new Error(`Run ${index + 1} is missing ${field}.`);
   }
+  const manifestTask = manifestTasks.get(row.taskId);
+  if (!manifestTask) throw new Error(`Run ${index + 1} has extra or unrecognized task ${JSON.stringify(row.taskId)}.`);
+  if (row.studyId !== manifest.studyId) throw new Error(`Run ${index + 1} does not match frozen studyId ${manifest.studyId}.`);
+  for (const field of ["environmentId", "contextId", "budgetId", "rubricId", "priceSheetId"]) {
+    if (typeof row[field] !== "string" || !row[field].trim()) throw new Error(`Run ${index + 1} has invalid ${field}.`);
+  }
+  if (contexts.has(row.contextId)) throw new Error(`Run ${index + 1} reuses contextId ${JSON.stringify(row.contextId)}.`);
+  contexts.add(row.contextId);
   if (!requiredArms.includes(row.arm)) throw new Error(`Run ${index + 1} has unknown arm ${JSON.stringify(row.arm)}.`);
+  if (typeof row.taskText !== "string" || row.taskText !== manifestTask.taskText) throw new Error(`Run ${index + 1} task text differs from the frozen manifest.`);
+  const computedTaskHash = sha256(row.taskText);
+  if (!/^[a-f0-9]{64}$/i.test(row.taskTextSha256) || computedTaskHash !== row.taskTextSha256.toLowerCase()) {
+    throw new Error(`Run ${index + 1} task-text hash does not match its task text.`);
+  }
+  if (row.taskTextSha256.toLowerCase() !== manifestTask.taskTextSha256.toLowerCase()) throw new Error(`Run ${index + 1} task-text hash does not match the frozen manifest.`);
+  if (row.repository !== manifestTask.repository || row.revision !== manifestTask.revision) throw new Error(`Run ${index + 1} does not use the frozen repository revision for ${row.taskId}.`);
+  if (row.model !== expectedModel || row.modelVersion !== expectedModelVersion) throw new Error(`Run ${index + 1} does not use the globally pinned model and version.`);
+  if (row.fixmapRevision !== expectedFixMapRevision) throw new Error(`Run ${index + 1} does not use the globally pinned FixMap revision.`);
+  if (row.tokenAccountingSource !== "provider-reported") throw new Error(`Run ${index + 1} token counters are not declared provider-reported.`);
   if (typeof row.transcript !== "string" || !row.transcript.trim()) throw new Error(`Run ${index + 1} has no transcript reference.`);
-  if (!/^[a-f0-9]{64}$/i.test(row.taskTextSha256)) throw new Error(`Run ${index + 1} has an invalid taskTextSha256.`);
-  if (!/^[a-f0-9]{64}$/i.test(row.transcriptSha256)) throw new Error(`Run ${index + 1} has an invalid transcriptSha256.`);
+  if (isAbsolute(row.transcript)) throw new Error(`Run ${index + 1} transcript must be relative to the run file.`);
+  const transcriptPath = resolve(inputDirectory, row.transcript);
+  const transcriptRelative = relative(inputDirectory, transcriptPath);
+  if (transcriptRelative === ".." || transcriptRelative.split(/[\\/]/)[0] === ".." || isAbsolute(transcriptRelative)) {
+    throw new Error(`Run ${index + 1} transcript escapes the run-data directory.`);
+  }
+  if (transcripts.has(transcriptPath)) throw new Error(`Run ${index + 1} reuses transcript ${JSON.stringify(row.transcript)}.`);
+  transcripts.add(transcriptPath);
+  let transcriptBytes;
+  try { transcriptBytes = await readFile(transcriptPath); } catch {
+    throw new Error(`Run ${index + 1} transcript file is missing: ${row.transcript}.`);
+  }
+  if (!/^[a-f0-9]{64}$/i.test(row.transcriptSha256) || sha256(transcriptBytes) !== row.transcriptSha256.toLowerCase()) {
+    throw new Error(`Run ${index + 1} transcript SHA-256 does not match ${row.transcript}.`);
+  }
   if (!runStatuses.includes(row.runStatus)) throw new Error(`Run ${index + 1} has invalid runStatus ${JSON.stringify(row.runStatus)}.`);
   if (!Number.isInteger(row.runOrder) || row.runOrder < 1 || row.runOrder > requiredArms.length) throw new Error(`Run ${index + 1} has an invalid runOrder.`);
   if (!Number.isFinite(row.timeoutMs) || row.timeoutMs <= 0) throw new Error(`Run ${index + 1} has an invalid timeoutMs.`);
@@ -79,16 +175,19 @@ for (const [index, row] of rows.entries()) {
   keys.add(key);
 }
 
-const tasks = [...new Set(rows.map((row) => row.taskId))];
-for (const task of tasks) {
-  const taskRows = rows.filter((row) => row.taskId === task);
-  const arms = taskRows.map((row) => row.arm).sort();
-  if (JSON.stringify(arms) !== JSON.stringify([...requiredArms].sort())) {
-    throw new Error(`Task ${task} does not contain exactly one run from every arm.`);
-  }
-  if (new Set(taskRows.map((row) => row.runOrder)).size !== requiredArms.length) throw new Error(`Task ${task} does not contain a unique randomized runOrder for every arm.`);
-  for (const field of ["taskTextSha256", "model", "modelVersion", "repository", "revision", "environmentId", "timeoutMs", "budgetId", "priceSheetId"]) {
-    if (new Set(taskRows.map((row) => row[field])).size !== 1) throw new Error(`Task ${task} does not hold ${field} constant.`);
+const expectedKeys = new Set(manifest.tasks.flatMap((task) => requiredArms.map((arm) => `${task.taskId}\0${arm}`)));
+const missingKeys = [...expectedKeys].filter((key) => !keys.has(key));
+if (missingKeys.length > 0) {
+  const formatted = missingKeys.map((key) => key.replace("\0", " / ")).join(", ");
+  throw new Error(`Study is missing required manifest task/arm runs: ${formatted}.`);
+}
+if (rows.length !== expectedKeys.size) throw new Error(`Study has ${rows.length} runs; expected exactly ${expectedKeys.size}.`);
+
+for (const task of manifest.tasks) {
+  const taskRows = rows.filter((row) => row.taskId === task.taskId);
+  if (new Set(taskRows.map((row) => row.runOrder)).size !== requiredArms.length) throw new Error(`Task ${task.taskId} does not contain a unique randomized runOrder for every arm.`);
+  for (const field of ["taskText", "taskTextSha256", "repository", "revision", "environmentId", "timeoutMs", "budgetId", "rubricId", "priceSheetId"]) {
+    if (new Set(taskRows.map((row) => row[field])).size !== 1) throw new Error(`Task ${task.taskId} does not hold ${field} constant.`);
   }
 }
 
@@ -104,10 +203,11 @@ const aggregate = Object.fromEntries(requiredArms.map((arm) => {
   return [arm, {
     runs: armRows.length,
     completedRuns: armRows.filter((row) => row.runStatus === "completed").length,
-    failedRuns: armRows.filter((row) => row.runStatus !== "completed").length,
-    taskResolutionRate: rate("taskResolved"),
-    acceptedPatchRate: rate("finalPatchAccepted"),
-    firstThreeFileRate: rate("correctFileInFirstThreeOpened"),
+    failedRuns: armRows.filter((row) => row.runStatus === "failed").length,
+    timedOutRuns: armRows.filter((row) => row.runStatus === "timed-out").length,
+    taskResolutionRateAllRuns: rate("taskResolved"),
+    acceptedPatchRateAllRuns: rate("finalPatchAccepted"),
+    firstThreeFileRateAllRuns: rate("correctFileInFirstThreeOpened"),
     medianToolCallsToRelevantFile: median("toolCallsToFirstRelevantFile"),
     medianTotalToolCalls: median("totalToolCalls"),
     medianRepositorySearchCalls: median("repositorySearchCalls"),
@@ -123,4 +223,19 @@ const aggregate = Object.fromEntries(requiredArms.map((arm) => {
     medianWallClockMs: median("wallClockMs")
   }];
 }));
-process.stdout.write(`${JSON.stringify({ protocolVersion: 2, tasks: tasks.length, aggregate }, null, 2)}\n`);
+
+process.stdout.write(`${JSON.stringify({
+  protocolVersion: protocol.protocolVersion,
+  manifestVersion: manifest.manifestVersion,
+  studyId: manifest.studyId,
+  taskManifestSha256: sha256(manifestBytes),
+  model: expectedModel,
+  modelVersion: expectedModelVersion,
+  fixmapRevision: expectedFixMapRevision,
+  tasks: manifest.tasks.length,
+  arms: requiredArms.length,
+  runs: rows.length,
+  publishableAggregate: true,
+  failuresRetained: true,
+  aggregate
+}, null, 2)}\n`);
diff --git a/scripts/evaluate-agent-study.test.mjs b/scripts/evaluate-agent-study.test.mjs
new file mode 100644
index 0000000..ea3b550
--- /dev/null
+++ b/scripts/evaluate-agent-study.test.mjs
@@ -0,0 +1,255 @@
+import assert from "node:assert/strict";
+import { execFile } from "node:child_process";
+import { createHash } from "node:crypto";
+import { mkdtemp, mkdir, readFile, rm, unlink, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { dirname, join, relative, resolve } from "node:path";
+import { promisify } from "node:util";
+import { fileURLToPath } from "node:url";
+import test from "node:test";
+
+const execFileAsync = promisify(execFile);
+const root = resolve(dirname(fileURLToPath(import.meta.url)), "..");
+const evaluatorPath = join(root, "scripts", "evaluate-agent-study.mjs");
+const manifest = JSON.parse(await readFile(join(root, "benchmarks", "agent-study", "tasks.json"), "utf8"));
+const arms = ["baseline", "fixmap-available", "fixmap-instructed", "fixmap-impact"];
+const model = "test-model";
+const modelVersion = "test-model-2026-08-20";
+const fixmapRevision = "bbc8469c937397fecac787c71c04521a6eb1a87d";
+const sha256 = (value) => createHash("sha256").update(value).digest("hex");
+
+async function createFixture(t, mutate = async () => {}) {
+  const directory = await mkdtemp(join(tmpdir(), "fixmap-agent-study-"));
+  t.after(() => rm(directory, { recursive: true, force: true }));
+  const transcriptDirectory = join(directory, "transcripts");
+  await mkdir(transcriptDirectory);
+  const rows = [];
+  const transcriptPaths = [];
+
+  for (const task of manifest.tasks) {
+    for (const [armIndex, arm] of arms.entries()) {
+      const transcriptPath = join(transcriptDirectory, `${task.taskId}-${arm}.txt`);
+      const transcript = `Transcript for ${task.taskId} / ${arm}\n`;
+      await writeFile(transcriptPath, transcript, "utf8");
+      transcriptPaths.push(transcriptPath);
+      rows.push({
+        studyId: manifest.studyId,
+        taskId: task.taskId,
+        taskText: task.taskText,
+        taskTextSha256: task.taskTextSha256,
+        arm,
+        runOrder: armIndex + 1,
+        model,
+        modelVersion,
+        fixmapRevision,
+        repository: task.repository,
+        revision: task.revision,
+        environmentId: "test-environment",
+        contextId: `context-${task.taskId}-${arm}`,
+        timeoutMs: 600000,
+        budgetId: "test-budget",
+        rubricId: `rubric-${task.taskId}`,
+        priceSheetId: "test-price-sheet",
+        tokenAccountingSource: "provider-reported",
+        transcript: relative(directory, transcriptPath).replaceAll("\\", "/"),
+        transcriptSha256: sha256(transcript),
+        runStatus: "completed",
+        failureReason: null,
+        taskResolved: true,
+        correctFileInFirstThreeOpened: true,
+        toolCallsToFirstRelevantFile: 2,
+        filesOpenedBeforeFirstEdit: 3,
+        incorrectFilesEdited: 0,
+        totalToolCalls: 8,
+        inputTokens: 100,
+        cachedInputTokens: null,
+        outputTokens: 25,
+        reasoningTokens: null,
+        totalTokens: 125,
+        modelCostUsd: 0.01,
+        turns: 4,
+        repositorySearchCalls: 2,
+        filesRead: 3,
+        sourceBytesRead: 4096,
+        wallClockMs: 120000,
+        testsSelectedCorrectly: true,
+        finalPatchAccepted: true,
+        fixmapPlanUsed: arm !== "baseline",
+        verifyUsefulWarnings: arm === "fixmap-instructed" || arm === "fixmap-impact"
+      });
+    }
+  }
+
+  await mutate({ directory, rows, transcriptPaths });
+  const runPath = join(directory, "runs.jsonl");
+  await writeFile(runPath, `${rows.map((row) => JSON.stringify(row)).join("\n")}\n`, "utf8");
+  return { directory, rows, runPath, transcriptPaths };
+}
+
+async function evaluate(runPath, extraArguments = []) {
+  return execFileAsync(process.execPath, [
+    evaluatorPath,
+    "--input", runPath,
+    "--model", model,
+    "--model-version", modelVersion,
+    "--fixmap-revision", fixmapRevision,
+    ...extraArguments
+  ], { cwd: root });
+}
+
+test("accepts only the complete frozen manifest cross product", async (t) => {
+  const fixture = await createFixture(t);
+  const { stdout } = await evaluate(fixture.runPath);
+  const result = JSON.parse(stdout);
+  assert.equal(result.publishableAggregate, true);
+  assert.equal(result.tasks, manifest.tasks.length);
+  assert.equal(result.runs, manifest.tasks.length * arms.length);
+  assert.equal(result.model, model);
+  assert.equal(result.modelVersion, modelVersion);
+  assert.equal(result.fixmapRevision, fixmapRevision);
+});
+
+test("rejects a missing manifest task", async (t) => {
+  const taskId = manifest.tasks[0].taskId;
+  const fixture = await createFixture(t, async ({ rows }) => {
+    for (let index = rows.length - 1; index >= 0; index -= 1) if (rows[index].taskId === taskId) rows.splice(index, 1);
+  });
+  await assert.rejects(() => evaluate(fixture.runPath), /missing required manifest task\/arm runs/);
+});
+
+test("rejects a missing arm", async (t) => {
+  const fixture = await createFixture(t, async ({ rows }) => {
+    rows.splice(rows.findIndex((row) => row.arm === "fixmap-impact"), 1);
+  });
+  await assert.rejects(() => evaluate(fixture.runPath), /missing required manifest task\/arm runs/);
+});
+
+test("rejects an extra or unrecognized task", async (t) => {
+  const fixture = await createFixture(t, async ({ directory, rows }) => {
+    const extraTranscript = join(directory, "transcripts", "extra-task.txt");
+    const content = "extra task transcript\n";
+    await writeFile(extraTranscript, content, "utf8");
+    rows.push({
+      ...rows[0],
+      taskId: "not-in-the-frozen-manifest",
+      transcript: relative(directory, extraTranscript).replaceAll("\\", "/"),
+      transcriptSha256: sha256(content)
+    });
+  });
+  await assert.rejects(() => evaluate(fixture.runPath), /extra or unrecognized task/);
+});
+
+test("rejects altered task text", async (t) => {
+  const fixture = await createFixture(t, async ({ rows }) => { rows[0].taskText += " altered"; });
+  await assert.rejects(() => evaluate(fixture.runPath), /task text differs from the frozen manifest/);
+});
+
+test("rejects a mismatched task hash", async (t) => {
+  const fixture = await createFixture(t, async ({ rows }) => { rows[0].taskTextSha256 = "0".repeat(64); });
+  await assert.rejects(() => evaluate(fixture.runPath), /task-text hash does not match/);
+});
+
+test("rejects a missing transcript file", async (t) => {
+  const fixture = await createFixture(t, async ({ transcriptPaths }) => { await unlink(transcriptPaths[0]); });
+  await assert.rejects(() => evaluate(fixture.runPath), /transcript file is missing/);
+});
+
+test("rejects a transcript modified after hashing", async (t) => {
+  const fixture = await createFixture(t, async ({ transcriptPaths }) => { await writeFile(transcriptPaths[0], "modified transcript\n", "utf8"); });
+  await assert.rejects(() => evaluate(fixture.runPath), /transcript SHA-256 does not match/);
+});
+
+test("rejects an incorrect transcript hash", async (t) => {
+  const fixture = await createFixture(t, async ({ rows }) => { rows[0].transcriptSha256 = "0".repeat(64); });
+  await assert.rejects(() => evaluate(fixture.runPath), /transcript SHA-256 does not match/);
+});
+
+test("rejects mixed model versions instead of pooling them", async (t) => {
+  const fixture = await createFixture(t, async ({ rows }) => { rows[0].modelVersion = "different-version"; });
+  await assert.rejects(() => evaluate(fixture.runPath), /globally pinned model and version/);
+});
+
+test("rejects duplicate task-arm runs", async (t) => {
+  const fixture = await createFixture(t, async ({ directory, rows }) => {
+    const duplicateTranscript = join(directory, "transcripts", "duplicate-run.txt");
+    const content = "duplicate run transcript\n";
+    await writeFile(duplicateTranscript, content, "utf8");
+    rows.push({
+      ...rows[0],
+      contextId: "context-duplicate-run",
+      transcript: relative(directory, duplicateTranscript).replaceAll("\\", "/"),
+      transcriptSha256: sha256(content)
+    });
+  });
+  await assert.rejects(() => evaluate(fixture.runPath), /Duplicate task\/arm run/);
+});
+
+test("rejects a reused context instead of treating it as a fresh run", async (t) => {
+  const fixture = await createFixture(t, async ({ rows }) => { rows[1].contextId = rows[0].contextId; });
+  await assert.rejects(() => evaluate(fixture.runPath), /reuses contextId/);
+});
+
+test("retains failed and timed-out runs in the publishable aggregate", async (t) => {
+  const fixture = await createFixture(t, async ({ rows }) => {
+    rows[0].runStatus = "failed";
+    rows[0].failureReason = "agent process exited";
+    rows[0].taskResolved = null;
+    rows[0].finalPatchAccepted = null;
+    rows[1].runStatus = "timed-out";
+    rows[1].failureReason = "study timeout reached";
+    rows[1].taskResolved = null;
+    rows[1].finalPatchAccepted = null;
+  });
+  const { stdout } = await evaluate(fixture.runPath);
+  const result = JSON.parse(stdout);
+  assert.equal(result.runs, manifest.tasks.length * arms.length);
+  assert.equal(result.aggregate.baseline.failedRuns, 1);
+  assert.equal(result.aggregate["fixmap-available"].timedOutRuns, 1);
+  assert.equal(result.aggregate.baseline.runs, manifest.tasks.length);
+  assert.equal(result.aggregate["fixmap-available"].runs, manifest.tasks.length);
+  assert.equal(result.aggregate.baseline.taskResolutionRateAllRuns, (manifest.tasks.length - 1) / manifest.tasks.length);
+  assert.equal(result.aggregate["fixmap-available"].taskResolutionRateAllRuns, (manifest.tasks.length - 1) / manifest.tasks.length);
+});
+
+test("rejects a discarded failed run as an incomplete task-arm cross product", async (t) => {
+  const fixture = await createFixture(t, async ({ rows }) => {
+    rows[0].runStatus = "failed";
+    rows[0].failureReason = "agent process exited";
+    rows.splice(0, 1);
+  });
+  await assert.rejects(() => evaluate(fixture.runPath), /missing required manifest task\/arm runs/);
+});
+
+test("keeps unsupported provider token counters null without estimating them", async (t) => {
+  const fixture = await createFixture(t, async ({ rows }) => {
+    for (const row of rows.filter((candidate) => candidate.arm === "baseline")) {
+      row.inputTokens = null;
+      row.cachedInputTokens = null;
+      row.outputTokens = null;
+      row.reasoningTokens = null;
+      row.totalTokens = null;
+    }
+  });
+  const { stdout } = await evaluate(fixture.runPath);
+  const result = JSON.parse(stdout);
+  assert.equal(result.aggregate.baseline.medianInputTokens, null);
+  assert.equal(result.aggregate.baseline.medianTotalTokens, null);
+});
+
+test("requires an explicit global model and model version", async (t) => {
+  const fixture = await createFixture(t);
+  await assert.rejects(
+    () => execFileAsync(process.execPath, [evaluatorPath, "--input", fixture.runPath], { cwd: root }),
+    /globally pinned --model, --model-version, and --fixmap-revision/
+  );
+});
+
+test("rejects a mixed FixMap revision", async (t) => {
+  const fixture = await createFixture(t, async ({ rows }) => { rows[0].fixmapRevision = "0".repeat(40); });
+  await assert.rejects(() => evaluate(fixture.runPath), /globally pinned FixMap revision/);
+});
+
+test("rejects token counters that are not declared provider-reported", async (t) => {
+  const fixture = await createFixture(t, async ({ rows }) => { rows[0].tokenAccountingSource = "estimated"; });
+  await assert.rejects(() => evaluate(fixture.runPath), /token counters are not declared provider-reported/);
+});