diff --git a/claude/.claude-plugin/plugin.json b/claude/.claude-plugin/plugin.json index e73f022..1159bcb 100644 --- a/claude/.claude-plugin/plugin.json +++ b/claude/.claude-plugin/plugin.json @@ -1,13 +1,13 @@ { "name": "reflection-cc", - "displayName": "Reflection (Claude Code)", "version": "0.1.0", - "description": "Re-prompts Claude Code when it stops prematurely — catches PERMISSION-SEEKING, STOPPED-WITH-TODOS, and FALSE-COMPLETE failure modes, and injects targeted recovery instructions.", - "author": { - "name": "dzianisv", - "url": "https://github.com/dzianisv" - }, - "repository": "https://github.com/dzianisv/opencode-plugins", + "description": "Re-prompts Claude Code when it stops prematurely due to failure modes like summary-drift-stop or tool-available-punt", + "author": "dzianisv", "license": "MIT", - "hooks": "./hooks/hooks.json" + "hooks": { + "stop": { + "command": "${CLAUDE_PLUGIN_ROOT}/bin/reflect.mjs", + "timeout": 30000 + } + } } diff --git a/claude/README.md b/claude/README.md index 9287e74..96129c3 100644 --- a/claude/README.md +++ b/claude/README.md @@ -4,28 +4,7 @@ Re-prompts Claude Code when it stops prematurely due to failure modes like summa ## Install -### Via `/plugin` marketplace (recommended) - -```bash -# 1. Register the marketplace (one-time per machine) -/plugin marketplace add dzianisv/opencode-plugins - -# 2. Install the plugin -/plugin install reflection-cc -``` - -Or in one step using the CLI: - -```bash -claude plugin marketplace add dzianisv/opencode-plugins -claude plugin install reflection-cc -``` - -This uses the `marketplace.json` at the repo root (`.claude-plugin/marketplace.json`) which points the `./claude` subdirectory as the plugin source. - -### Manual (settings-based install — always works) - -Add the Stop hook directly to `~/.claude/settings.json`: +**Recommended (works today, CC v2.x):** add the Stop hook directly to `~/.claude/settings.json`: ```json { @@ -45,9 +24,9 @@ Add the Stop hook directly to `~/.claude/settings.json`: } ``` -**One-session try:** write the JSON above to a file and pass `--settings ./reflect-settings.json`. +The plugin manifest under `.claude-plugin/` is included for future marketplace publication, but in CC v2.1.150 `--plugin-dir` and the `enabledPlugins` config path do NOT activate `Stop` hooks for headless `-p` sessions. The settings-based install above is the authoritative path until that gap closes. -> Note: the Stop hook event name is `"Stop"` (capital S) — lowercase `"stop"` is silently ignored by Claude Code. +**One-session try:** `claude --settings ''` ... or write the JSON to a file and pass `--settings ./reflect-settings.json`. ## Failure Categories diff --git a/claude/bin/reflect.mjs b/claude/bin/reflect.mjs index 7c22e4b..e99d1d0 100755 --- a/claude/bin/reflect.mjs +++ b/claude/bin/reflect.mjs @@ -357,17 +357,16 @@ export function buildStopContext(stopPayload, transcriptTail) { } } - // Derive final assistant text: prefer CC's `last_assistant_message` field (the - // documented Stop hook field name as of CC v2.x — NOT `response`), fall back - // to the last assistant entry's text content from the transcript tail. - let final_assistant_text = (stopPayload?.last_assistant_message ?? stopPayload?.response ?? '').trim(); + // Derive final assistant text: prefer CC's `response` field (it IS the last turn), + // fall back to the last assistant entry's text content from the tail. + let final_assistant_text = (stopPayload?.response ?? '').trim(); if (!final_assistant_text) { // Walk tail in reverse, find last assistant entry with a text block for (let i = transcriptTail.length - 1; i >= 0; i--) { const entry = transcriptTail[i]; if (entry.type !== 'assistant') continue; const content = entry?.message?.content; - if (!Array.isArray(content)) break; + if (!Array.isArray(content)) continue; const textBlocks = content.filter((c) => c?.type === 'text'); if (textBlocks.length > 0) { final_assistant_text = textBlocks.map((b) => b.text).join('\n').trim(); @@ -447,27 +446,6 @@ async function main() { // uncaughtException handler exits 0 (fail-safe: no inject, no fs ops). const cwd = sanitizeCwd(payload?.cwd ?? process.cwd()); - // ── 1.5. SESSION-SCOPED DISABLED CHECK ────────────────────────────────── - // Write current session ID so agents can reference it without knowing it upfront: - // cat .reflection/current_session - // Disable this session: - // echo "SESSION_ID" >> .reflection/disabled - // Enable: - // grep -v "SESSION_ID" .reflection/disabled > .reflection/disabled.tmp && mv .reflection/disabled.tmp .reflection/disabled - const reflDir = path.join(cwd, '.reflection'); - fs.mkdirSync(reflDir, { recursive: true }); - fs.writeFileSync(path.join(reflDir, 'current_session'), session_id, 'utf8'); - - const disabledFlag = path.join(reflDir, 'disabled'); - try { - const disabledIds = fs.readFileSync(disabledFlag, 'utf8') - .split('\n').map(l => l.trim()).filter(Boolean); - if (disabledIds.includes(session_id)) { - debug({ msg: 'disabled_for_session', session_id }, cwd); - process.exit(0); - } - } catch { /* file absent = not disabled */ } - // ── 2. ATTEMPT CAP ──────────────────────────────────────────────────────── const attempts = readAttempts(session_id, cwd); if (attempts >= MAX_ATTEMPTS) { diff --git a/claude/hooks/hooks.json b/claude/hooks/hooks.json index bbac244..7e27ad3 100644 --- a/claude/hooks/hooks.json +++ b/claude/hooks/hooks.json @@ -1,15 +1,8 @@ { "hooks": { - "Stop": [ - { - "hooks": [ - { - "type": "command", - "command": "${CLAUDE_PLUGIN_ROOT}/bin/reflect.mjs", - "timeout": 30 - } - ] - } - ] + "stop": { + "command": "${CLAUDE_PLUGIN_ROOT}/bin/reflect.mjs", + "timeout": 30000 + } } } diff --git a/claude/lib/judge.mjs b/claude/lib/judge.mjs index eb1b551..d3e3a34 100644 --- a/claude/lib/judge.mjs +++ b/claude/lib/judge.mjs @@ -17,9 +17,8 @@ */ import { readFileSync } from 'node:fs'; -import { homedir, platform } from 'node:os'; +import { homedir } from 'node:os'; import { join } from 'node:path'; -import { execFileSync } from 'node:child_process'; // --------------------------------------------------------------------------- // Constants @@ -67,76 +66,33 @@ function sanitizeError(text) { // --------------------------------------------------------------------------- /** - * Reads the Claude Code OAuth credentials JSON from its platform store. - * On macOS, Claude Code keeps credentials in the login keychain (generic - * password "Claude Code-credentials"), NOT in a file — so the file read on - * darwin almost always fails and the keychain is the real source. On - * Linux/Windows the credentials live at ~/.claude/.credentials.json. + * Loads the OAuth access token from ~/.claude/.credentials.json. + * Throws a sentinel error (prefixed "judge:") if the file is missing, + * unreadable, or the token is absent/empty — caller treats this as no-inject. * - * Returns the parsed object ({ claudeAiOauth: { accessToken, ... } }) or null - * if no source is available / parseable. - * - * @returns {object | null} + * @returns {string} access token */ -function readOauthCredentials() { - // 1. File (Linux/Windows, and macOS installs that opted out of keychain). +function loadOAuthToken() { const credPath = join(homedir(), '.claude', '.credentials.json'); + let raw; try { - return JSON.parse(readFileSync(credPath, 'utf8')); - } catch { - /* fall through to keychain on macOS */ - } - - // 2. macOS keychain. - if (platform() === 'darwin') { - try { - const out = execFileSync( - 'security', - ['find-generic-password', '-s', 'Claude Code-credentials', '-w'], - { encoding: 'utf8', timeout: 5_000, stdio: ['ignore', 'pipe', 'ignore'] }, - ); - return JSON.parse(out.trim()); - } catch { - /* no keychain item, or not parseable */ - } - } - - return null; -} - -/** - * Loads auth credentials for the Anthropic API, trying sources in order: - * 1. ANTHROPIC_API_KEY env var (x-api-key header, no beta header needed) - * 2. OAuth token from ~/.claude/.credentials.json (Linux/Windows) or the - * macOS login keychain ("Claude Code-credentials") — Bearer + - * oauth-2025-04-20 beta. - * - * Returns { type: 'apikey' | 'oauth', value: string }. - * Throws a sentinel error (prefixed "judge:") if neither is available. - * - * @returns {{ type: 'apikey' | 'oauth', value: string }} - */ -function loadAuth() { - // 1. Explicit API key env var - const apiKey = process.env.ANTHROPIC_API_KEY; - if (apiKey && apiKey.trim()) { - return { type: 'apikey', value: apiKey.trim() }; + raw = readFileSync(credPath, 'utf8'); + } catch (err) { + throw new Error(`judge: cannot read credentials file: ${err.message}`); } - // 2. OAuth token from credentials file or macOS keychain - const obj = readOauthCredentials(); - if (!obj) { - throw new Error( - 'judge: no ANTHROPIC_API_KEY set and no Claude Code OAuth credentials found ' + - '(checked ~/.claude/.credentials.json and the macOS keychain)', - ); + let obj; + try { + obj = JSON.parse(raw); + } catch (err) { + throw new Error(`judge: credentials file is not valid JSON: ${err.message}`); } const token = obj?.claudeAiOauth?.accessToken; if (!token) { - throw new Error('judge: OAuth credentials present but missing claudeAiOauth.accessToken'); + throw new Error('judge: no claudeAiOauth.accessToken in ~/.claude/.credentials.json'); } - return { type: 'oauth', value: token }; + return token; } // --------------------------------------------------------------------------- @@ -174,11 +130,11 @@ function buildPrompt(ctx) { return `You classify how a Claude Code assistant ended a turn. Pick ONE category. CATEGORIES: -- complete: task is done; assistant delivered the answer or finished the requested work WITH evidence. -- waiting_for_user_legitimate: assistant asks a question that ONLY the user can answer (OAuth/2FA/captcha/credential retrieval, or a genuine preference the user must supply). -- tool_available_punt: assistant punts to the user about something the available tools could resolve. The assistant has tools like Bash, WebFetch, browser MCP, etc., yet asks the user instead of trying. -- summary_drift_stop: assistant wrote a summary/plan with a "next step" and STOPPED before doing it. e.g., "I've created the file. Next step: run the tests." (without running them.) -- genuinely_stuck: assistant stopped mid-thought or without clear conclusion; no question, no summary, just halted. Often a short response. +- complete: task is done; assistant delivered the answer or finished the requested work. +- waiting_for_user_legitimate: assistant asks a question that ONLY the user can answer (preference, missing info no tool can fetch). +- tool_available_punt: assistant punts to the user about something the available tools could resolve. The assistant has access to tools like Bash, WebFetch, browser MCP, etc., yet asks the user instead of trying. +- summary_drift_stop: assistant wrote a summary or plan with a "next step" and STOPPED before doing the next step. e.g., "I've created the file. Next step: run the tests." (without running them.) +- genuinely_stuck: assistant stopped mid-thought or without clear conclusion; no question, no summary, just halted. Often short. - working: rarely a stop; only assign if the final turn is clearly mid-action (e.g., "Running tests now...") with no closure. TOOLS THE ASSISTANT HAD: ${tools || '(none recorded)'} @@ -189,18 +145,8 @@ ${userMsgs || '(none)'} FINAL ASSISTANT TEXT: ${finalText} -PREMATURE-STOP ANTIPATTERNS (mined from 227 real agent stops where the user replied; 78% were premature — the user said "go"/"continue"/"yes do it" or corrected the agent). Use these to sharpen category assignments: - -- PERMISSION-SEEKING (most common, ~40%): the response ends by asking to do work it can already do — "Want me to…?", "Would you like me to…?", "Should I…?", "Shall I proceed?", or "Try running it now"/"Please run X and confirm" (deferring a check the agent could run itself). DECISIVE TEST: if the final turn is a yes/no or "want me to X?" question AND X is something the agent can do with its own tools AND X carries no irreversible risk → classify as tool_available_punt. Asking is only legitimate before a destructive/irreversible action (delete prod data, force-push, send an irreversible external message) → classify as waiting_for_user_legitimate. - -- STOPPED-WITH-TODOS (~30%): the response lists "Remaining Tasks"/"Next steps"/"Still TODO"/"What I did NOT do" or names a verify/run/check/create-PR step as "next" — then stops without doing it. Listing remaining work does not complete it → classify as summary_drift_stop. - -- FALSE-COMPLETE: claims "done"/"complete"/"ready"/"all tasks complete" but the CORE requested action never happened, a required check was skipped, or there is no evidence. An empty/no-text response on an action task is NEVER complete. For an "add a " task, writing files is not enough — code must be wired in AND verified (test/build/run); "ready to use" with no integration is incomplete → classify as summary_drift_stop (not complete). - -- LEGITIMATE STOP (do NOT flag as premature): genuine human-only block (OAuth consent, 2FA code, credential/API-key retrieval, captcha) → waiting_for_user_legitimate. Genuine completion WITH evidence (commands+output, tests passing, PR/CI verified) → complete; do not invent missing work. - Respond ONLY with a JSON object on a single line, no markdown fence, no prose: -{"category": "", "reason": "", "confidence": <0.0-1.0>}`; +{"category": "", "reason": "", "confidence": <0.0-1.0>}`; } // --------------------------------------------------------------------------- @@ -286,23 +232,14 @@ export async function classifyStop(stopContext, opts = {}) { const model = opts.model ?? DEFAULT_MODEL; const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS; - // Test escape hatch: REFLECTION_CC_FAKE_JUDGE=: (e.g. - // "summary_drift_stop:0.9") returns a hardcoded verdict without an API call. - // Only active when the env var is set — never in production. - const fakeJudge = process.env.REFLECTION_CC_FAKE_JUDGE; - if (fakeJudge) { - const [cat, conf] = fakeJudge.split(':'); - const category = CATEGORIES.includes(cat) ? cat : 'complete'; - return { - category, - reason: `[fake judge] ${fakeJudge}`, - confidence: parseFloat(conf ?? '0.9') || 0.9, - }; + // Load token — throws "judge: ..." on failure (caller treats as no-inject) + let token; + try { + token = loadOAuthToken(); + } catch (err) { + throw err; // already prefixed with "judge:" } - // Load auth — throws "judge: ..." on failure (caller treats as no-inject) - const auth = loadAuth(); - const prompt = buildPrompt(stopContext); const body = JSON.stringify({ @@ -312,36 +249,30 @@ export async function classifyStop(stopContext, opts = {}) { messages: [{ role: 'user', content: prompt }], }); - // Build request headers depending on auth type. - // - API key: x-api-key header, no beta header needed - // - OAuth: Bearer token + anthropic-beta oauth header - const headers = { - 'anthropic-version': ANTHROPIC_VERSION, - 'content-type': 'application/json', - }; - if (auth.type === 'apikey') { - headers['x-api-key'] = auth.value; - } else { - headers['authorization'] = `Bearer ${auth.value}`; - headers['anthropic-beta'] = ANTHROPIC_BETA; - } - // Compose abort signal: hard timeout + optional caller signal const timeoutController = new AbortController(); const timerId = setTimeout(() => timeoutController.abort(), timeoutMs); // Merge caller signal if provided + let signal = timeoutController.signal; if (opts.signal) { + // If either aborts, abort both opts.signal.addEventListener('abort', () => timeoutController.abort(), { once: true }); + // We still use timeoutController.signal — it fires on timeout OR on opts.signal abort } let res; try { res = await fetch(API_URL, { method: 'POST', - headers, + headers: { + 'anthropic-version': ANTHROPIC_VERSION, + 'anthropic-beta': ANTHROPIC_BETA, + 'authorization': `Bearer ${token}`, + 'content-type': 'application/json', + }, body, - signal: timeoutController.signal, + signal, }); } catch (err) { clearTimeout(timerId); diff --git a/claude/test/e2e-cc.mjs b/claude/test/e2e-cc.mjs index 64c26cb..538b2d2 100644 --- a/claude/test/e2e-cc.mjs +++ b/claude/test/e2e-cc.mjs @@ -49,13 +49,7 @@ function loadOAuthToken() { } } -// Lazy-load TOKEN — only called when running scenarios that need real API access -// (scenarios 1-3). Scenario 4 (direct-pipe) does not need this. -let _token; -function getToken() { - if (!_token) _token = loadOAuthToken(); - return _token; -} +const TOKEN = loadOAuthToken(); // -------------------------------------------------------------------------- // Scenarios @@ -148,21 +142,17 @@ function runDirectPipeScenario() { transcript_path: tFile, cwd: sandbox, hook_event_name: "Stop", - last_assistant_message: "I've created factorial.py and test_factorial.py. Next step: run `python -m pytest test_factorial.py -v` to verify the tests pass.", + response: "I've created factorial.py and test_factorial.py. Next step: run `python -m pytest test_factorial.py -v` to verify the tests pass.", stop_hook_active: false, }; - // Use REFLECTION_CC_FAKE_JUDGE so this test exercises the full hook wiring - // (stdin parsing, loop guard, attempt counter, feedback builder, stdout JSON) - // without a real API call. The mock returns summary_drift_stop:0.95, which - // the feedback builder maps to a block decision — exactly the inject path. const startTime = Date.now(); const result = spawnSync("node", [join(PLUGIN_DIR, "bin", "reflect.mjs")], { input: JSON.stringify(payload), cwd: sandbox, timeout: 30_000, encoding: "utf8", - env: { ...process.env, REFLECTION_CC_DEBUG: "1", REFLECTION_CC_FAKE_JUDGE: "summary_drift_stop:0.95" }, + env: { ...process.env, REFLECTION_CC_DEBUG: "1" }, }); const elapsed = ((Date.now() - startTime) / 1000).toFixed(1); @@ -298,7 +288,7 @@ Respond ONLY with a JSON object on a single line, no markdown fence: headers: { "anthropic-version": "2023-06-01", "anthropic-beta": "oauth-2025-04-20", - "authorization": `Bearer ${getToken()}`, + "authorization": `Bearer ${TOKEN}`, "content-type": "application/json", }, body: JSON.stringify({ @@ -404,312 +394,8 @@ function runScenario(scenario) { return { scenario, result, sandbox, transcriptPath, evidenceDir, elapsed }; } -// -------------------------------------------------------------------------- -// Scenario 5: direct-pipe complete — fake judge returns "complete", verify -// NO block is emitted (exit 0, stdout is empty or not a block decision). -// -------------------------------------------------------------------------- - -function runDirectPipeCompleteScenario() { - const id = 5; - const name = "direct_pipe_complete_no_inject"; - const sandbox = join(tmpdir(), "cc-reflect-e2e", `s${id}-${Date.now()}`); - mkdirSync(sandbox, { recursive: true, mode: 0o700 }); - const evidenceDir = join(EVIDENCE_DIR, `scenario-${id}-${name}`); - mkdirSync(evidenceDir, { recursive: true }); - - process.stderr.write(`\n[s${id}] ${name}\n`); - process.stderr.write(` sandbox : ${sandbox}\n`); - process.stderr.write(` evidence : ${evidenceDir}\n`); - - const fakeSessionId = "test-complete-" + Date.now(); - const tFile = join(sandbox, `transcript-${fakeSessionId}.jsonl`); - const entries = [ - { type: "user", uuid: "u1", sessionId: fakeSessionId, message: { role: "user", content: "What is 2 + 2?" } }, - { type: "assistant", uuid: "a1", sessionId: fakeSessionId, message: { role: "assistant", content: [{ type: "text", text: "4" }] } }, - ]; - writeFileSync(tFile, entries.map(e => JSON.stringify(e)).join("\n") + "\n"); - - const payload = { - session_id: fakeSessionId, - transcript_path: tFile, - cwd: sandbox, - hook_event_name: "Stop", - last_assistant_message: "4", - stop_hook_active: false, - }; - - const startTime = Date.now(); - const result = spawnSync("node", [join(PLUGIN_DIR, "bin", "reflect.mjs")], { - input: JSON.stringify(payload), - cwd: sandbox, - timeout: 30_000, - encoding: "utf8", - // Fake judge returns "complete" → plugin must NOT emit a block decision - env: { ...process.env, REFLECTION_CC_DEBUG: "1", REFLECTION_CC_FAKE_JUDGE: "complete:0.99" }, - }); - const elapsed = ((Date.now() - startTime) / 1000).toFixed(1); - - writeFileSync(join(evidenceDir, "stdin.json"), JSON.stringify(payload, null, 2)); - writeFileSync(join(evidenceDir, "stdout.txt"), result.stdout ?? ""); - writeFileSync(join(evidenceDir, "stderr.txt"), result.stderr ?? ""); - - let stdout = {}; - try { stdout = JSON.parse(result.stdout ?? "{}"); } catch {} - - const didBlock = stdout.decision === "block"; - let verdict = "FAIL"; - let reason; - if (result.status !== 0) { - reason = `reflect.mjs exited non-zero: ${result.status}`; - } else if (didBlock) { - reason = `false positive: plugin blocked a 'complete' verdict (reason: ${stdout.reason?.slice(0, 80)})`; - } else { - verdict = "PASS"; - reason = "no block emitted on complete verdict (correct)"; - } - - process.stderr.write(` exit=${result.status} elapsed=${elapsed}s\n`); - process.stderr.write(` verdict : ${verdict} — ${reason}\n`); - - writeFileSync(join(evidenceDir, "verdict.json"), JSON.stringify({ - scenario: name, - verdict, - reason, - actual_stdout: stdout, - exit_code: result.status, - elapsed_s: elapsed, - }, null, 2)); - - if (!KEEP) { - try { rmSync(sandbox, { recursive: true, force: true }); } catch {} - } - - return { - scenario: name, - expectsInject: false, - injects: didBlock ? 1 : 0, - verdict, - reason, - elapsed_s: elapsed, - }; -} - -// -------------------------------------------------------------------------- -// Scenario 6: disabled session is skipped — session ID in .reflection/disabled, -// hook must exit 0 without emitting a block even on a drift transcript. -// -------------------------------------------------------------------------- - -function runToggleDisabledSkipsScenario() { - const id = 6; - const name = "toggle_disabled_session_skips"; - const sandbox = join(tmpdir(), "cc-reflect-e2e", `s${id}-${Date.now()}`); - mkdirSync(sandbox, { recursive: true, mode: 0o700 }); - const evidenceDir = join(EVIDENCE_DIR, `scenario-${id}-${name}`); - mkdirSync(evidenceDir, { recursive: true }); - - process.stderr.write(`\n[s${id}] ${name}\n`); - process.stderr.write(` sandbox : ${sandbox}\n`); - - const sessionId = "disabled-session-" + Date.now(); - - // Write the session ID into .reflection/disabled before the hook fires. - const reflDir = join(sandbox, ".reflection"); - mkdirSync(reflDir, { recursive: true }); - writeFileSync(join(reflDir, "disabled"), sessionId + "\n"); - - const tFile = join(sandbox, `transcript-${sessionId}.jsonl`); - const entries = [ - { type: "user", uuid: "u1", sessionId, message: { role: "user", content: "Write a file and run tests." } }, - { type: "assistant", uuid: "a1", sessionId, message: { role: "assistant", content: [{ type: "text", text: "I wrote the file. Next step: run pytest to verify." }] } }, - ]; - writeFileSync(tFile, entries.map(e => JSON.stringify(e)).join("\n") + "\n"); - - const payload = { - session_id: sessionId, - transcript_path: tFile, - cwd: sandbox, - last_assistant_message: "I wrote the file. Next step: run pytest to verify.", - stop_hook_active: false, - }; - - const startTime = Date.now(); - const result = spawnSync("node", [join(PLUGIN_DIR, "bin", "reflect.mjs")], { - input: JSON.stringify(payload), - cwd: sandbox, - timeout: 30_000, - encoding: "utf8", - // FAKE_JUDGE would return summary_drift_stop, but the disabled check must - // fire BEFORE the judge is ever called — so block must NOT be emitted. - env: { ...process.env, REFLECTION_CC_DEBUG: "1", REFLECTION_CC_FAKE_JUDGE: "summary_drift_stop:0.95" }, - }); - const elapsed = ((Date.now() - startTime) / 1000).toFixed(1); - - writeFileSync(join(evidenceDir, "stdin.json"), JSON.stringify(payload, null, 2)); - writeFileSync(join(evidenceDir, "stdout.txt"), result.stdout ?? ""); - writeFileSync(join(evidenceDir, "stderr.txt"), result.stderr ?? ""); - for (const f of readdirSync(reflDir)) { - try { writeFileSync(join(evidenceDir, f), readFileSync(join(reflDir, f))); } catch {} - } - - let stdout = {}; - try { stdout = JSON.parse(result.stdout ?? "{}"); } catch {} - - const blocked = stdout.decision === "block"; - let verdict, reason; - if (result.status !== 0) { - verdict = "FAIL"; reason = `reflect.mjs exited non-zero: ${result.status}`; - } else if (blocked) { - verdict = "FAIL"; reason = "plugin injected despite session being in disabled list"; - } else { - verdict = "PASS"; reason = "hook skipped (no block) for disabled session ID"; - } - - process.stderr.write(` exit=${result.status} elapsed=${elapsed}s\n`); - process.stderr.write(` verdict : ${verdict} — ${reason}\n`); - writeFileSync(join(evidenceDir, "verdict.json"), JSON.stringify({ scenario: name, verdict, reason, exit_code: result.status, elapsed_s: elapsed }, null, 2)); - if (!KEEP) try { rmSync(sandbox, { recursive: true, force: true }); } catch {} - return { scenario: name, expectsInject: false, injects: blocked ? 1 : 0, verdict, reason, elapsed_s: elapsed }; -} - -// -------------------------------------------------------------------------- -// Scenario 7: other session is unaffected — session A is in .reflection/disabled, -// session B fires with a drift transcript and MUST still receive a block. -// -------------------------------------------------------------------------- - -function runToggleOtherSessionUnaffectedScenario() { - const id = 7; - const name = "toggle_other_session_unaffected"; - const sandbox = join(tmpdir(), "cc-reflect-e2e", `s${id}-${Date.now()}`); - mkdirSync(sandbox, { recursive: true, mode: 0o700 }); - const evidenceDir = join(EVIDENCE_DIR, `scenario-${id}-${name}`); - mkdirSync(evidenceDir, { recursive: true }); - - process.stderr.write(`\n[s${id}] ${name}\n`); - process.stderr.write(` sandbox : ${sandbox}\n`); - - const disabledSessionId = "disabled-other-" + Date.now(); - const activeSessionId = "active-session-" + Date.now(); - - // Only the OTHER session is disabled; the active session must still run. - const reflDir = join(sandbox, ".reflection"); - mkdirSync(reflDir, { recursive: true }); - writeFileSync(join(reflDir, "disabled"), disabledSessionId + "\n"); - - const tFile = join(sandbox, `transcript-${activeSessionId}.jsonl`); - const entries = [ - { type: "user", uuid: "u1", sessionId: activeSessionId, message: { role: "user", content: "Write a Python factorial and test it." } }, - { type: "assistant", uuid: "a1", sessionId: activeSessionId, message: { role: "assistant", content: [{ type: "text", text: "Created factorial.py. Next: run pytest." }] } }, - ]; - writeFileSync(tFile, entries.map(e => JSON.stringify(e)).join("\n") + "\n"); - - const payload = { - session_id: activeSessionId, - transcript_path: tFile, - cwd: sandbox, - last_assistant_message: "Created factorial.py. Next: run pytest.", - stop_hook_active: false, - }; - - const startTime = Date.now(); - const result = spawnSync("node", [join(PLUGIN_DIR, "bin", "reflect.mjs")], { - input: JSON.stringify(payload), - cwd: sandbox, - timeout: 30_000, - encoding: "utf8", - env: { ...process.env, REFLECTION_CC_DEBUG: "1", REFLECTION_CC_FAKE_JUDGE: "summary_drift_stop:0.95" }, - }); - const elapsed = ((Date.now() - startTime) / 1000).toFixed(1); - - writeFileSync(join(evidenceDir, "stdin.json"), JSON.stringify(payload, null, 2)); - writeFileSync(join(evidenceDir, "stdout.txt"), result.stdout ?? ""); - writeFileSync(join(evidenceDir, "stderr.txt"), result.stderr ?? ""); - - let stdout = {}; - try { stdout = JSON.parse(result.stdout ?? "{}"); } catch {} - - const blocked = stdout.decision === "block"; - let verdict, reason; - if (result.status !== 0) { - verdict = "FAIL"; reason = `reflect.mjs exited non-zero: ${result.status}`; - } else if (!blocked) { - verdict = "FAIL"; reason = "active session was not blocked — disabled list for OTHER session leaked over"; - } else { - verdict = "PASS"; reason = "active session blocked correctly; disabled list for other session had no effect"; - } - - process.stderr.write(` exit=${result.status} elapsed=${elapsed}s\n`); - process.stderr.write(` verdict : ${verdict} — ${reason}\n`); - writeFileSync(join(evidenceDir, "verdict.json"), JSON.stringify({ scenario: name, verdict, reason, exit_code: result.status, elapsed_s: elapsed }, null, 2)); - if (!KEEP) try { rmSync(sandbox, { recursive: true, force: true }); } catch {} - return { scenario: name, expectsInject: true, injects: blocked ? 1 : 0, verdict, reason, elapsed_s: elapsed }; -} - -// -------------------------------------------------------------------------- -// Scenario 8: .reflection/current_session written — after the hook fires for -// a session, the file must contain that session's ID so agents can reference it. -// -------------------------------------------------------------------------- - -function runToggleCurrentSessionFileScenario() { - const id = 8; - const name = "toggle_current_session_file_written"; - const sandbox = join(tmpdir(), "cc-reflect-e2e", `s${id}-${Date.now()}`); - mkdirSync(sandbox, { recursive: true, mode: 0o700 }); - const evidenceDir = join(EVIDENCE_DIR, `scenario-${id}-${name}`); - mkdirSync(evidenceDir, { recursive: true }); - - process.stderr.write(`\n[s${id}] ${name}\n`); - process.stderr.write(` sandbox : ${sandbox}\n`); - - const sessionId = "csf-session-" + Date.now(); - const tFile = join(sandbox, `transcript-${sessionId}.jsonl`); - const entries = [ - { type: "user", uuid: "u1", sessionId, message: { role: "user", content: "What is 2+2?" } }, - { type: "assistant", uuid: "a1", sessionId, message: { role: "assistant", content: [{ type: "text", text: "4" }] } }, - ]; - writeFileSync(tFile, entries.map(e => JSON.stringify(e)).join("\n") + "\n"); - - const payload = { - session_id: sessionId, - transcript_path: tFile, - cwd: sandbox, - last_assistant_message: "4", - stop_hook_active: false, - }; - - spawnSync("node", [join(PLUGIN_DIR, "bin", "reflect.mjs")], { - input: JSON.stringify(payload), - cwd: sandbox, - timeout: 30_000, - encoding: "utf8", - env: { ...process.env, REFLECTION_CC_DEBUG: "1", REFLECTION_CC_FAKE_JUDGE: "complete:0.99" }, - }); - - const currentSessionFile = join(sandbox, ".reflection", "current_session"); - let writtenId = null; - try { writtenId = readFileSync(currentSessionFile, "utf8").trim(); } catch {} - - writeFileSync(join(evidenceDir, "current_session.txt"), writtenId ?? "(not written)"); - - const verdict = writtenId === sessionId ? "PASS" : "FAIL"; - const reason = writtenId === sessionId - ? `current_session file contains correct session ID (${sessionId.slice(0, 16)}…)` - : `expected ${sessionId}, got ${writtenId ?? "(nothing)"}`; - - process.stderr.write(` verdict : ${verdict} — ${reason}\n`); - writeFileSync(join(evidenceDir, "verdict.json"), JSON.stringify({ scenario: name, verdict, reason, written_id: writtenId, expected_id: sessionId }, null, 2)); - if (!KEEP) try { rmSync(sandbox, { recursive: true, force: true }); } catch {} - return { scenario: name, expectsInject: false, injects: 0, verdict, reason, elapsed_s: "0.0" }; -} - async function main() { - const allScenarios = [ - ...SCENARIOS, - { id: 4, name: "direct_pipe_summary_drift", _direct: true }, - { id: 5, name: "direct_pipe_complete_no_inject", _direct5: true }, - { id: 6, name: "toggle_disabled_session_skips", _s6: true }, - { id: 7, name: "toggle_other_session_unaffected", _s7: true }, - { id: 8, name: "toggle_current_session_file_written", _s8: true }, - ]; + const allScenarios = [...SCENARIOS, { id: 4, name: "direct_pipe_summary_drift", _direct: true }]; const toRun = ONLY ? allScenarios.filter(s => s.id === ONLY) : allScenarios; if (toRun.length === 0) { process.stderr.write(`No scenario with id ${ONLY}\n`); @@ -724,22 +410,6 @@ async function main() { summary.push(runDirectPipeScenario()); continue; } - if (scenario._direct5) { - summary.push(runDirectPipeCompleteScenario()); - continue; - } - if (scenario._s6) { - summary.push(runToggleDisabledSkipsScenario()); - continue; - } - if (scenario._s7) { - summary.push(runToggleOtherSessionUnaffectedScenario()); - continue; - } - if (scenario._s8) { - summary.push(runToggleCurrentSessionFileScenario()); - continue; - } const run = runScenario(scenario); const transcript = run.transcriptPath && existsSync(run.transcriptPath) ? readFileSync(run.transcriptPath, "utf8")