From 2c1b62510436996d351c1de5ba04bbce7d340516 Mon Sep 17 00:00:00 2001 From: Dean Sharon Date: Fri, 3 Jul 2026 01:36:42 +0300 Subject: [PATCH 01/26] feat(hooks): phase 0 pre-work for dream system simplification - dream-lock: replace BSD-only `stat -f %m` with portable get_mtime, falling back to treating an existing lock as maximally stale (not a hard crash) when get_mtime isn't sourced yet by the caller. - hooks.ts: add optional `matcher` field to HookMatcher (PostToolUse hook registration support for a later AskUserQuestion capture hook). - json-helper.cjs: make merge-observation self-locking on .devflow/dream/.observations.lock, mirroring assign-anchor's internal locking on .decisions.lock. Removes the caller lock-dance requirement. Part of the dream system simplification (Phase 0 of 2-coder sequence). --- scripts/hooks/dream-lock | 19 +++- scripts/hooks/json-helper.cjs | 163 ++++++++++++++++++---------------- src/cli/utils/hooks.ts | 2 + 3 files changed, 106 insertions(+), 78 deletions(-) diff --git a/scripts/hooks/dream-lock b/scripts/hooks/dream-lock index a13a7f2b..648de636 100755 --- a/scripts/hooks/dream-lock +++ b/scripts/hooks/dream-lock @@ -1,5 +1,12 @@ #!/bin/bash -# mkdir-based locking for macOS bash 3.2 (no flock) +# mkdir-based locking, portable across BSD (macOS) and GNU (Linux) stat. +# +# Source-order requirement: source get-mtime (scripts/hooks/get-mtime) BEFORE +# sourcing this file so get_mtime() is defined and stale-lock detection uses a +# real mtime. If get_mtime is not yet defined when a lock_dir already exists, +# this file falls back to treating the lock as maximally stale (immediate +# break) rather than crashing on a BSD-only stat flag — safe-by-construction, +# never a hard failure, but source get-mtime first for accurate staleness. dream_lock_acquire() { local lock_dir="$1" @@ -9,8 +16,14 @@ dream_lock_acquire() { # Break stale locks if [ -d "$lock_dir" ]; then - local lock_age - lock_age=$(( $(date +%s) - $(stat -f %m "$lock_dir" 2>/dev/null || echo 0) )) + local lock_age lock_mtime + if declare -F get_mtime >/dev/null 2>&1; then + lock_mtime=$(get_mtime "$lock_dir") + else + lock_mtime="" + fi + lock_mtime="${lock_mtime:-0}" + lock_age=$(( $(date +%s) - lock_mtime )) if [ "$lock_age" -gt "$stale_threshold" ]; then rmdir "$lock_dir" 2>/dev/null || true fi diff --git a/scripts/hooks/json-helper.cjs b/scripts/hooks/json-helper.cjs index adc260fb..22d84040 100755 --- a/scripts/hooks/json-helper.cjs +++ b/scripts/hooks/json-helper.cjs @@ -536,12 +536,12 @@ try { // If the id is new, insert a new entry with LLM-provided fields verbatim. // D11: ID collision with different-type entry → suffix _b to avoid trampling. // D12: evidence array capped at 10 (FIFO). - // D53: merge-observation is locked EXTERNALLY by the caller (dream agent acquires/ - // releases .devflow/dream/.observations.lock around the Bash subshell call), while - // assign-anchor self-locks INTERNALLY via .decisions.lock. These are two distinct lock - // domains — merge-observation itself never acquires a lock; it relies on the caller to - // serialize concurrent writes. This is intentional: the subshell pattern in the Dream - // agent acquires the lock, invokes this op, and releases — all in a single Bash call. + // D53: merge-observation self-locks INTERNALLY on .devflow/dream/.observations.lock, + // mirroring assign-anchor's internal locking on .decisions.lock. These remain two + // distinct lock domains — a caller that also needs .decisions.lock (e.g. immediately + // promoting via assign-anchor) must acquire it only AFTER this call returns; never hold + // both simultaneously (ADR-017). Callers must NOT wrap this op in their own + // .observations.lock mkdir — doing so would nest against this self-lock and time out. // ------------------------------------------------------------------------- case 'merge-observation': { const logFile = safePath(args[0]); @@ -552,79 +552,92 @@ try { process.exit(1); } - // D54 (E1): self-create parent directory on first write (fresh project, file+dir absent). - // The prior batch-merge op created the dir; merge-observation matches that so callers - // do not need to pre-create the log directory. - const logDir = path.dirname(logFile); - if (!fs.existsSync(logDir)) { - fs.mkdirSync(logDir, { recursive: true }); - } + const moProjectRoot = process.cwd(); + const moLockDir = getObservationsLockDir(moProjectRoot); + fs.mkdirSync(path.dirname(moLockDir), { recursive: true }); - let logEntries = []; - if (fs.existsSync(logFile)) { - logEntries = parseJsonl(logFile); + if (!acquireMkdirLock(moLockDir, 30000, 60000)) { + process.stderr.write(`merge-observation: timeout acquiring lock at ${moLockDir}\n`); + process.exit(1); } - const logMap = new Map(logEntries.map(e => [e.id, e])); - const nowIso = new Date().toISOString().replace(/\.\d{3}Z$/, 'Z'); - - // ID-keyed lookup: match by obs_xxx id only - const existing = logMap.get(newObs.id); - let merged = false; - - if (existing) { - // Reinforce: increment count, merge evidence (FIFO cap 10), update last_seen. - // Store LLM-provided confidence/status/quality_ok verbatim (no recalculation). - const newCount = (existing.observations || 0) + 1; - existing.observations = newCount; - existing.evidence = mergeEvidence(existing.evidence || [], newObs.evidence || []); - if (typeof newObs.confidence === 'number') existing.confidence = newObs.confidence; - if (newObs.status) existing.status = newObs.status; - existing.last_seen = nowIso; - if (newObs.pattern) existing.pattern = newObs.pattern; - if (newObs.details) existing.details = newObs.details; - if (newObs.quality_ok === true) existing.quality_ok = true; - // Passthrough new ledger fields from incoming obs (if LLM sets them) - if (newObs.anchor_id !== undefined) existing.anchor_id = newObs.anchor_id; - if (newObs.date !== undefined) existing.date = newObs.date; - if (newObs.decisions_status !== undefined) existing.decisions_status = newObs.decisions_status; - if (newObs.amendments !== undefined) existing.amendments = newObs.amendments; - if (newObs.raw_body !== undefined) existing.raw_body = newObs.raw_body; - - merged = true; - learningLog(`merge-observation: merged into ${existing.id} (count=${newCount})`); - } else { - // D11: ID collision recovery - let newId = newObs.id; - if (logMap.has(newId)) { - newId = newId + '_b'; - learningLog(`merge-observation: ID collision resolved: ${newObs.id} -> ${newId}`); + + try { + // D54 (E1): self-create parent directory on first write (fresh project, file+dir absent). + // The prior batch-merge op created the dir; merge-observation matches that so callers + // do not need to pre-create the log directory. + const logDir = path.dirname(logFile); + if (!fs.existsSync(logDir)) { + fs.mkdirSync(logDir, { recursive: true }); } - const entry = { - id: newId, - type: newObs.type, - pattern: newObs.pattern, - confidence: typeof newObs.confidence === 'number' ? newObs.confidence : 0, - observations: 1, - first_seen: nowIso, - last_seen: nowIso, - status: newObs.status || 'observing', - evidence: (newObs.evidence || []).slice(0, 10), - details: newObs.details || '', - quality_ok: newObs.quality_ok === true, - }; - // Passthrough new ledger fields if present on the new obs - if (newObs.anchor_id !== undefined) entry.anchor_id = newObs.anchor_id; - if (newObs.date !== undefined) entry.date = newObs.date; - if (newObs.decisions_status !== undefined) entry.decisions_status = newObs.decisions_status; - if (newObs.amendments !== undefined) entry.amendments = newObs.amendments; - if (newObs.raw_body !== undefined) entry.raw_body = newObs.raw_body; - - logMap.set(newId, entry); - learningLog(`merge-observation: new entry ${newId} confidence=${entry.confidence}`); - } - writeJsonlAtomic(logFile, Array.from(logMap.values())); - console.log(JSON.stringify({ merged, id: existing ? existing.id : newObs.id })); + let logEntries = []; + if (fs.existsSync(logFile)) { + logEntries = parseJsonl(logFile); + } + const logMap = new Map(logEntries.map(e => [e.id, e])); + const nowIso = new Date().toISOString().replace(/\.\d{3}Z$/, 'Z'); + + // ID-keyed lookup: match by obs_xxx id only + const existing = logMap.get(newObs.id); + let merged = false; + + if (existing) { + // Reinforce: increment count, merge evidence (FIFO cap 10), update last_seen. + // Store LLM-provided confidence/status/quality_ok verbatim (no recalculation). + const newCount = (existing.observations || 0) + 1; + existing.observations = newCount; + existing.evidence = mergeEvidence(existing.evidence || [], newObs.evidence || []); + if (typeof newObs.confidence === 'number') existing.confidence = newObs.confidence; + if (newObs.status) existing.status = newObs.status; + existing.last_seen = nowIso; + if (newObs.pattern) existing.pattern = newObs.pattern; + if (newObs.details) existing.details = newObs.details; + if (newObs.quality_ok === true) existing.quality_ok = true; + // Passthrough new ledger fields from incoming obs (if LLM sets them) + if (newObs.anchor_id !== undefined) existing.anchor_id = newObs.anchor_id; + if (newObs.date !== undefined) existing.date = newObs.date; + if (newObs.decisions_status !== undefined) existing.decisions_status = newObs.decisions_status; + if (newObs.amendments !== undefined) existing.amendments = newObs.amendments; + if (newObs.raw_body !== undefined) existing.raw_body = newObs.raw_body; + + merged = true; + learningLog(`merge-observation: merged into ${existing.id} (count=${newCount})`); + } else { + // D11: ID collision recovery + let newId = newObs.id; + if (logMap.has(newId)) { + newId = newId + '_b'; + learningLog(`merge-observation: ID collision resolved: ${newObs.id} -> ${newId}`); + } + const entry = { + id: newId, + type: newObs.type, + pattern: newObs.pattern, + confidence: typeof newObs.confidence === 'number' ? newObs.confidence : 0, + observations: 1, + first_seen: nowIso, + last_seen: nowIso, + status: newObs.status || 'observing', + evidence: (newObs.evidence || []).slice(0, 10), + details: newObs.details || '', + quality_ok: newObs.quality_ok === true, + }; + // Passthrough new ledger fields if present on the new obs + if (newObs.anchor_id !== undefined) entry.anchor_id = newObs.anchor_id; + if (newObs.date !== undefined) entry.date = newObs.date; + if (newObs.decisions_status !== undefined) entry.decisions_status = newObs.decisions_status; + if (newObs.amendments !== undefined) entry.amendments = newObs.amendments; + if (newObs.raw_body !== undefined) entry.raw_body = newObs.raw_body; + + logMap.set(newId, entry); + learningLog(`merge-observation: new entry ${newId} confidence=${entry.confidence}`); + } + + writeJsonlAtomic(logFile, Array.from(logMap.values())); + console.log(JSON.stringify({ merged, id: existing ? existing.id : newObs.id })); + } finally { + releaseLock(moLockDir); + } break; } diff --git a/src/cli/utils/hooks.ts b/src/cli/utils/hooks.ts index ed1c328c..7ab7419a 100644 --- a/src/cli/utils/hooks.ts +++ b/src/cli/utils/hooks.ts @@ -13,6 +13,8 @@ export interface HookEntry { } export interface HookMatcher { + /** Tool-name filter for PostToolUse/PreToolUse hooks (e.g. "AskUserQuestion"). Absent = matches all. */ + matcher?: string; hooks: HookEntry[]; } From 1c21defc750c0179e9e003b52b7cfa37e9b5cd05 Mon Sep 17 00:00:00 2001 From: Dean Sharon Date: Fri, 3 Jul 2026 01:54:13 +0300 Subject: [PATCH 02/26] feat(hooks): phase 1 capture layer for dream system simplification MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New hooks (dual-append to memory + dream queues, independently gated): - queue-append: shared JSONL-append helper (umask-077 creation, 200->100 overflow truncation under lock, one-fork combined memory+decisions config read via queue_read_gates). - capture-prompt (UserPromptSubmit): dual-append user turn, modeled on dream-dispatch which stays untouched. - capture-turn (Stop): dual-append assistant turn + decisions usage scanner (D29 grep-first), modeled on dream-capture which stays untouched. Never spawns a process. - capture-question (PostToolUse, AskUserQuestion): one {role:"qa"} row per question, Q/A independently truncated at 1000 chars. Payload shape pinned against a real scratch-project PostToolUse probe plus mined transcript samples (multi-question success + a real InputValidationError case) — see the handoff doc for specifics. - memory-worker (Stop): 120s-throttle + nohup-spawn block lifted verbatim from dream-capture, registered after capture-turn so append-before-spawn ordering holds by array position. Modified (additive): - background-memory-update: qa rows now emit a "Q&A:" stanza into TURNS_TEXT and count as content-bearing in the orphan-only guard (renamed _HAS_ASSISTANT -> _HAS_CONTENT); added a DEVFLOW_BG_DREAM re-entrancy guard. - session-start-memory: added DEVFLOW_BG_UPDATER/DEVFLOW_BG_DREAM re-entrancy guards (latent pre-existing gap); added a cold-path recovery for orphaned .pending-turns.processing (mirrors dream-recover's logic, duplicated rather than sourced since dream-recover is slated for removal in the follow-up cutover). Part of the dream system simplification (Phase 1 of 2-coder sequence). --- scripts/hooks/background-memory-update | 34 +++++- scripts/hooks/capture-prompt | 95 +++++++++++++++ scripts/hooks/capture-question | 157 +++++++++++++++++++++++++ scripts/hooks/capture-turn | 119 +++++++++++++++++++ scripts/hooks/memory-worker | 103 ++++++++++++++++ scripts/hooks/queue-append | 126 ++++++++++++++++++++ scripts/hooks/session-start-memory | 38 ++++++ 7 files changed, 666 insertions(+), 6 deletions(-) create mode 100755 scripts/hooks/capture-prompt create mode 100755 scripts/hooks/capture-question create mode 100755 scripts/hooks/capture-turn create mode 100755 scripts/hooks/memory-worker create mode 100755 scripts/hooks/queue-append diff --git a/scripts/hooks/background-memory-update b/scripts/hooks/background-memory-update index 3f054596..19850de9 100755 --- a/scripts/hooks/background-memory-update +++ b/scripts/hooks/background-memory-update @@ -22,6 +22,16 @@ set -e +# Re-entrancy guard: the dream worker's own claude -p session (DEVFLOW_BG_DREAM=1 +# in its env) fires Stop hooks on its own turns. If those hooks ever chain into +# spawning this worker, bail out immediately rather than cascading a memory +# refresh out of a nested dream-agent session (belt-and-suspenders alongside the +# equivalent guard in memory-worker, which is the intended sole spawn site). +if [ "${DEVFLOW_BG_DREAM:-}" = "1" ]; then + echo "background-memory-update: EXIT: bg_dream" >&2 + exit 0 +fi + CWD="$1" if [ -z "$CWD" ] || [ ! -d "$CWD" ]; then echo "background-memory-update: CWD missing or not a directory: '$CWD'" >&2 @@ -127,26 +137,29 @@ if ! acquire_lock; then exit 0 fi -# --- Orphan-only auto-clean: if queue has no assistant turn, truncate and exit --- +# --- Orphan-only auto-clean: if queue has no assistant/qa turn, truncate and exit --- # This prevents fabrication-prone LLM runs with only user turns in the queue. +# A qa row (captured Q&A pair) counts as content-bearing here too — it carries +# the same synthesis-worthy signal as an assistant turn (AC-F10). This check +# and the TURNS_TEXT extraction loop below must agree on that. # When neither jq nor node is available (_JSON_AVAILABLE=false) we skip the check # and allow the run to proceed — conservative: better to attempt than to truncate blindly. if [ -f "$QUEUE_FILE" ] && [ -s "$QUEUE_FILE" ] && [ "$_JSON_AVAILABLE" = "true" ]; then if [ "$_HAS_JQ" = "true" ]; then - _HAS_ASSISTANT=$(jq -r 'select(.role=="assistant") | .role' "$QUEUE_FILE" 2>/dev/null | head -1 || echo "") + _HAS_CONTENT=$(jq -r 'select(.role=="assistant" or .role=="qa") | .role' "$QUEUE_FILE" 2>/dev/null | head -1 || echo "") else # SECURITY: pass path via argv, never interpolate into node -e source (avoids shell injection # for repo paths containing quotes or special characters — applies ADR-008 plumbing principle) - _HAS_ASSISTANT=$(node -e ' + _HAS_CONTENT=$(node -e ' const f = process.argv[1]; const lines = require("fs").readFileSync(f, "utf8").split("\n"); for (const l of lines) { - try { const o = JSON.parse(l); if (o.role === "assistant") { process.stdout.write("assistant"); break; } } catch {} + try { const o = JSON.parse(l); if (o.role === "assistant" || o.role === "qa") { process.stdout.write(o.role); break; } } catch {} } ' "$QUEUE_FILE" 2>/dev/null || echo "") fi - if [ -z "$_HAS_ASSISTANT" ]; then - log "User-only queue (no assistant turn) — truncating without LLM run" + if [ -z "$_HAS_CONTENT" ]; then + log "User-only queue (no assistant/qa turn) — truncating without LLM run" rm -f "$QUEUE_FILE" 2>/dev/null || true exit 0 fi @@ -250,6 +263,15 @@ Turn ${TURN_COUNT}: Assistant: ${CONTENT} " fi + elif [ "$ROLE" = "qa" ]; then + # Captured AskUserQuestion Q&A pair — a self-contained stanza, not paired + # with CURRENT_USER (a qa row can arrive mid-turn, between a user prompt + # and its eventual assistant reply; leave CURRENT_USER buffered untouched). + TURN_COUNT=$(( TURN_COUNT + 1 )) + TURNS_TEXT="${TURNS_TEXT} +Turn ${TURN_COUNT}: +Q&A: ${CONTENT} +" fi done <&2; exit 1; } +if [ "$_JSON_AVAILABLE" = "false" ]; then dbg "EXIT: no json"; exit 0; fi + +INPUT=$(cat) + +FIELDS=$(printf '%s' "$INPUT" | json_extract_cwd_prompt) +CWD="${FIELDS%%$'\001'*}" +PROMPT="${FIELDS#*$'\001'}" + +dbg "CWD=$CWD PROMPT_LENGTH=${#PROMPT}" + +if [ -z "$CWD" ] || [ ! -d "$CWD" ]; then dbg "EXIT: bad CWD"; exit 0; fi + +devflow_debug_set_cwd "$CWD" + +# Anchor .devflow/ to the project root (prevents a stray nested .devflow/ when this +# hook runs with a CWD inside .devflow/...). Empty → fall back to CWD (old behavior). +source "$SCRIPT_DIR/resolve-project-root" 2>/dev/null || true +PROJECT_ROOT="$(df_resolve_root "$CWD" 2>/dev/null || true)" +[ -n "$PROJECT_ROOT" ] || PROJECT_ROOT="$CWD" + +DEVFLOW_DIR="$PROJECT_ROOT/.devflow" +MEMORY_DIR="$DEVFLOW_DIR/memory" +DREAM_DIR="$DEVFLOW_DIR/dream" + +if [ -z "$PROMPT" ]; then + dbg "EXIT: empty PROMPT" + exit 0 +fi + +# Truncate to 2000 chars (AC-F1) +if [ ${#PROMPT} -gt 2000 ]; then + PROMPT="${PROMPT:0:2000}... [truncated]" +fi + +source "$SCRIPT_DIR/queue-append" || { echo "capture-prompt: failed to source queue-append" >&2; exit 1; } + +# --- AC-P1: exactly ONE config-read fork, fetching both memory + decisions fields --- +queue_read_gates "$DREAM_DIR/config.json" "$DEVFLOW_DIR/decisions/.disabled" +MEMORY_ENABLED="$_QG_MEMORY" +DECISIONS_ENABLED="$_QG_DECISIONS" + +dbg "MEMORY_ENABLED=$MEMORY_ENABLED DECISIONS_ENABLED=$DECISIONS_ENABLED" + +if [ "$MEMORY_ENABLED" != "true" ] && [ "$DECISIONS_ENABLED" != "true" ]; then + dbg "EXIT: both features disabled" + exit 0 +fi + +# Normal logging (only paid for once we know at least one feature is enabled) +source "$SCRIPT_DIR/hook-log-init" "capture-prompt" + +# Auto-create .devflow/ and ensure .gitignore entries +source "$SCRIPT_DIR/ensure-devflow-init" "$CWD" || exit 0 + +source "$SCRIPT_DIR/get-mtime" || { echo "capture-prompt: failed to source get-mtime" >&2; exit 1; } +source "$SCRIPT_DIR/dream-lock" || { echo "capture-prompt: failed to source dream-lock" >&2; exit 1; } + +mkdir -p "$MEMORY_DIR" "$DREAM_DIR" 2>/dev/null || true + +TS=$(date +%s) +queue_append_both \ + "$MEMORY_DIR/.pending-turns.jsonl" "$DREAM_DIR/.pending-turns.jsonl" \ + "$MEMORY_ENABLED" "$DECISIONS_ENABLED" \ + "user" "$PROMPT" "$TS" + +log "Captured user turn (${#PROMPT} chars)" +dbg "=== HOOK COMPLETE ===" +exit 0 diff --git a/scripts/hooks/capture-question b/scripts/hooks/capture-question new file mode 100755 index 00000000..952a60d8 --- /dev/null +++ b/scripts/hooks/capture-question @@ -0,0 +1,157 @@ +#!/bin/bash + +# Dream System: capture-question (PostToolUse Hook, matcher: AskUserQuestion) +# Captures each answered question as a {role:"qa"} row into BOTH the memory and +# dream queues -- high decision-signal content that free-running Stop-hook +# turns miss (the model's own paraphrase of an answer is lossy; the raw Q&A +# pair is not). +# +# PAYLOAD SHAPE (pinned empirically -- see .devflow/docs/handoff-*.md for the +# mining method and exact captured samples): +# Envelope (captured via a scratch-project PostToolUse probe against the +# CURRENT harness version): session_id, transcript_path, cwd, prompt_id, +# permission_mode, effort, hook_event_name, tool_name, tool_input, +# tool_response, tool_use_id, duration_ms. Verified: tool_response is +# byte-identical to the toolUseResult recorded in the transcript JSONL for +# the same tool call. +# tool_input.questions: [{question, header, options:[{label,description}], multiSelect}] +# tool_response (success): {questions:[...], answers:{: }, annotations:{}} +# tool_response (validation error, e.g. missing "questions"): a PLAIN STRING, +# e.g. "InputValidationError: [...]" -- a real sample mined from +# ~/.claude/projects. +# No genuine cancelled/interrupted or free-text ("Other") sample was found +# despite an exhaustive machine-wide search; free text would use the SAME +# answers[question]=string shape as a listed option (confirmed structurally +# from the observed schema, not invented), so no special case is needed. +# +# Handling contract (regardless of shape): one qa row per question; cancelled/ +# absent/errored tool_response -> exit 0, zero writes; non-AskUserQuestion +# tool_name -> exit 0 (defensive, even though the hook is matcher-scoped). + +dbg() { :; } + +set -e + +if [ "${DEVFLOW_BG_UPDATER:-}" = "1" ]; then dbg "EXIT: bg_updater"; exit 0; fi +if [ "${DEVFLOW_BG_DREAM:-}" = "1" ]; then dbg "EXIT: bg_dream"; exit 0; fi + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" + +source "$SCRIPT_DIR/hook-bootstrap" "capture-question" + +source "$SCRIPT_DIR/json-parse" || { echo "capture-question: failed to source json-parse" >&2; exit 1; } +if [ "$_JSON_AVAILABLE" = "false" ]; then dbg "EXIT: no json"; exit 0; fi + +INPUT=$(cat) + +TOOL_NAME=$(printf '%s' "$INPUT" | json_field "tool_name" "") +if [ "$TOOL_NAME" != "AskUserQuestion" ]; then + dbg "EXIT: not AskUserQuestion (tool_name=$TOOL_NAME)" + exit 0 +fi + +CWD=$(printf '%s' "$INPUT" | json_field "cwd" "") +if [ -z "$CWD" ] || [ ! -d "$CWD" ]; then dbg "EXIT: bad CWD"; exit 0; fi + +devflow_debug_set_cwd "$CWD" + +source "$SCRIPT_DIR/resolve-project-root" 2>/dev/null || true +PROJECT_ROOT="$(df_resolve_root "$CWD" 2>/dev/null || true)" +[ -n "$PROJECT_ROOT" ] || PROJECT_ROOT="$CWD" + +DEVFLOW_DIR="$PROJECT_ROOT/.devflow" +MEMORY_DIR="$DEVFLOW_DIR/memory" +DREAM_DIR="$DEVFLOW_DIR/dream" + +# --- Parse questions + answers into "questionanswer" rows (one subprocess) --- +# Defensive against: tool_response absent or a plain string (error case), missing +# questions/answers, multiSelect array answers (joined with "; "), non-object +# tool_input. Tabs/newlines within a question or answer are collapsed to spaces +# so the TAB-delimited row format below stays unambiguous (mirrors +# background-memory-update's own gsub("\n";" ") convention for the same reason). +if [ "$_HAS_JQ" = "true" ]; then + QA_ROWS=$(printf '%s' "$INPUT" | jq -r ' + (.tool_response // {}) as $tr + | if ($tr|type) != "object" then + empty + else + (.tool_input.questions // [])[] as $q + | ($tr.answers // {})[$q.question] as $a + | select($a != null) + | (($q.question // "") | gsub("\n";" ") | gsub("\t";" ")) as $qtext + | ((if ($a|type)=="array" then ($a|join("; ")) else ($a|tostring) end) | gsub("\n";" ") | gsub("\t";" ")) as $atext + | $qtext + "\t" + $atext + end + ' 2>/dev/null || echo "") +else + QA_ROWS=$(printf '%s' "$INPUT" | node -e ' + let raw = ""; + try { raw = require("fs").readFileSync("/dev/stdin", "utf8"); } catch { process.exit(0); } + let input; + try { input = JSON.parse(raw); } catch { process.exit(0); } + const tr = input.tool_response; + if (!tr || typeof tr !== "object") process.exit(0); + const questions = Array.isArray(input.tool_input && input.tool_input.questions) ? input.tool_input.questions : []; + const answers = (tr.answers && typeof tr.answers === "object") ? tr.answers : {}; + const clean = (s) => String(s).replace(/\n/g, " ").replace(/\t/g, " "); + const lines = []; + for (const q of questions) { + if (!q || typeof q.question !== "string") continue; + const a = answers[q.question]; + if (a === undefined || a === null) continue; + const aStr = Array.isArray(a) ? a.join("; ") : String(a); + lines.push(clean(q.question) + "\t" + clean(aStr)); + } + process.stdout.write(lines.join("\n")); + ' 2>/dev/null || echo "") +fi + +if [ -z "$QA_ROWS" ]; then + dbg "EXIT: no answered questions found (cancelled/errored/absent tool_response, or non-object shape)" + exit 0 +fi + +source "$SCRIPT_DIR/queue-append" || { echo "capture-question: failed to source queue-append" >&2; exit 1; } + +# --- AC-P1-style: ONE config fork reads both memory + decisions fields --- +queue_read_gates "$DREAM_DIR/config.json" "$DEVFLOW_DIR/decisions/.disabled" +MEMORY_ENABLED="$_QG_MEMORY" +DECISIONS_ENABLED="$_QG_DECISIONS" + +if [ "$MEMORY_ENABLED" != "true" ] && [ "$DECISIONS_ENABLED" != "true" ]; then + dbg "EXIT: both features disabled" + exit 0 +fi + +source "$SCRIPT_DIR/hook-log-init" "capture-question" +source "$SCRIPT_DIR/ensure-devflow-init" "$CWD" || exit 0 +source "$SCRIPT_DIR/get-mtime" || { echo "capture-question: failed to source get-mtime" >&2; exit 1; } +source "$SCRIPT_DIR/dream-lock" || { echo "capture-question: failed to source dream-lock" >&2; exit 1; } + +mkdir -p "$MEMORY_DIR" "$DREAM_DIR" 2>/dev/null || true + +_QUESTION_COUNT=0 +while IFS=$'\t' read -r Q A; do + [ -z "$Q" ] && continue + + # Q and A truncated INDEPENDENTLY at 1000 chars each so a long question can't + # swallow the answer. + if [ ${#Q} -gt 1000 ]; then Q="${Q:0:1000}... [truncated]"; fi + if [ ${#A} -gt 1000 ]; then A="${A:0:1000}... [truncated]"; fi + + CONTENT="Q: ${Q} +A: ${A}" + TS=$(date +%s) + queue_append_both \ + "$MEMORY_DIR/.pending-turns.jsonl" "$DREAM_DIR/.pending-turns.jsonl" \ + "$MEMORY_ENABLED" "$DECISIONS_ENABLED" \ + "qa" "$CONTENT" "$TS" + _QUESTION_COUNT=$(( _QUESTION_COUNT + 1 )) +done <&2; exit 1; } +if [ "$_JSON_AVAILABLE" = "false" ]; then dbg "EXIT: no json"; exit 0; fi + +INPUT=$(cat) +dbg "INPUT length=${#INPUT}" + +# Resolve project directory and assistant response in one subprocess. +# Uses ASCII SOH (0x01) as delimiter; split with parameter expansion (bash 3.2 safe). +# Mirrors dream-capture's own CWD/ASSISTANT_MSG split. +if [ "$_HAS_JQ" = "true" ]; then + _FIELDS=$(printf '%s' "$INPUT" | jq -r '(.cwd // "") + "" + (.last_assistant_message // "")') || { dbg "EXIT: jq failed"; exit 0; } +else + _FIELDS=$(printf '%s' "$INPUT" | node -e " + const j=JSON.parse(require('fs').readFileSync('/dev/stdin','utf8')); + process.stdout.write((j.cwd||'')+'\x01'+(j.last_assistant_message||''))") || { dbg "EXIT: node failed"; exit 0; } +fi +CWD="${_FIELDS%%$'\001'*}" +ASSISTANT_MSG="${_FIELDS#*$'\001'}" + +if [ -z "$CWD" ] || [ ! -d "$CWD" ]; then dbg "EXIT: bad CWD"; exit 0; fi + +devflow_debug_set_cwd "$CWD" +dbg "CWD=$CWD" +# SECURITY: Never log ASSISTANT_MSG or INPUT content -- may contain secrets. +dbg "ASSISTANT_MSG length=${#ASSISTANT_MSG}" + +# Anchor .devflow/ to the project root (prevents a stray nested .devflow/ when this +# hook runs with a CWD inside .devflow/...). Empty → fall back to CWD (old behavior). +source "$SCRIPT_DIR/resolve-project-root" 2>/dev/null || true +PROJECT_ROOT="$(df_resolve_root "$CWD" 2>/dev/null || true)" +[ -n "$PROJECT_ROOT" ] || PROJECT_ROOT="$CWD" + +DEVFLOW_DIR="$PROJECT_ROOT/.devflow" +MEMORY_DIR="$DEVFLOW_DIR/memory" +DREAM_DIR="$DEVFLOW_DIR/dream" + +# Skip if empty response +if [ -z "$ASSISTANT_MSG" ]; then + dbg "EXIT: empty ASSISTANT_MSG" + exit 0 +fi + +# Truncate to 2000 chars +if [ ${#ASSISTANT_MSG} -gt 2000 ]; then + ASSISTANT_MSG="${ASSISTANT_MSG:0:2000}... [truncated]" +fi + +source "$SCRIPT_DIR/queue-append" || { echo "capture-turn: failed to source queue-append" >&2; exit 1; } + +# --- AC-P1: exactly ONE config-read fork, fetching both memory + decisions fields --- +queue_read_gates "$DREAM_DIR/config.json" "$DEVFLOW_DIR/decisions/.disabled" +MEMORY_ENABLED="$_QG_MEMORY" +DECISIONS_ENABLED="$_QG_DECISIONS" + +dbg "MEMORY_ENABLED=$MEMORY_ENABLED DECISIONS_ENABLED=$DECISIONS_ENABLED" + +# --- Decisions usage scanner (independent of the memory/dream queue gates below) --- +# D29: Grep-first reorder -- cheap in-process citation check gates the scanner call. +SCANNER="$SCRIPT_DIR/decisions-usage-scan.cjs" +if [ -f "$SCANNER" ] && printf '%s' "$ASSISTANT_MSG" | grep -qE 'ADR-[0-9]+|PF-[0-9]+'; then + if [ "$DECISIONS_ENABLED" = "true" ]; then + dbg "Running decisions usage scanner" + printf '%s' "$ASSISTANT_MSG" | node "$SCANNER" --cwd "$PROJECT_ROOT" 2>/dev/null || true + fi +fi + +if [ "$MEMORY_ENABLED" != "true" ] && [ "$DECISIONS_ENABLED" != "true" ]; then + dbg "EXIT: both features disabled" + exit 0 +fi + +# Logging (needs CWD and at-least-one-feature-enabled path — source after the +# gate check so the log-paths subprocess overhead is only paid when needed) +source "$SCRIPT_DIR/hook-log-init" "capture-turn" + +# Auto-create .devflow/ and ensure .gitignore entries +source "$SCRIPT_DIR/ensure-devflow-init" "$CWD" || exit 0 + +source "$SCRIPT_DIR/get-mtime" || { echo "capture-turn: failed to source get-mtime" >&2; exit 1; } +source "$SCRIPT_DIR/dream-lock" || { echo "capture-turn: failed to source dream-lock" >&2; exit 1; } + +mkdir -p "$MEMORY_DIR" "$DREAM_DIR" 2>/dev/null || true + +TS=$(date +%s) +queue_append_both \ + "$MEMORY_DIR/.pending-turns.jsonl" "$DREAM_DIR/.pending-turns.jsonl" \ + "$MEMORY_ENABLED" "$DECISIONS_ENABLED" \ + "assistant" "$ASSISTANT_MSG" "$TS" + +log "Captured assistant turn (${#ASSISTANT_MSG} chars)" +dbg "=== HOOK COMPLETE ===" +exit 0 diff --git a/scripts/hooks/memory-worker b/scripts/hooks/memory-worker new file mode 100755 index 00000000..2aa9664f --- /dev/null +++ b/scripts/hooks/memory-worker @@ -0,0 +1,103 @@ +#!/bin/bash + +# Dream System: memory-worker (Stop Hook) +# Lifted from dream-capture's 120s-throttle + nohup-spawn block. Registered +# AFTER capture-turn in the Stop hook array so append-before-spawn ordering is +# preserved by array position. This hook does NOT append to any queue itself -- +# capture-turn already did that earlier in the same Stop event. + +# Safe no-op fallback: must exist before set -e and before hook-bootstrap is sourced. +dbg() { :; } + +set -e + +if [ "${DEVFLOW_BG_UPDATER:-}" = "1" ]; then dbg "EXIT: bg_updater"; exit 0; fi +if [ "${DEVFLOW_BG_DREAM:-}" = "1" ]; then dbg "EXIT: bg_dream"; exit 0; fi + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" + +source "$SCRIPT_DIR/hook-bootstrap" "memory-worker" + +source "$SCRIPT_DIR/json-parse" || { echo "memory-worker: failed to source json-parse" >&2; exit 1; } +if [ "$_JSON_AVAILABLE" = "false" ]; then dbg "EXIT: no json"; exit 0; fi + +INPUT=$(cat) + +CWD=$(printf '%s' "$INPUT" | json_field "cwd" "") +if [ -z "$CWD" ] || [ ! -d "$CWD" ]; then dbg "EXIT: bad CWD"; exit 0; fi + +devflow_debug_set_cwd "$CWD" +dbg "CWD=$CWD" + +source "$SCRIPT_DIR/resolve-project-root" 2>/dev/null || true +PROJECT_ROOT="$(df_resolve_root "$CWD" 2>/dev/null || true)" +[ -n "$PROJECT_ROOT" ] || PROJECT_ROOT="$CWD" + +DEVFLOW_DIR="$PROJECT_ROOT/.devflow" +MEMORY_DIR="$DEVFLOW_DIR/memory" +DREAM_DIR="$DEVFLOW_DIR/dream" + +# Read dream config -- memory:false gates this hook entirely. +DREAM_CONFIG="$DREAM_DIR/config.json" +MEMORY_ENABLED="true" +if [ -f "$DREAM_CONFIG" ]; then + MEMORY_ENABLED=$(json_field_file "$DREAM_CONFIG" "memory" "true") +fi + +dbg "MEMORY_ENABLED=$MEMORY_ENABLED" + +if [ "$MEMORY_ENABLED" != "true" ]; then + dbg "EXIT: memory disabled" + exit 0 +fi + +source "$SCRIPT_DIR/hook-log-init" "memory-worker" +source "$SCRIPT_DIR/ensure-devflow-init" "$CWD" || exit 0 +source "$SCRIPT_DIR/get-mtime" || { echo "memory-worker: failed to source get-mtime" >&2; exit 1; } + +# --- Spawn detached worker if throttle expired --- +# Throttle key: .working-memory-last-trigger mtime (120s window). We touch it +# BEFORE spawning to prevent a second concurrent Stop hook from double-spawning +# within the same 120s window. +TRIGGER_FILE="$MEMORY_DIR/.working-memory-last-trigger" +NOW=$(date +%s) +LAST_TRIGGER=0 +if [ -f "$TRIGGER_FILE" ]; then + LAST_TRIGGER=$(get_mtime "$TRIGGER_FILE") +fi +LAST_TRIGGER="${LAST_TRIGGER:-0}" +TRIGGER_AGE=$(( NOW - LAST_TRIGGER )) + +if [ "$TRIGGER_AGE" -lt 120 ]; then + log "Throttled: worker spawned ${TRIGGER_AGE}s ago" + dbg "EXIT: throttled TRIGGER_AGE=${TRIGGER_AGE}s" + exit 0 +fi + +# Resolve worker and claude binary +UPDATER="$SCRIPT_DIR/background-memory-update" +CLAUDE_BIN=$(command -v claude 2>/dev/null || true) + +if [ -z "$CLAUDE_BIN" ]; then + log "SKIP: claude binary not found — worker not spawned (queue intact)" + dbg "EXIT: no claude binary" + exit 0 +fi + +if [ ! -x "$UPDATER" ]; then + log "SKIP: background-memory-update not found or not executable: $UPDATER" + dbg "EXIT: worker missing" + exit 0 +fi + +dbg "Throttle passed: TRIGGER_AGE=${TRIGGER_AGE}s >= 120s — spawning worker" + +# Touch trigger BEFORE spawning (prevents concurrent Stop hooks from double-spawning) +touch "$TRIGGER_FILE" 2>/dev/null || true + +# Spawn detached worker — nohup + disown so it survives the Stop hook process exit +nohup "$UPDATER" "$CWD" >/dev/null 2>&1 & disown + +log "Spawned background-memory-update worker (CWD=$CWD)" +dbg "=== HOOK COMPLETE ===" +exit 0 diff --git a/scripts/hooks/queue-append b/scripts/hooks/queue-append new file mode 100755 index 00000000..8fe76b5b --- /dev/null +++ b/scripts/hooks/queue-append @@ -0,0 +1,126 @@ +#!/bin/bash +# queue-append -- shared queue-append helper (sourced, not executed). +# +# Single implementation of: umask-077 file creation, JSONL emit (jq --arg / node +# JSON.stringify -- never string concatenation), and 200->100 overflow truncation +# under a lock. Used by capture-prompt, capture-turn, and capture-question so the +# per-feature dual-queue-append logic exists in exactly one place. +# +# Source-order requirement: source json-parse (for _HAS_JQ) and dream-lock (for +# dream_lock_acquire/dream_lock_release, which itself requires get-mtime) BEFORE +# calling queue_append_row or queue_append_both. Sourcing this file itself only +# defines functions -- no side effects until they are called. +# +# queue_append_row +# Appends one JSONL row {role, content, ts} to . Creates the file +# with mode 0600 (umask 077) if absent. After appending, truncates from 200 to +# the newest 100 lines under a lock (dream_lock_acquire on ".lock", +# 2s timeout) -- mirrors dream-capture's own overflow guard. The append itself +# is intentionally lock-free (accepted-class race shared with the pre-existing +# memory design -- see the design doc's "Append-vs-claim race" note). +# +# queue_append_both +# Calls queue_append_row for each queue whose *_enabled flag is "true". Each +# queue is gated independently -- callers compute memory_enabled/dream_enabled +# from dream config themselves (see queue_read_gates below for the one-fork +# combined read that keeps this to a single config subprocess per hook). +# +# queue_read_gates +# Reads BOTH the "memory" and "decisions" fields from dream config.json in a +# SINGLE subprocess fork (AC-P1 -- exactly one config-read fork per capture +# hook, not two). Sets _QG_MEMORY and _QG_DECISIONS ("true"/"false") in the +# caller's scope. Missing config file -> both default "true". decisions is +# further forced "false" when the sentinel path exists (dual-signal gate, +# mirroring dream-capture/dream-evaluate's existing convention). The two +# values are newline-separated rather than using a control-character +# delimiter: they are always the literal strings "true"/"false", never +# arbitrary content, so a plain newline split is unambiguous and easy to +# review (no invisible bytes hiding in the source). + +queue_append_row() { + local _qar_file="$1" _qar_role="$2" _qar_content="$3" _qar_ts="$4" + + if [ ! -f "$_qar_file" ]; then + (umask 077 && touch "$_qar_file") 2>/dev/null || true + fi + + if [ "$_HAS_JQ" = "true" ]; then + jq -n -c --arg role "$_qar_role" --arg content "$_qar_content" --argjson ts "$_qar_ts" \ + '{role: $role, content: $content, ts: $ts}' >> "$_qar_file" + else + node -e "process.stdout.write(JSON.stringify({role:process.argv[1],content:process.argv[2],ts:parseInt(process.argv[3])})+String.fromCharCode(10))" \ + -- "$_qar_role" "$_qar_content" "$_qar_ts" >> "$_qar_file" + fi + + # Overflow safety: >200 lines -> keep newest 100, locked to prevent a multi-session race. + if [ -f "$_qar_file" ]; then + local _qar_lines + _qar_lines=$(wc -l < "$_qar_file" | tr -d ' ') + if [ "$_qar_lines" -gt 200 ]; then + local _qar_lock="${_qar_file}.lock" + if dream_lock_acquire "$_qar_lock" 2; then + _qar_lines=$(wc -l < "$_qar_file" | tr -d ' ') + if [ "$_qar_lines" -gt 200 ]; then + local _qar_tmp="${_qar_file}.tmp.$$" + tail -100 "$_qar_file" > "$_qar_tmp" && mv "$_qar_tmp" "$_qar_file" || rm -f "$_qar_tmp" + log "Queue overflow: truncated from $_qar_lines to 100 lines ($(basename "$_qar_file"))" + dbg "Queue overflow: truncated from $_qar_lines to 100 lines ($_qar_file)" + fi + dream_lock_release "$_qar_lock" + fi + fi + fi +} + +queue_append_both() { + local _qab_memory_queue="$1" _qab_dream_queue="$2" + local _qab_memory_enabled="$3" _qab_dream_enabled="$4" + local _qab_role="$5" _qab_content="$6" _qab_ts="$7" + + if [ "$_qab_memory_enabled" = "true" ]; then + queue_append_row "$_qab_memory_queue" "$_qab_role" "$_qab_content" "$_qab_ts" + fi + if [ "$_qab_dream_enabled" = "true" ]; then + queue_append_row "$_qab_dream_queue" "$_qab_role" "$_qab_content" "$_qab_ts" + fi +} + +queue_read_gates() { + local _qg_config="$1" _qg_sentinel="$2" + _QG_MEMORY="true" + _QG_DECISIONS="true" + + if [ -f "$_qg_config" ]; then + local _qg_fields + if [ "$_HAS_JQ" = "true" ]; then + # if/then/else (not //) preserves an explicit `false` -- mirrors json_field_file's + # own documented rationale (jq's // would replace a real `false` with the default). + # The comma produces two raw-output lines (newline-separated) in one jq process. + _qg_fields=$(jq -r ' + (if (.memory | type) == "null" then "true" else (.memory | tostring) end), + (if (.decisions | type) == "null" then "true" else (.decisions | tostring) end) + ' "$_qg_config" 2>/dev/null) || _qg_fields="" + else + _qg_fields=$(node -e " + const j = JSON.parse(require('fs').readFileSync(process.argv[1], 'utf8')); + const m = j.memory === undefined ? 'true' : String(j.memory); + const d = j.decisions === undefined ? 'true' : String(j.decisions); + process.stdout.write(m + String.fromCharCode(10) + d); + " -- "$_qg_config" 2>/dev/null) || _qg_fields="" + fi + if [ -n "$_qg_fields" ]; then + _QG_MEMORY="${_qg_fields%%$'\n'*}" + _QG_DECISIONS="${_qg_fields#*$'\n'}" + fi + fi + + if [ -f "$_qg_sentinel" ]; then + _QG_DECISIONS="false" + fi + # Explicit return: this function's meaning is "populate _QG_* outputs," not a + # success/failure signal, so its exit status must never leak the truthiness of + # the last internal test into the caller (a bare `[ -f x ] && y=z` as the final + # statement would return 1 whenever the sentinel is absent — the common case — + # aborting any `set -e` caller that invokes this function as a plain statement). + return 0 +} diff --git a/scripts/hooks/session-start-memory b/scripts/hooks/session-start-memory index cbf98967..8f65edb6 100644 --- a/scripts/hooks/session-start-memory +++ b/scripts/hooks/session-start-memory @@ -12,6 +12,13 @@ dbg() { :; } set -e +# Re-entrancy guards — before hook-bootstrap to minimize background session +# overhead. A latent pre-existing gap: this hook had no guard against firing +# from within the memory or dream worker's own nested claude -p session +# (SessionStart fires on every session, including a detached worker's). +if [ "${DEVFLOW_BG_UPDATER:-}" = "1" ]; then dbg "EXIT: bg_updater"; exit 0; fi +if [ "${DEVFLOW_BG_DREAM:-}" = "1" ]; then dbg "EXIT: bg_dream"; exit 0; fi + # JSON parsing (jq with node fallback) — silently no-op if neither available SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" @@ -54,6 +61,37 @@ if [ -f "$DREAM_CONFIG" ]; then fi fi +# --- D56c cold-path recovery: orphaned .pending-turns.processing --- +# Mirrors dream-recover's dream_recover_stale .pending-turns.processing logic +# (memory stale threshold 300s), duplicated here (not sourced) so this hook has +# no dependency on dream-recover — which is scheduled for removal once the old +# Section-2 pending-work flow is retired. background-memory-update is the +# PRIMARY recovery owner: it re-merges a leftover .processing under its own +# lock on every spawn. This is the COLD-PATH fallback for when the worker +# never re-spawns (memory disabled mid-flight, host offline, etc.): a +# SessionStart arriving >300s after the crash with no intervening Stop hook. +# Non-clobber: only recovers when .pending-turns.jsonl does NOT already exist, +# so a concurrent session's fresh queue is never overwritten. +source "$SCRIPT_DIR/get-mtime" || { echo "session-start-memory: failed to source get-mtime" >&2; exit 1; } +_SSM_PT_PROC="$MEMORY_DIR/.pending-turns.processing" +_SSM_PT_JSONL="$MEMORY_DIR/.pending-turns.jsonl" +if [ -f "$_SSM_PT_PROC" ]; then + _SSM_PT_MTIME=$(get_mtime "$_SSM_PT_PROC" 2>/dev/null || true) + _SSM_PT_MTIME="${_SSM_PT_MTIME:-0}" + _SSM_NOW_PT=$(date +%s) + _SSM_PT_AGE=$(( _SSM_NOW_PT - _SSM_PT_MTIME )) + if [ "$_SSM_PT_AGE" -gt 300 ]; then + if [ ! -f "$_SSM_PT_JSONL" ]; then + mv "$_SSM_PT_PROC" "$_SSM_PT_JSONL" 2>/dev/null || true + dbg "Recovered orphaned .pending-turns.processing (age ${_SSM_PT_AGE}s)" + log "Recovered orphaned .pending-turns.processing → .pending-turns.jsonl (cold path)" + else + dbg "Stale .pending-turns.processing skipped — .pending-turns.jsonl already exists" + log "Stale .pending-turns.processing skipped (cold path): .pending-turns.jsonl already exists" + fi + fi +fi + CONTEXT="" # --- Section 1: Working Memory with git-reconciled header --- From bf34dabf9e0c1b5d6c39b0a2faddf4ed9f09fabd Mon Sep 17 00:00:00 2001 From: Dean Sharon Date: Fri, 3 Jul 2026 02:00:01 +0300 Subject: [PATCH 03/26] feat(hooks): phase 2 detached dream worker for dream system simplification MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New: - dream-procedure.md: combined decisions-detection + curation procedure, read directly by the dream worker's claude -p session (not a skill -- skills do not load in claude -p). Carries over the abstain-by-default bar, NOT-a-decision/NOT-a-pitfall lists, ADR-XOR-PF rule, dedup-first discipline, and curation bounds (<=5 changes, 7-day protection window) from shared/skills/dream-decisions + dream-curation, adapted for self-locking ops and worker-owned lifecycle (no marker claim/heartbeat/merge ceremony). Ends with rotate-observations then touch .last-dream-ok; writes last-run-summary only when the ledger changed. - background-dream-update: detached worker skeleton adapted from background-memory-update — mkdir lock on .worker.lock (900s stale, fail-fast ~2s acquire, unlike the memory worker's 90s wait), dual- signal decisions re-check, leftover .processing merge + rename-claim, model resolved from project then global decisions.json (default opus), claude -p with a short path-pointing prompt (the agent reads the claimed queue snapshot itself -- no raw content ever appears in the prompt or argv), 600s watchdog, and a strictly-newer-than- pre-spawn-baseline .last-dream-ok stamp as the success gate. - spawn-dream-worker (SessionStart): re-entrancy guards, dual-signal decisions gate, (queue non-empty OR leftover .processing) check with no jq parse, claude-on-PATH check, nohup spawn. No stdout directive. Modified (additive only): - session-start-context: added DEVFLOW_BG_DREAM/DEVFLOW_BG_UPDATER re-entrancy guards; added AC-F15 last-run-summary injection next to the decisions TL;DR (deletes the file after injecting). Section 2 (old pending-work directive flow) is untouched -- its removal is coupled to retiring dream-collect-tasks/dream-recover in a later cutover. - pre-compact-memory: added the same re-entrancy guards (latent pre-existing gap). Part of the dream system simplification (Phase 2 of 2-coder sequence). --- scripts/hooks/background-dream-update | 305 ++++++++++++++++++++++++++ scripts/hooks/dream-procedure.md | 184 ++++++++++++++++ scripts/hooks/pre-compact-memory | 7 + scripts/hooks/session-start-context | 31 +++ scripts/hooks/spawn-dream-worker | 89 ++++++++ 5 files changed, 616 insertions(+) create mode 100755 scripts/hooks/background-dream-update create mode 100644 scripts/hooks/dream-procedure.md create mode 100755 scripts/hooks/spawn-dream-worker diff --git a/scripts/hooks/background-dream-update b/scripts/hooks/background-dream-update new file mode 100755 index 00000000..dd81c4f2 --- /dev/null +++ b/scripts/hooks/background-dream-update @@ -0,0 +1,305 @@ +#!/bin/bash + +# background-dream-update — Detached dream worker (decisions detection + curation) +# Spawned as a detached background process by spawn-dream-worker (SessionStart hook) +# whenever the dream queue is non-empty or a leftover .processing batch exists. +# Claims the queue, spawns `claude -p` pointed at dream-procedure.md, and verifies +# success via the .last-dream-ok stamp — mirrors background-memory-update's proven +# skeleton (lock, leftover-merge, rename-claim, watchdog, stamp-advance success gate), +# adapted for the dream feature's own bounds and gate signals. +# +# Applies ADR-008 (LLM-vs-plumbing): the AGENT (per dream-procedure.md) does ALL the +# detection/curation work and calls the ledger ops itself via its own Bash tool. This +# script is plumbing only: claim the queue, spawn the agent, verify the stamp advanced. +# +# Usage: background-dream-update +# +# Success: removes .pending-turns.processing; .last-dream-ok mtime advanced +# (written by the agent itself, per dream-procedure.md) +# Failure: leaves .pending-turns.processing. Unlike the memory worker, there is +# no cold-path recovery for this queue: the NEXT spawn-dream-worker +# invocation (any subsequent SessionStart) sees the leftover .processing +# via `[ -f .pending-turns.processing ]` and re-triggers a spawn, which +# merges it back in below. An orphaned .processing with decisions +# disabled has no path back (accepted — swept by `devflow decisions --clear`). + +set -e + +# Re-entrancy guard: the dream worker's own claude -p session (DEVFLOW_BG_DREAM=1 in +# its env) fires SessionStart hooks on its own turns; without this, every dream run +# would recursively spawn another worker. spawn-dream-worker already guards its own +# spawn decision, but this worker also guards itself defensively (belt-and-suspenders, +# matching background-memory-update's equivalent DEVFLOW_BG_DREAM guard). +if [ "${DEVFLOW_BG_DREAM:-}" = "1" ]; then + echo "background-dream-update: EXIT: bg_dream" >&2 + exit 0 +fi +if [ "${DEVFLOW_BG_UPDATER:-}" = "1" ]; then + echo "background-dream-update: EXIT: bg_updater" >&2 + exit 0 +fi + +CWD="$1" +if [ -z "$CWD" ] || [ ! -d "$CWD" ]; then + echo "background-dream-update: CWD missing or not a directory: '$CWD'" >&2 + exit 1 +fi + +# Resolve SCRIPT_DIR early — needed for sourcing helpers +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" + +# Source JSON parsing helpers (jq with node fallback) +source "$SCRIPT_DIR/json-parse" || { echo "background-dream-update: failed to source json-parse" >&2; exit 1; } + +# Source hook-log-init for log() and size guard (requires $SCRIPT_DIR and $CWD, both set +# above; does NOT pull in hook-bootstrap so the detached-worker invariant is preserved) +source "$SCRIPT_DIR/hook-log-init" "background-dream-update" \ + || { echo "background-dream-update: failed to source hook-log-init" >&2; exit 1; } + +# Source get-mtime for mtime comparisons +source "$SCRIPT_DIR/get-mtime" || { echo "background-dream-update: failed to source get-mtime" >&2; exit 1; } + +log "Starting (CWD=$CWD)" + +# --- Resolve paths --- +# Anchor .devflow/ to the project root (prevents a stray nested .devflow/ when the +# worker is spawned with a CWD inside .devflow/...). Empty → fall back to CWD. +source "$SCRIPT_DIR/resolve-project-root" 2>/dev/null || true +PROJECT_ROOT="$(df_resolve_root "$CWD" 2>/dev/null || true)" +[ -n "$PROJECT_ROOT" ] || PROJECT_ROOT="$CWD" +DEVFLOW_DIR="$PROJECT_ROOT/.devflow" +DREAM_DIR="$DEVFLOW_DIR/dream" +DECISIONS_DIR_DATA="$DEVFLOW_DIR/decisions" +QUEUE_FILE="$DREAM_DIR/.pending-turns.jsonl" +PROCESSING_FILE="$DREAM_DIR/.pending-turns.processing" +OK_FILE="$DREAM_DIR/.last-dream-ok" +LOCK_DIR="$DREAM_DIR/.worker.lock" +DREAM_CONFIG="$DREAM_DIR/config.json" +DECISIONS_CONFIG_PROJECT="$DECISIONS_DIR_DATA/decisions.json" +DECISIONS_CONFIG_GLOBAL="$HOME/.devflow/decisions.json" +PROCEDURE_FILE="$SCRIPT_DIR/dream-procedure.md" + +# --- Re-check decisions dual-signal at runtime (defense-in-depth: feature may have +# been disabled since spawn-dream-worker made its spawn decision) --- +DECISIONS_ENABLED="true" +if [ -f "$DREAM_CONFIG" ]; then + DECISIONS_ENABLED=$(json_field_file "$DREAM_CONFIG" "decisions" "true") +fi +if [ -f "$DECISIONS_DIR_DATA/.disabled" ]; then + DECISIONS_ENABLED="false" +fi +if [ "$DECISIONS_ENABLED" != "true" ]; then + log "ABORT: decisions disabled in dream config/sentinel (disabled after spawn)" + exit 0 +fi + +# --- Resolve claude binary --- +CLAUDE_BIN=$(command -v claude 2>/dev/null || true) +if [ -z "$CLAUDE_BIN" ]; then + log "SKIP: claude binary not found on PATH" + exit 0 +fi + +if [ ! -f "$PROCEDURE_FILE" ]; then + log "SKIP: dream-procedure.md not found: $PROCEDURE_FILE" + exit 0 +fi + +# --- Worker-level lock (900s stale-break, FAIL-FAST ~2s acquire) --- +# Unlike background-memory-update's 90s wait, a waiter here has nothing to +# contribute once the holder has already claimed the queue — the NEXT +# spawn-dream-worker invocation (any subsequent SessionStart) will simply try +# again later. Do not copy the memory worker's long wait. +# +# INVARIANT: STALE_THRESHOLD (900s) must always exceed the watchdog total +# in-flight time (WATCHDOG_SECS + WATCHDOG_KILL_GRACE_SECS = 600+5 = 605s, plus +# margin). A live worker holding the lock will have been watchdog-killed within +# ~605s, so a 900s stale-break is safe-by-construction. If either bound is +# raised, this invariant MUST be re-verified (see the runtime assertion below). +STALE_THRESHOLD=900 # 15 minutes + +break_stale_lock() { + if [ ! -d "$LOCK_DIR" ]; then return; fi + local _lock_mtime _now _age + _lock_mtime=$(get_mtime "$LOCK_DIR") + _now=$(date +%s) + _age=$(( _now - _lock_mtime )) + if [ "$_age" -gt "$STALE_THRESHOLD" ]; then + log "Breaking stale worker lock (age: ${_age}s, threshold: ${STALE_THRESHOLD}s)" + rmdir "$LOCK_DIR" 2>/dev/null || true + fi +} + +acquire_lock() { + local _timeout=2 + local _waited=0 + while ! mkdir "$LOCK_DIR" 2>/dev/null; do + if [ "$_waited" -ge "$_timeout" ]; then + return 1 + fi + sleep 1 + _waited=$(( _waited + 1 )) + done + return 0 +} + +cleanup_lock() { + rmdir "$LOCK_DIR" 2>/dev/null || true +} +trap cleanup_lock EXIT + +break_stale_lock + +if ! acquire_lock; then + log "SKIP: lock held by another worker (fail-fast) — queue preserved for next spawn" + trap - EXIT + exit 0 +fi + +# --- Claim queue atomically. If a leftover .processing exists from a previous +# crash, merge new queue entries into it (same primary-recovery model as +# background-memory-update). --- +if [ -f "$PROCESSING_FILE" ]; then + log "Found leftover .processing from previous crash — merging" + if [ -f "$QUEUE_FILE" ]; then + cat "$QUEUE_FILE" >> "$PROCESSING_FILE" 2>/dev/null || true + rm -f "$QUEUE_FILE" 2>/dev/null || true + log "Merged new queue entries into recovery batch" + fi + # Cap to prevent unbounded growth (same bounds as the memory worker's own guard) + PROC_OVERFLOW_CAP=200 + PROC_OVERFLOW_TARGET=100 + _PROC_LINES=$(wc -l < "$PROCESSING_FILE" | tr -d ' ') + if [ "$_PROC_LINES" -gt "$PROC_OVERFLOW_CAP" ]; then + _PROC_TMP="$PROCESSING_FILE.tmp.$$" + tail -"$PROC_OVERFLOW_TARGET" "$PROCESSING_FILE" > "$_PROC_TMP" && mv "$_PROC_TMP" "$PROCESSING_FILE" || rm -f "$_PROC_TMP" 2>/dev/null || true + log "Processing overflow: truncated from $_PROC_LINES to $PROC_OVERFLOW_TARGET lines" + fi +elif [ -f "$QUEUE_FILE" ]; then + mv "$QUEUE_FILE" "$PROCESSING_FILE" 2>/dev/null || { log "SKIP: failed to claim queue (race condition — another worker got it)"; exit 0; } +else + log "SKIP: no queue file to process" + exit 0 +fi + +TOTAL_LINES=$(wc -l < "$PROCESSING_FILE" | tr -d ' ') +log "Claimed $TOTAL_LINES queued entries" + +if [ "$TOTAL_LINES" -eq 0 ]; then + rm -f "$PROCESSING_FILE" 2>/dev/null || true + log "Processing file empty — skipping" + exit 0 +fi + +# --- Resolve model: project decisions.json, then global, then "opus" default. --- +# NOTE: this "opus" default is intentionally DIFFERENT from DecisionsConfig's own +# TS-side default ("sonnet" — src/cli/utils/decisions-config.ts, used elsewhere for +# unconfigured projects); the plan calls for this worker specifically to default to +# opus. An explicit `model` set by a prior `devflow decisions --configure` still wins +# either way, since it is read before the default is applied. +MODEL="" +if [ -f "$DECISIONS_CONFIG_PROJECT" ]; then + MODEL=$(json_field_file "$DECISIONS_CONFIG_PROJECT" "model" "") +fi +if [ -z "$MODEL" ] && [ -f "$DECISIONS_CONFIG_GLOBAL" ]; then + MODEL=$(json_field_file "$DECISIONS_CONFIG_GLOBAL" "model" "") +fi +MODEL="${MODEL:-opus}" +log "Resolved model: $MODEL" + +# --- Build prompt (short pointer, not a data dump — the agent reads the claimed +# queue snapshot and decisions files itself via its own Read tool. Untrusted +# conversation content therefore never appears in this prompt or in argv.) --- +PROMPT="Read ${PROCEDURE_FILE} and follow it exactly. + +Inputs (read-only unless the procedure says otherwise): +- Claimed queue snapshot: ${PROCESSING_FILE} +- Observation log: ${DECISIONS_DIR_DATA}/decisions-log.jsonl +- Rendered decisions: ${DECISIONS_DIR_DATA}/decisions.md +- Rendered pitfalls: ${DECISIONS_DIR_DATA}/pitfalls.md +- Citation usage: ${DECISIONS_DIR_DATA}/.decisions-usage.json + +Project root: ${PROJECT_ROOT}" + +log "Spawning claude -p (model=$MODEL)" + +# --- Success-gate baseline: capture .last-dream-ok's mtime BEFORE spawning so +# success can be verified as "strictly newer than this baseline" (file-vs-file, +# not file-vs-wallclock — matches background-memory-update's own PRE_UPDATE_MTIME +# pattern, avoiding a false-negative when the agent touches the stamp in the same +# second as worker start). --- +PRE_DREAM_MTIME=0 +if [ -f "$OK_FILE" ]; then + PRE_DREAM_MTIME=$(get_mtime "$OK_FILE") +fi + +# --- Watchdog timeout (overridable for tests: DEVFLOW_BG_WATCHDOG_SECS=2) --- +WATCHDOG_SECS="${DEVFLOW_BG_WATCHDOG_SECS:-600}" +WATCHDOG_KILL_GRACE_SECS=5 + +# NASA/JPL Rule 5 — runtime assertion: STALE_THRESHOLD must exceed the watchdog's +# total in-flight time so the lock stale-break can never evict a still-live worker. +# Checked before spawning claude so a bad override is caught early. +_WATCHDOG_TOTAL=$(( WATCHDOG_SECS + WATCHDOG_KILL_GRACE_SECS )) +if [ "$STALE_THRESHOLD" -le "$_WATCHDOG_TOTAL" ]; then + echo "[background-dream-update] FATAL: STALE_THRESHOLD ($STALE_THRESHOLD) must exceed watchdog total (${WATCHDOG_SECS}+${WATCHDOG_KILL_GRACE_SECS}=${_WATCHDOG_TOTAL})" >&2 + exit 1 +fi + +# --- Run claude -p with watchdog --- +# D37: Enable job control (set -m) only around the spawn so bash gives claude its OWN +# process group (PGID == CLAUDE_PID) even in this non-interactive script — see +# background-memory-update for the full rationale (self-kill-bug avoidance). +set -m +DEVFLOW_BG_DREAM=1 "$CLAUDE_BIN" -p \ + --model "$MODEL" \ + --allowedTools 'Read,Grep,Glob,Write,Edit,Bash' \ + --output-format text \ + <<< "$PROMPT" \ + >> "$LOG_FILE" 2>&1 & +CLAUDE_PID=$! +set +m + +( + sleep "$WATCHDOG_SECS" + kill -TERM -"$CLAUDE_PID" 2>/dev/null || true + sleep "$WATCHDOG_KILL_GRACE_SECS" + kill -KILL -"$CLAUDE_PID" 2>/dev/null || true +) & +WD_PID=$! + +if wait "$CLAUDE_PID" 2>/dev/null; then + CLAUDE_EXIT=0 +else + CLAUDE_EXIT=$? +fi + +# Cancel watchdog (avoids stray kill hitting a recycled PID after this process exits) +kill "$WD_PID" 2>/dev/null || true +wait "$WD_PID" 2>/dev/null || true + +if [ "$CLAUDE_EXIT" -gt 128 ]; then + log "FAIL: claude -p killed by watchdog after ${WATCHDOG_SECS}s" + exit 0 +elif [ "$CLAUDE_EXIT" -ne 0 ]; then + log "FAIL: claude -p exited with code $CLAUDE_EXIT" + exit 0 +fi + +# --- Verify success: .last-dream-ok mtime strictly greater than pre-spawn baseline --- +UPDATED="false" +if [ -f "$OK_FILE" ]; then + NEW_OK_MTIME=$(get_mtime "$OK_FILE") + if [ "$NEW_OK_MTIME" -gt "$PRE_DREAM_MTIME" ]; then + UPDATED="true" + fi +fi + +if [ "$UPDATED" = "true" ]; then + rm -f "$PROCESSING_FILE" 2>/dev/null || true + log "SUCCESS: dream run completed, .last-dream-ok advanced" +else + log "FAIL: .last-dream-ok did not advance past pre-spawn baseline — leaving .processing for recovery" +fi + +exit 0 diff --git a/scripts/hooks/dream-procedure.md b/scripts/hooks/dream-procedure.md new file mode 100644 index 00000000..94b44f2c --- /dev/null +++ b/scripts/hooks/dream-procedure.md @@ -0,0 +1,184 @@ +# Dream Worker Procedure + +## Iron Law + +> **assign-anchor OWNS NUMBERING; render OWNS THE .md; NEVER HAND-EDIT decisions.md or pitfalls.md** +> +> ADR and PF numbers are assigned exclusively by `assign-anchor`. The `.md` files are written +> exclusively by `render-decisions.cjs` (invoked internally by `assign-anchor`/`retire-anchor`). +> One `assign-anchor` invocation claims one number and re-renders both files atomically. To +> deprecate, supersede, or retire an entry, call `retire-anchor ` — never +> edit the `.md` files directly. + +This procedure is read directly (via the Read tool) by the detached dream worker's +`claude -p` session — spawned by `spawn-dream-worker`/`background-dream-update` whenever the +dream queue is non-empty or a leftover `.processing` batch needs re-attempting. It is NOT a +Claude Code skill (skills do not load in `claude -p` sessions). You have Read, Grep, Glob, +Write, Edit, and Bash. Do everything below yourself — no other script validates or applies +your output. + +## Inputs (read-only, except where noted) + +The worker script's invocation prompt tells you the exact paths to use. Within the project's +`.devflow/decisions/` and `.devflow/dream/` directories, read: + +- The claimed queue snapshot (conversation turns + qa rows since the last run) +- `decisions-log.jsonl` — full observation history (for dedup and recurrence patterns) +- `decisions.md` and `pitfalls.md` — the rendered, currently-active entries +- `.decisions-usage.json` — citation counts, keyed by anchor ID (`ADR-NNN`/`PF-NNN`) + +You do NOT need to claim, heartbeat, or merge any marker files — the worker script already +claimed the queue atomically (rename-to-claim) before spawning you. There is nothing left to +claim. + +## Part 1 — Decision & pitfall detection + +Read the queue snapshot in full (cap at the last 30 dialog-worthy entries if the file is very +large). Read `decisions-log.jsonl` in full for dedup. + +**LLM judgment — creation bar (abstain-by-default)**: + +Most runs produce nothing. If unsure, record nothing. Only capture what a future contributor +would need and could not reconstruct from the code. + +**NOT a decision**: bug fix, one-off UX tweak, routine refactor, applying an existing pattern, +dependency bump, or anything already covered by an existing ADR in the log. + +**NOT a pitfall**: typo, transient flake, mistake with no general lesson, or a problem fully +prevented by existing tooling. + +**Positive bar**: +- Decision = a deliberate architectural choice or trade-off with rationale that constrains + future work. It must be a real fork in the road, not an obvious choice. +- Pitfall = a non-obvious failure mode with a transferable lesson that the next contributor + cannot recover from the code alone. + +**ADR-XOR-PF (hard rule)**: one incident yields exactly one of an ADR or a PF — never both. +Concrete failure → PF; forward-looking architectural choice → ADR. + +**Dedup before creating (read the log first)**: if an existing row (any status, including +Retired) already covers this concern, reinforce it (reuse its `obs_` id via +`merge-observation`) instead of creating a new entry. Duplication is worse than silence. + +For each pattern that clears the creation bar: +1. Scan the log first for a matching existing entry. REUSE its `obs_` id if found. +2. Estimate `confidence` honestly — this is curation metadata only, NOT a gate. Estimate what + the evidence actually supports; do not inflate it. +3. Author a full `details` string: `"context: X; decision: Y; rationale: Z"` (decision) or + `"area: X; issue: Y; impact: Z; resolution: W"` (pitfall). + +Write (or reinforce) each observation directly — `merge-observation` is self-locking (it +acquires and releases `.devflow/dream/.observations.lock` internally), so call it plainly: + +```bash +node "$HOME/.devflow/scripts/hooks/json-helper.cjs" merge-observation \ + ".devflow/decisions/decisions-log.jsonl" \ + '{"id":"obs_xxx","type":"decision","pattern":"...","evidence":["..."],"details":"context: ...; decision: ...; rationale: ...","confidence":0.8,"status":"observing","quality_ok":true}' +``` + +Do NOT wrap this call in your own lock acquisition — the op self-locks; an external lock +around it would nest against the internal one and time out. + +**If promoting** (quality_ok=true, pattern recurs or is clearly significant after clearing the +creation bar above): promote via `assign-anchor` (self-locking on `.decisions.lock`, atomic — +claims the number and re-renders both `.md` files in one call): + +```bash +node "$HOME/.devflow/scripts/hooks/json-helper.cjs" assign-anchor "decision" "obs_xxx" +node "$HOME/.devflow/scripts/hooks/json-helper.cjs" assign-anchor "pitfall" "obs_xxx" +``` + +NEVER hand-edit `decisions.md` or `pitfalls.md`. NEVER invent an ADR-NNN/PF-NNN number +yourself — `assign-anchor` is the only source of numbering. + +## Part 2 — Curation + +Periodic housekeeping of the decisions ledger and rendered `.md` files. Bounds: **≤5 curation +changes per run**. **7-day protection window** — never touch any entry whose `date` field in +the ledger is within the past 7 days. The window key is the ledger row's `date` field +(YYYY-MM-DD), not anything in the `.md` file. + +Read active entry counts: + +```bash +node "$HOME/.devflow/scripts/hooks/json-helper.cjs" count-active "decision" +node "$HOME/.devflow/scripts/hooks/json-helper.cjs" count-active "pitfall" +``` + +Cite counts come from `.decisions-usage.json` directly (no scan step here — +`decisions-usage-scan.cjs` is a write-path tool triggered by conversation capture, not a +curation reporter). + +**Rotate stale observations first** (before selecting curation candidates) — self-locking on +`.observations.lock`: + +```bash +node "$HOME/.devflow/scripts/hooks/json-helper.cjs" rotate-observations +``` + +This archives `observing` rows older than 30 days to `decisions-log.archive.jsonl` +(gitignored). It never touches anchored (`anchor_id` set) or `created`/`ready` rows. + +**Staleness signal** (run once per curation pass, before selecting candidates): + +```bash +node "$HOME/.devflow/scripts/hooks/lib/staleness.cjs" \ + ".devflow/decisions/decisions-log.jsonl" "$(pwd)" +``` + +Entries flagged `mayBeStale: true` (their referenced files no longer exist) are preferred +retirement candidates, within the 7-day window and ≤5-change bounds above — a signal to +prefer, not an automatic retirement. + +**LLM judgment — identify entries to retire or merge**: + +Retire an entry when it is: +- Superseded by a newer, more precise entry on the same topic +- Contradicted by evidence in recent sessions +- Never cited (0 cites) AND older than 30 days AND low-confidence in the log + +**ADR-XOR-PF awareness**: if curation finds two entries covering the same incident (one ADR, +one PF), consolidate to the more accurate type and retire the other. + +**Dedup awareness**: before retiring, check whether two near-duplicate entries could be +consolidated. Retire the less specific one and update the surviving entry's `pattern` to +absorb the key insight from the retired entry. + +**RETIRE BY STATUS — never hand-edit the .md**: + +```bash +node "$HOME/.devflow/scripts/hooks/json-helper.cjs" retire-anchor +# status ∈ Deprecated | Superseded | Retired +``` + +`retire-anchor` self-locks on `.decisions.lock` across the full ledger-write + render critical +section. Atomic and idempotent. Call it once per entry — never hold a lock across multiple +invocations yourself. + +**Citation preservation**: if an entry being retired has inbound `applies ADR-NNN` citations +in other entries, update those entries' `pattern`/`details` to reference the surviving entry +instead (edit those ledger rows directly, then re-render via +`node "$HOME/.devflow/scripts/hooks/lib/render-decisions.cjs" render "$(pwd)"`). + +**Cap enforcement**: stop after 5 changes regardless of remaining candidates. + +**Transparency**: if you retired or merged anything this run, say so briefly in your final +output. + +## Run visibility + +If (and only if) this run changed the ledger (created, promoted, retired, or merged +anything), write a 1-3 line summary to `.devflow/dream/last-run-summary` describing what +changed. If nothing changed, do NOT create this file. `session-start-context` injects this +file's content once at the next session start, then deletes it — you do not need to manage +its lifecycle beyond writing it. + +## Finishing + +Regardless of whether Part 1 or Part 2 found anything to do: + +1. Run `rotate-observations` if you have not already run it this pass (Part 2 already covers + this — do not run it twice). +2. Touch `.devflow/dream/.last-dream-ok` LAST, after all other writes are complete. This is + the worker script's success signal — it will not be reached if you error out or are killed + by the watchdog, which is the correct (safe) outcome for a partial run. diff --git a/scripts/hooks/pre-compact-memory b/scripts/hooks/pre-compact-memory index 98b80d57..7cabddf1 100644 --- a/scripts/hooks/pre-compact-memory +++ b/scripts/hooks/pre-compact-memory @@ -12,6 +12,13 @@ dbg() { :; } set -e +# Re-entrancy guards — before hook-bootstrap to minimize background session +# overhead. A latent pre-existing gap: this hook had no guard against firing +# from within the memory or dream worker's own nested claude -p session +# (PreCompact can fire mid-session on a long-running detached worker too). +if [ "${DEVFLOW_BG_UPDATER:-}" = "1" ]; then dbg "EXIT: bg_updater"; exit 0; fi +if [ "${DEVFLOW_BG_DREAM:-}" = "1" ]; then dbg "EXIT: bg_dream"; exit 0; fi + SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" source "$SCRIPT_DIR/hook-bootstrap" "pre-compact-memory" diff --git a/scripts/hooks/session-start-context b/scripts/hooks/session-start-context index a7192e38..f5f5c168 100755 --- a/scripts/hooks/session-start-context +++ b/scripts/hooks/session-start-context @@ -13,6 +13,15 @@ dbg() { :; } # set -e intentionally omitted: sections 1.5 and 2 are independent — # a failure in one must not prevent the others from running. +# Re-entrancy guards — before hook-bootstrap to minimize background session +# overhead. The dream worker's own claude -p session fires SessionStart hooks; +# without this guard every dream run would recursively re-trigger this hook's +# own directive-emitting logic (Section 2) from inside its own nested session. +# A latent pre-existing gap for DEVFLOW_BG_UPDATER too (this hook previously +# had no re-entrancy guard at all). +if [ "${DEVFLOW_BG_DREAM:-}" = "1" ]; then dbg "EXIT: bg_dream"; exit 0; fi +if [ "${DEVFLOW_BG_UPDATER:-}" = "1" ]; then dbg "EXIT: bg_updater"; exit 0; fi + # JSON parsing (jq with node fallback) — silently no-op if neither available SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" @@ -86,6 +95,28 @@ ${DECISIONS_SECTION}" fi fi fi + + # --- AC-F15: dream worker last-run-summary (agent-written, injected once) --- + # The dream worker (background-dream-update) writes this file only when it + # changed the ledger. Inject its content next to the decisions TL;DR, then + # delete it — no file means no injection and no error. + _SC_LAST_RUN_SUMMARY="$DREAM_DIR/last-run-summary" + if [ -f "$_SC_LAST_RUN_SUMMARY" ]; then + _SC_SUMMARY_CONTENT=$(cat "$_SC_LAST_RUN_SUMMARY" 2>/dev/null || true) + if [ -n "$_SC_SUMMARY_CONTENT" ]; then + _SC_SUMMARY_SECTION="--- DREAM LAST RUN --- +$_SC_SUMMARY_CONTENT" + if [ -n "$CONTEXT" ]; then + CONTEXT="${CONTEXT} + +${_SC_SUMMARY_SECTION}" + else + CONTEXT="$_SC_SUMMARY_SECTION" + fi + dbg "last-run-summary injected" + fi + rm -f "$_SC_LAST_RUN_SUMMARY" 2>/dev/null || true + fi fi # --- Section 2: Dream Pending-Work --- diff --git a/scripts/hooks/spawn-dream-worker b/scripts/hooks/spawn-dream-worker new file mode 100755 index 00000000..c9c839f5 --- /dev/null +++ b/scripts/hooks/spawn-dream-worker @@ -0,0 +1,89 @@ +#!/bin/bash + +# Dream System: spawn-dream-worker (SessionStart Hook) +# Always-registered. Gate: decisions enabled (config + .disabled sentinel) AND +# (dream queue non-empty OR leftover .processing) AND claude on PATH. When all +# hold, spawns a detached background-dream-update worker. No stdout directive -- +# unlike the old session-start-context Section 2 flow, this hook does not ask +# the main model to do anything; it silently spawns the worker itself (the +# proven memory-worker pattern). + +# Safe no-op fallback: must exist before set -e and before hook-bootstrap is sourced. +dbg() { :; } + +set -e + +# Re-entrancy guards FIRST: the worker's own claude -p session fires SessionStart +# hooks; without this guard every dream run would recursively spawn another worker. +if [ "${DEVFLOW_BG_DREAM:-}" = "1" ]; then dbg "EXIT: bg_dream"; exit 0; fi +if [ "${DEVFLOW_BG_UPDATER:-}" = "1" ]; then dbg "EXIT: bg_updater"; exit 0; fi + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" + +source "$SCRIPT_DIR/hook-bootstrap" "spawn-dream-worker" + +source "$SCRIPT_DIR/json-parse" || { echo "spawn-dream-worker: failed to source json-parse" >&2; exit 1; } +if [ "$_JSON_AVAILABLE" = "false" ]; then dbg "EXIT: no json"; exit 0; fi + +INPUT=$(cat) + +CWD=$(printf '%s' "$INPUT" | json_field "cwd" "") +if [ -z "$CWD" ] || [ ! -d "$CWD" ]; then dbg "EXIT: bad CWD"; exit 0; fi + +devflow_debug_set_cwd "$CWD" + +# Anchor .devflow/ to the project root (prevents a stray nested .devflow/ when this +# hook runs with a CWD inside .devflow/...). Empty → fall back to CWD (old behavior). +source "$SCRIPT_DIR/resolve-project-root" 2>/dev/null || true +PROJECT_ROOT="$(df_resolve_root "$CWD" 2>/dev/null || true)" +[ -n "$PROJECT_ROOT" ] || PROJECT_ROOT="$CWD" + +DEVFLOW_DIR="$PROJECT_ROOT/.devflow" +DREAM_DIR="$DEVFLOW_DIR/dream" +DECISIONS_DIR_DATA="$DEVFLOW_DIR/decisions" + +source "$SCRIPT_DIR/hook-log-init" "spawn-dream-worker" + +# --- Dual-signal decisions gate --- +DREAM_CONFIG="$DREAM_DIR/config.json" +DECISIONS_ENABLED="true" +if [ -f "$DREAM_CONFIG" ]; then + DECISIONS_ENABLED=$(json_field_file "$DREAM_CONFIG" "decisions" "true") +fi +[ -f "$DECISIONS_DIR_DATA/.disabled" ] && DECISIONS_ENABLED="false" + +if [ "$DECISIONS_ENABLED" != "true" ]; then + dbg "EXIT: decisions disabled" + exit 0 +fi + +# --- Queue non-empty OR leftover .processing (synchronous, no jq parse — AC-P2) --- +QUEUE_FILE="$DREAM_DIR/.pending-turns.jsonl" +PROCESSING_FILE="$DREAM_DIR/.pending-turns.processing" + +if [ ! -s "$QUEUE_FILE" ] && [ ! -f "$PROCESSING_FILE" ]; then + dbg "EXIT: no pending dream work" + exit 0 +fi + +# --- claude-on-PATH check --- +CLAUDE_BIN=$(command -v claude 2>/dev/null || true) +if [ -z "$CLAUDE_BIN" ]; then + log "SKIP: claude binary not found — dream worker not spawned (queue intact)" + dbg "EXIT: no claude binary" + exit 0 +fi + +WORKER="$SCRIPT_DIR/background-dream-update" +if [ ! -x "$WORKER" ]; then + log "SKIP: background-dream-update not found or not executable: $WORKER" + dbg "EXIT: worker missing" + exit 0 +fi + +# Spawn detached worker — nohup + disown so it survives this hook process exit +nohup "$WORKER" "$CWD" >/dev/null 2>&1 & disown + +log "Spawned background-dream-update worker (CWD=$CWD)" +dbg "=== HOOK COMPLETE ===" +exit 0 From 693ae0ce2834a2182b3e5cbf3480a1cefdee2d1b Mon Sep 17 00:00:00 2001 From: Dean Sharon Date: Fri, 3 Jul 2026 02:13:54 +0300 Subject: [PATCH 04/26] test(hooks): add coverage for the dream system simplification phases 0-2 New dedicated test files: - tests/queue-append.test.ts: queue_append_row/queue_append_both/ queue_read_gates -- 0600 creation, escaping fuzz (quotes, newlines, command substitution, unicode), 200->100 overflow under lock, degraded no-jq path, 20-parallel-append interleave, truncate-vs- append race (asserts no corruption, not zero data loss -- an accepted-class race shared with the pre-existing memory design). - tests/capture-hooks.test.ts: capture-prompt, capture-turn (incl. AC-F5 never-spawns), capture-question (real mined multi-question + errored AskUserQuestion fixtures), memory-worker, spawn-dream-worker. - tests/background-dream-update.test.ts: DW1 happy path (real json-helper.cjs ops), DW3 prompt path-pointing, DW4 malformed rows tolerated, DW5 leftover-.processing merge, DW6 watchdog/invariant/ stamp-not-advanced, DW7 lock fail-fast/stale-evict, DW8 CLI contract (no skip-permissions, DEVFLOW_BG_DREAM=1, no hostile argv/stdin), and AC-C3 dream-procedure.md contract pinning. Additive-only changes to existing files: - tests/sentinel.test.ts: new DEVFLOW_BG_UPDATER/DEVFLOW_BG_DREAM guard cases for session-start-context, session-start-memory, and pre-compact-memory (existing guard describes untouched). - tests/eager-memory-refresh.test.ts: S18 (qa rows in background-memory-update -- orphan gate + TURNS_TEXT agreement) and S19 (session-start-memory cold-path .pending-turns.processing recovery). - tests/learning/merge-observation.test.ts: AC-C4 self-locking concurrency test (15 parallel invocations, no corruption, CLI signature unchanged). - tests/shell-hooks.test.ts: HOOK_SCRIPTS bash -n entries for the 7 new scripts. 1913 tests passing (1794 baseline + 119 new), 0 new failures. --- tests/background-dream-update.test.ts | 629 ++++++++++++++++++++++ tests/capture-hooks.test.ts | 656 +++++++++++++++++++++++ tests/eager-memory-refresh.test.ts | 196 +++++++ tests/learning/merge-observation.test.ts | 97 +++- tests/queue-append.test.ts | 392 ++++++++++++++ tests/sentinel.test.ts | 105 ++++ tests/shell-hooks.test.ts | 7 + 7 files changed, 2081 insertions(+), 1 deletion(-) create mode 100644 tests/background-dream-update.test.ts create mode 100644 tests/capture-hooks.test.ts create mode 100644 tests/queue-append.test.ts diff --git a/tests/background-dream-update.test.ts b/tests/background-dream-update.test.ts new file mode 100644 index 00000000..d3b7d20c --- /dev/null +++ b/tests/background-dream-update.test.ts @@ -0,0 +1,629 @@ +/** + * tests/background-dream-update.test.ts + * + * Tests for scripts/hooks/background-dream-update (the detached dream worker) + * and scripts/hooks/dream-procedure.md (the procedure it points its agent at). + * + * Harness idioms follow eager-memory-refresh.test.ts: fake-claude PATH shim, + * temp dirs, DEVFLOW_BG_WATCHDOG_SECS override, execSync. Real `claude` is + * never invoked from these tests. + * + * DEVIATION NOTE (see handoff): "DW3 stdin dump contains full claimed queue" + * is interpreted as "the prompt delivered via stdin contains the PATH to the + * claimed queue snapshot, from which the full queue is reachable" — matching + * the plan's own explicit prompt template ("Read dream-procedure.md and + * follow it; inputs at "), rather than inlining raw queue content into + * the prompt. The agent reads the queue file itself via its own Read tool. + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import { execSync } from 'child_process'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as os from 'os'; + +const HOOKS_DIR = path.resolve(__dirname, '..', 'scripts', 'hooks'); +const JSON_HELPER = path.join(HOOKS_DIR, 'json-helper.cjs'); +const WORKER = path.join(HOOKS_DIR, 'background-dream-update'); +const PROCEDURE = path.join(HOOKS_DIR, 'dream-procedure.md'); + +// --------------------------------------------------------------------------- +// Harness helpers +// --------------------------------------------------------------------------- + +function runWorker( + projectDir: string, + homeDir: string, + shimDir: string, + extraEnv: Record = {}, +): { exitCode: number } { + try { + execSync(`bash "${WORKER}" "${projectDir}"`, { + env: { + ...process.env, + HOME: homeDir, + PATH: `${shimDir}:${process.env.PATH ?? '/usr/bin:/bin'}`, + ...extraEnv, + }, + stdio: 'ignore', + }); + return { exitCode: 0 }; + } catch (e: unknown) { + const err = e as { status?: number }; + return { exitCode: err.status ?? 1 }; + } +} + +function workerLogPath(projectDir: string, homeDir: string): string { + const slug = projectDir.replace(/^\//, '').replace(/\//g, '-'); + return path.join(homeDir, '.devflow', 'logs', slug, '.background-dream-update.log'); +} + +/** A fake claude that dumps its stdin + argv, then acts as a scripted "agent" + * calling the REAL json-helper.cjs ops before touching .last-dream-ok. */ +function createRealAgentShim(shimDir: string, projectDir: string, opts: { skipStamp?: boolean } = {}): { stdinCapture: string; argvCapture: string } { + const stdinCapture = path.join(shimDir, 'stdin-captured.txt'); + const argvCapture = path.join(shimDir, 'argv-captured.txt'); + const stampLine = opts.skipStamp ? '' : `touch "${projectDir}/.devflow/dream/.last-dream-ok"`; + fs.writeFileSync( + path.join(shimDir, 'claude'), + `#!/bin/bash +echo "$@" > "${argvCapture}" +cat > "${stdinCapture}" +cd "${projectDir}" +node "${JSON_HELPER}" merge-observation \\ + ".devflow/decisions/decisions-log.jsonl" \\ + '{"id":"obs_dw1","type":"pitfall","pattern":"test pitfall","evidence":["e1"],"details":"area: x; issue: y; impact: z; resolution: w","confidence":0.7,"status":"ready","quality_ok":true}' +node "${JSON_HELPER}" assign-anchor "pitfall" "obs_dw1" +echo "Created PF-001 for test pitfall" > .devflow/dream/last-run-summary +node "${JSON_HELPER}" rotate-observations || true +${stampLine} +exit 0 +`, + ); + fs.chmodSync(path.join(shimDir, 'claude'), 0o755); + return { stdinCapture, argvCapture }; +} + +function seedDreamQueue(projectDir: string, content = 'assistant reply mentioning a workaround for widget caching'): void { + fs.writeFileSync( + path.join(projectDir, '.devflow', 'dream', '.pending-turns.jsonl'), + JSON.stringify({ role: 'assistant', content, ts: Math.floor(Date.now() / 1000) }) + '\n', + ); +} + +describe('shell hook syntax: background-dream-update passes bash -n', () => { + it('bash -n succeeds', () => { + expect(() => { + execSync(`bash -n "${WORKER}"`, { stdio: 'pipe' }); + }).not.toThrow(); + }); +}); + +// ============================================================================= +// DW1 — happy path +// ============================================================================= +describe('DW1: happy path — claim, spawn, real ledger ops, success gate', () => { + let projectDir: string; + let homeDir: string; + let shimDir: string; + + beforeEach(() => { + projectDir = fs.mkdtempSync(path.join(os.tmpdir(), 'bdu-dw1-')); + homeDir = fs.mkdtempSync(path.join(os.tmpdir(), 'bdu-dw1-home-')); + shimDir = fs.mkdtempSync(path.join(os.tmpdir(), 'bdu-dw1-shim-')); + fs.mkdirSync(path.join(projectDir, '.devflow', 'dream'), { recursive: true }); + fs.mkdirSync(path.join(projectDir, '.devflow', 'decisions'), { recursive: true }); + seedDreamQueue(projectDir); + }); + + afterEach(() => { + fs.rmSync(projectDir, { recursive: true, force: true }); + fs.rmSync(homeDir, { recursive: true, force: true }); + fs.rmSync(shimDir, { recursive: true, force: true }); + }); + + it('assigns an anchor, renders .md, deletes .processing, touches .last-dream-ok', () => { + createRealAgentShim(shimDir, projectDir); + + const { exitCode } = runWorker(projectDir, homeDir, shimDir); + expect(exitCode).toBe(0); + + const ledger = fs + .readFileSync(path.join(projectDir, '.devflow', 'decisions', 'decisions-ledger.jsonl'), 'utf-8') + .trim() + .split('\n') + .map((l) => JSON.parse(l)); + expect(ledger).toHaveLength(1); + expect(ledger[0]).toMatchObject({ anchor_id: 'PF-001', decisions_status: 'Active' }); + + const pitfallsMd = fs.readFileSync(path.join(projectDir, '.devflow', 'decisions', 'pitfalls.md'), 'utf-8'); + expect(pitfallsMd).toContain('PF-001'); + + expect(fs.existsSync(path.join(projectDir, '.devflow', 'dream', '.pending-turns.processing'))).toBe(false); + expect(fs.existsSync(path.join(projectDir, '.devflow', 'dream', '.last-dream-ok'))).toBe(true); + expect(fs.existsSync(path.join(projectDir, '.devflow', 'dream', 'last-run-summary'))).toBe(true); + }); + + it('no .worker.lock left behind on success', () => { + createRealAgentShim(shimDir, projectDir); + runWorker(projectDir, homeDir, shimDir); + expect(fs.existsSync(path.join(projectDir, '.devflow', 'dream', '.worker.lock'))).toBe(false); + }); +}); + +// ============================================================================= +// DW3 — prompt (delivered via stdin) points at the full claimed queue +// ============================================================================= +describe('DW3: stdin prompt contains a path from which the full claimed queue is reachable', () => { + let projectDir: string; + let homeDir: string; + let shimDir: string; + + beforeEach(() => { + projectDir = fs.mkdtempSync(path.join(os.tmpdir(), 'bdu-dw3-')); + homeDir = fs.mkdtempSync(path.join(os.tmpdir(), 'bdu-dw3-home-')); + shimDir = fs.mkdtempSync(path.join(os.tmpdir(), 'bdu-dw3-shim-')); + fs.mkdirSync(path.join(projectDir, '.devflow', 'dream'), { recursive: true }); + fs.mkdirSync(path.join(projectDir, '.devflow', 'decisions'), { recursive: true }); + }); + + afterEach(() => { + fs.rmSync(projectDir, { recursive: true, force: true }); + fs.rmSync(homeDir, { recursive: true, force: true }); + fs.rmSync(shimDir, { recursive: true, force: true }); + }); + + it('stdin contains the claimed .pending-turns.processing path, and reading it yields the full queue', () => { + const SENTINEL = 'DW3_UNIQUE_QUEUE_MARKER_widget_caching_decision'; + seedDreamQueue(projectDir, SENTINEL); + + // Snapshot the processing file's content from WITHIN the shim, before the + // worker's own success-path cleanup (`rm -f "$PROCESSING_FILE"`) removes it. + const stdinCapture = path.join(shimDir, 'stdin.txt'); + const queueSnapshot = path.join(shimDir, 'queue-snapshot.txt'); + const processingPath = path.join(projectDir, '.devflow', 'dream', '.pending-turns.processing'); + fs.writeFileSync( + path.join(shimDir, 'claude'), + `#!/bin/bash +cat > "${stdinCapture}" +cp "${processingPath}" "${queueSnapshot}" +touch "${projectDir}/.devflow/dream/.last-dream-ok" +exit 0 +`, + ); + fs.chmodSync(path.join(shimDir, 'claude'), 0o755); + + runWorker(projectDir, homeDir, shimDir); + + const prompt = fs.readFileSync(stdinCapture, 'utf-8'); + expect(prompt).toContain(processingPath); + expect(prompt).toContain(path.join(projectDir, '.devflow', 'decisions', 'decisions-log.jsonl')); + expect(prompt).toContain('dream-procedure.md'); + + // The full queue was reachable at the path named in the prompt while the + // agent ran (the claim renamed .pending-turns.jsonl -> .pending-turns.processing; + // it is deleted only AFTER a successful run, which is why we snapshot it above). + expect(fs.readFileSync(queueSnapshot, 'utf-8')).toContain(SENTINEL); + }); + + it('prompt does not inline raw queue content directly (path-pointing design)', () => { + const SENTINEL = 'DW3_SHOULD_NOT_BE_INLINE_abcdef123456'; + seedDreamQueue(projectDir, SENTINEL); + const stdinCapture = path.join(shimDir, 'stdin.txt'); + fs.writeFileSync( + path.join(shimDir, 'claude'), + `#!/bin/bash\ncat > "${stdinCapture}"\ntouch "${projectDir}/.devflow/dream/.last-dream-ok"\nexit 0\n`, + ); + fs.chmodSync(path.join(shimDir, 'claude'), 0o755); + + runWorker(projectDir, homeDir, shimDir); + + const prompt = fs.readFileSync(stdinCapture, 'utf-8'); + expect(prompt).not.toContain(SENTINEL); + }); +}); + +// ============================================================================= +// DW4 — malformed rows tolerated (the worker never parses the queue as JSON) +// ============================================================================= +describe('DW4: malformed queue rows are tolerated by the worker script', () => { + let projectDir: string; + let homeDir: string; + let shimDir: string; + + beforeEach(() => { + projectDir = fs.mkdtempSync(path.join(os.tmpdir(), 'bdu-dw4-')); + homeDir = fs.mkdtempSync(path.join(os.tmpdir(), 'bdu-dw4-home-')); + shimDir = fs.mkdtempSync(path.join(os.tmpdir(), 'bdu-dw4-shim-')); + fs.mkdirSync(path.join(projectDir, '.devflow', 'dream'), { recursive: true }); + fs.mkdirSync(path.join(projectDir, '.devflow', 'decisions'), { recursive: true }); + }); + + afterEach(() => { + fs.rmSync(projectDir, { recursive: true, force: true }); + fs.rmSync(homeDir, { recursive: true, force: true }); + fs.rmSync(shimDir, { recursive: true, force: true }); + }); + + it('claim + spawn + success gate all succeed even with malformed JSON lines in the queue', () => { + fs.writeFileSync( + path.join(projectDir, '.devflow', 'dream', '.pending-turns.jsonl'), + ['not valid json at all', '{"role":"assistant","content":"valid row","ts":1}', '{{{broken', ''].join('\n'), + ); + fs.writeFileSync( + path.join(shimDir, 'claude'), + `#!/bin/bash\ncat > /dev/null\ntouch "${projectDir}/.devflow/dream/.last-dream-ok"\nexit 0\n`, + ); + fs.chmodSync(path.join(shimDir, 'claude'), 0o755); + + const { exitCode } = runWorker(projectDir, homeDir, shimDir); + expect(exitCode).toBe(0); + expect(fs.existsSync(path.join(projectDir, '.devflow', 'dream', '.last-dream-ok'))).toBe(true); + expect(fs.existsSync(path.join(projectDir, '.devflow', 'dream', '.pending-turns.processing'))).toBe(false); + }); +}); + +// ============================================================================= +// DW5 — leftover .processing merged with new queue entries +// ============================================================================= +describe('DW5: leftover .processing from a previous crash is merged with new queue entries', () => { + let projectDir: string; + let homeDir: string; + let shimDir: string; + + beforeEach(() => { + projectDir = fs.mkdtempSync(path.join(os.tmpdir(), 'bdu-dw5-')); + homeDir = fs.mkdtempSync(path.join(os.tmpdir(), 'bdu-dw5-home-')); + shimDir = fs.mkdtempSync(path.join(os.tmpdir(), 'bdu-dw5-shim-')); + fs.mkdirSync(path.join(projectDir, '.devflow', 'dream'), { recursive: true }); + fs.mkdirSync(path.join(projectDir, '.devflow', 'decisions'), { recursive: true }); + }); + + afterEach(() => { + fs.rmSync(projectDir, { recursive: true, force: true }); + fs.rmSync(homeDir, { recursive: true, force: true }); + fs.rmSync(shimDir, { recursive: true, force: true }); + }); + + it('both the leftover .processing batch and the new queue reach the claimed file', () => { + const processingFile = path.join(projectDir, '.devflow', 'dream', '.pending-turns.processing'); + const queueFile = path.join(projectDir, '.devflow', 'dream', '.pending-turns.jsonl'); + fs.writeFileSync(processingFile, JSON.stringify({ role: 'assistant', content: 'PRIOR-CRASHED-TURN', ts: 1 }) + '\n'); + fs.writeFileSync(queueFile, JSON.stringify({ role: 'assistant', content: 'NEW-QUEUE-TURN', ts: 2 }) + '\n'); + + const stdinCapture = path.join(shimDir, 'stdin.txt'); + fs.writeFileSync( + path.join(shimDir, 'claude'), + `#!/bin/bash\ncat > "${stdinCapture}"\ntouch "${projectDir}/.devflow/dream/.last-dream-ok"\nexit 0\n`, + ); + fs.chmodSync(path.join(shimDir, 'claude'), 0o755); + + runWorker(projectDir, homeDir, shimDir); + + // The queue file must be gone (merged in), and the (still-existing until + // success) processing file contains both batches. + expect(fs.existsSync(queueFile)).toBe(false); + // On success .processing is removed — but the CONTENT it held before removal + // is what the agent's stdin pointed at; verify via the captured prompt path + // and a pre-removal snapshot check using a shim that reads before touching ok. + }); + + it('merged content is visible to the agent before .last-dream-ok is touched', () => { + const processingFile = path.join(projectDir, '.devflow', 'dream', '.pending-turns.processing'); + const queueFile = path.join(projectDir, '.devflow', 'dream', '.pending-turns.jsonl'); + fs.writeFileSync(processingFile, JSON.stringify({ role: 'assistant', content: 'PRIOR-CRASHED-TURN', ts: 1 }) + '\n'); + fs.writeFileSync(queueFile, JSON.stringify({ role: 'assistant', content: 'NEW-QUEUE-TURN', ts: 2 }) + '\n'); + + const mergedSnapshot = path.join(shimDir, 'merged-snapshot.txt'); + fs.writeFileSync( + path.join(shimDir, 'claude'), + `#!/bin/bash +cat > /dev/null +cp "${processingFile}" "${mergedSnapshot}" +touch "${projectDir}/.devflow/dream/.last-dream-ok" +exit 0 +`, + ); + fs.chmodSync(path.join(shimDir, 'claude'), 0o755); + + runWorker(projectDir, homeDir, shimDir); + + const merged = fs.readFileSync(mergedSnapshot, 'utf-8'); + expect(merged).toContain('PRIOR-CRASHED-TURN'); + expect(merged).toContain('NEW-QUEUE-TURN'); + }); + + it('200->100 overflow cap applies to the merged .processing file', () => { + const processingFile = path.join(projectDir, '.devflow', 'dream', '.pending-turns.processing'); + const queueFile = path.join(projectDir, '.devflow', 'dream', '.pending-turns.jsonl'); + const processingLines = Array.from({ length: 150 }, (_, i) => JSON.stringify({ role: 'assistant', content: `old-${i}`, ts: i })); + const queueLines = Array.from({ length: 60 }, (_, i) => JSON.stringify({ role: 'assistant', content: `new-${i}`, ts: 200 + i })); + fs.writeFileSync(processingFile, processingLines.join('\n') + '\n'); + fs.writeFileSync(queueFile, queueLines.join('\n') + '\n'); + + const snapshot = path.join(shimDir, 'snapshot.txt'); + fs.writeFileSync( + path.join(shimDir, 'claude'), + `#!/bin/bash\ncat > /dev/null\ncp "${processingFile}" "${snapshot}"\ntouch "${projectDir}/.devflow/dream/.last-dream-ok"\nexit 0\n`, + ); + fs.chmodSync(path.join(shimDir, 'claude'), 0o755); + + runWorker(projectDir, homeDir, shimDir); + + const lines = fs.readFileSync(snapshot, 'utf-8').trim().split('\n').filter(Boolean); + expect(lines).toHaveLength(100); + }); +}); + +// ============================================================================= +// DW6 — watchdog kill + invariant assertion + stamp-not-advanced +// ============================================================================= +describe('DW6: watchdog kill, runtime invariant, and stamp-not-advanced all leave .processing', () => { + let projectDir: string; + let homeDir: string; + let shimDir: string; + + beforeEach(() => { + projectDir = fs.mkdtempSync(path.join(os.tmpdir(), 'bdu-dw6-')); + homeDir = fs.mkdtempSync(path.join(os.tmpdir(), 'bdu-dw6-home-')); + shimDir = fs.mkdtempSync(path.join(os.tmpdir(), 'bdu-dw6-shim-')); + fs.mkdirSync(path.join(projectDir, '.devflow', 'dream'), { recursive: true }); + fs.mkdirSync(path.join(projectDir, '.devflow', 'decisions'), { recursive: true }); + seedDreamQueue(projectDir); + }); + + afterEach(() => { + fs.rmSync(projectDir, { recursive: true, force: true }); + fs.rmSync(homeDir, { recursive: true, force: true }); + fs.rmSync(shimDir, { recursive: true, force: true }); + }); + + it('hanging claude is killed by the watchdog; worker exits 0; .processing retained', () => { + fs.writeFileSync(path.join(shimDir, 'claude'), '#!/bin/bash\ncat > /dev/null\nsleep 300\n'); + fs.chmodSync(path.join(shimDir, 'claude'), 0o755); + + const { exitCode } = runWorker(projectDir, homeDir, shimDir, { DEVFLOW_BG_WATCHDOG_SECS: '2' }); + + expect(exitCode).toBe(0); + expect(fs.existsSync(path.join(projectDir, '.devflow', 'dream', '.last-dream-ok'))).toBe(false); + expect(fs.existsSync(path.join(projectDir, '.devflow', 'dream', '.pending-turns.processing'))).toBe(true); + }, 20000); + + it('runtime invariant assertion fails loud when WATCHDOG_TOTAL >= STALE_THRESHOLD (900)', () => { + // 896 + 5 grace = 901 > 900 stale threshold -> the FATAL assertion must fire. + fs.writeFileSync(path.join(shimDir, 'claude'), '#!/bin/bash\ncat > /dev/null\nexit 0\n'); + fs.chmodSync(path.join(shimDir, 'claude'), 0o755); + + let exitCode = 0; + try { + execSync(`bash "${WORKER}" "${projectDir}"`, { + env: { ...process.env, HOME: homeDir, PATH: `${shimDir}:${process.env.PATH}`, DEVFLOW_BG_WATCHDOG_SECS: '896' }, + stdio: 'pipe', + }); + } catch (e: unknown) { + exitCode = (e as { status?: number }).status ?? 1; + } + expect(exitCode).toBe(1); + }); + + it('claude exits 0 but .last-dream-ok is not advanced -> .processing retained (no-op agent)', () => { + // Agent runs and exits cleanly but forgets to touch the stamp (or a stale + // stamp already exists and is not advanced past the pre-spawn baseline). + fs.writeFileSync(path.join(shimDir, 'claude'), '#!/bin/bash\ncat > /dev/null\nexit 0\n'); + fs.chmodSync(path.join(shimDir, 'claude'), 0o755); + + const { exitCode } = runWorker(projectDir, homeDir, shimDir); + + expect(exitCode).toBe(0); + expect(fs.existsSync(path.join(projectDir, '.devflow', 'dream', '.pending-turns.processing'))).toBe(true); + }); + + it('pre-existing stale .last-dream-ok that is NOT touched again -> still treated as failure', () => { + const okFile = path.join(projectDir, '.devflow', 'dream', '.last-dream-ok'); + fs.writeFileSync(okFile, ''); + const old = new Date(Date.now() - 3600 * 1000); + fs.utimesSync(okFile, old, old); + const baselineMtime = fs.statSync(okFile).mtimeMs; + + fs.writeFileSync(path.join(shimDir, 'claude'), '#!/bin/bash\ncat > /dev/null\nexit 0\n'); // does not touch the stamp + fs.chmodSync(path.join(shimDir, 'claude'), 0o755); + + runWorker(projectDir, homeDir, shimDir); + + expect(fs.statSync(okFile).mtimeMs).toBe(baselineMtime); + expect(fs.existsSync(path.join(projectDir, '.devflow', 'dream', '.pending-turns.processing'))).toBe(true); + }); +}); + +// ============================================================================= +// DW7 — lock discipline +// ============================================================================= +describe('DW7: .worker.lock discipline (fail-fast acquire, 900s stale-evict)', () => { + let projectDir: string; + let homeDir: string; + let shimDir: string; + + beforeEach(() => { + projectDir = fs.mkdtempSync(path.join(os.tmpdir(), 'bdu-dw7-')); + homeDir = fs.mkdtempSync(path.join(os.tmpdir(), 'bdu-dw7-home-')); + shimDir = fs.mkdtempSync(path.join(os.tmpdir(), 'bdu-dw7-shim-')); + fs.mkdirSync(path.join(projectDir, '.devflow', 'dream'), { recursive: true }); + fs.mkdirSync(path.join(projectDir, '.devflow', 'decisions'), { recursive: true }); + seedDreamQueue(projectDir); + }); + + afterEach(() => { + fs.rmSync(projectDir, { recursive: true, force: true }); + fs.rmSync(homeDir, { recursive: true, force: true }); + fs.rmSync(shimDir, { recursive: true, force: true }); + }); + + it('a foreign FRESH lock blocks the worker immediately (<=3s measured), queue untouched', () => { + fs.writeFileSync(path.join(shimDir, 'claude'), '#!/bin/bash\necho "SHOULD NOT RUN" >&2\nexit 0\n'); + fs.chmodSync(path.join(shimDir, 'claude'), 0o755); + fs.mkdirSync(path.join(projectDir, '.devflow', 'dream', '.worker.lock')); + + const start = Date.now(); + const { exitCode } = runWorker(projectDir, homeDir, shimDir); + const elapsed = Date.now() - start; + + expect(exitCode).toBe(0); + expect(elapsed).toBeLessThan(3000); + // Queue was never claimed — still present as .pending-turns.jsonl, not renamed. + expect(fs.existsSync(path.join(projectDir, '.devflow', 'dream', '.pending-turns.jsonl'))).toBe(true); + }); + + it('a backdated (>900s) lock is evicted, allowing the worker to proceed', () => { + fs.writeFileSync( + path.join(shimDir, 'claude'), + `#!/bin/bash\ncat > /dev/null\ntouch "${projectDir}/.devflow/dream/.last-dream-ok"\nexit 0\n`, + ); + fs.chmodSync(path.join(shimDir, 'claude'), 0o755); + const lockDir = path.join(projectDir, '.devflow', 'dream', '.worker.lock'); + fs.mkdirSync(lockDir); + const old = new Date(Date.now() - 1000 * 1000); // > 900s + fs.utimesSync(lockDir, old, old); + + const { exitCode } = runWorker(projectDir, homeDir, shimDir); + + expect(exitCode).toBe(0); + expect(fs.existsSync(path.join(projectDir, '.devflow', 'dream', '.last-dream-ok'))).toBe(true); + }); + + it('lock is released after a normal run completes', () => { + fs.writeFileSync( + path.join(shimDir, 'claude'), + `#!/bin/bash\ncat > /dev/null\ntouch "${projectDir}/.devflow/dream/.last-dream-ok"\nexit 0\n`, + ); + fs.chmodSync(path.join(shimDir, 'claude'), 0o755); + runWorker(projectDir, homeDir, shimDir); + expect(fs.existsSync(path.join(projectDir, '.devflow', 'dream', '.worker.lock'))).toBe(false); + }); +}); + +// ============================================================================= +// DW8 — CLI contract +// ============================================================================= +describe('DW8: CLI contract — allowedTools without skip-permissions, env flag, no hostile argv', () => { + let projectDir: string; + let homeDir: string; + let shimDir: string; + + beforeEach(() => { + projectDir = fs.mkdtempSync(path.join(os.tmpdir(), 'bdu-dw8-')); + homeDir = fs.mkdtempSync(path.join(os.tmpdir(), 'bdu-dw8-home-')); + shimDir = fs.mkdtempSync(path.join(os.tmpdir(), 'bdu-dw8-shim-')); + fs.mkdirSync(path.join(projectDir, '.devflow', 'dream'), { recursive: true }); + fs.mkdirSync(path.join(projectDir, '.devflow', 'decisions'), { recursive: true }); + }); + + afterEach(() => { + fs.rmSync(projectDir, { recursive: true, force: true }); + fs.rmSync(homeDir, { recursive: true, force: true }); + fs.rmSync(shimDir, { recursive: true, force: true }); + }); + + it('argv contains --allowedTools with the expected tool list and NEVER --dangerously-skip-permissions', () => { + seedDreamQueue(projectDir); + const { argvCapture } = createRealAgentShim(shimDir, projectDir); + runWorker(projectDir, homeDir, shimDir); + + const argv = fs.readFileSync(argvCapture, 'utf-8'); + expect(argv).toContain('--allowedTools'); + expect(argv).toContain('Read,Grep,Glob,Write,Edit,Bash'); + expect(argv).not.toContain('--dangerously-skip-permissions'); + }); + + it('DEVFLOW_BG_DREAM=1 is present in the child claude env', () => { + seedDreamQueue(projectDir); + const envCapture = path.join(shimDir, 'env-captured.txt'); + fs.writeFileSync( + path.join(shimDir, 'claude'), + `#!/bin/bash\ncat > /dev/null\nprintf '%s' "$DEVFLOW_BG_DREAM" > "${envCapture}"\ntouch "${projectDir}/.devflow/dream/.last-dream-ok"\nexit 0\n`, + ); + fs.chmodSync(path.join(shimDir, 'claude'), 0o755); + + runWorker(projectDir, homeDir, shimDir); + + expect(fs.readFileSync(envCapture, 'utf-8')).toBe('1'); + }); + + it('hostile queue content never appears in argv (path-pointing prompt design)', () => { + const HOSTILE = 'HOSTILE_ARGV_LEAK_TEST $(rm -rf /) `evil` --dangerously-skip-permissions'; + seedDreamQueue(projectDir, HOSTILE); + const { argvCapture } = createRealAgentShim(shimDir, projectDir); + + runWorker(projectDir, homeDir, shimDir); + + const argv = fs.readFileSync(argvCapture, 'utf-8'); + expect(argv).not.toContain('HOSTILE_ARGV_LEAK_TEST'); + }); + + it('hostile queue content never appears in the prompt (stdin) either', () => { + const HOSTILE = 'HOSTILE_STDIN_LEAK_TEST_marker_9f3a7b'; + seedDreamQueue(projectDir, HOSTILE); + const stdinCapture = path.join(shimDir, 'stdin.txt'); + fs.writeFileSync( + path.join(shimDir, 'claude'), + `#!/bin/bash\ncat > "${stdinCapture}"\ntouch "${projectDir}/.devflow/dream/.last-dream-ok"\nexit 0\n`, + ); + fs.chmodSync(path.join(shimDir, 'claude'), 0o755); + + runWorker(projectDir, homeDir, shimDir); + + expect(fs.readFileSync(stdinCapture, 'utf-8')).not.toContain(HOSTILE); + }); +}); + +// ============================================================================= +// AC-C3 — dream-procedure.md contract (pinned) +// ============================================================================= +describe('AC-C3: dream-procedure.md contract', () => { + const content = fs.readFileSync(PROCEDURE, 'utf-8'); + + it('exists and is readable', () => { + expect(fs.existsSync(PROCEDURE)).toBe(true); + }); + + it('is not a skill (no allowed-tools frontmatter — skills do not load in claude -p)', () => { + expect(content.startsWith('---')).toBe(false); + }); + + it('instructs read-only inputs: queue snapshot, decisions-log, decisions.md, pitfalls.md, usage file', () => { + expect(content).toContain('decisions-log.jsonl'); + expect(content).toContain('decisions.md'); + expect(content).toContain('pitfalls.md'); + expect(content).toContain('.decisions-usage.json'); + }); + + it('references only the sanctioned write ops: merge-observation, assign-anchor, retire-anchor, rotate-observations', () => { + expect(content).toContain('merge-observation'); + expect(content).toContain('assign-anchor'); + expect(content).toContain('retire-anchor'); + expect(content).toContain('rotate-observations'); + }); + + it('references .last-dream-ok and last-run-summary lifecycle', () => { + expect(content).toContain('.last-dream-ok'); + expect(content).toContain('last-run-summary'); + }); + + it('contains the Iron-Law strings: never hand-edit, ADR-XOR-PF, abstain-by-default', () => { + expect(content.toLowerCase()).toContain('never hand-edit'); + expect(content).toContain('ADR-XOR-PF'); + expect(content).toContain('abstain-by-default'); + }); + + it('contains the curation bounds: <=5 changes, 7-day protection window', () => { + expect(content).toContain('7-day protection window'); + // \s+ between every word tolerates an incidental mid-sentence line wrap in + // the markdown source (a literal newline, not just a space). + expect(content.toLowerCase()).toMatch(/(≤5|<=5)\s+curation\s+changes/); + }); + + it('does NOT instruct marker claim/heartbeat/multi-marker-merge ceremony (worker script owns lifecycle)', () => { + expect(content).not.toContain('.processing files (dream-recover'); + expect(content.toLowerCase()).toContain('do not need to claim, heartbeat, or merge'); + }); +}); diff --git a/tests/capture-hooks.test.ts b/tests/capture-hooks.test.ts new file mode 100644 index 00000000..9cbe5017 --- /dev/null +++ b/tests/capture-hooks.test.ts @@ -0,0 +1,656 @@ +/** + * tests/capture-hooks.test.ts + * + * Tests for the Phase 1/2 capture + dispatch layer of the dream system + * simplification: capture-prompt, capture-turn, capture-question, + * memory-worker, and spawn-dream-worker. A dedicated file (mirroring + * eager-memory-refresh.test.ts's precedent for a redesign-scoped test file) + * rather than adding to the already-large shell-hooks.test.ts. + * + * Harness idioms follow eager-memory-refresh.test.ts: fake-claude PATH shim, + * temp dirs, DEVFLOW_BG_WATCHDOG_SECS override, execSync + JSON stdin. Real + * `claude` is never invoked from these tests. + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import { execSync } from 'child_process'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as os from 'os'; + +const HOOKS_DIR = path.resolve(__dirname, '..', 'scripts', 'hooks'); +const CAPTURE_PROMPT = path.join(HOOKS_DIR, 'capture-prompt'); +const CAPTURE_TURN = path.join(HOOKS_DIR, 'capture-turn'); +const CAPTURE_QUESTION = path.join(HOOKS_DIR, 'capture-question'); +const MEMORY_WORKER = path.join(HOOKS_DIR, 'memory-worker'); +const SPAWN_DREAM_WORKER = path.join(HOOKS_DIR, 'spawn-dream-worker'); + +// --------------------------------------------------------------------------- +// Harness helpers (mirrors eager-memory-refresh.test.ts) +// --------------------------------------------------------------------------- + +function runHook( + hookPath: string, + input: object, + homeDir: string, + extraEnv: Record = {}, +): { stdout: string; stderr: string; exitCode: number } { + try { + const result = execSync(`bash "${hookPath}"`, { + input: JSON.stringify(input), + env: { ...process.env, HOME: homeDir, ...extraEnv }, + stdio: ['pipe', 'pipe', 'pipe'], + }); + return { stdout: result.toString(), stderr: '', exitCode: 0 }; + } catch (e: unknown) { + const err = e as { stdout?: Buffer; stderr?: Buffer; status?: number }; + return { stdout: err.stdout?.toString() ?? '', stderr: err.stderr?.toString() ?? '', exitCode: err.status ?? 1 }; + } +} + +function runHookWithPath( + hookPath: string, + input: object, + homeDir: string, + shimDir: string, + extraEnv: Record = {}, +): { stdout: string; stderr: string; exitCode: number } { + return runHook(hookPath, input, homeDir, { + PATH: `${shimDir}:${process.env.PATH ?? '/usr/bin:/bin'}`, + ...extraEnv, + }); +} + +function createFakeClaudeShim(shimDir: string, memFile: string): void { + const bin = path.join(shimDir, 'claude'); + fs.writeFileSync( + bin, + `#!/bin/bash +echo "" > "${memFile}" +echo "## Now" >> "${memFile}" +exit 0 +`, + ); + fs.chmodSync(bin, 0o755); +} + +function backdateMtime(filePath: string, secondsAgo: number): void { + const past = new Date(Date.now() - secondsAgo * 1000); + fs.utimesSync(filePath, past, past); +} + +function readJsonl(file: string): Record[] { + if (!fs.existsSync(file)) return []; + return fs.readFileSync(file, 'utf-8').trim().split('\n').filter(Boolean).map((l) => JSON.parse(l)); +} + +function writeDreamConfig(projectDir: string, fields: Record): void { + const dir = path.join(projectDir, '.devflow', 'dream'); + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync(path.join(dir, 'config.json'), JSON.stringify(fields)); +} + +function workerLogPath(projectDir: string, homeDir: string, hookName: string): string { + const slug = projectDir.replace(/^\//, '').replace(/\//g, '-'); + return path.join(homeDir, '.devflow', 'logs', slug, `.${hookName}.log`); +} + +// ============================================================================= +// capture-prompt +// ============================================================================= +describe('capture-prompt', () => { + let projectDir: string; + let homeDir: string; + + beforeEach(() => { + projectDir = fs.mkdtempSync(path.join(os.tmpdir(), 'cap-prompt-')); + homeDir = fs.mkdtempSync(path.join(os.tmpdir(), 'cap-prompt-home-')); + }); + + afterEach(() => { + fs.rmSync(projectDir, { recursive: true, force: true }); + fs.rmSync(homeDir, { recursive: true, force: true }); + }); + + it('AC-F1: both features enabled (no config) -> one {role:"user"} row to BOTH queues', () => { + runHook(CAPTURE_PROMPT, { cwd: projectDir, prompt: 'hello world' }, homeDir); + const mem = readJsonl(path.join(projectDir, '.devflow', 'memory', '.pending-turns.jsonl')); + const dream = readJsonl(path.join(projectDir, '.devflow', 'dream', '.pending-turns.jsonl')); + expect(mem).toEqual([{ role: 'user', content: 'hello world', ts: expect.any(Number) }]); + expect(dream).toEqual([{ role: 'user', content: 'hello world', ts: expect.any(Number) }]); + }); + + it('AC-F1: content truncated at 2000 chars', () => { + const longPrompt = 'x'.repeat(2500); + runHook(CAPTURE_PROMPT, { cwd: projectDir, prompt: longPrompt }, homeDir); + const mem = readJsonl(path.join(projectDir, '.devflow', 'memory', '.pending-turns.jsonl')); + expect((mem[0].content as string).length).toBe(2000 + '... [truncated]'.length); + }); + + it('empty prompt -> zero appends, no .devflow scaffolding', () => { + runHook(CAPTURE_PROMPT, { cwd: projectDir, prompt: '' }, homeDir); + expect(fs.existsSync(path.join(projectDir, '.devflow'))).toBe(false); + }); + + it('AC-F4: memory:false -> no memory-queue append (dream append unaffected)', () => { + writeDreamConfig(projectDir, { memory: false }); + runHook(CAPTURE_PROMPT, { cwd: projectDir, prompt: 'test' }, homeDir); + expect(fs.existsSync(path.join(projectDir, '.devflow', 'memory', '.pending-turns.jsonl'))).toBe(false); + expect(readJsonl(path.join(projectDir, '.devflow', 'dream', '.pending-turns.jsonl'))).toHaveLength(1); + }); + + it('AC-F4: decisions disabled via config field -> no dream-queue append (memory unaffected)', () => { + writeDreamConfig(projectDir, { decisions: false }); + runHook(CAPTURE_PROMPT, { cwd: projectDir, prompt: 'test' }, homeDir); + expect(readJsonl(path.join(projectDir, '.devflow', 'memory', '.pending-turns.jsonl'))).toHaveLength(1); + expect(fs.existsSync(path.join(projectDir, '.devflow', 'dream', '.pending-turns.jsonl'))).toBe(false); + }); + + it('AC-F4: decisions disabled via .disabled sentinel -> no dream-queue append (memory unaffected)', () => { + fs.mkdirSync(path.join(projectDir, '.devflow', 'decisions'), { recursive: true }); + fs.writeFileSync(path.join(projectDir, '.devflow', 'decisions', '.disabled'), ''); + runHook(CAPTURE_PROMPT, { cwd: projectDir, prompt: 'test' }, homeDir); + expect(readJsonl(path.join(projectDir, '.devflow', 'memory', '.pending-turns.jsonl'))).toHaveLength(1); + expect(fs.existsSync(path.join(projectDir, '.devflow', 'dream', '.pending-turns.jsonl'))).toBe(false); + }); + + it('both disabled -> zero appends, no scaffolding', () => { + writeDreamConfig(projectDir, { memory: false, decisions: false }); + runHook(CAPTURE_PROMPT, { cwd: projectDir, prompt: 'test' }, homeDir); + expect(fs.existsSync(path.join(projectDir, '.devflow', 'memory', '.pending-turns.jsonl'))).toBe(false); + expect(fs.existsSync(path.join(projectDir, '.devflow', 'dream', '.pending-turns.jsonl'))).toBe(false); + }); + + it('AC-F14: DEVFLOW_BG_UPDATER=1 -> exit 0, zero filesystem writes', () => { + const { exitCode } = runHook(CAPTURE_PROMPT, { cwd: projectDir, prompt: 'test' }, homeDir, { DEVFLOW_BG_UPDATER: '1' }); + expect(exitCode).toBe(0); + expect(fs.existsSync(path.join(projectDir, '.devflow'))).toBe(false); + }); + + it('AC-F14: DEVFLOW_BG_DREAM=1 -> exit 0, zero filesystem writes', () => { + const { exitCode } = runHook(CAPTURE_PROMPT, { cwd: projectDir, prompt: 'test' }, homeDir, { DEVFLOW_BG_DREAM: '1' }); + expect(exitCode).toBe(0); + expect(fs.existsSync(path.join(projectDir, '.devflow'))).toBe(false); + }); +}); + +// ============================================================================= +// capture-turn +// ============================================================================= +describe('capture-turn', () => { + let projectDir: string; + let homeDir: string; + + beforeEach(() => { + projectDir = fs.mkdtempSync(path.join(os.tmpdir(), 'cap-turn-')); + homeDir = fs.mkdtempSync(path.join(os.tmpdir(), 'cap-turn-home-')); + }); + + afterEach(() => { + fs.rmSync(projectDir, { recursive: true, force: true }); + fs.rmSync(homeDir, { recursive: true, force: true }); + }); + + it('AC-F2: last_assistant_message present -> one {role:"assistant"} row to both queues', () => { + runHook(CAPTURE_TURN, { cwd: projectDir, session_id: 't', last_assistant_message: 'response text' }, homeDir); + expect(readJsonl(path.join(projectDir, '.devflow', 'memory', '.pending-turns.jsonl'))).toEqual([ + { role: 'assistant', content: 'response text', ts: expect.any(Number) }, + ]); + expect(readJsonl(path.join(projectDir, '.devflow', 'dream', '.pending-turns.jsonl'))).toEqual([ + { role: 'assistant', content: 'response text', ts: expect.any(Number) }, + ]); + }); + + it('AC-F2: empty message -> zero appends', () => { + runHook(CAPTURE_TURN, { cwd: projectDir, session_id: 't', last_assistant_message: '' }, homeDir); + expect(fs.existsSync(path.join(projectDir, '.devflow'))).toBe(false); + }); + + it('AC-F5: capture-turn NEVER spawns a process', () => { + // No claude shim on PATH at all, and no trigger/throttle files involved. + // If capture-turn tried to spawn anything, .working-memory.lock/ or a + // worker log line would appear; assert their total absence. + fs.mkdirSync(path.join(projectDir, '.devflow', 'memory'), { recursive: true }); + runHook(CAPTURE_TURN, { cwd: projectDir, session_id: 't', last_assistant_message: 'hi' }, homeDir, { + PATH: '/usr/bin:/bin', // deliberately excludes any claude shim + }); + expect(fs.existsSync(path.join(projectDir, '.devflow', 'memory', '.working-memory.lock'))).toBe(false); + expect(fs.existsSync(path.join(projectDir, '.devflow', 'memory', '.working-memory-last-trigger'))).toBe(false); + const logFile = workerLogPath(projectDir, homeDir, 'background-memory-update'); + expect(fs.existsSync(logFile)).toBe(false); + }); + + it('capture-turn never spawns even with a stale trigger + populated queue + no claude shim', () => { + fs.mkdirSync(path.join(projectDir, '.devflow', 'memory'), { recursive: true }); + const triggerFile = path.join(projectDir, '.devflow', 'memory', '.working-memory-last-trigger'); + fs.writeFileSync(triggerFile, ''); + backdateMtime(triggerFile, 600); // "throttle expired" — would matter for memory-worker, not capture-turn + const beforeMtime = fs.statSync(triggerFile).mtimeMs; + + runHook(CAPTURE_TURN, { cwd: projectDir, session_id: 't', last_assistant_message: 'hello' }, homeDir); + + // capture-turn must not touch the trigger file (that's memory-worker's job) + expect(fs.statSync(triggerFile).mtimeMs).toBe(beforeMtime); + const logFile = workerLogPath(projectDir, homeDir, 'background-memory-update'); + expect(fs.existsSync(logFile)).toBe(false); + }); + + it('AC-F4: gating independent per queue (memory:false, decisions enabled)', () => { + writeDreamConfig(projectDir, { memory: false }); + runHook(CAPTURE_TURN, { cwd: projectDir, session_id: 't', last_assistant_message: 'x' }, homeDir); + expect(fs.existsSync(path.join(projectDir, '.devflow', 'memory', '.pending-turns.jsonl'))).toBe(false); + expect(readJsonl(path.join(projectDir, '.devflow', 'dream', '.pending-turns.jsonl'))).toHaveLength(1); + }); + + it('decisions usage scanner still runs when memory is disabled (matches dream-capture behavior)', () => { + writeDreamConfig(projectDir, { memory: false }); + // decisions-usage-scan.cjs itself no-ops when .devflow/memory/ is absent + // (its own guard) — pre-create it, matching the established convention in + // sentinel.test.ts's mkMemoryDir helper for the equivalent dream-capture test. + fs.mkdirSync(path.join(projectDir, '.devflow', 'memory'), { recursive: true }); + fs.mkdirSync(path.join(projectDir, '.devflow', 'decisions'), { recursive: true }); + const usagePath = path.join(projectDir, '.devflow', 'decisions', '.decisions-usage.json'); + fs.writeFileSync(usagePath, JSON.stringify({ version: 1, entries: { 'ADR-001': { cites: 0, last_cited: null } } })); + runHook(CAPTURE_TURN, { cwd: projectDir, session_id: 't', last_assistant_message: 'applies ADR-001' }, homeDir); + const updated = JSON.parse(fs.readFileSync(usagePath, 'utf-8')); + expect(updated.entries['ADR-001'].cites).toBe(1); + }); + + it('AC-F14: DEVFLOW_BG_DREAM=1 -> exit 0, zero filesystem writes', () => { + const { exitCode } = runHook( + CAPTURE_TURN, + { cwd: projectDir, session_id: 't', last_assistant_message: 'hi' }, + homeDir, + { DEVFLOW_BG_DREAM: '1' }, + ); + expect(exitCode).toBe(0); + expect(fs.existsSync(path.join(projectDir, '.devflow'))).toBe(false); + }); +}); + +// ============================================================================= +// capture-question +// ============================================================================= +describe('capture-question', () => { + let projectDir: string; + let homeDir: string; + + beforeEach(() => { + projectDir = fs.mkdtempSync(path.join(os.tmpdir(), 'cap-question-')); + homeDir = fs.mkdtempSync(path.join(os.tmpdir(), 'cap-question-home-')); + }); + + afterEach(() => { + fs.rmSync(projectDir, { recursive: true, force: true }); + fs.rmSync(homeDir, { recursive: true, force: true }); + }); + + // Fixture mined from a real transcript (~/.claude/projects/-Users-dean-Sandbox-devflow), + // structurally identical to the captured toolUseResult shape (verified byte-identical + // to the PostToolUse tool_response field via a scratch-project probe). + const REAL_MULTI_QUESTION_PAYLOAD = { + session_id: 'test', + hook_event_name: 'PostToolUse', + tool_name: 'AskUserQuestion', + tool_input: { + questions: [ + { + question: 'How should I handle the Phase 5 Scrutinizer review?', + header: 'Scrutinizer', + multiSelect: false, + options: [{ label: 'Re-run, inert probes only' }, { label: 'Skip Scrutinizer entirely' }], + }, + { + question: 'Going forward, how do you want me to handle subagents?', + header: 'Shell policy', + multiSelect: false, + options: [{ label: 'Flag before running' }], + }, + ], + }, + tool_response: { + questions: [], + answers: { + 'How should I handle the Phase 5 Scrutinizer review?': 'Re-run, inert probes only', + 'Going forward, how do you want me to handle subagents?': 'Flag before running', + }, + annotations: {}, + }, + }; + + // Real errored sample mined from ~/.claude/projects (a different project's + // transcript): an AskUserQuestion call with a missing required `questions` param. + const REAL_ERRORED_PAYLOAD = { + session_id: 'test', + hook_event_name: 'PostToolUse', + tool_name: 'AskUserQuestion', + tool_input: {}, + tool_response: + 'InputValidationError: [\n {\n "expected": "array",\n "code": "invalid_type",\n "path": [\n "questions"\n ],\n "message": "Invalid input: expected array, received undefined"\n }\n]', + }; + + it('AC-F3: one {role:"qa"} row PER QUESTION, to both queues', () => { + runHook(CAPTURE_QUESTION, { ...REAL_MULTI_QUESTION_PAYLOAD, cwd: projectDir }, homeDir); + const mem = readJsonl(path.join(projectDir, '.devflow', 'memory', '.pending-turns.jsonl')); + const dream = readJsonl(path.join(projectDir, '.devflow', 'dream', '.pending-turns.jsonl')); + expect(mem).toHaveLength(2); + expect(dream).toHaveLength(2); + expect(mem[0]).toMatchObject({ + role: 'qa', + content: 'Q: How should I handle the Phase 5 Scrutinizer review?\nA: Re-run, inert probes only', + }); + expect(mem[1]).toMatchObject({ + role: 'qa', + content: 'Q: Going forward, how do you want me to handle subagents?\nA: Flag before running', + }); + }); + + it('non-AskUserQuestion tool_name -> zero appends (even with a matcher, defensively)', () => { + runHook( + CAPTURE_QUESTION, + { cwd: projectDir, tool_name: 'Read', tool_input: { file_path: '/tmp/x' }, tool_response: { type: 'text' } }, + homeDir, + ); + expect(fs.existsSync(path.join(projectDir, '.devflow'))).toBe(false); + }); + + it('malformed/errored tool_response (real InputValidationError sample) -> exit 0, zero writes', () => { + const { exitCode } = runHook(CAPTURE_QUESTION, { ...REAL_ERRORED_PAYLOAD, cwd: projectDir }, homeDir); + expect(exitCode).toBe(0); + expect(fs.existsSync(path.join(projectDir, '.devflow'))).toBe(false); + }); + + it('absent tool_response -> exit 0, zero writes', () => { + const { exitCode } = runHook( + CAPTURE_QUESTION, + { cwd: projectDir, tool_name: 'AskUserQuestion', tool_input: { questions: [{ question: 'q?' }] } }, + homeDir, + ); + expect(exitCode).toBe(0); + expect(fs.existsSync(path.join(projectDir, '.devflow'))).toBe(false); + }); + + it('multiSelect array answer -> joined with "; "', () => { + runHook( + CAPTURE_QUESTION, + { + cwd: projectDir, + tool_name: 'AskUserQuestion', + tool_input: { questions: [{ question: 'Pick colors', multiSelect: true, options: [{ label: 'Red' }, { label: 'Blue' }] }] }, + tool_response: { answers: { 'Pick colors': ['Red', 'Blue'] } }, + }, + homeDir, + ); + const mem = readJsonl(path.join(projectDir, '.devflow', 'memory', '.pending-turns.jsonl')); + expect(mem[0].content).toBe('Q: Pick colors\nA: Red; Blue'); + }); + + it('truncation: Q and A are capped at 1000 chars INDEPENDENTLY', () => { + const longQ = 'Q'.repeat(1500); + const longA = 'A'.repeat(1500); + runHook( + CAPTURE_QUESTION, + { + cwd: projectDir, + tool_name: 'AskUserQuestion', + tool_input: { questions: [{ question: longQ, multiSelect: false, options: [] }] }, + tool_response: { answers: { [longQ]: longA } }, + }, + homeDir, + ); + const mem = readJsonl(path.join(projectDir, '.devflow', 'memory', '.pending-turns.jsonl')); + const content = mem[0].content as string; + const [qPart, aPart] = content.split('\nA: '); + expect(qPart.replace('Q: ', '')).toHaveLength(1000 + '... [truncated]'.length); + expect(aPart).toHaveLength(1000 + '... [truncated]'.length); + }); + + it('hostile answer content is safely escaped (quotes, $(...), newlines collapsed)', () => { + const hostileAnswer = 'yes "quoted" $(rm -rf /) `backtick`'; + runHook( + CAPTURE_QUESTION, + { + cwd: projectDir, + tool_name: 'AskUserQuestion', + tool_input: { questions: [{ question: 'proceed?', multiSelect: false, options: [] }] }, + tool_response: { answers: { 'proceed?': hostileAnswer } }, + }, + homeDir, + ); + const mem = readJsonl(path.join(projectDir, '.devflow', 'memory', '.pending-turns.jsonl')); + expect(mem[0].content).toBe(`Q: proceed?\nA: ${hostileAnswer}`); + }); + + it('AC-F4: gating independent per queue (decisions disabled, memory enabled)', () => { + writeDreamConfig(projectDir, { decisions: false }); + runHook(CAPTURE_QUESTION, { ...REAL_MULTI_QUESTION_PAYLOAD, cwd: projectDir }, homeDir); + expect(readJsonl(path.join(projectDir, '.devflow', 'memory', '.pending-turns.jsonl'))).toHaveLength(2); + expect(fs.existsSync(path.join(projectDir, '.devflow', 'dream', '.pending-turns.jsonl'))).toBe(false); + }); + + it('AC-F14: both BG guards -> exit 0, zero writes', () => { + const r1 = runHook(CAPTURE_QUESTION, { ...REAL_MULTI_QUESTION_PAYLOAD, cwd: projectDir }, homeDir, { DEVFLOW_BG_UPDATER: '1' }); + expect(r1.exitCode).toBe(0); + expect(fs.existsSync(path.join(projectDir, '.devflow'))).toBe(false); + + const r2 = runHook(CAPTURE_QUESTION, { ...REAL_MULTI_QUESTION_PAYLOAD, cwd: projectDir }, homeDir, { DEVFLOW_BG_DREAM: '1' }); + expect(r2.exitCode).toBe(0); + expect(fs.existsSync(path.join(projectDir, '.devflow'))).toBe(false); + }); +}); + +// ============================================================================= +// memory-worker +// ============================================================================= +describe('memory-worker', () => { + let projectDir: string; + let homeDir: string; + let shimDir: string; + + beforeEach(() => { + projectDir = fs.mkdtempSync(path.join(os.tmpdir(), 'mem-worker-')); + homeDir = fs.mkdtempSync(path.join(os.tmpdir(), 'mem-worker-home-')); + shimDir = fs.mkdtempSync(path.join(os.tmpdir(), 'mem-worker-shim-')); + fs.mkdirSync(path.join(projectDir, '.devflow', 'memory'), { recursive: true }); + fs.mkdirSync(path.join(projectDir, '.devflow', 'dream'), { recursive: true }); + }); + + afterEach(() => { + fs.rmSync(projectDir, { recursive: true, force: true }); + fs.rmSync(homeDir, { recursive: true, force: true }); + fs.rmSync(shimDir, { recursive: true, force: true }); + }); + + it('120s throttle honored: fresh trigger -> no spawn', () => { + const triggerFile = path.join(projectDir, '.devflow', 'memory', '.working-memory-last-trigger'); + fs.writeFileSync(triggerFile, ''); + const beforeMtime = fs.statSync(triggerFile).mtimeMs; + + runHookWithPath(MEMORY_WORKER, { cwd: projectDir }, homeDir, shimDir); + + expect(fs.statSync(triggerFile).mtimeMs).toBe(beforeMtime); + }); + + it('touch-before-spawn: stale trigger -> trigger touched, worker spawned', () => { + const memFile = path.join(projectDir, '.devflow', 'memory', 'WORKING-MEMORY.md'); + createFakeClaudeShim(shimDir, memFile); + const triggerFile = path.join(projectDir, '.devflow', 'memory', '.working-memory-last-trigger'); + fs.writeFileSync(triggerFile, ''); + backdateMtime(triggerFile, 600); + + runHookWithPath(MEMORY_WORKER, { cwd: projectDir }, homeDir, shimDir); + + expect(fs.statSync(triggerFile).mtimeMs).toBeGreaterThan(Date.now() - 15000); + }); + + it('spawn happens: worker log shows Starting (fake claude shim on PATH)', async () => { + const memFile = path.join(projectDir, '.devflow', 'memory', 'WORKING-MEMORY.md'); + createFakeClaudeShim(shimDir, memFile); + fs.writeFileSync( + path.join(projectDir, '.devflow', 'memory', '.pending-turns.jsonl'), + JSON.stringify({ role: 'user', content: 'hi', ts: 1 }) + '\n' + JSON.stringify({ role: 'assistant', content: 'hey', ts: 2 }) + '\n', + ); + const triggerFile = path.join(projectDir, '.devflow', 'memory', '.working-memory-last-trigger'); + fs.writeFileSync(triggerFile, ''); + backdateMtime(triggerFile, 600); + + runHookWithPath(MEMORY_WORKER, { cwd: projectDir }, homeDir, shimDir); + + // Poll briefly for the detached worker's log (nohup-spawned, async) + const logFile = workerLogPath(projectDir, homeDir, 'background-memory-update'); + const deadline = Date.now() + 5000; + while (Date.now() < deadline && !fs.existsSync(logFile)) { + await new Promise((r) => setTimeout(r, 100)); + } + expect(fs.existsSync(logFile)).toBe(true); + expect(fs.readFileSync(logFile, 'utf-8')).toContain('Starting (CWD='); + }); + + it('both BG guards prevent spawn', () => { + const memFile = path.join(projectDir, '.devflow', 'memory', 'WORKING-MEMORY.md'); + createFakeClaudeShim(shimDir, memFile); + const triggerFile = path.join(projectDir, '.devflow', 'memory', '.working-memory-last-trigger'); + fs.writeFileSync(triggerFile, ''); + backdateMtime(triggerFile, 600); + + runHookWithPath(MEMORY_WORKER, { cwd: projectDir }, homeDir, shimDir, { DEVFLOW_BG_UPDATER: '1' }); + expect(fs.statSync(triggerFile).mtimeMs).toBeLessThan(Date.now() - 590 * 1000 + 15000); + + backdateMtime(triggerFile, 600); + runHookWithPath(MEMORY_WORKER, { cwd: projectDir }, homeDir, shimDir, { DEVFLOW_BG_DREAM: '1' }); + // Trigger must still be stale — guard fired before the throttle check even ran + const age = Date.now() - fs.statSync(triggerFile).mtimeMs; + expect(age).toBeGreaterThan(590 * 1000); + }); + + it('memory:false -> no spawn attempted, no trigger touch', () => { + writeDreamConfig(projectDir, { memory: false }); + const triggerFile = path.join(projectDir, '.devflow', 'memory', '.working-memory-last-trigger'); + fs.writeFileSync(triggerFile, ''); + backdateMtime(triggerFile, 600); + + runHookWithPath(MEMORY_WORKER, { cwd: projectDir }, homeDir, shimDir); + + const age = Date.now() - fs.statSync(triggerFile).mtimeMs; + expect(age).toBeGreaterThan(590 * 1000); + }); +}); + +// ============================================================================= +// spawn-dream-worker +// ============================================================================= +describe('spawn-dream-worker', () => { + let projectDir: string; + let homeDir: string; + let shimDir: string; + + beforeEach(() => { + projectDir = fs.mkdtempSync(path.join(os.tmpdir(), 'sdw-')); + homeDir = fs.mkdtempSync(path.join(os.tmpdir(), 'sdw-home-')); + shimDir = fs.mkdtempSync(path.join(os.tmpdir(), 'sdw-shim-')); + fs.mkdirSync(path.join(projectDir, '.devflow', 'dream'), { recursive: true }); + fs.mkdirSync(path.join(projectDir, '.devflow', 'decisions'), { recursive: true }); + }); + + afterEach(() => { + fs.rmSync(projectDir, { recursive: true, force: true }); + fs.rmSync(homeDir, { recursive: true, force: true }); + fs.rmSync(shimDir, { recursive: true, force: true }); + }); + + function fakeClaudeShim(): void { + const bin = path.join(shimDir, 'claude'); + fs.writeFileSync(bin, `#!/bin/bash\nsleep 0.2\n`); + fs.chmodSync(bin, 0o755); + } + + async function waitForWorkerLog(): Promise { + const logFile = workerLogPath(projectDir, homeDir, 'background-dream-update'); + const deadline = Date.now() + 5000; + while (Date.now() < deadline && !fs.existsSync(logFile)) { + await new Promise((r) => setTimeout(r, 100)); + } + return fs.existsSync(logFile) ? fs.readFileSync(logFile, 'utf-8') : ''; + } + + it('queue non-empty + claude present -> spawns background-dream-update', async () => { + fakeClaudeShim(); + fs.writeFileSync(path.join(projectDir, '.devflow', 'dream', '.pending-turns.jsonl'), JSON.stringify({ role: 'assistant', content: 'x', ts: 1 }) + '\n'); + + runHookWithPath(SPAWN_DREAM_WORKER, { cwd: projectDir }, homeDir, shimDir); + + const log = await waitForWorkerLog(); + expect(log).toContain('Starting (CWD='); + }); + + it('empty queue + no .processing -> no spawn', async () => { + runHookWithPath(SPAWN_DREAM_WORKER, { cwd: projectDir }, homeDir, shimDir); + await new Promise((r) => setTimeout(r, 300)); + expect(fs.existsSync(workerLogPath(projectDir, homeDir, 'background-dream-update'))).toBe(false); + }); + + it('leftover .processing alone (empty queue) -> spawns', async () => { + fakeClaudeShim(); + fs.writeFileSync(path.join(projectDir, '.devflow', 'dream', '.pending-turns.processing'), JSON.stringify({ role: 'assistant', content: 'x', ts: 1 }) + '\n'); + + runHookWithPath(SPAWN_DREAM_WORKER, { cwd: projectDir }, homeDir, shimDir); + + const log = await waitForWorkerLog(); + expect(log).toContain('Starting (CWD='); + }); + + it('config decisions:false blocks spawn independently', async () => { + writeDreamConfig(projectDir, { decisions: false }); + fs.writeFileSync(path.join(projectDir, '.devflow', 'dream', '.pending-turns.jsonl'), JSON.stringify({ role: 'assistant', content: 'x', ts: 1 }) + '\n'); + + runHookWithPath(SPAWN_DREAM_WORKER, { cwd: projectDir }, homeDir, shimDir); + await new Promise((r) => setTimeout(r, 300)); + expect(fs.existsSync(workerLogPath(projectDir, homeDir, 'background-dream-update'))).toBe(false); + }); + + it('.disabled sentinel blocks spawn independently of the config field', async () => { + fs.writeFileSync(path.join(projectDir, '.devflow', 'decisions', '.disabled'), ''); + fs.writeFileSync(path.join(projectDir, '.devflow', 'dream', '.pending-turns.jsonl'), JSON.stringify({ role: 'assistant', content: 'x', ts: 1 }) + '\n'); + + runHookWithPath(SPAWN_DREAM_WORKER, { cwd: projectDir }, homeDir, shimDir); + await new Promise((r) => setTimeout(r, 300)); + expect(fs.existsSync(workerLogPath(projectDir, homeDir, 'background-dream-update'))).toBe(false); + }); + + it('claude absent from PATH -> clean no-op, no error', () => { + fs.writeFileSync(path.join(projectDir, '.devflow', 'dream', '.pending-turns.jsonl'), JSON.stringify({ role: 'assistant', content: 'x', ts: 1 }) + '\n'); + const { exitCode } = runHook(SPAWN_DREAM_WORKER, { cwd: projectDir }, homeDir, { PATH: '/usr/bin:/bin' }); + expect(exitCode).toBe(0); + }); + + it('never emits stdout, regardless of outcome', () => { + fakeClaudeShim(); + fs.writeFileSync(path.join(projectDir, '.devflow', 'dream', '.pending-turns.jsonl'), JSON.stringify({ role: 'assistant', content: 'x', ts: 1 }) + '\n'); + const { stdout } = runHookWithPath(SPAWN_DREAM_WORKER, { cwd: projectDir }, homeDir, shimDir); + expect(stdout.trim()).toBe(''); + }); + + it('AC-F14: both BG guards -> exit 0, no spawn', async () => { + fakeClaudeShim(); + fs.writeFileSync(path.join(projectDir, '.devflow', 'dream', '.pending-turns.jsonl'), JSON.stringify({ role: 'assistant', content: 'x', ts: 1 }) + '\n'); + + const r1 = runHookWithPath(SPAWN_DREAM_WORKER, { cwd: projectDir }, homeDir, shimDir, { DEVFLOW_BG_DREAM: '1' }); + expect(r1.exitCode).toBe(0); + const r2 = runHookWithPath(SPAWN_DREAM_WORKER, { cwd: projectDir }, homeDir, shimDir, { DEVFLOW_BG_UPDATER: '1' }); + expect(r2.exitCode).toBe(0); + + await new Promise((r) => setTimeout(r, 300)); + expect(fs.existsSync(workerLogPath(projectDir, homeDir, 'background-dream-update'))).toBe(false); + }); + + it('AC-P2: synchronous portion completes quickly (no jq parse of the queue)', () => { + // A large queue file should not slow down the gate check itself, since it + // only ever does `test -s` (existence + non-empty), never a jq/node parse. + const bigQueue = Array.from({ length: 500 }, (_, i) => JSON.stringify({ role: 'assistant', content: `line ${i}`, ts: i })).join('\n') + '\n'; + fs.writeFileSync(path.join(projectDir, '.devflow', 'dream', '.pending-turns.jsonl'), bigQueue); + const start = Date.now(); + runHook(SPAWN_DREAM_WORKER, { cwd: projectDir }, homeDir, { PATH: '/usr/bin:/bin' }); // no claude -> returns fast regardless + const elapsed = Date.now() - start; + expect(elapsed).toBeLessThan(2000); + }); +}); diff --git a/tests/eager-memory-refresh.test.ts b/tests/eager-memory-refresh.test.ts index b07e4ad3..97cc117c 100644 --- a/tests/eager-memory-refresh.test.ts +++ b/tests/eager-memory-refresh.test.ts @@ -1464,3 +1464,199 @@ describe('S12: install survival — background-memory-update not deleted by init expect(mode & 0o100).toBeGreaterThan(0); }); }); + +// ============================================================================= +// S18 — AC-F10: qa rows flow into memory synthesis (orphan gate + TURNS_TEXT agree) +// +// A qa row (captured AskUserQuestion Q&A pair) must count as content-bearing for +// the orphan-only auto-clean guard — the same way an assistant row does — and +// must appear in the prompt fed to claude as its own "Q&A:" stanza. +// ============================================================================= +describe('S18: AC-F10 — qa rows in background-memory-update (orphan gate + TURNS_TEXT)', () => { + let projectDir: string; + let homeDir: string; + let shimDir: string; + + beforeEach(() => { + projectDir = fs.mkdtempSync(path.join(os.tmpdir(), 'emr-s18-')); + homeDir = fs.mkdtempSync(path.join(os.tmpdir(), 'emr-s18-home-')); + shimDir = fs.mkdtempSync(path.join(os.tmpdir(), 'emr-s18-shim-')); + fs.mkdirSync(path.join(projectDir, '.devflow', 'memory'), { recursive: true }); + fs.mkdirSync(path.join(projectDir, '.devflow', 'dream'), { recursive: true }); + initGitRepo(projectDir); + }); + + afterEach(() => { + fs.rmSync(projectDir, { recursive: true, force: true }); + fs.rmSync(homeDir, { recursive: true, force: true }); + fs.rmSync(shimDir, { recursive: true, force: true }); + }); + + it('user + qa (no assistant) is NOT truncated as user-only — a real run is attempted', () => { + const memFile = path.join(projectDir, '.devflow', 'memory', 'WORKING-MEMORY.md'); + createFakeClaudeShim(shimDir, memFile); + + const ts = Math.floor(Date.now() / 1000); + fs.writeFileSync( + path.join(projectDir, '.devflow', 'memory', '.pending-turns.jsonl'), + [ + JSON.stringify({ role: 'user', content: 'what should I pick?', ts }), + JSON.stringify({ role: 'qa', content: 'Q: pick one\nA: option B', ts: ts + 1 }), + ].join('\n') + '\n' + ); + + runWorker(projectDir, homeDir, shimDir); + + // WORKING-MEMORY.md written proves the orphan gate did NOT truncate the queue + // (the "no assistant turn" auto-clean path never invokes claude at all). + expect(fs.existsSync(memFile)).toBe(true); + const processingFile = path.join(projectDir, '.devflow', 'memory', '.pending-turns.processing'); + expect(fs.existsSync(processingFile)).toBe(false); + }); + + it('qa content appears in the prompt fed to claude as a "Q&A:" stanza', () => { + const memFile = path.join(projectDir, '.devflow', 'memory', 'WORKING-MEMORY.md'); + const stdinCapture = path.join(shimDir, 'stdin-captured.txt'); + const claudeBin = path.join(shimDir, 'claude'); + fs.writeFileSync( + claudeBin, + `#!/bin/bash +cat > "${stdinCapture}" +echo "" > "${memFile}" +echo "## Now" >> "${memFile}" +exit 0 +` + ); + fs.chmodSync(claudeBin, 0o755); + + const ts = Math.floor(Date.now() / 1000); + fs.writeFileSync( + path.join(projectDir, '.devflow', 'memory', '.pending-turns.jsonl'), + [ + JSON.stringify({ role: 'user', content: 'need a decision', ts }), + JSON.stringify({ role: 'qa', content: 'Q: ship now or wait?\nA: ship now', ts: ts + 1 }), + ].join('\n') + '\n' + ); + + const { exitCode } = runWorker(projectDir, homeDir, shimDir); + expect(exitCode).toBe(0); + + expect(fs.existsSync(stdinCapture)).toBe(true); + const capturedStdin = fs.readFileSync(stdinCapture, 'utf-8'); + expect(capturedStdin).toContain('Q&A:'); + expect(capturedStdin).toContain('ship now or wait?'); + expect(capturedStdin).toContain('ship now'); + }); + + it('regression: pure user-only queue (no qa, no assistant) is STILL truncated without an LLM run', () => { + const memFile = path.join(projectDir, '.devflow', 'memory', 'WORKING-MEMORY.md'); + createFakeClaudeShim(shimDir, memFile); + + const ts = Math.floor(Date.now() / 1000); + fs.writeFileSync( + path.join(projectDir, '.devflow', 'memory', '.pending-turns.jsonl'), + JSON.stringify({ role: 'user', content: 'just a question', ts }) + '\n' + ); + + runWorker(projectDir, homeDir, shimDir); + + // Orphan gate must still fire for a genuinely user-only queue — claude never invoked. + expect(fs.existsSync(memFile)).toBe(false); + expect(fs.existsSync(path.join(projectDir, '.devflow', 'memory', '.pending-turns.jsonl'))).toBe(false); + }); + + it('qa-only queue (no user, no assistant) is NOT truncated as user-only', () => { + const memFile = path.join(projectDir, '.devflow', 'memory', 'WORKING-MEMORY.md'); + createFakeClaudeShim(shimDir, memFile); + + const ts = Math.floor(Date.now() / 1000); + fs.writeFileSync( + path.join(projectDir, '.devflow', 'memory', '.pending-turns.jsonl'), + JSON.stringify({ role: 'qa', content: 'Q: only question\nA: only answer', ts }) + '\n' + ); + + runWorker(projectDir, homeDir, shimDir); + + expect(fs.existsSync(memFile)).toBe(true); + }); +}); + +// ============================================================================= +// S19 — session-start-memory cold-path recovery for orphaned .pending-turns.processing +// +// Mirrors dream-recover's dream_recover_stale logic (duplicated, not sourced — +// session-start-memory has no dependency on dream-recover). Age >300s + no +// existing .jsonl -> recovered; .jsonl present -> left alone (non-clobber). +// ============================================================================= +describe('S19: session-start-memory cold-path .pending-turns.processing recovery', () => { + let projectDir: string; + let homeDir: string; + + beforeEach(() => { + projectDir = fs.mkdtempSync(path.join(os.tmpdir(), 'emr-s19-')); + homeDir = fs.mkdtempSync(path.join(os.tmpdir(), 'emr-s19-home-')); + fs.mkdirSync(path.join(projectDir, '.devflow', 'memory'), { recursive: true }); + fs.mkdirSync(path.join(projectDir, '.devflow', 'dream'), { recursive: true }); + }); + + afterEach(() => { + fs.rmSync(projectDir, { recursive: true, force: true }); + fs.rmSync(homeDir, { recursive: true, force: true }); + }); + + it('stale (>300s) orphaned .processing with no .jsonl present is recovered', () => { + const proc = path.join(projectDir, '.devflow', 'memory', '.pending-turns.processing'); + fs.writeFileSync(proc, JSON.stringify({ role: 'user', content: 'orphaned', ts: 1 }) + '\n'); + backdateMtime(proc, 600); + + runHook(SESSION_START_MEMORY_HOOK, { cwd: projectDir }, homeDir); + + expect(fs.existsSync(proc)).toBe(false); + const jsonl = path.join(projectDir, '.devflow', 'memory', '.pending-turns.jsonl'); + expect(fs.existsSync(jsonl)).toBe(true); + expect(fs.readFileSync(jsonl, 'utf-8')).toContain('orphaned'); + }); + + it('fresh (<300s) .processing is left alone (not yet stale)', () => { + const proc = path.join(projectDir, '.devflow', 'memory', '.pending-turns.processing'); + fs.writeFileSync(proc, JSON.stringify({ role: 'user', content: 'fresh', ts: 1 }) + '\n'); + // No backdate — mtime is "now" + + runHook(SESSION_START_MEMORY_HOOK, { cwd: projectDir }, homeDir); + + expect(fs.existsSync(proc)).toBe(true); + expect(fs.existsSync(path.join(projectDir, '.devflow', 'memory', '.pending-turns.jsonl'))).toBe(false); + }); + + it('non-clobber: stale .processing is left in place when .pending-turns.jsonl already exists', () => { + const proc = path.join(projectDir, '.devflow', 'memory', '.pending-turns.processing'); + fs.writeFileSync(proc, JSON.stringify({ role: 'user', content: 'orphaned', ts: 1 }) + '\n'); + backdateMtime(proc, 600); + const jsonl = path.join(projectDir, '.devflow', 'memory', '.pending-turns.jsonl'); + fs.writeFileSync(jsonl, JSON.stringify({ role: 'user', content: 'fresh-queue', ts: 2 }) + '\n'); + + runHook(SESSION_START_MEMORY_HOOK, { cwd: projectDir }, homeDir); + + expect(fs.existsSync(proc)).toBe(true); + expect(fs.readFileSync(jsonl, 'utf-8')).toContain('fresh-queue'); + expect(fs.readFileSync(jsonl, 'utf-8')).not.toContain('orphaned'); + }); + + it('no .processing at all — hook proceeds normally (no error, no spurious .jsonl)', () => { + const { exitCode } = runHook(SESSION_START_MEMORY_HOOK, { cwd: projectDir }, homeDir); + expect(exitCode).toBe(0); + expect(fs.existsSync(path.join(projectDir, '.devflow', 'memory', '.pending-turns.jsonl'))).toBe(false); + }); + + it('recovery is skipped entirely when memory is disabled via dream config', () => { + writeDreamConfig(projectDir, { memory: false }); + const proc = path.join(projectDir, '.devflow', 'memory', '.pending-turns.processing'); + fs.writeFileSync(proc, JSON.stringify({ role: 'user', content: 'orphaned', ts: 1 }) + '\n'); + backdateMtime(proc, 600); + + runHook(SESSION_START_MEMORY_HOOK, { cwd: projectDir }, homeDir); + + // memory:false gates the whole hook (including the new recovery block) — .processing untouched + expect(fs.existsSync(proc)).toBe(true); + }); +}); diff --git a/tests/learning/merge-observation.test.ts b/tests/learning/merge-observation.test.ts index 78d56e4a..76bebf37 100644 --- a/tests/learning/merge-observation.test.ts +++ b/tests/learning/merge-observation.test.ts @@ -6,10 +6,11 @@ // D12: evidence capped at 10 (FIFO). import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import { execSync, spawn } from 'child_process'; import * as fs from 'fs'; import * as path from 'path'; import * as os from 'os'; -import { runHelper } from './helpers.js'; +import { runHelper, JSON_HELPER } from './helpers.js'; function readLog(logPath: string): Record[] { if (!fs.existsSync(logPath)) return []; @@ -283,3 +284,97 @@ describe('merge-observation — LLM-provided fields stored verbatim', () => { expect(entries[0].status).toBe('ready'); }); }); + +// AC-C4: merge-observation self-locks internally on .devflow/dream/.observations.lock +// (D53). N parallel invocations against the SAME log file must never corrupt it — +// every row must survive as valid JSON with no torn/interleaved writes — and the CLI +// signature (`merge-observation `) must remain unchanged. +describe('merge-observation — self-locking concurrency (AC-C4)', () => { + let tmpDir: string; + let logFile: string; + + beforeEach(() => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'merge-obs-concurrency-')); + logFile = path.join(tmpDir, 'decisions-log.jsonl'); + }); + + afterEach(() => { + fs.rmSync(tmpDir, { recursive: true, force: true }); + }); + + it('N=15 parallel invocations (distinct ids) all persist with no corruption', () => { + const N = 15; + const cwd = tmpDir; // process.cwd() for each child -> shared .devflow/dream/.observations.lock under tmpDir + + const results = Promise.all( + Array.from({ length: N }, (_, i) => { + const obs = JSON.stringify({ + id: `obs_concurrent_${i}`, + type: 'pitfall', + pattern: `pattern ${i}`, + evidence: [`evidence ${i}`], + details: `area: x; issue: y-${i}; impact: z; resolution: w`, + confidence: 0.5, + status: 'observing', + quality_ok: true, + }); + return new Promise<{ code: number | null; stdout: string; stderr: string }>((resolve) => { + const proc = spawn('node', [JSON_HELPER, 'merge-observation', logFile, obs], { + cwd, + stdio: ['ignore', 'pipe', 'pipe'], + }); + let stdout = ''; + let stderr = ''; + proc.stdout?.on('data', (d) => { stdout += d.toString(); }); + proc.stderr?.on('data', (d) => { stderr += d.toString(); }); + proc.on('close', (code) => resolve({ code, stdout, stderr })); + }); + }), + ); + + return results.then((outcomes) => { + // Every invocation must exit 0 — self-locking must never make a caller fail + // outright (only rotate-observations/assign-anchor's OWN callers get a hard + // timeout error; a fresh, uncontended log file should never hit the 30s cap + // with only 15 short-lived writers). + for (const o of outcomes) { + expect(o.code, `stderr: ${o.stderr}`).toBe(0); + } + + // No corruption: every line must parse as valid JSON, and all N distinct + // ids must be present exactly once (no torn writes, no lost updates). + const lines = fs.readFileSync(logFile, 'utf8').trim().split('\n').filter(Boolean); + expect(lines).toHaveLength(N); + const parsed = lines.map((l) => JSON.parse(l)); + const ids = new Set(parsed.map((e) => e.id)); + expect(ids.size).toBe(N); + for (let i = 0; i < N; i++) { + expect(ids.has(`obs_concurrent_${i}`)).toBe(true); + } + + // No stray lock directory left behind. + const lockDir = path.join(tmpDir, '.devflow', 'dream', '.observations.lock'); + expect(fs.existsSync(lockDir)).toBe(false); + }); + }, 20000); + + it('CLI signature unchanged: merge-observation still returns {merged, id}', () => { + const newObs = JSON.stringify({ + id: 'obs_sig_check', + type: 'decision', + pattern: 'signature check', + evidence: ['e'], + details: 'context: x; decision: y; rationale: z', + confidence: 0.6, + status: 'observing', + quality_ok: true, + }); + const result = JSON.parse( + execSync(`node "${JSON_HELPER}" merge-observation "${logFile}" '${newObs}'`, { + cwd: tmpDir, + encoding: 'utf8', + }).trim(), + ); + expect(result).toEqual({ merged: false, id: 'obs_sig_check' }); + }); +}); diff --git a/tests/queue-append.test.ts b/tests/queue-append.test.ts new file mode 100644 index 00000000..e4869b4b --- /dev/null +++ b/tests/queue-append.test.ts @@ -0,0 +1,392 @@ +/** + * tests/queue-append.test.ts + * + * Tests for scripts/hooks/queue-append — the shared dual-queue-append helper + * used by capture-prompt, capture-turn, and capture-question. + * + * Harness note: queue_append_row/queue_append_both/queue_read_gates are bash + * functions (sourced, not standalone executables), so each test sources + * json-parse -> get-mtime -> dream-lock -> queue-append and then calls the + * function(s) under test via a small inline bash script executed with `bash -c`. + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import { execSync } from 'child_process'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as os from 'os'; + +const HOOKS_DIR = path.resolve(__dirname, '..', 'scripts', 'hooks'); +const QUEUE_APPEND = path.join(HOOKS_DIR, 'queue-append'); + +/** Source the full dependency chain queue-append needs, then run `script`. */ +function runWithQueueAppend(script: string): { stdout: string; stderr: string; exitCode: number } { + const full = ` +set -e +log() { :; } +dbg() { :; } +source "${path.join(HOOKS_DIR, 'json-parse')}" +source "${path.join(HOOKS_DIR, 'get-mtime')}" +source "${path.join(HOOKS_DIR, 'dream-lock')}" +source "${QUEUE_APPEND}" +${script} +`; + try { + const result = execSync(`bash -c '${full.replace(/'/g, "'\\''")}'`, { stdio: ['pipe', 'pipe', 'pipe'] }); + return { stdout: result.toString(), stderr: '', exitCode: 0 }; + } catch (e: unknown) { + const err = e as { stdout?: Buffer; stderr?: Buffer; status?: number }; + return { stdout: err.stdout?.toString() ?? '', stderr: err.stderr?.toString() ?? '', exitCode: err.status ?? 1 }; + } +} + +function readJsonl(file: string): Record[] { + if (!fs.existsSync(file)) return []; + return fs + .readFileSync(file, 'utf-8') + .trim() + .split('\n') + .filter(Boolean) + .map((l) => JSON.parse(l)); +} + +describe('shell hook syntax: queue-append passes bash -n', () => { + it('bash -n succeeds', () => { + expect(() => { + execSync(`bash -n "${QUEUE_APPEND}"`, { stdio: 'pipe' }); + }).not.toThrow(); + }); +}); + +describe('queue_append_row', () => { + let tmpDir: string; + + beforeEach(() => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'queue-append-row-')); + }); + + afterEach(() => { + fs.rmSync(tmpDir, { recursive: true, force: true }); + }); + + it('creates the queue file with mode 0600 on first write', () => { + const q = path.join(tmpDir, 'q.jsonl'); + runWithQueueAppend(`queue_append_row "${q}" "user" "hello" "1000000000"`); + expect(fs.existsSync(q)).toBe(true); + const mode = fs.statSync(q).mode & 0o777; + expect(mode).toBe(0o600); + }); + + it('appends a valid {role, content, ts} JSON row', () => { + const q = path.join(tmpDir, 'q.jsonl'); + runWithQueueAppend(`queue_append_row "${q}" "assistant" "hi there" "1234"`); + const rows = readJsonl(q); + expect(rows).toEqual([{ role: 'assistant', content: 'hi there', ts: 1234 }]); + }); + + it('appends multiple rows without truncating existing content', () => { + const q = path.join(tmpDir, 'q.jsonl'); + runWithQueueAppend(` + queue_append_row "${q}" "user" "one" "1" + queue_append_row "${q}" "assistant" "two" "2" + `); + const rows = readJsonl(q); + expect(rows).toHaveLength(2); + expect(rows[0].content).toBe('one'); + expect(rows[1].content).toBe('two'); + }); + + describe('escaping fuzz (quotes, newlines, command substitution, unicode)', () => { + const cases: Array<{ name: string; content: string }> = [ + { name: 'double quotes', content: 'she said "hello" to me' }, + { name: 'single quotes', content: "it's a test" }, + { name: 'newlines', content: 'line one\nline two\nline three' }, + { name: 'command substitution syntax', content: 'run $(rm -rf /) or `echo pwned`' }, + { name: 'unicode', content: 'café 日本語 \u{1f600}' }, + { name: 'backslashes', content: 'C:\\Users\\test\\path' }, + { name: 'mixed', content: '"$(nested \'quotes\')" \n with \\backslash and émoji \u{1f600}' }, + ]; + + for (const { name, content } of cases) { + it(`round-trips ${name} exactly`, () => { + const q = path.join(tmpDir, 'q.jsonl'); + const tmpContentFile = path.join(tmpDir, 'content.txt'); + fs.writeFileSync(tmpContentFile, content); + // Pass content via a file + command substitution to avoid the TEST + // harness's own shell-escaping concerns; queue_append_row itself + // receives it as a plain positional argument, exactly like a real + // hook would pass $PROMPT/$ASSISTANT_MSG. + const { exitCode } = runWithQueueAppend(` + CONTENT="$(cat "${tmpContentFile}")" + queue_append_row "${q}" "user" "$CONTENT" "1" + `); + expect(exitCode).toBe(0); + const rows = readJsonl(q); + expect(rows).toHaveLength(1); + expect(rows[0].content).toBe(content); + }); + } + }); + + describe('overflow: 200 -> 100 truncation under lock', () => { + it('truncates to newest 100 lines once the file exceeds 200', () => { + const q = path.join(tmpDir, 'q.jsonl'); + const lines: string[] = []; + for (let i = 0; i < 205; i++) { + lines.push(JSON.stringify({ role: 'user', content: `line-${i}`, ts: i })); + } + fs.writeFileSync(q, lines.join('\n') + '\n'); + + runWithQueueAppend(`queue_append_row "${q}" "user" "final-row" "9999"`); + + // 205 pre-seeded + 1 appended = 206 transiently, then truncated to the + // newest 100 (which includes the just-appended row, since it's newest). + const rows = readJsonl(q); + expect(rows).toHaveLength(100); + expect(rows[rows.length - 1].content).toBe('final-row'); + // Oldest surviving row should be from near the tail of the original 205 + expect((rows[0].content as string).startsWith('line-')).toBe(true); + }); + + it('does not truncate when the file is at or below 200 lines', () => { + const q = path.join(tmpDir, 'q.jsonl'); + const lines: string[] = []; + for (let i = 0; i < 150; i++) { + lines.push(JSON.stringify({ role: 'user', content: `line-${i}`, ts: i })); + } + fs.writeFileSync(q, lines.join('\n') + '\n'); + + runWithQueueAppend(`queue_append_row "${q}" "user" "extra" "9999"`); + + const rows = readJsonl(q); + expect(rows).toHaveLength(151); + }); + + it('no lock directory left behind after truncation', () => { + const q = path.join(tmpDir, 'q.jsonl'); + const lines: string[] = []; + for (let i = 0; i < 205; i++) lines.push(JSON.stringify({ role: 'user', content: `l${i}`, ts: i })); + fs.writeFileSync(q, lines.join('\n') + '\n'); + + runWithQueueAppend(`queue_append_row "${q}" "user" "x" "1"`); + + expect(fs.existsSync(`${q}.lock`)).toBe(false); + }); + }); + + describe('degraded no-jq path (node fallback)', () => { + it('still produces valid JSONL when _HAS_JQ is forced false', () => { + const q = path.join(tmpDir, 'q.jsonl'); + const { exitCode } = runWithQueueAppend(` + _HAS_JQ=false + queue_append_row "${q}" "user" "no jq here: \\"quoted\\"" "42" + `); + expect(exitCode).toBe(0); + const rows = readJsonl(q); + expect(rows).toEqual([{ role: 'user', content: 'no jq here: "quoted"', ts: 42 }]); + }); + }); + + describe('20-parallel-append interleave (no corruption)', () => { + it('20 concurrent appends to the SAME queue file all survive as valid JSON lines', async () => { + const q = path.join(tmpDir, 'q.jsonl'); + const N = 20; + const scriptPath = path.join(tmpDir, 'append-one.sh'); + fs.writeFileSync( + scriptPath, + `#!/bin/bash +set -e +log() { :; } +dbg() { :; } +source "${path.join(HOOKS_DIR, 'json-parse')}" +source "${path.join(HOOKS_DIR, 'get-mtime')}" +source "${path.join(HOOKS_DIR, 'dream-lock')}" +source "${QUEUE_APPEND}" +queue_append_row "$1" "user" "row-$2" "$2" +`, + ); + fs.chmodSync(scriptPath, 0o755); + + const { spawn } = await import('child_process'); + const runs = Array.from({ length: N }, (_, i) => { + return new Promise((resolve) => { + const proc = spawn('bash', [scriptPath, q, String(i)], { stdio: 'ignore' }); + proc.on('close', (code) => resolve(code)); + }); + }); + const codes = await Promise.all(runs); + for (const c of codes) expect(c).toBe(0); + + const rows = readJsonl(q); + // Every line must be valid, parseable JSON (readJsonl would throw otherwise) + // and all N distinct row identifiers must be present exactly once. + expect(rows).toHaveLength(N); + const contents = new Set(rows.map((r) => r.content)); + expect(contents.size).toBe(N); + for (let i = 0; i < N; i++) { + expect(contents.has(`row-${i}`)).toBe(true); + } + }, 15000); + }); + + describe('truncate-vs-append race', () => { + // ACCEPTED RISK (documented in the design): the truncate path is + // read-then-replace (`tail -100 file > tmp && mv tmp file`), not an + // in-place lock-held write. A concurrent lock-free append landing on the + // original file AFTER the `tail` snapshot but BEFORE the `mv` replaces it + // can be silently dropped by the replace — the same class of race the + // pre-existing dream-capture/dream-dispatch queue-overflow logic already + // has (this helper extracts, not changes, that behavior). The guarantee + // this test actually pins is NO CORRUPTION (every surviving line remains + // valid, parseable JSON) — not zero data loss under a genuine race. + it('races a truncation without ever corrupting the file (parseable JSONL, sane length)', async () => { + const q = path.join(tmpDir, 'q.jsonl'); + const lines: string[] = []; + for (let i = 0; i < 205; i++) lines.push(JSON.stringify({ role: 'user', content: `seed-${i}`, ts: i })); + fs.writeFileSync(q, lines.join('\n') + '\n'); + + const scriptPath = path.join(tmpDir, 'append-race.sh'); + fs.writeFileSync( + scriptPath, + `#!/bin/bash +set -e +log() { :; } +dbg() { :; } +source "${path.join(HOOKS_DIR, 'json-parse')}" +source "${path.join(HOOKS_DIR, 'get-mtime')}" +source "${path.join(HOOKS_DIR, 'dream-lock')}" +source "${QUEUE_APPEND}" +queue_append_row "$1" "user" "race-$2" "$2" +`, + ); + fs.chmodSync(scriptPath, 0o755); + + const { spawn } = await import('child_process'); + const runs = [0, 1, 2].map((i) => { + return new Promise((resolve) => { + const proc = spawn('bash', [scriptPath, q, String(i)], { stdio: 'ignore' }); + proc.on('close', (code) => resolve(code)); + }); + }); + const codes = await Promise.all(runs); + for (const c of codes) expect(c).toBe(0); + + // No corruption: readJsonl throws on any malformed line, so reaching + // this point at all already proves every surviving line is valid JSON. + const rows = readJsonl(q); + // Sane length: truncation keeps the newest 100 plus whatever raced in + // after the last truncate pass — never near-zero, never wildly over. + expect(rows.length).toBeGreaterThanOrEqual(100); + expect(rows.length).toBeLessThanOrEqual(103); + + // Best-effort (not guaranteed under the accepted race): most or all of + // the 3 racing rows typically survive — assert at least one did, so a + // total-loss regression (e.g. a bug that drops ALL concurrent appends) + // would still be caught. + const contents = new Set(rows.map((r) => r.content)); + const survived = ['race-0', 'race-1', 'race-2'].filter((r) => contents.has(r)); + expect(survived.length).toBeGreaterThanOrEqual(1); + }, 15000); + }); +}); + +describe('queue_append_both', () => { + let tmpDir: string; + + beforeEach(() => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'queue-append-both-')); + }); + + afterEach(() => { + fs.rmSync(tmpDir, { recursive: true, force: true }); + }); + + it('writes to both queues when both flags are true', () => { + const mem = path.join(tmpDir, 'mem.jsonl'); + const dream = path.join(tmpDir, 'dream.jsonl'); + runWithQueueAppend(`queue_append_both "${mem}" "${dream}" "true" "true" "user" "hi" "1"`); + expect(readJsonl(mem)).toHaveLength(1); + expect(readJsonl(dream)).toHaveLength(1); + }); + + it('writes only to the memory queue when dream_enabled is false', () => { + const mem = path.join(tmpDir, 'mem.jsonl'); + const dream = path.join(tmpDir, 'dream.jsonl'); + runWithQueueAppend(`queue_append_both "${mem}" "${dream}" "true" "false" "user" "hi" "1"`); + expect(readJsonl(mem)).toHaveLength(1); + expect(fs.existsSync(dream)).toBe(false); + }); + + it('writes only to the dream queue when memory_enabled is false', () => { + const mem = path.join(tmpDir, 'mem.jsonl'); + const dream = path.join(tmpDir, 'dream.jsonl'); + runWithQueueAppend(`queue_append_both "${mem}" "${dream}" "false" "true" "user" "hi" "1"`); + expect(fs.existsSync(mem)).toBe(false); + expect(readJsonl(dream)).toHaveLength(1); + }); + + it('writes to neither queue when both flags are false', () => { + const mem = path.join(tmpDir, 'mem.jsonl'); + const dream = path.join(tmpDir, 'dream.jsonl'); + runWithQueueAppend(`queue_append_both "${mem}" "${dream}" "false" "false" "user" "hi" "1"`); + expect(fs.existsSync(mem)).toBe(false); + expect(fs.existsSync(dream)).toBe(false); + }); +}); + +describe('queue_read_gates', () => { + let tmpDir: string; + + beforeEach(() => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'queue-read-gates-')); + }); + + afterEach(() => { + fs.rmSync(tmpDir, { recursive: true, force: true }); + }); + + function readGates(config: Record | null, sentinelExists: boolean): { memory: string; decisions: string; exitCode: number } { + const configPath = path.join(tmpDir, 'config.json'); + if (config !== null) fs.writeFileSync(configPath, JSON.stringify(config)); + const sentinelPath = path.join(tmpDir, '.disabled'); + if (sentinelExists) fs.writeFileSync(sentinelPath, ''); + + const { stdout, exitCode } = runWithQueueAppend(` + queue_read_gates "${configPath}" "${sentinelPath}" + echo "MEMORY=$_QG_MEMORY" + echo "DECISIONS=$_QG_DECISIONS" + `); + const memMatch = stdout.match(/MEMORY=(\S*)/); + const decMatch = stdout.match(/DECISIONS=(\S*)/); + return { memory: memMatch?.[1] ?? '', decisions: decMatch?.[1] ?? '', exitCode }; + } + + it('both default to true when config is missing', () => { + const r = readGates(null, false); + expect(r).toMatchObject({ memory: 'true', decisions: 'true', exitCode: 0 }); + }); + + it('reads both explicit fields in one pass', () => { + const r = readGates({ memory: true, decisions: false }, false); + expect(r).toMatchObject({ memory: 'true', decisions: 'false' }); + }); + + it('memory:false, decisions field absent -> memory false, decisions defaults true', () => { + const r = readGates({ memory: false }, false); + expect(r).toMatchObject({ memory: 'false', decisions: 'true' }); + }); + + it('sentinel forces decisions false independent of the config field', () => { + const r = readGates({ decisions: true }, true); + expect(r).toMatchObject({ decisions: 'false' }); + }); + + it('never exits non-zero even when the sentinel is absent (set -e safety)', () => { + // Regression guard: queue_read_gates's last statement must not leak a bare + // `[ -f sentinel ] && x=y` truthiness as the function's own return code — + // that would abort any `set -e` caller whenever the sentinel is absent + // (the common case). + const r = readGates({}, false); + expect(r.exitCode).toBe(0); + }); +}); diff --git a/tests/sentinel.test.ts b/tests/sentinel.test.ts index 1d7a1e70..71a758b8 100644 --- a/tests/sentinel.test.ts +++ b/tests/sentinel.test.ts @@ -545,3 +545,108 @@ describe('manageSentinel utility', () => { }); }); +// ─── Part F: DEVFLOW_BG_* re-entrancy guards (AC-F14) ────────────────────── +// +// session-start-context, session-start-memory, and pre-compact-memory had no +// re-entrancy guard at all (a latent pre-existing gap): the memory or dream +// worker's own nested claude -p session fires these SessionStart/PreCompact +// hooks too. Guards were added additively — these are NEW test cases; the +// existing guard describes above (dream-dispatch, dream-capture, +// dream-evaluate) are untouched. + +describe('sentinel guard: session-start-context DEVFLOW_BG_* re-entrancy', () => { + const HOOK = path.join(HOOKS_DIR, 'session-start-context'); + let tmpDir: string; + + beforeEach(() => { tmpDir = mkTmpDir(); }); + afterEach(() => { fs.rmSync(tmpDir, { recursive: true, force: true }); }); + + it('outputs nothing when DEVFLOW_BG_UPDATER=1, even with a decisions TL;DR present', () => { + mkMemoryDir(tmpDir); + const decisionsDir = path.join(tmpDir, '.devflow', 'decisions'); + fs.writeFileSync(path.join(decisionsDir, 'decisions.md'), '\n# Decisions\n'); + const input = sessionInput(tmpDir); + const output = execSync(`DEVFLOW_BG_UPDATER=1 bash "${HOOK}"`, { input, stdio: ['pipe', 'pipe', 'pipe'] }).toString().trim(); + expect(output).toBe(''); + }); + + it('outputs nothing when DEVFLOW_BG_DREAM=1, even with a decisions TL;DR present', () => { + mkMemoryDir(tmpDir); + const decisionsDir = path.join(tmpDir, '.devflow', 'decisions'); + fs.writeFileSync(path.join(decisionsDir, 'decisions.md'), '\n# Decisions\n'); + const input = sessionInput(tmpDir); + const output = execSync(`DEVFLOW_BG_DREAM=1 bash "${HOOK}"`, { input, stdio: ['pipe', 'pipe', 'pipe'] }).toString().trim(); + expect(output).toBe(''); + }); + + it('DEVFLOW_BG_DREAM=1 makes zero filesystem writes (no .gitignore/.devflow scaffolding)', () => { + // A fresh tmpDir with no .devflow/ at all — the guard must fire before + // ensure-root-gitignore or any other write-side-effect runs. + const input = sessionInput(tmpDir); + execSync(`DEVFLOW_BG_DREAM=1 bash "${HOOK}"`, { input, stdio: ['pipe', 'pipe', 'pipe'] }); + expect(fs.existsSync(path.join(tmpDir, '.devflow'))).toBe(false); + }); +}); + +describe('sentinel guard: session-start-memory DEVFLOW_BG_* re-entrancy', () => { + const HOOK = path.join(HOOKS_DIR, 'session-start-memory'); + let tmpDir: string; + + beforeEach(() => { tmpDir = mkTmpDir(); }); + afterEach(() => { fs.rmSync(tmpDir, { recursive: true, force: true }); }); + + it('outputs nothing when DEVFLOW_BG_UPDATER=1, even with WORKING-MEMORY.md present', () => { + mkMemoryDir(tmpDir); + fs.writeFileSync(path.join(tmpDir, '.devflow', 'memory', 'WORKING-MEMORY.md'), '## Now\n- testing'); + const input = sessionInput(tmpDir); + const output = execSync(`DEVFLOW_BG_UPDATER=1 bash "${HOOK}"`, { input, stdio: ['pipe', 'pipe', 'pipe'] }).toString().trim(); + expect(output).toBe(''); + }); + + it('outputs nothing when DEVFLOW_BG_DREAM=1, even with WORKING-MEMORY.md present', () => { + mkMemoryDir(tmpDir); + fs.writeFileSync(path.join(tmpDir, '.devflow', 'memory', 'WORKING-MEMORY.md'), '## Now\n- testing'); + const input = sessionInput(tmpDir); + const output = execSync(`DEVFLOW_BG_DREAM=1 bash "${HOOK}"`, { input, stdio: ['pipe', 'pipe', 'pipe'] }).toString().trim(); + expect(output).toBe(''); + }); + + it('DEVFLOW_BG_DREAM=1 does not recover a stale .pending-turns.processing (guard fires before the cold-path check)', () => { + mkMemoryDir(tmpDir); + const proc = path.join(tmpDir, '.devflow', 'memory', '.pending-turns.processing'); + fs.writeFileSync(proc, JSON.stringify({ role: 'user', content: 'x', ts: 1 }) + '\n'); + const old = new Date(Date.now() - 600 * 1000); + fs.utimesSync(proc, old, old); + const input = sessionInput(tmpDir); + execSync(`DEVFLOW_BG_DREAM=1 bash "${HOOK}"`, { input, stdio: ['pipe', 'pipe', 'pipe'] }); + expect(fs.existsSync(proc)).toBe(true); + expect(fs.existsSync(path.join(tmpDir, '.devflow', 'memory', '.pending-turns.jsonl'))).toBe(false); + }); +}); + +describe('sentinel guard: pre-compact-memory DEVFLOW_BG_* re-entrancy', () => { + const HOOK = path.join(HOOKS_DIR, 'pre-compact-memory'); + let tmpDir: string; + + beforeEach(() => { tmpDir = mkTmpDir(); }); + afterEach(() => { fs.rmSync(tmpDir, { recursive: true, force: true }); }); + + it('does not write backup.json when DEVFLOW_BG_UPDATER=1', () => { + mkMemoryDir(tmpDir); + const input = sessionInput(tmpDir); + expect(() => { + execSync(`DEVFLOW_BG_UPDATER=1 bash "${HOOK}"`, { input, stdio: ['pipe', 'pipe', 'pipe'] }); + }).not.toThrow(); + expect(fs.existsSync(path.join(tmpDir, '.devflow', 'memory', 'backup.json'))).toBe(false); + }); + + it('does not write backup.json when DEVFLOW_BG_DREAM=1', () => { + mkMemoryDir(tmpDir); + const input = sessionInput(tmpDir); + expect(() => { + execSync(`DEVFLOW_BG_DREAM=1 bash "${HOOK}"`, { input, stdio: ['pipe', 'pipe', 'pipe'] }); + }).not.toThrow(); + expect(fs.existsSync(path.join(tmpDir, '.devflow', 'memory', 'backup.json'))).toBe(false); + }); +}); + diff --git a/tests/shell-hooks.test.ts b/tests/shell-hooks.test.ts index 7bbf64b2..6dad1839 100644 --- a/tests/shell-hooks.test.ts +++ b/tests/shell-hooks.test.ts @@ -31,6 +31,13 @@ const HOOK_SCRIPTS = [ 'dream-dispatch', 'eval-helpers', 'eval-decisions', + 'queue-append', + 'capture-prompt', + 'capture-turn', + 'capture-question', + 'memory-worker', + 'spawn-dream-worker', + 'background-dream-update', ]; describe('shell hook syntax checks', () => { From 5217b6c51d6e58932f7f0fcdc0b7feb64cdfb8ee Mon Sep 17 00:00:00 2001 From: Dean Sharon Date: Fri, 3 Jul 2026 03:20:28 +0300 Subject: [PATCH 05/26] =?UTF-8?q?feat(hooks)!:=20simplify=20dream=20system?= =?UTF-8?q?=20=E2=80=94=20unified=20capture=20+=20detached=20dream=20worke?= =?UTF-8?q?r?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the marker-pipeline decisions/curation flow (SessionEnd eval-* modules writing per-session .json markers, consumed by a SessionStart Dream subagent via session-start-context's directive-emission + dream-collect-tasks/dream-recover) with a hook-spawned detached `claude -p` worker that reads scripts/hooks/dream-procedure.md directly. Queue-append is unified across memory and dream into a shared capture-prompt/capture-turn/capture-question bundle (capture.ts), independent of the Stop-hook throttle/spawn logic (now memory-worker, for memory) and the SessionStart spawn gate (now spawn-dream-worker + dream.ts, for decisions). Deletions: scripts/hooks/{dream-capture,dream-dispatch,dream-evaluate,eval-helpers, eval-decisions,eval-curation,dream-collect-tasks,dream-recover,lib/transcript-filter.cjs, lib/dream-ops.cjs}, shared/agents/dream.md, shared/skills/{dream-decisions,dream-curation}/, json-helper.cjs's read-dream dispatch, session-start-context's Section 2 (DREAM MAINTENANCE directive). TS/CLI: new capture.ts (capture-prompt/capture-turn/capture-question, always-on) and dream.ts (spawn-dream-worker, always-on); memory.ts rebased to a 3-hook bundle (Stop/SessionStart/PreCompact) with dream-dispatch/dream-capture/dream-evaluate added to the legacy sweep; init.ts wires the capture+dream bundles unconditionally in the order required for Stop=[capture-turn, memory-worker] and SessionStart=[session-start-memory, session-start-context, spawn-dream-worker]; uninstall.ts strips the new bundles; decisions-config.ts drops max_daily_runs/ throttle_minutes (the new worker has no daily cap — it's gated by queue non-emptiness) and defaults model to opus; decisions.ts's --clear/--reset now resolve the git root explicitly (fixes a subdirectory-cwd bug) and skip a project entirely while .devflow/dream/.worker.lock is held; project-paths.ts adds the new dream queue/stamp/lock path helpers; plugins.ts removes the dream agent and dream-decisions/dream-curation skills from core-skills/ambient with LEGACY array entries for upgrade cleanup. BREAKING CHANGES: SessionEnd hook removed entirely. Marker files (decisions.*.json/curation.*.json and their .processing/.retries/.failed variants) are no longer read or written — a migration sweeps legacy state separately. DecisionsConfig.max_daily_runs/throttle_minutes dropped (silently ignored if present in on-disk config). Decisions model default changed from sonnet to opus. The dream agent and its 2 per-task skills are removed. Co-Authored-By: Claude --- .../.claude-plugin/plugin.json | 3 +- .../.claude-plugin/plugin.json | 8 +- scripts/hooks/dream-capture | 211 --- scripts/hooks/dream-collect-tasks | 256 --- scripts/hooks/dream-dispatch | 85 - scripts/hooks/dream-evaluate | 141 -- scripts/hooks/dream-recover | 185 -- scripts/hooks/eval-curation | 81 - scripts/hooks/eval-decisions | 79 - scripts/hooks/eval-helpers | 70 - scripts/hooks/json-helper.cjs | 9 - scripts/hooks/lib/dream-ops.cjs | 54 - scripts/hooks/lib/project-paths.cjs | 30 +- scripts/hooks/lib/transcript-filter.cjs | 207 --- scripts/hooks/run-hook | 8 + scripts/hooks/session-start-context | 131 +- shared/agents/dream.md | 89 - shared/skills/dream-curation/SKILL.md | 139 -- shared/skills/dream-decisions/SKILL.md | 116 -- src/cli/commands/capture.ts | 161 ++ src/cli/commands/decisions.ts | 121 +- src/cli/commands/dream.ts | 92 + src/cli/commands/init.ts | 42 +- src/cli/commands/memory.ts | 39 +- src/cli/commands/uninstall.ts | 4 + src/cli/plugins.ts | 37 +- src/cli/utils/decisions-config.ts | 27 +- src/cli/utils/project-paths.ts | 25 +- src/templates/settings.json | 29 +- tests/background-dream-update.test.ts | 70 + tests/capture-hooks.test.ts | 63 + tests/capture.test.ts | 260 +++ tests/decisions/cli-subcommands.test.ts | 67 +- tests/decisions/config.test.ts | 92 +- tests/decisions/decisions-format.test.ts | 28 +- tests/decisions/dream-curation.test.ts | 149 +- tests/eager-memory-refresh.test.ts | 253 +-- tests/learning/filter.test.ts | 268 --- tests/memory.test.ts | 182 +- tests/project-paths.test.ts | 30 +- tests/sentinel.test.ts | 239 +-- tests/shell-hooks.test.ts | 1599 +---------------- 42 files changed, 1330 insertions(+), 4449 deletions(-) delete mode 100755 scripts/hooks/dream-capture delete mode 100644 scripts/hooks/dream-collect-tasks delete mode 100755 scripts/hooks/dream-dispatch delete mode 100755 scripts/hooks/dream-evaluate delete mode 100644 scripts/hooks/dream-recover delete mode 100644 scripts/hooks/eval-curation delete mode 100644 scripts/hooks/eval-decisions delete mode 100644 scripts/hooks/eval-helpers delete mode 100644 scripts/hooks/lib/dream-ops.cjs delete mode 100644 scripts/hooks/lib/transcript-filter.cjs delete mode 100644 shared/agents/dream.md delete mode 100644 shared/skills/dream-curation/SKILL.md delete mode 100644 shared/skills/dream-decisions/SKILL.md create mode 100644 src/cli/commands/capture.ts create mode 100644 src/cli/commands/dream.ts create mode 100644 tests/capture.test.ts delete mode 100644 tests/learning/filter.test.ts diff --git a/plugins/devflow-ambient/.claude-plugin/plugin.json b/plugins/devflow-ambient/.claude-plugin/plugin.json index 28d5c1c9..32019962 100644 --- a/plugins/devflow-ambient/.claude-plugin/plugin.json +++ b/plugins/devflow-ambient/.claude-plugin/plugin.json @@ -27,8 +27,7 @@ "resolver", "designer", "knowledge", - "researcher", - "dream" + "researcher" ], "skills": [ "review-methodology", diff --git a/plugins/devflow-core-skills/.claude-plugin/plugin.json b/plugins/devflow-core-skills/.claude-plugin/plugin.json index 51fda8f3..b57543df 100644 --- a/plugins/devflow-core-skills/.claude-plugin/plugin.json +++ b/plugins/devflow-core-skills/.claude-plugin/plugin.json @@ -16,9 +16,7 @@ "enforcement", "foundation" ], - "agents": [ - "dream" - ], + "agents": [], "skills": [ "apply-decisions", "apply-feature-knowledge", @@ -28,9 +26,7 @@ "boundary-validation", "test-driven-development", "testing", - "dependency-research", - "dream-decisions", - "dream-curation" + "dependency-research" ], "rules": [ "security", diff --git a/scripts/hooks/dream-capture b/scripts/hooks/dream-capture deleted file mode 100755 index dc747ad7..00000000 --- a/scripts/hooks/dream-capture +++ /dev/null @@ -1,211 +0,0 @@ -#!/bin/bash - -# Dream System: dream-capture (Stop Hook) -# Captures assistant responses to .devflow/memory/.pending-turns.jsonl queue. -# After the 120s throttle, spawns a detached `claude -p` worker (background-memory-update) -# to rewrite WORKING-MEMORY.md. No dream marker is written for memory — memory refresh -# now happens eagerly in the background, not via the SessionStart Dream subagent. -# On failure: does nothing (stale memory is better than fake data). - -# Safe no-op fallback: must exist before set -e and before hook-bootstrap is sourced. -dbg() { :; } - -set -e - -# Feedback-loop guards — before hook-bootstrap to minimize background session overhead -if [ "${DEVFLOW_BG_UPDATER:-}" = "1" ]; then dbg "EXIT: bg_updater"; exit 0; fi -if [ "${DEVFLOW_BG_LEARNER:-}" = "1" ]; then dbg "EXIT: bg_learner"; exit 0; fi -if [ "${DEVFLOW_BG_KNOWLEDGE_REFRESH:-}" = "1" ]; then dbg "EXIT: bg_knowledge"; exit 0; fi - -# Resolve script directory once -SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" - -source "$SCRIPT_DIR/hook-bootstrap" "dream-capture" - -# JSON parsing (jq with node fallback) -source "$SCRIPT_DIR/json-parse" || { echo "dream-capture: failed to source json-parse" >&2; exit 1; } -if [ "$_JSON_AVAILABLE" = "false" ]; then dbg "EXIT: no json"; exit 0; fi - -dbg "json available, has_jq=$_HAS_JQ" - -INPUT=$(cat) -dbg "INPUT length=${#INPUT}" -if [ "${DEVFLOW_HOOK_DEBUG:-}" = "1" ]; then dbg "INPUT keys=$(printf '%s' "$INPUT" | jq -r 'keys | join(", ")' 2>/dev/null || echo 'jq-failed')"; fi - -# Resolve project directory and assistant response in one subprocess. -# Stop hook fields: cwd, last_assistant_message, session_id, transcript_path, etc. -# Uses ASCII SOH (0x01) as delimiter; split with parameter expansion (bash 3.2 safe). -if [ "$_HAS_JQ" = "true" ]; then - _FIELDS=$(printf '%s' "$INPUT" | jq -r '(.cwd // "") + "" + (.last_assistant_message // "")') || { dbg "EXIT: jq failed"; exit 0; } -else - _FIELDS=$(printf '%s' "$INPUT" | node -e " - const j=JSON.parse(require('fs').readFileSync('/dev/stdin','utf8')); - process.stdout.write((j.cwd||'')+'\x01'+(j.last_assistant_message||''))") || { dbg "EXIT: node failed"; exit 0; } -fi -# Split on the first SOH with parameter expansion (newline-safe). NOT cut: cut is -# line-oriented, so a multi-line last_assistant_message leaks its body lines into CWD, -# corrupting the -d directory check below (silent drop of every multi-line turn). -# Mirrors dream-dispatch's CWD/PROMPT split. -CWD="${_FIELDS%%$'\001'*}" -ASSISTANT_MSG="${_FIELDS#*$'\001'}" - -if [ -z "$CWD" ] || [ ! -d "$CWD" ]; then dbg "EXIT: bad CWD"; exit 0; fi - -devflow_debug_set_cwd "$CWD" -dbg "CWD=$CWD" -# SECURITY: Never log ASSISTANT_MSG or INPUT content — may contain secrets. -# Only log metadata (length, keys, presence checks). -dbg "ASSISTANT_MSG length=${#ASSISTANT_MSG}" - -# Anchor .devflow/ to the project root (prevents a stray nested .devflow/ when this -# hook runs with a CWD inside .devflow/...). Empty → fall back to CWD (old behavior). -source "$SCRIPT_DIR/resolve-project-root" 2>/dev/null || true -PROJECT_ROOT="$(df_resolve_root "$CWD" 2>/dev/null || true)" -[ -n "$PROJECT_ROOT" ] || PROJECT_ROOT="$CWD" - -DEVFLOW_DIR="$PROJECT_ROOT/.devflow" -MEMORY_DIR="$DEVFLOW_DIR/memory" -DREAM_DIR="$DEVFLOW_DIR/dream" - -# Read dream config — memory:false gates memory-specific sections only. -# The decisions usage scanner runs independently of memory state. -DREAM_CONFIG="$DREAM_DIR/config.json" -MEMORY_ENABLED="true" -if [ -f "$DREAM_CONFIG" ]; then - MEMORY_ENABLED=$(json_field_file "$DREAM_CONFIG" "memory" "true") -fi - -dbg "MEMORY_ENABLED=$MEMORY_ENABLED" - -# Skip if empty response -if [ -z "$ASSISTANT_MSG" ]; then - dbg "EXIT: empty ASSISTANT_MSG" - exit 0 -fi - -dbg "PASSED all checks, proceeding to capture" - -# Truncate to 2000 chars -if [ ${#ASSISTANT_MSG} -gt 2000 ]; then - ASSISTANT_MSG="${ASSISTANT_MSG:0:2000}... [truncated]" -fi - -# --- Decisions usage scanner (independent of memory state) --- -# D29: Grep-first reorder — cheap in-process citation check gates the expensive config parse. -# Dual-signal: skip when decisions is disabled via config OR sentinel (mirror of dream-evaluate). -# On the common case (no ADR/PF citations) the grep exits false and the config subprocess is -# never forked, preserving the D29 optimization (~5-20ms jq / ~50-100ms node per turn saved). -SCANNER="$SCRIPT_DIR/decisions-usage-scan.cjs" -if [ -f "$SCANNER" ] && printf '%s' "$ASSISTANT_MSG" | grep -qE 'ADR-[0-9]+|PF-[0-9]+'; then - _DEC_ENABLED_CAPTURE="true" - if [ -f "$DREAM_CONFIG" ]; then - _DEC_ENABLED_CAPTURE=$(json_field_file "$DREAM_CONFIG" "decisions" "true") - fi - [ -f "$DEVFLOW_DIR/decisions/.disabled" ] && _DEC_ENABLED_CAPTURE="false" - if [ "$_DEC_ENABLED_CAPTURE" = "true" ]; then - dbg "Running decisions usage scanner" - printf '%s' "$ASSISTANT_MSG" | node "$SCANNER" --cwd "$PROJECT_ROOT" 2>/dev/null || true - fi -fi - -# --- Memory-specific sections (gated by memory:false in dream config) --- -if [ "$MEMORY_ENABLED" = "false" ]; then - dbg "EXIT: memory disabled" - exit 0 -fi - -# Logging (needs CWD and memory-enabled path — source after memory gate so log-paths -# subprocess overhead (~15-20ms) is only paid when memory is actually enabled) -source "$SCRIPT_DIR/hook-log-init" "dream-capture" - -# Auto-create .devflow/ and ensure .gitignore entries -source "$SCRIPT_DIR/ensure-devflow-init" "$CWD" || exit 0 - -source "$SCRIPT_DIR/get-mtime" || { echo "dream-capture: failed to source get-mtime" >&2; exit 1; } -source "$SCRIPT_DIR/dream-lock" || { echo "dream-capture: failed to source dream-lock" >&2; exit 1; } - -# --- Append to queue --- -QUEUE_FILE="$MEMORY_DIR/.pending-turns.jsonl" - -# Ensure queue file has restricted permissions (600) if not yet created. -if [ ! -f "$QUEUE_FILE" ]; then - (umask 077 && touch "$QUEUE_FILE") 2>/dev/null || true -fi - -TS=$(date +%s) -if [ "$_HAS_JQ" = "true" ]; then - jq -n -c --arg role "assistant" --arg content "$ASSISTANT_MSG" --argjson ts "$TS" \ - '{role: $role, content: $content, ts: $ts}' >> "$QUEUE_FILE" -else - node -e "process.stdout.write(JSON.stringify({role:'assistant',content:process.argv[1],ts:parseInt(process.argv[2])})+'\n')" \ - -- "$ASSISTANT_MSG" "$TS" >> "$QUEUE_FILE" -fi - -dbg "Appended to queue: QUEUE_FILE=$QUEUE_FILE" - -# Queue overflow safety: if >200 lines, keep last 100 (locked to prevent multi-session race) -if [ -f "$QUEUE_FILE" ]; then - LINE_COUNT=$(wc -l < "$QUEUE_FILE" | tr -d ' ') - if [ "$LINE_COUNT" -gt 200 ]; then - _QUEUE_LOCK="$MEMORY_DIR/.pending-turns.lock" - if dream_lock_acquire "$_QUEUE_LOCK" 2; then - LINE_COUNT=$(wc -l < "$QUEUE_FILE" | tr -d ' ') - if [ "$LINE_COUNT" -gt 200 ]; then - _TRUNC_TMP="$QUEUE_FILE.tmp.$$" && tail -100 "$QUEUE_FILE" > "$_TRUNC_TMP" && mv "$_TRUNC_TMP" "$QUEUE_FILE" || rm -f "$_TRUNC_TMP" - log "Queue overflow: truncated from $LINE_COUNT to 100 lines" - dbg "Queue overflow: truncated from $LINE_COUNT to 100 lines" - fi - dream_lock_release "$_QUEUE_LOCK" - fi - fi -fi - -log "Captured assistant turn (${#ASSISTANT_MSG} chars)" - -# --- Spawn detached worker if throttle expired --- -# Throttle key: .working-memory-last-trigger mtime (120s window). -# We touch it BEFORE spawning to prevent a second concurrent Stop hook from -# double-spawning within the same 120s window. -TRIGGER_FILE="$MEMORY_DIR/.working-memory-last-trigger" -NOW=$(date +%s) -LAST_TRIGGER=0 -if [ -f "$TRIGGER_FILE" ]; then - LAST_TRIGGER=$(get_mtime "$TRIGGER_FILE") -fi -LAST_TRIGGER="${LAST_TRIGGER:-0}" -TRIGGER_AGE=$(( NOW - LAST_TRIGGER )) - -if [ "$TRIGGER_AGE" -lt 120 ]; then - log "Throttled: worker spawned ${TRIGGER_AGE}s ago" - dbg "EXIT: throttled TRIGGER_AGE=${TRIGGER_AGE}s" - exit 0 -fi - -# Resolve worker and claude binary -UPDATER="$SCRIPT_DIR/background-memory-update" -CLAUDE_BIN=$(command -v claude 2>/dev/null || true) - -if [ -z "$CLAUDE_BIN" ]; then - log "SKIP: claude binary not found — worker not spawned (queue intact)" - dbg "EXIT: no claude binary" - exit 0 -fi - -if [ ! -x "$UPDATER" ]; then - log "SKIP: background-memory-update not found or not executable: $UPDATER" - dbg "EXIT: worker missing" - exit 0 -fi - -dbg "Throttle passed: TRIGGER_AGE=${TRIGGER_AGE}s >= 120s — spawning worker" - -# Touch trigger BEFORE spawning (prevents concurrent Stop hooks from double-spawning) -touch "$TRIGGER_FILE" 2>/dev/null || true - -# Spawn detached worker — nohup + disown so it survives the Stop hook process exit -nohup "$UPDATER" "$CWD" >/dev/null 2>&1 & disown - -log "Spawned background-memory-update worker (CWD=$CWD)" -dbg "=== HOOK COMPLETE ===" - -exit 0 diff --git a/scripts/hooks/dream-collect-tasks b/scripts/hooks/dream-collect-tasks deleted file mode 100644 index 6777702a..00000000 --- a/scripts/hooks/dream-collect-tasks +++ /dev/null @@ -1,256 +0,0 @@ -#!/bin/bash -# Dream helper: dream-collect-tasks (sourced helper) -# Collects pending dream task types from .devflow/dream/*.json markers. -# -# Source this file to get the dream_collect_tasks and dream_build_spawn_directive functions. -# -# Function: dream_collect_tasks DREAM_DIR DECISIONS_ENABLED -# -# Inputs (positional args): -# $1 DREAM_DIR — path to .devflow/dream/ -# $2 DECISIONS_ENABLED — "true"/"false" -# -# Outputs (exported variable): -# _DREAM_TASKS — comma-separated, deduped, sorted list of pending task types. -# Empty string if no pending tasks. -# NOTE: "memory" never appears — memory markers are unconditionally swept -# (background-memory-update worker handles refresh from dream-capture). -# -# Behaviour: -# - Skips config.json (reserved). -# - Pass 1: unconditional sweep — deletes learning.* and memory.* markers always -# (both pipelines removed from Dream subagent); deletes disabled decisions -# markers. When decisions is disabled, curation markers are also swept (curation -# depends on decisions data and should not run when decisions is disabled). -# Unknown types pass through unchanged. -# - Pass 1 runs over ALL markers regardless of cap, so no orphan survives by -# being pushed past position 50. -# - COLLECT_LIMIT=50: when >50 kept markers remain after pass 1, orders by -# mtime OLDEST-FIRST (FIFO) and takes the first 50. get_mtime is invoked -# ONLY on this overflow path — zero stat subprocesses for <=50 markers. -# - Markers beyond the cap (overflow path) are RETAINED (never deleted). -# - Requires: get_mtime() (source get-mtime before calling), _HAS_JQ, dbg(), log() -# - Uses JUST_RECOVERED (if set) to preserve .retries files for just-recovered markers. -# -# No set -e: every path exits cleanly. - -# _DREAM_TASKS is the output variable -_DREAM_TASKS="" - -# _derive_marker_type BASENAME — strips .json suffix then returns the component before the first dot. -# Examples: "decisions.abc123.json" → "decisions", "memory.json" → "memory" -# Called from both pass 1 and pass 2 so the naming rule has one source of truth. -_derive_marker_type() { - local _fbn="${1%.json}" - case "$_fbn" in - *.*) printf '%s' "${_fbn%%.*}" ;; - *) printf '%s' "$_fbn" ;; - esac -} - -dream_collect_tasks() { - local dream_dir="${1:?dream_collect_tasks: dream_dir required}" - local dec_enabled="${2:-true}" - - _DREAM_TASKS="" - - # Guard: dream dir must exist - if [ ! -d "$dream_dir" ]; then - dbg "dream_collect_tasks: no dream dir — skip" - return 0 - fi - - # Guard: no .json markers at all — fast exit - if ! compgen -G "$dream_dir/*.json" > /dev/null 2>&1; then - dbg "dream_collect_tasks: no .json markers — skip" - return 0 - fi - - local collect_limit=50 - - # --------------------------------------------------------------------------- - # Pass 1: single scan over all .json markers. - # - Skip config.json. - # - Derive type via parameter expansion (no basename subprocess). - # - Deletion (unconditional sweep — runs regardless of cap): - # learning → rm -f always (orphan sweep, R1) - # memory → rm -f always (no longer a Dream subagent task) - # decisions → rm -f if dec_enabled != true - # curation / unknown → KEEP (pass through; no flag gates them) - # - Kept markers appended to newline-separated `candidates`; count tracked. - # --------------------------------------------------------------------------- - local candidates="" - local count=0 - local _f _base _type _full_basename - - for _f in "$dream_dir"/*.json; do - [ -f "$_f" ] || continue - _base="${_f##*/}" # filename only — no basename subprocess - # Skip config - case "$_base" in config.json) continue ;; esac - # Derive type: "decisions.abc123.json" → "decisions" (via _derive_marker_type) - _type=$(_derive_marker_type "$_base") - - # Deletion sweep (runs for every marker — not capped) - case "$_type" in - learning) - # Learning pipeline removed — delete any orphaned learning markers on sight (R1) - rm -f "$_f" 2>/dev/null || true - dbg "dream_collect_tasks: deleted orphaned learning marker: $_base" - continue - ;; - memory) - # Memory is no longer a Dream subagent task — background-memory-update worker - # handles refresh directly from dream-capture. Delete any stale memory.* markers - # unconditionally (same treatment as learning.*). - rm -f "$_f" 2>/dev/null || true - dbg "dream_collect_tasks: deleted stale memory marker: $_base" - continue - ;; - decisions) - if [ "$dec_enabled" != "true" ]; then - rm -f "$_f" 2>/dev/null || true - dbg "dream_collect_tasks: deleted disabled-feature marker: $_base" - continue - fi - ;; - curation) - # Curation depends on decisions data — sweep when decisions is disabled - # so stray curation markers don't trigger Dream agent spawns when disabled. - if [ "$dec_enabled" != "true" ]; then - rm -f "$_f" 2>/dev/null || true - dbg "dream_collect_tasks: deleted disabled-feature curation marker: $_base" - continue - fi - ;; - # unknown types: pass through unchanged - esac - - # Marker kept — accumulate - if [ -n "$candidates" ]; then - candidates="${candidates} -${_f}" - else - candidates="${_f}" - fi - count=$(( count + 1 )) - done - - if [ "$count" -eq 0 ]; then - dbg "dream_collect_tasks: no enabled markers found" - return 0 - fi - - # --------------------------------------------------------------------------- - # Cap selection: if count <= collect_limit, use all candidates directly. - # get_mtime is invoked ONLY when count > collect_limit (overflow path). - # --------------------------------------------------------------------------- - local selected - if [ "$count" -le "$collect_limit" ]; then - selected="$candidates" - else - # Overflow: compute mtime per candidate, sort oldest-first, take first 50. - # Format: MTIMEFILEPATH → sort -k1,1n → head -n 50 → strip mtime field. - local mtime_list="" _mt - while IFS= read -r _f; do - [ -f "$_f" ] || continue - _mt=$(get_mtime "$_f" 2>/dev/null || true) - _mt="${_mt:-0}" - if [ -n "$mtime_list" ]; then - mtime_list="${mtime_list} -${_mt} ${_f}" - else - mtime_list="${_mt} ${_f}" - fi - done </dev/null || true ;; - esac - - # Accumulate type - if [ -n "$raw_tasks" ]; then - raw_tasks="${raw_tasks} -${_sel_type}" - else - raw_tasks="${_sel_type}" - fi - done </dev/null || printf '%s\n' "$raw_tasks" | sort -u | tr '\n' ',' | sed 's/,$//') - dbg "dream_collect_tasks: _DREAM_TASKS=$_DREAM_TASKS" - return 0 -} - -# _DREAM_DIRECTIVE is the output variable for dream_build_spawn_directive -_DREAM_DIRECTIVE="" - -# dream_build_spawn_directive TASKS -# Builds the DREAM MAINTENANCE per-task spawn directive for a comma-separated -# task list and assigns it to the global _DREAM_DIRECTIVE (empty if no known -# task types are present). Communicates via a global — mirroring the -# dream_collect_tasks → _DREAM_TASKS contract — rather than stdout, so the -# exact directive bytes (including trailing newlines) survive intact; command -# substitution would strip them. -# -# Hardcoded task→model map: decisions=opus, curation=opus. -# decisions+curation co-pending → exactly ONE opus spawn that -# runs decisions THEN curation sequentially (prevents .decisions.lock contention). -# Unknown task types are skipped (belt-and-suspenders — dream_collect_tasks -# should never emit them). -dream_build_spawn_directive() { - local _dbsd_tasks="$1" - _DREAM_DIRECTIVE="" - - local _dbsd_dec="false" _dbsd_cur="false" - case ",$_dbsd_tasks," in *,decisions,*) _dbsd_dec="true" ;; esac - case ",$_dbsd_tasks," in *,curation,*) _dbsd_cur="true" ;; esac - - local _dbsd_lines="" - if [ "$_dbsd_dec" = "true" ] && [ "$_dbsd_cur" = "true" ]; then - _dbsd_lines="${_dbsd_lines}Agent(subagent_type=\"Dream\", model=\"opus\", run_in_background: true, prompt: \"Process pending decisions and curation marker(s): claim each per your plumbing, then load devflow:dream-decisions and follow it, THEN load devflow:dream-curation and follow it (sequentially, never concurrently).\") -" - elif [ "$_dbsd_dec" = "true" ]; then - _dbsd_lines="${_dbsd_lines}Agent(subagent_type=\"Dream\", model=\"opus\", run_in_background: true, prompt: \"Process pending 'decisions' marker(s): claim per your plumbing, then load devflow:dream-decisions and follow it.\") -" - elif [ "$_dbsd_cur" = "true" ]; then - _dbsd_lines="${_dbsd_lines}Agent(subagent_type=\"Dream\", model=\"opus\", run_in_background: true, prompt: \"Process pending 'curation' marker(s): claim per your plumbing, then load devflow:dream-curation and follow it.\") -" - fi - - # No known task types → leave _DREAM_DIRECTIVE empty so the caller skips emission. - [ -z "$_dbsd_lines" ] && return 0 - - _DREAM_DIRECTIVE="--- DREAM MAINTENANCE (pending from previous session) --- -Pending background maintenance: ${_dbsd_tasks}. -Spawn the following background agents (one non-blocking call each, in order). Do not narrate, summarize, or ask — just spawn them, then continue. -${_dbsd_lines}" - return 0 -} diff --git a/scripts/hooks/dream-dispatch b/scripts/hooks/dream-dispatch deleted file mode 100755 index 364e8265..00000000 --- a/scripts/hooks/dream-dispatch +++ /dev/null @@ -1,85 +0,0 @@ -#!/bin/bash - -# Dream System: dream-dispatch (UserPromptSubmit Hook) -# Capture-only: appends the user turn to .devflow/memory/.pending-turns.jsonl. -# Marker collection and directive emission have moved to session-start-context (Section 2). - -# Safe no-op fallback: must exist before set -e and before hook-bootstrap is sourced. -dbg() { :; } - -set -e - -# Feedback-loop guards — before hook-bootstrap to minimize background session overhead -if [ "${DEVFLOW_BG_UPDATER:-}" = "1" ]; then dbg "EXIT: bg_updater"; exit 0; fi -if [ "${DEVFLOW_BG_LEARNER:-}" = "1" ]; then dbg "EXIT: bg_learner"; exit 0; fi -if [ "${DEVFLOW_BG_KNOWLEDGE_REFRESH:-}" = "1" ]; then dbg "EXIT: bg_knowledge"; exit 0; fi - -SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" - -source "$SCRIPT_DIR/hook-bootstrap" "dream-dispatch" - -source "$SCRIPT_DIR/json-parse" || { echo "dream-dispatch: failed to source json-parse" >&2; exit 1; } -if [ "$_JSON_AVAILABLE" = "false" ]; then dbg "EXIT: no json"; exit 0; fi - -INPUT=$(cat) - -FIELDS=$(printf '%s' "$INPUT" | json_extract_cwd_prompt) -CWD="${FIELDS%%$'\001'*}" -PROMPT="${FIELDS#*$'\001'}" - -dbg "CWD=$CWD PROMPT_LENGTH=${#PROMPT}" - -if [ -z "$CWD" ] || [ ! -d "$CWD" ]; then dbg "EXIT: bad CWD"; exit 0; fi - -devflow_debug_set_cwd "$CWD" - -# Anchor .devflow/ to the project root (prevents a stray nested .devflow/ when this -# hook runs with a CWD inside .devflow/...). Empty → fall back to CWD (old behavior). -source "$SCRIPT_DIR/resolve-project-root" 2>/dev/null || true -PROJECT_ROOT="$(df_resolve_root "$CWD" 2>/dev/null || true)" -[ -n "$PROJECT_ROOT" ] || PROJECT_ROOT="$CWD" - -DEVFLOW_DIR="$PROJECT_ROOT/.devflow" -MEMORY_DIR="$DEVFLOW_DIR/memory" -DREAM_DIR="$DEVFLOW_DIR/dream" - -# Normal logging -source "$SCRIPT_DIR/hook-log-init" "dream-dispatch" - -# Read dream config — memory:false skips user turn capture -DREAM_CONFIG="$DREAM_DIR/config.json" -MEMORY_ENABLED="true" -if [ -f "$DREAM_CONFIG" ]; then - MEMORY_ENABLED=$(json_field_file "$DREAM_CONFIG" "memory" "true") -fi - -dbg "memory=$MEMORY_ENABLED" - -# --- Capture user turn --- -if [ "$MEMORY_ENABLED" = "true" ] && [ -n "$PROMPT" ]; then - source "$SCRIPT_DIR/ensure-devflow-init" "$CWD" || exit 0 - - # Truncate to 2000 chars - if [ ${#PROMPT} -gt 2000 ]; then - PROMPT="${PROMPT:0:2000}... [truncated]" - fi - - QUEUE_FILE="$MEMORY_DIR/.pending-turns.jsonl" - - # Ensure queue file is created with restricted permissions (600) before first write. - if [ ! -f "$QUEUE_FILE" ]; then - (umask 077 && touch "$QUEUE_FILE") 2>/dev/null || true - fi - - TS=$(date +%s) - if [ "$_HAS_JQ" = "true" ]; then - jq -n -c --arg role "user" --arg content "$PROMPT" --argjson ts "$TS" \ - '{role: $role, content: $content, ts: $ts}' >> "$QUEUE_FILE" - else - node -e "process.stdout.write(JSON.stringify({role:'user',content:process.argv[1],ts:parseInt(process.argv[2])})+'\n')" \ - -- "$PROMPT" "$TS" >> "$QUEUE_FILE" - fi -fi - -dbg "=== HOOK COMPLETE ===" -exit 0 diff --git a/scripts/hooks/dream-evaluate b/scripts/hooks/dream-evaluate deleted file mode 100755 index 1ba11196..00000000 --- a/scripts/hooks/dream-evaluate +++ /dev/null @@ -1,141 +0,0 @@ -#!/bin/bash - -# Dream System: dream-evaluate (SessionEnd Hook) -# Evaluates session-level features: decisions detection and curation. -# Writes marker files to .devflow/dream/ for dream-dispatch to pick up. -# -# Orchestrator: sources eval-helpers + 2 feature modules after shared setup. - -# Safe no-op fallback: must exist before set -e and before hook-bootstrap is sourced. -dbg() { :; } - -set -e - -# Feedback-loop guards — before hook-bootstrap to minimize background session overhead -# Order: UPDATER, LEARNER, KNOWLEDGE — consistent with dream-capture and dream-dispatch. -# dbg() is the pre-bootstrap no-op here; annotations are safe but silent without debug active. -if [ "${DEVFLOW_BG_UPDATER:-}" = "1" ]; then dbg "EXIT: bg_updater"; exit 0; fi -if [ "${DEVFLOW_BG_LEARNER:-}" = "1" ]; then dbg "EXIT: bg_learner"; exit 0; fi -if [ "${DEVFLOW_BG_KNOWLEDGE_REFRESH:-}" = "1" ]; then dbg "EXIT: bg_knowledge"; exit 0; fi - -SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" - -source "$SCRIPT_DIR/hook-bootstrap" "dream-evaluate" - -source "$SCRIPT_DIR/json-parse" || { echo "dream-evaluate: failed to source json-parse" >&2; exit 1; } -if [ "$_JSON_AVAILABLE" = "false" ]; then dbg "EXIT: no json"; exit 0; fi - -INPUT=$(cat) - -CWD=$(printf '%s' "$INPUT" | json_field "cwd" "") -if [ -z "$CWD" ] || [ ! -d "$CWD" ]; then dbg "EXIT: bad CWD"; exit 0; fi - -devflow_debug_set_cwd "$CWD" -dbg "CWD=$CWD" - -# Anchor .devflow/ to the project root (prevents a stray nested .devflow/ when this -# hook runs with a CWD inside .devflow/...). Empty → fall back to CWD (old behavior). -source "$SCRIPT_DIR/resolve-project-root" 2>/dev/null || true -PROJECT_ROOT="$(df_resolve_root "$CWD" 2>/dev/null || true)" -[ -n "$PROJECT_ROOT" ] || PROJECT_ROOT="$CWD" - -DEVFLOW_DIR="$PROJECT_ROOT/.devflow" -[ ! -d "$DEVFLOW_DIR" ] && exit 0 - -MEMORY_DIR="$DEVFLOW_DIR/memory" -DREAM_DIR="$DEVFLOW_DIR/dream" -DECISIONS_DIR_DATA="$DEVFLOW_DIR/decisions" - -# Read dream config -DREAM_CONFIG="$DREAM_DIR/config.json" -DECISIONS_ENABLED="true" - -if [ -f "$DREAM_CONFIG" ]; then - DECISIONS_ENABLED=$(json_field_file "$DREAM_CONFIG" "decisions" "true") -fi - -# Dual-signal gate: config OR sentinel disables decisions (and transitively curation). -# This ensures session-start-context and dream-evaluate honour the same signal so -# stray markers are never written when decisions is disabled via the sentinel alone. -[ -f "$DECISIONS_DIR_DATA/.disabled" ] && DECISIONS_ENABLED="false" - -# Log + dream-lock -source "$SCRIPT_DIR/hook-log-init" "dream-evaluate" -source "$SCRIPT_DIR/dream-lock" || { log "failed to source dream-lock"; exit 1; } - -log "SessionEnd hook triggered" -dbg "DECISIONS=$DECISIONS_ENABLED" - -# --- Find transcript --- -ENCODED_CWD=$(echo "$CWD" | sed 's|^/||' | tr '/' '-') -PROJECTS_DIR="$HOME/.claude/projects/-${ENCODED_CWD}" - -SESSION_ID=$(printf '%s' "$INPUT" | json_field "session_id" "") -dbg "SESSION_ID=$SESSION_ID" -TRANSCRIPT="" - -if [ -d "$PROJECTS_DIR" ]; then - # Validate session_id before use in path — guard against path traversal (../ sequences) - if [ -n "$SESSION_ID" ] && echo "$SESSION_ID" | grep -qE '^[a-zA-Z0-9_-]+$' && [ -f "$PROJECTS_DIR/${SESSION_ID}.jsonl" ]; then - TRANSCRIPT="$PROJECTS_DIR/${SESSION_ID}.jsonl" - else - # Fallback: sort all session JSONL files by mtime and pick the newest. - # Assumption: PROJECTS_DIR contains O(hundreds) of files at most — bounded by - # Claude Code's own session retention policy, so ls -t overhead is acceptable. - TRANSCRIPT=$(ls -t "$PROJECTS_DIR"/*.jsonl 2>/dev/null | head -1 || true) - [ -z "$TRANSCRIPT" ] || [ ! -f "$TRANSCRIPT" ] && TRANSCRIPT="" - [ -n "$TRANSCRIPT" ] && SESSION_ID=$(basename "$TRANSCRIPT" .jsonl) - fi -fi - -# Session depth check (min 3 user turns) -SESSION_DEEP="false" -if [ -n "$TRANSCRIPT" ] && [ -f "$TRANSCRIPT" ]; then - # Use jq for accurate top-level type matching; grep may over-count when the literal appears in content - if [ "$_HAS_JQ" = "true" ]; then - USER_TURNS=$(jq -c 'select(.type == "user")' "$TRANSCRIPT" 2>/dev/null | wc -l | tr -d ' ') - else - USER_TURNS=$(grep -c '"type":"user"' "$TRANSCRIPT" 2>/dev/null || echo "0") - fi - if [ "$USER_TURNS" -ge 3 ]; then - SESSION_DEEP="true" - log "Session depth: $USER_TURNS turns" - dbg "SESSION_DEEP=true USER_TURNS=$USER_TURNS TRANSCRIPT=$TRANSCRIPT" - else - log "Shallow session ($USER_TURNS turns)" - dbg "SESSION_DEEP=false USER_TURNS=$USER_TURNS" - fi -else - log "No transcript found" - dbg "No transcript found" -fi - -mkdir -p "$DREAM_DIR" - -NOW=$(date +%s) -TODAY=$(date '+%Y-%m-%d') -FILTER_LIB="$SCRIPT_DIR/lib/transcript-filter.cjs" -MARKER_SUFFIX="${SESSION_ID:-$$}" - -# --- Feature modules --- -# -# Orchestrator-exported contract: the following shell variables are set before -# eval-* modules are sourced and are consumed by one or more of them. -# NOW — epoch seconds (date +%s) -# TODAY — YYYY-MM-DD (date '+%Y-%m-%d') -# SESSION_ID — transcript session identifier (string) -# TRANSCRIPT — absolute path to session JSONL file (may be empty) -# SESSION_DEEP — "true" if session has ≥3 user turns -# FILTER_LIB — absolute path to transcript-filter.cjs -# MARKER_SUFFIX — suffix for per-session marker filenames (SESSION_ID or $$) -# DREAM_DIR — absolute path to .devflow/dream/ -# DECISIONS_DIR_DATA — absolute path to .devflow/decisions/ -# DECISIONS_ENABLED — "true"/"false" -# _HAS_JQ — "true" if jq is available on PATH -# -source "$SCRIPT_DIR/eval-helpers" -source "$SCRIPT_DIR/eval-decisions" -source "$SCRIPT_DIR/eval-curation" - -dbg "=== HOOK COMPLETE ===" -exit 0 diff --git a/scripts/hooks/dream-recover b/scripts/hooks/dream-recover deleted file mode 100644 index 03058c82..00000000 --- a/scripts/hooks/dream-recover +++ /dev/null @@ -1,185 +0,0 @@ -#!/bin/bash -# Dream helper: dream-recover (sourced helper) -# Recovers stale .processing markers back to .json for retry. -# -# Source this file to get the dream_recover_stale function. -# -# Function: dream_recover_stale DREAM_DIR MEMORY_DIR -# -# Inputs (positional args): -# $1 DREAM_DIR — path to .devflow/dream/ -# $2 MEMORY_DIR — path to .devflow/memory/ (for pending-turns.processing recovery) -# -# Outputs (exported variable): -# JUST_RECOVERED — space-prefixed space-separated list of full basenames (without .processing) -# that were recovered in this invocation. -# Consumed by dream_collect_tasks to preserve .retries files. -# -# Behaviour: -# - PROC_LIMIT=10: cap on .processing files examined per run. -# - MAX_RETRIES=3: on 3rd failed attempt, rename to .failed (do NOT delete). -# - .failed cleanup only when older than 24h. -# - JUST_RECOVERED guard: recovered markers have their .retries preserved (not reset). -# - Per-type stale thresholds: -# memory = 300s (5 minutes) -# decisions = 1800s (30 minutes) -# unknown = 1800s (default) -# - Also recovers orphaned .pending-turns.processing: -# If MEMORY_DIR/.pending-turns.processing exists and is older than 300s, -# rename it back to .pending-turns.jsonl ONLY if .pending-turns.jsonl -# does not already exist. If .jsonl already exists, leave .processing -# for manual/next-run handling (non-destructive). -# - Requires: get_mtime() (source get-mtime before calling), dbg(), log() -# - No set -e: every path exits cleanly. -# -# D56a: Per-type stale thresholds. -# memory = 300s (5 min) because the memory agent is fast — if it is still -# running after 5 minutes the session almost certainly crashed. All other -# types (decisions, curation) run the LLM and may take up to 30 minutes on -# large logs or slow models; 1800s avoids yanking an actively-running Dream agent. -# -# D56b: JUST_RECOVERED guard. -# When dream_recover_stale renames a .processing back to .json, it records -# the basename in JUST_RECOVERED. dream_collect_tasks reads this list and -# preserves the existing .retries counter for recovered markers instead of -# resetting it. Without this guard, a marker that has failed 2/3 times would -# have its counter silently reset to 0 on recovery, defeating MAX_RETRIES. -# -# D56c: Orphaned .pending-turns.processing non-clobber recovery. -# The memory agent renames .pending-turns.jsonl → .pending-turns.processing -# as an atomic claim. If the session crashes mid-processing, .jsonl is gone -# and .processing is orphaned. Recovery renames it back ONLY when .jsonl does -# not already exist — this prevents clobbering a fresh .jsonl that a concurrent -# session may have written while the prior session's .processing was stale. - -# JUST_RECOVERED is the output variable — consumed by dream_collect_tasks -JUST_RECOVERED="" - -# _dream_stale_threshold TYPE → echoes threshold in seconds for that marker type -_dream_stale_threshold() { - case "${1:-}" in - memory) echo "300" ;; - *) echo "1800" ;; - esac -} - -dream_recover_stale() { - local dream_dir="${1:?dream_recover_stale: dream_dir required}" - local memory_dir="${2:?dream_recover_stale: memory_dir required}" - - JUST_RECOVERED="" - - local now - now=$(date +%s) - - # --- Clean up .failed markers older than 24h --- - if compgen -G "$dream_dir/*.failed" > /dev/null 2>&1; then - for _failed_file in "$dream_dir"/*.failed; do - [ -f "$_failed_file" ] || continue - local _failed_mtime - _failed_mtime=$(get_mtime "$_failed_file" 2>/dev/null || true) - _failed_mtime="${_failed_mtime:-0}" - local _failed_age=$(( now - _failed_mtime )) - if [ "$_failed_age" -gt 86400 ]; then - rm -f "$_failed_file" 2>/dev/null || true - dbg "dream_recover_stale: removed old .failed: $(basename "$_failed_file")" - fi - done - fi - - # --- Recover stale .processing markers --- - if ! compgen -G "$dream_dir/*.processing" > /dev/null 2>&1; then - dbg "dream_recover_stale: no .processing markers" - else - local proc_count=0 - local proc_limit=10 - local max_retries=3 - for _proc_file in "$dream_dir"/*.processing; do - [ -f "$_proc_file" ] || continue - proc_count=$(( proc_count + 1 )) - [ "$proc_count" -gt "$proc_limit" ] && break - - local _proc_mtime - _proc_mtime=$(get_mtime "$_proc_file" 2>/dev/null || true) - _proc_mtime="${_proc_mtime:-0}" - local _proc_age=$(( now - _proc_mtime )) - - # Derive type from filename: {type}.{session}.processing - local _proc_basename - _proc_basename=$(basename "$_proc_file" .processing) - local _proc_type - case "$_proc_basename" in - *.*) _proc_type="${_proc_basename%%.*}" ;; - *) _proc_type="$_proc_basename" ;; - esac - - local _threshold - _threshold=$(_dream_stale_threshold "$_proc_type") - - if [ "$_proc_age" -gt "$_threshold" ]; then - local _base="${_proc_file%.processing}" - local _retry_file="${_base}.retries" - local _retry_count=0 - if [ -f "$_retry_file" ]; then - _retry_count=$(cat "$_retry_file" 2>/dev/null | tr -dc '0-9' || true) - _retry_count="${_retry_count:-0}" - fi - - if [ "$_retry_count" -ge "$max_retries" ]; then - # Too many retries — mark as permanently failed - mv "$_proc_file" "${_base}.failed" 2>/dev/null || true - rm -f "$_retry_file" 2>/dev/null || true - dbg "dream_recover_stale: permanently failed after $_retry_count retries: $(basename "$_base")" - log "Stale marker permanently failed after $_retry_count retries: $(basename "$_base")" - else - # Bump retry counter and recover back to .json - echo $(( _retry_count + 1 )) > "$_retry_file" - mv "$_proc_file" "${_base}.json" 2>/dev/null || true - JUST_RECOVERED="${JUST_RECOVERED} ${_proc_basename}" - dbg "dream_recover_stale: recovered (retry $(( _retry_count + 1 ))/$max_retries): $(basename "$_base")" - log "Stale marker recovered (retry $(( _retry_count + 1 ))/$max_retries): $(basename "$_base")" - fi - fi - done - fi - - # --- Recover orphaned .pending-turns.processing --- - local _pt_proc="$memory_dir/.pending-turns.processing" - local _pt_jsonl="$memory_dir/.pending-turns.jsonl" - if [ -f "$_pt_proc" ]; then - local _pt_mtime - _pt_mtime=$(get_mtime "$_pt_proc" 2>/dev/null || true) - _pt_mtime="${_pt_mtime:-0}" - local _pt_age=$(( now - _pt_mtime )) - local _pt_threshold - _pt_threshold=$(_dream_stale_threshold "memory") - if [ "$_pt_age" -gt "$_pt_threshold" ]; then - # Cold-path fallback: recovery is intentionally lock-free (no .working-memory.lock - # acquisition here). The worker (background-memory-update) is the PRIMARY recovery - # owner: it runs under .working-memory.lock and re-merges a leftover .processing - # into its own batch. This fallback fires only when the worker never re-spawned - # (i.e., SessionStart arrives >300s after the crash with no intervening Stop hook). - # - # Safe-by-construction: STALE_THRESHOLD (300s) strictly exceeds the worker's - # watchdog total (WATCHDOG_SECS 120s + WATCHDOG_KILL_GRACE_SECS 5s + margin ≈ 125s). - # By the time we can act, any live worker's claude -p has already been watchdog-killed - # and the worker has exited — the .working-memory.lock is gone. - # - # INVARIANT: if STALE_THRESHOLD is ever raised to ≥ worker-watchdog-total, the - # lock-free assumption breaks and a race window opens between this recovery and - # the worker's merge block. Preserve STALE_THRESHOLD > watchdog-total at all times. - if [ ! -f "$_pt_jsonl" ]; then - # Safe to recover — no .jsonl exists to be clobbered - mv "$_pt_proc" "$_pt_jsonl" 2>/dev/null || true - dbg "dream_recover_stale: recovered orphaned .pending-turns.processing → .pending-turns.jsonl" - log "Recovered orphaned .pending-turns.processing → .pending-turns.jsonl" - else - # .jsonl already exists — leave .processing for manual/next-run handling - dbg "dream_recover_stale: .pending-turns.processing stale but .pending-turns.jsonl exists — leaving for next run" - log "Stale .pending-turns.processing skipped: .pending-turns.jsonl already exists" - fi - fi - fi - - return 0 -} diff --git a/scripts/hooks/eval-curation b/scripts/hooks/eval-curation deleted file mode 100644 index 663abc1e..00000000 --- a/scripts/hooks/eval-curation +++ /dev/null @@ -1,81 +0,0 @@ -#!/bin/bash -# dream-evaluate: Curation evaluation module. -# Writes a throttled curation marker at most once per 7 days. -# Curation is gated by DECISIONS_ENABLED — when decisions is disabled (via -# dream config or .disabled sentinel), curation is also skipped so that -# disabling decisions does not stamp .curation-last and cause a 7-day -# suppression window after re-enable. -# Source from dream-evaluate orchestrator after eval-helpers. -# -# Requires (from orchestrator namespace): -# DECISIONS_ENABLED, DREAM_DIR, NOW, MARKER_SUFFIX, -# _HAS_JQ, log(), dbg() -# -# D58: 7-day curation throttle. -# Curation (deprecating/merging ADR/PF entries) is expensive LLM work that -# produces at most 5 changes per run. Running it every session would burn -# tokens without meaningful benefit — decisions files rarely need pruning more -# than once a week. The 7-day (604800s) throttle is stored in .curation-last -# (epoch timestamp, atomic write via temp+mv). The throttle is written -# OPTIMISTICALLY at marker-write time (before the LLM actually runs the -# curation), so subsequent sessions in the same window are skipped without -# needing to check whether the LLM task completed. If the LLM task fails, the -# next window will pick it up. This is intentional: a failed curation is less -# costly than repeatedly spawning curation work in a retry storm. - -: "${DECISIONS_ENABLED:?eval-curation: DECISIONS_ENABLED must be set by orchestrator}" -: "${DREAM_DIR:?eval-curation: DREAM_DIR must be set by orchestrator}" -: "${NOW:?eval-curation: NOW must be set by orchestrator}" -: "${MARKER_SUFFIX:?eval-curation: MARKER_SUFFIX must be set by orchestrator}" -: "${_HAS_JQ:?eval-curation: _HAS_JQ must be set by orchestrator}" - -# Gate: curation inherits decisions enabled-state. When decisions is disabled, -# skip immediately — BEFORE reading or writing .curation-last — so disabling -# decisions does not burn a 7-day suppression window. -# Use return (not exit): this file is sourced under set -e in dream-evaluate; -# exit would kill the whole orchestrator process. -if [ "$DECISIONS_ENABLED" != "true" ]; then - log "Curation skipped: decisions disabled" - dbg "eval-curation: DECISIONS_ENABLED=$DECISIONS_ENABLED — returning" - return 0 -fi - -# Throttle: at most once per 7 days (604800 seconds). -_CUR_THROTTLE=604800 -_CUR_LAST_FILE="$DREAM_DIR/.curation-last" - -_CUR_LAST=0 -if [ -f "$_CUR_LAST_FILE" ]; then - _CUR_LAST=$(tr -dc '0-9' < "$_CUR_LAST_FILE" 2>/dev/null || true) - _CUR_LAST="${_CUR_LAST:-0}" -fi -_CUR_AGE=$(( NOW - _CUR_LAST )) - -dbg "Curation: AGE=${_CUR_AGE}s (throttle=${_CUR_THROTTLE}s)" - -if [ "$_CUR_AGE" -lt "$_CUR_THROTTLE" ]; then - log "Curation throttled: last run ${_CUR_AGE}s ago" - dbg "Curation throttled: AGE=${_CUR_AGE}s" -else - # Write curation marker (per-session name for concurrency safety) - _CUR_MARKER="$DREAM_DIR/curation.${MARKER_SUFFIX}.json" - _CUR_TMP="${_CUR_MARKER}.tmp.$$" - - if [ "$_HAS_JQ" = "true" ]; then - jq -n \ - --argjson timestamp "$NOW" \ - '{timestamp: $timestamp}' \ - > "$_CUR_TMP" && mv "$_CUR_TMP" "$_CUR_MARKER" || rm -f "$_CUR_TMP" - else - node -e " - process.stdout.write(JSON.stringify({timestamp: parseInt(process.argv[1])}) + '\n') - " -- "$NOW" > "$_CUR_TMP" && mv "$_CUR_TMP" "$_CUR_MARKER" || rm -f "$_CUR_TMP" - fi - - # Optimistically update throttle marker so subsequent sessions in the same 7-day - # window are skipped — the dream agent does the actual curation work. - printf '%s' "$NOW" > "$_CUR_LAST_FILE" 2>/dev/null || true - - log "Wrote curation marker" - dbg "Wrote curation marker: $_CUR_MARKER" -fi diff --git a/scripts/hooks/eval-decisions b/scripts/hooks/eval-decisions deleted file mode 100644 index f033c19c..00000000 --- a/scripts/hooks/eval-decisions +++ /dev/null @@ -1,79 +0,0 @@ -#!/bin/bash -# dream-evaluate: Decisions evaluation module. -# Extracts dialog pairs from transcript and writes decisions marker. -# Source from dream-evaluate orchestrator after eval-helpers. -# -# Requires (from orchestrator namespace): -# DECISIONS_ENABLED, SESSION_DEEP, DREAM_DIR, DECISIONS_DIR_DATA, -# FILTER_LIB, TRANSCRIPT, NOW, TODAY, MARKER_SUFFIX, -# _HAS_JQ, json_field_file(), log(), dbg(), -# read_daily_cap(), atomic_increment_daily(), load_existing_ids() - -: "${DREAM_DIR:?eval-decisions: DREAM_DIR must be set by orchestrator}" -: "${DECISIONS_DIR_DATA:?eval-decisions: DECISIONS_DIR_DATA must be set by orchestrator}" -: "${NOW:?eval-decisions: NOW must be set by orchestrator}" -: "${TODAY:?eval-decisions: TODAY must be set by orchestrator}" -: "${MARKER_SUFFIX:?eval-decisions: MARKER_SUFFIX must be set by orchestrator}" -: "${_HAS_JQ:?eval-decisions: _HAS_JQ must be set by orchestrator}" - -if [ "$DECISIONS_ENABLED" = "true" ] && [ "$SESSION_DEEP" = "true" ]; then - log "Evaluating decisions..." - - # Daily cap check - _DEC_RUNS_FILE="$DREAM_DIR/.decisions-runs-today" - _DEC_RUNS_TODAY=$(read_daily_cap "$_DEC_RUNS_FILE" 0) - - _DEC_MAX_RUNS=3 - _DEC_CONFIG="$DECISIONS_DIR_DATA/decisions.json" - if [ -f "$_DEC_CONFIG" ]; then - _DEC_MAX_RUNS=$(json_field_file "$_DEC_CONFIG" "max_daily_runs" "3") - fi - _DEC_MAX_RUNS=$(echo "$_DEC_MAX_RUNS" | tr -dc '0-9') - _DEC_MAX_RUNS="${_DEC_MAX_RUNS:-3}" - - dbg "Decisions: RUNS_TODAY=$_DEC_RUNS_TODAY MAX_RUNS=$_DEC_MAX_RUNS" - if [ "$_DEC_RUNS_TODAY" -ge "$_DEC_MAX_RUNS" ]; then - log "Decisions daily cap reached ($_DEC_RUNS_TODAY/$_DEC_MAX_RUNS)" - dbg "Decisions daily cap reached — skipping" - else - # Extract dialog pairs from transcript - _DEC_DIALOG_PAIRS="" - if [ -f "$FILTER_LIB" ] && [ -n "$TRANSCRIPT" ] && [ -f "$TRANSCRIPT" ]; then - _DEC_DIALOG_PAIRS=$(node "$FILTER_LIB" dialog-pairs "$TRANSCRIPT" 2>/dev/null || true) - fi - - if [ -n "$_DEC_DIALOG_PAIRS" ]; then - # Load existing observation IDs - _DEC_EXISTING_IDS=$(load_existing_ids "$DECISIONS_DIR_DATA/decisions-log.jsonl") - - # Write decisions marker (per-session name for concurrency safety) - _DEC_MARKER="$DREAM_DIR/decisions.${MARKER_SUFFIX}.json" - _DEC_TMP="${_DEC_MARKER}.tmp.$$" - if [ "$_HAS_JQ" = "true" ]; then - jq -n \ - --arg dialogPairs "$_DEC_DIALOG_PAIRS" \ - --argjson existingObservationIds "$_DEC_EXISTING_IDS" \ - --argjson timestamp "$NOW" \ - '{dialogPairs: $dialogPairs, existingObservationIds: $existingObservationIds, timestamp: $timestamp}' \ - > "$_DEC_TMP" && mv "$_DEC_TMP" "$_DEC_MARKER" || rm -f "$_DEC_TMP" - else - node -e " - process.stdout.write(JSON.stringify({ - dialogPairs: process.argv[1], - existingObservationIds: JSON.parse(process.argv[2]), - timestamp: parseInt(process.argv[3]) - }) + '\n') - " -- "$_DEC_DIALOG_PAIRS" "$_DEC_EXISTING_IDS" "$NOW" > "$_DEC_TMP" && mv "$_DEC_TMP" "$_DEC_MARKER" || rm -f "$_DEC_TMP" - fi - - # Update daily cap - atomic_increment_daily "$_DEC_RUNS_FILE" "$TODAY" - - log "Wrote decisions marker" - dbg "Wrote decisions marker: $_DEC_MARKER" - else - log "Decisions: no dialog pairs extracted" - dbg "Decisions: no dialog pairs in transcript" - fi - fi -fi diff --git a/scripts/hooks/eval-helpers b/scripts/hooks/eval-helpers deleted file mode 100644 index a7be3dda..00000000 --- a/scripts/hooks/eval-helpers +++ /dev/null @@ -1,70 +0,0 @@ -#!/bin/bash -# Shared utility functions for dream-evaluate feature modules. -# Source this file from the dream-evaluate orchestrator before sourcing eval-* modules. -# -# Exports: read_daily_cap(), atomic_increment_daily(), load_existing_ids(), -# _eval_release_lock() -# -# Requires: $TODAY, dream_lock_acquire/dream_lock_release (from dream-lock), -# $_HAS_JQ, json_field_file (from json-parse). - -# read_daily_cap FILE DEFAULT → echoes today's run count (0 if file absent or dated yesterday) -read_daily_cap() { - local runs_file="$1" default="${2:-0}" - local count=0 - if [ -f "$runs_file" ]; then - local date_field - date_field=$(cut -f1 "$runs_file" 2>/dev/null || true) - if [ "$date_field" = "$TODAY" ]; then - if grep -q "$(printf '\t')" "$runs_file" 2>/dev/null; then - count=$(cut -f2 "$runs_file" 2>/dev/null | tr -dc '0-9') - count="${count:-$default}" - fi - fi - fi - echo "$count" -} - -# atomic_increment_daily RUNS_FILE TODAY_VAL — increment daily counter under lock -atomic_increment_daily() { - local runs_file="$1" today_val="$2" - local lock_dir="${runs_file}.lock" - if dream_lock_acquire "$lock_dir" 3; then - # Re-read under lock for accurate count - local actual - actual=$(read_daily_cap "$runs_file" 0) - printf '%s\t%d\n' "$today_val" "$((actual + 1))" > "$runs_file" - dream_lock_release "$lock_dir" - fi -} - -# load_existing_ids LOG_FILE → echoes JSON array of .id values ([] when absent/empty) -load_existing_ids() { - local log_file="$1" - if [ ! -f "$log_file" ]; then - echo "[]" - return - fi - if [ "$_HAS_JQ" = "true" ]; then - # Stream line-by-line then collect — avoids slurping entire file into memory - jq -c '.id // empty' "$log_file" 2>/dev/null | jq -s '.' 2>/dev/null || echo "[]" - else - node -e " - const readline=require('readline'); - const fs=require('fs'); - const rl=readline.createInterface({input:fs.createReadStream(process.argv[1]),crlfDelay:Infinity}); - const ids=[]; - rl.on('line',l=>{if(!l.trim())return;try{const id=JSON.parse(l).id;if(id)ids.push(id);}catch{}}); - rl.on('close',()=>process.stdout.write(JSON.stringify(ids))); - " -- "$log_file" 2>/dev/null || echo "[]" - fi -} - -# _eval_release_lock LOCK_DIR — shared EXIT trap target for eval-* modules. -# Each module sets: trap '_eval_release_lock "$DREAM_DIR/..lock"' EXIT -# and clears with: trap - EXIT in its normal path. -# Using a shared function prevents the second sourced module from silently -# overwriting the first module's lock-specific trap string. -_eval_release_lock() { - dream_lock_release "${1:-}" -} diff --git a/scripts/hooks/json-helper.cjs b/scripts/hooks/json-helper.cjs index 22d84040..a495fda4 100755 --- a/scripts/hooks/json-helper.cjs +++ b/scripts/hooks/json-helper.cjs @@ -26,7 +26,6 @@ // backup-construct Build pre-compact backup JSON from --arg pairs // merge-observation Reinforce existing observation by id (D14) // decisions-usage-scan Scan session context for ADR/PF cite counts -// read-dream Read field from dream JSON (allowed fields only; returns [] on any error) 'use strict'; @@ -328,14 +327,6 @@ function parseArgs(argList) { if (require.main === module) { try { - // Route to domain modules first; fall through to the main switch if not handled. - if (op === 'read-dream') { - const dreamOps = require('./lib/dream-ops.cjs'); - if (dreamOps.handle(op, args, process.cwd())) { - process.exit(0); - } - } - switch (op) { case 'get-field': { const input = JSON.parse(readStdin()); diff --git a/scripts/hooks/lib/dream-ops.cjs b/scripts/hooks/lib/dream-ops.cjs deleted file mode 100644 index e2cfc20d..00000000 --- a/scripts/hooks/lib/dream-ops.cjs +++ /dev/null @@ -1,54 +0,0 @@ -'use strict'; - -const fs = require('fs'); -const { safePath } = require('./safe-path.cjs'); - -const ALLOWED_FIELDS = new Set(['referencedFiles', 'description']); - -/** - * Handle dream-related operations. - * Returns true if the command was handled, false otherwise. - * - * read-dream : - * Only fields in ALLOWED_FIELDS are permitted. - * Array fields → JSON-stringified array (string elements only) - * String fields → raw string value - * Other/missing/disallowed → '[]' - * - * @param {string} command - * @param {string[]} args - * @param {string} cwd — working directory used as safePath root - * @returns {boolean} - */ -function handle(command, args, cwd) { - if (command !== 'read-dream') return false; - - if (!args[0] || !args[1]) { - console.log('[]'); - return true; - } - - const field = args[1]; - if (!ALLOWED_FIELDS.has(field)) { - console.log('[]'); - return true; - } - - try { - const dreamFile = safePath(args[0], cwd); - const data = JSON.parse(fs.readFileSync(dreamFile, 'utf8')); - const value = data[field]; - if (Array.isArray(value)) { - console.log(JSON.stringify(value.filter(v => typeof v === 'string'))); - } else if (typeof value === 'string') { - console.log(value); - } else { - console.log('[]'); - } - } catch { - console.log('[]'); - } - return true; -} - -module.exports = { handle }; diff --git a/scripts/hooks/lib/project-paths.cjs b/scripts/hooks/lib/project-paths.cjs index 94974fa1..77cfa3df 100644 --- a/scripts/hooks/lib/project-paths.cjs +++ b/scripts/hooks/lib/project-paths.cjs @@ -55,6 +55,26 @@ function getDreamConfigPath(projectRoot) { return path.join(projectRoot, '.devflow', 'dream', 'config.json'); } +/** .devflow/dream/.pending-turns.jsonl — decisions detection queue (dual-write with memory queue) */ +function getDreamPendingTurnsPath(projectRoot) { + return path.join(projectRoot, '.devflow', 'dream', '.pending-turns.jsonl'); +} + +/** .devflow/dream/.pending-turns.processing — atomic claim during background-dream-update processing */ +function getDreamPendingTurnsProcessingPath(projectRoot) { + return path.join(projectRoot, '.devflow', 'dream', '.pending-turns.processing'); +} + +/** .devflow/dream/.last-dream-ok — touched by the agent on successful dream worker completion */ +function getDreamLastOkPath(projectRoot) { + return path.join(projectRoot, '.devflow', 'dream', '.last-dream-ok'); +} + +/** .devflow/dream/.worker.lock — mkdir-based lock directory held by a live background-dream-update run */ +function getDreamWorkerLockDir(projectRoot) { + return path.join(projectRoot, '.devflow', 'dream', '.worker.lock'); +} + // --------------------------------------------------------------------------- // Decisions files // --------------------------------------------------------------------------- @@ -124,11 +144,6 @@ function getDecisionsNotificationsPath(projectRoot) { return path.join(projectRoot, '.devflow', 'decisions', '.decisions-notifications.json'); } -/** .devflow/decisions/.decisions-runs-today */ -function getDecisionsRunsTodayPath(projectRoot) { - return path.join(projectRoot, '.devflow', 'decisions', '.decisions-runs-today'); -} - /** .devflow/decisions/.decisions-batch-ids */ function getDecisionsBatchIdsPath(projectRoot) { return path.join(projectRoot, '.devflow', 'decisions', '.decisions-batch-ids'); @@ -213,6 +228,10 @@ module.exports = { getDocsDir, // Dream files getDreamConfigPath, + getDreamPendingTurnsPath, + getDreamPendingTurnsProcessingPath, + getDreamLastOkPath, + getDreamWorkerLockDir, // Decisions files getDecisionsFilePath, getPitfallsFilePath, @@ -227,7 +246,6 @@ module.exports = { getDecisionsUsagePath, getDecisionsUsageLockDir, getDecisionsNotificationsPath, - getDecisionsRunsTodayPath, getDecisionsBatchIdsPath, // Memory files getWorkingMemoryPath, diff --git a/scripts/hooks/lib/transcript-filter.cjs b/scripts/hooks/lib/transcript-filter.cjs deleted file mode 100644 index 51afbeb6..00000000 --- a/scripts/hooks/lib/transcript-filter.cjs +++ /dev/null @@ -1,207 +0,0 @@ -// scripts/hooks/lib/transcript-filter.cjs -// -// Channel-based transcript filter for the self-learning pipeline. -// -// DESIGN: D1 — two-channel filter separates USER_SIGNALS (workflow/procedural detection) -// from DIALOG_PAIRS (decision/pitfall detection). These two channels serve different -// upstream purposes: USER_SIGNALS need only clean user text; DIALOG_PAIRS need both -// the preceding assistant context AND the user correction to identify pitfalls and -// decisions with rationale. -// -// DESIGN: D2 — filter rules reject five classes of pollution: -// 1. isMeta:true — hook/system messages -// 2. sourceToolUseID / toolUseResult — tool invocation scaffolding -// 3. Wrapped framework noise (, , etc.) -// 4. tool_result content items in user turns -// 5. Empty turns (<5 chars after trim) -// -// DESIGN: D10 — this module is pure data transformation (no I/O). Called once per batch. -// Kept in a separate testable CJS module so unit tests can import it directly -// without spawning a shell process. - -'use strict'; - -/** - * Regex for framework-injected XML wrappers we must reject. - * Covers: , , , - */ -const FRAMEWORK_NOISE_RE = /^\s*<(command-|local-command-|system-reminder|example)/; - -const CAP_TURNS = 80; -const CAP_TEXT_CHARS = 1200; -const MIN_TEXT_CHARS = 5; - -/** - * Returns true if a string contains framework-injected noise. - * @param {string} text - * @returns {boolean} - */ -function isNoisyText(text) { - return FRAMEWORK_NOISE_RE.test(text); -} - -/** - * Cap text to the per-turn character limit. - * @param {string} text - * @returns {string} - */ -function capText(text) { - return text.length > CAP_TEXT_CHARS ? text.slice(0, CAP_TEXT_CHARS) : text; -} - -/** - * Cleans text content from a user turn. - * For string content: reject if noisy. - * For array content: filter out tool_result items and noisy text items, join remainder. - * - * @param {unknown} content - raw content field from transcript JSON - * @returns {{ ok: boolean, text: string }} - */ -function cleanContent(content) { - if (typeof content === 'string') { - if (isNoisyText(content)) return { ok: false, text: '' }; - const trimmed = content.trim(); - if (trimmed.length < MIN_TEXT_CHARS) return { ok: false, text: '' }; - return { ok: true, text: trimmed }; - } - - if (Array.isArray(content)) { - // Reject entire turn if any item is a tool_result - if (content.some(item => item && item.type === 'tool_result')) { - return { ok: false, text: '' }; - } - // Join text items, excluding noisy ones - const texts = content - .filter(item => item && item.type === 'text' && typeof item.text === 'string') - .map(item => item.text) - .filter(t => !isNoisyText(t)) - .join('\n') - .trim(); - - if (texts.length < MIN_TEXT_CHARS) return { ok: false, text: '' }; - return { ok: true, text: texts }; - } - - return { ok: false, text: '' }; -} - -/** - * Checks whether a transcript line represents a polluted source we should reject. - * DESIGN: D2 — pollution sources listed here must be kept in sync with the spec. - * - * @param {object} entry - parsed JSONL entry - * @returns {boolean} true if the entry should be skipped entirely - */ -function isRejectedEntry(entry) { - if (!entry || typeof entry !== 'object') return true; - // Reject meta/system lines - if (entry.isMeta === true) return true; - // Reject tool scaffolding - if (entry.sourceToolUseID != null) return true; - if (entry.toolUseResult != null) return true; - return false; -} - -/** - * extractChannels — main export. - * - * Parses JSONL transcript content and returns two channels: - * - userSignals: clean user-turn texts (for workflow/procedural detection) - * - dialogPairs: [{prior, user}] tuples (for decision/pitfall detection) - * - * Processing: - * 1. Parse each JSONL line, reject polluted entries (D2) - * 2. Collect user/assistant turns with clean text content - * 3. Cap to last 80 turns, 1200 chars per turn text - * 4. Build USER_SIGNALS from user turns only - * 5. Build DIALOG_PAIRS from (assistant, user) adjacent pairs in the tail - * - * @param {string} jsonlContent - raw JSONL string from transcript file(s) - * @returns {{ userSignals: string[], dialogPairs: Array<{prior: string, user: string}> }} - */ -function extractChannels(jsonlContent) { - const lines = jsonlContent.split('\n').filter(line => line.trim().length > 0); - - /** @type {Array<{role: 'user'|'assistant', text: string, turnId: number}>} */ - const turns = []; - let turnId = 0; - - for (const line of lines) { - let entry; - try { - entry = JSON.parse(line); - } catch { - continue; - } - - if (isRejectedEntry(entry)) continue; - - // Extract the actual message from transcript envelope format - // Transcripts may have: { type, message: { role, content } } - // or direct: { type, content } - const messageType = entry.type; - const message = entry.message || entry; - const role = message.role || messageType; - const content = message.content; - - if (role === 'user') { - const { ok, text } = cleanContent(content); - if (!ok) continue; - turns.push({ role: 'user', text: capText(text), turnId: ++turnId }); - } else if (role === 'assistant') { - // For assistant turns: accept string content or text-array content - const { ok, text } = cleanContent(content); - if (!ok) continue; - // Assistant turn inherits current turnId (not incremented) - turns.push({ role: 'assistant', text: capText(text), turnId }); - } - } - - // Cap to last 80 turns - const tail = turns.length > CAP_TURNS ? turns.slice(turns.length - CAP_TURNS) : turns; - - // Build USER_SIGNALS: texts from user turns only - const userSignals = tail.filter(t => t.role === 'user').map(t => t.text); - - // Build DIALOG_PAIRS: adjacent (assistant, user) pairs in tail - /** @type {Array<{prior: string, user: string}>} */ - const dialogPairs = []; - for (let i = 1; i < tail.length; i++) { - if (tail[i].role === 'user' && tail[i - 1].role === 'assistant') { - dialogPairs.push({ prior: tail[i - 1].text, user: tail[i].text }); - } - } - - return { userSignals, dialogPairs }; -} - -module.exports = { extractChannels }; - -if (require.main === module) { - const argv = process.argv.slice(2); - const subcommand = argv[0]; - const filePath = argv[1]; - - if (!subcommand || !filePath || !['user-signals', 'dialog-pairs'].includes(subcommand)) { - process.stderr.write('Usage: node transcript-filter.cjs \n'); - process.exit(1); - } - - const fs = require('fs'); - let content = ''; - try { - content = fs.readFileSync(filePath, 'utf8'); - } catch { - // Output nothing so shell [ -n "$VAR" ] emptiness checks work correctly. - process.exit(0); - } - - const { userSignals, dialogPairs } = extractChannels(content); - - // Output nothing when results are empty so shell [ -n "$VAR" ] checks work correctly. - // JSON.stringify([]) produces "[]" which is a non-empty string that fools shell emptiness checks. - const results = subcommand === 'user-signals' ? userSignals : dialogPairs; - if (results.length > 0) { - process.stdout.write(JSON.stringify(results) + '\n'); - } -} diff --git a/scripts/hooks/run-hook b/scripts/hooks/run-hook index d61b10cd..5b9437e4 100755 --- a/scripts/hooks/run-hook +++ b/scripts/hooks/run-hook @@ -20,4 +20,12 @@ CMDBLOCK SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" HOOK_NAME="$1" shift +# Exit 0 (never fail the host event) when the named script is absent — covers the +# init upgrade-swap window and hand-edited settings.json entries pointing at a +# hook that was renamed or removed (e.g. an old dream-* registration surviving +# past a cutover before `devflow init` re-runs and sweeps it via LEGACY_HOOK_FILES). +if [ ! -f "$SCRIPT_DIR/${HOOK_NAME}" ]; then + echo "run-hook: '${HOOK_NAME}' not found in $SCRIPT_DIR — skipping" >&2 + exit 0 +fi exec bash "$SCRIPT_DIR/${HOOK_NAME}" "$@" diff --git a/scripts/hooks/session-start-context b/scripts/hooks/session-start-context index f5f5c168..ee0f3143 100755 --- a/scripts/hooks/session-start-context +++ b/scripts/hooks/session-start-context @@ -4,21 +4,20 @@ # Always-on hook that injects decisions TL;DR as additionalContext. # Sections are independently gated by feature sentinels — this hook itself is never disabled. # -# Section 1.5: Decisions TL;DR (skipped if .devflow/decisions/.disabled) -# Section 2: Dream pending-work (emits directive when pending tasks detected) +# Section 1.5: Decisions TL;DR (skipped if .devflow/decisions/.disabled), plus the +# detached dream worker's optional last-run-summary (see background-dream-update / +# dream-procedure.md — spawned separately by the spawn-dream-worker hook). # Safe no-op fallback: must exist before hook-bootstrap is sourced. dbg() { :; } -# set -e intentionally omitted: sections 1.5 and 2 are independent — -# a failure in one must not prevent the others from running. +# set -e intentionally omitted: a failure in this section must not crash the hook. # Re-entrancy guards — before hook-bootstrap to minimize background session -# overhead. The dream worker's own claude -p session fires SessionStart hooks; -# without this guard every dream run would recursively re-trigger this hook's -# own directive-emitting logic (Section 2) from inside its own nested session. -# A latent pre-existing gap for DEVFLOW_BG_UPDATER too (this hook previously -# had no re-entrancy guard at all). +# overhead. The memory worker's and dream worker's own claude -p sessions fire +# SessionStart hooks; without this guard a nested session would re-inject its +# own context. A latent pre-existing gap for DEVFLOW_BG_UPDATER too (this hook +# previously had no re-entrancy guard at all). if [ "${DEVFLOW_BG_DREAM:-}" = "1" ]; then dbg "EXIT: bg_dream"; exit 0; fi if [ "${DEVFLOW_BG_UPDATER:-}" = "1" ]; then dbg "EXIT: bg_updater"; exit 0; fi @@ -57,7 +56,6 @@ PROJECT_ROOT="$(df_resolve_root "$CWD" 2>/dev/null || true)" CONTEXT="" DEVFLOW_DIR="$PROJECT_ROOT/.devflow" -MEMORY_DIR="$DEVFLOW_DIR/memory" DREAM_DIR="$DEVFLOW_DIR/dream" DECISIONS_CONTENT_DIR="$DEVFLOW_DIR/decisions" @@ -119,119 +117,6 @@ ${_SC_SUMMARY_SECTION}" fi fi -# --- Section 2: Dream Pending-Work --- -# Detects pending dream tasks and emits a directive to spawn a background Dream agent. -# Guards: processor-spawn throttle (120s window) prevents duplicate emissions on rapid -# SessionStart events (e.g., /clear or compact). -# Requires: dream-recover + dream-collect-tasks helpers (sourced inline below). -# No set -e: any failure must not affect prior sections. -# -# D57a: Per-session .processing suffix (collision fix). -# Each eval-* module writes marker files as {type}.{session_id}.json, where -# session_id is the Claude session ID passed to the hook. When the Dream agent -# claims a marker it renames it to {type}.{session_id}.processing. This suffix -# prevents two concurrent Dream agent instances (from two simultaneous SessionStart -# events) from racing on the same marker file — each session only claims its own -# markers. -# -# D57b: 120s processor-spawn throttle. -# SessionStart fires on /clear, compact, and every new window open. Without -# a throttle, three rapid /clear commands would spawn three background agents -# all processing the same pending markers. The 120s window (.processor-spawned-at) -# ensures only one Dream agent is spawned per burst. The throttle is written -# atomically (temp+mv) so concurrent SessionStart events read a consistent value. -# -# D57c: Heartbeat rationale. -# The Dream agent touches each in-flight .processing file at the start of -# every phase. dream-recover uses the mtime to determine staleness (1800s -# threshold for non-memory types). The touch refreshes the mtime so a legitimate -# long-running Dream agent is not yanked mid-phase. Pairs with D56a thresholds. - -# Source helpers — soft-fail if absent (e.g., on older installs before rebuild). -# All sourced helpers follow the no-set-e discipline and exit cleanly on failure. -_SC2_HELPERS_OK="false" -if [ -f "$SCRIPT_DIR/get-mtime" ] && [ -f "$SCRIPT_DIR/dream-recover" ] && [ -f "$SCRIPT_DIR/dream-collect-tasks" ]; then - source "$SCRIPT_DIR/get-mtime" 2>/dev/null || true - source "$SCRIPT_DIR/dream-lock" 2>/dev/null || true - source "$SCRIPT_DIR/dream-recover" 2>/dev/null || true - source "$SCRIPT_DIR/dream-collect-tasks" 2>/dev/null || true - _SC2_HELPERS_OK="true" -fi - -if [ "$_SC2_HELPERS_OK" = "true" ]; then - # Determine feature enabled-state. - # Primary: dream/config.json fields; secondary: runtime sentinel for decisions. - # Only decisions is gated here (per ADR-016; avoids PF-009). - _SC2_DEC_EN="true" - - _SC2_DREAM_CONFIG="$DREAM_DIR/config.json" - if [ -f "$_SC2_DREAM_CONFIG" ]; then - _SC2_DEC_EN=$(json_field_file "$_SC2_DREAM_CONFIG" "decisions" "true") - fi - # Sentinel override for decisions - [ -f "$DEVFLOW_DIR/decisions/.disabled" ] && _SC2_DEC_EN="false" - - dbg "Section2: dec=$_SC2_DEC_EN" - - # Step 1: recover stale .processing markers so they become collectable in step 2 - if [ -d "$DREAM_DIR" ]; then - dream_recover_stale "$DREAM_DIR" "$MEMORY_DIR" || true - fi - - # Step 2: collect pending marker task types - dream_collect_tasks "$DREAM_DIR" "$_SC2_DEC_EN" || true - - # Emit directive if tasks pending, subject to processor-spawn throttle. - if [ -n "$_DREAM_TASKS" ]; then - _SC2_SPAWNED_AT_FILE="$DREAM_DIR/.processor-spawned-at" - _SC2_THROTTLE=120 - _SC2_EMIT="true" - _SC2_NOW=$(date +%s) - - if [ -f "$_SC2_SPAWNED_AT_FILE" ]; then - _SC2_LAST_SPAWN=$(cat "$_SC2_SPAWNED_AT_FILE" 2>/dev/null | tr -dc '0-9' || true) - _SC2_LAST_SPAWN="${_SC2_LAST_SPAWN:-0}" - _SC2_SPAWN_AGE=$(( _SC2_NOW - _SC2_LAST_SPAWN )) - if [ "$_SC2_SPAWN_AGE" -lt "$_SC2_THROTTLE" ]; then - _SC2_EMIT="false" - dbg "Section2: processor-spawn throttled (spawned ${_SC2_SPAWN_AGE}s ago)" - log "Dream processor-spawn throttled (${_SC2_SPAWN_AGE}s < ${_SC2_THROTTLE}s): tasks=$_DREAM_TASKS" - fi - fi - - if [ "$_SC2_EMIT" = "true" ]; then - # Write/refresh .processor-spawned-at atomically - _SC2_SPAWN_TMP="$DREAM_DIR/.processor-spawned-at.tmp.$$" - printf '%s' "$_SC2_NOW" > "$_SC2_SPAWN_TMP" 2>/dev/null && mv "$_SC2_SPAWN_TMP" "$_SC2_SPAWNED_AT_FILE" 2>/dev/null || rm -f "$_SC2_SPAWN_TMP" 2>/dev/null || true - - # Build the per-task spawn directive (one Agent() call per pending task with a - # hardcoded task→model map). Logic lives in dream_build_spawn_directive - # (sourced from dream-collect-tasks); it sets _DREAM_DIRECTIVE, mirroring the - # dream_collect_tasks → _DREAM_TASKS contract. Empty result → no known types. - dream_build_spawn_directive "$_DREAM_TASKS" - - if [ -n "$_DREAM_DIRECTIVE" ]; then - if [ -n "$CONTEXT" ]; then - CONTEXT="${CONTEXT} - -${_DREAM_DIRECTIVE}" - else - CONTEXT="$_DREAM_DIRECTIVE" - fi - dbg "Section2: emitted per-task dream directive (tasks=$_DREAM_TASKS)" - log "Dream per-task directive emitted: tasks=$_DREAM_TASKS" - else - dbg "Section2: tasks present but no known types to spawn — skipping directive" - log "Dream directive skipped: no known task types in tasks=$_DREAM_TASKS" - fi - fi - else - dbg "Section2: no pending dream tasks" - fi -else - dbg "Section2: helpers not available — skip" -fi - # --- Output --- # Only output if we have something to inject diff --git a/shared/agents/dream.md b/shared/agents/dream.md deleted file mode 100644 index 4e33acdd..00000000 --- a/shared/agents/dream.md +++ /dev/null @@ -1,89 +0,0 @@ ---- -name: Dream -description: Background maintenance agent — processes ONE pending task type named in its prompt (decisions, curation). Spawned per-task by session-start-context; loads the matching per-task skill via the Skill tool. -model: sonnet -tools: - - Read - - Bash - - Write - - Edit - - Glob - - Grep -skills: - - devflow:apply-decisions - - devflow:dream-decisions - - devflow:dream-curation ---- - -# Dream Agent - -You are spawned for the ONE task named in your prompt (or "decisions then curation" for the -combined Opus spawn). Your role: claim markers atomically, do real LLM work via the matching -per-task skill, write results through plumbing ops, clean up. - -> **Model note**: The `model: sonnet` frontmatter is the fallback default. In practice -> `session-start-context` overrides the model per spawn: opus for the combined -> `decisions then curation` spawn. When you see "Opus spawn" in this document, -> that refers to the model assigned by the orchestrator at spawn time. - -## Environment - -Installed scripts live at `$HOME/.devflow/scripts/hooks/`. -All node invocations use these paths: -- `node "$HOME/.devflow/scripts/hooks/json-helper.cjs" ...` -- `node "$HOME/.devflow/scripts/hooks/lib/decisions-index.cjs" ...` - -Project root is your current working directory (`.`). All `.devflow/` paths are relative to it. - -## Step 0 — Identify your task - -Your prompt names the task type(s) to process: `decisions`, `curation`, -or `decisions then curation` (the combined Opus spawn). Process ONLY the task(s) named. - -## Step 1 — Claim markers atomically - -List markers for your task type(s): -```bash -ls .devflow/dream/*.json 2>/dev/null | grep -v config.json -``` - -For each marker `{type}.{session}.json` matching your task, claim via atomic rename — preserving -the session suffix: -```bash -mv ".devflow/dream/{type}.{session}.json" ".devflow/dream/{type}.{session}.processing" 2>/dev/null -``` - -If `mv` fails (another Dream agent already claimed it), skip that marker. -If ALL markers for a task type fail to claim, skip that task entirely. - -**Heartbeat rule**: At the start of every phase, `touch` each in-flight `.processing` file to -refresh its mtime. This prevents dream-recover from reclaiming a file that is actively -being processed (recovery threshold is 1800s; a long decisions or curation pass can take time). - -**Concurrency rule**: Never hold a lock across tool calls. All log/decisions writes go through -the plumbing ops in the per-task skills — they hold locks internally for one atomic read-modify-write. - -**Multi-marker merge**: When multiple `{type}.{session}.processing` files exist for one type, -read them all, then union/concat their payloads before processing: -- `decisions`: concatenate `dialogPairs` strings; union `existingObservationIds` arrays -- `curation`: single marker only - -**Input cap**: Process only the last **30** dialog-pairs (truncate oldest if more). -This bounds token cost and keeps each run predictable. - -## Step 2 — Process each task via its skill - -For each task type you claimed markers for, load the matching skill and follow its procedure: - -- **decisions** → load `devflow:dream-decisions` via the Skill tool and follow its procedure exactly. -- **curation** → load `devflow:dream-curation` via the Skill tool and follow its procedure exactly. - -For the combined "decisions then curation" spawn: run decisions fully (claim + process + cleanup) -THEN run curation fully (claim + process + cleanup). Sequential — never concurrent. - -## Error discipline - -- On success for any task: delete all `.processing` files for that task type. -- On failure for any task: leave `.processing` files in place — dream-recover will retry them. -- Never delete `.processing` files for a task that did not fully complete. -- Each task type is independent — a failure in one must not prevent processing of others. diff --git a/shared/skills/dream-curation/SKILL.md b/shared/skills/dream-curation/SKILL.md deleted file mode 100644 index 15d5125e..00000000 --- a/shared/skills/dream-curation/SKILL.md +++ /dev/null @@ -1,139 +0,0 @@ ---- -name: dream-curation -description: "Dream agent per-task procedure for the 'curation' task. Loaded EXPLICITLY by the Dream agent via the Skill tool when the agent is spawned for a curation task — not auto-activated. Handles periodic housekeeping of the decisions ledger and observation log." -allowed-tools: Read, Bash, Write, Edit, Glob, Grep ---- - -# Dream Task: curation - -## Iron Law - -> **RETIRE BY STATUS — THE LEDGER IS THE SOURCE OF TRUTH** -> -> The `.md` files are rendered views of the ledger — they are never hand-edited. -> To deprecate, supersede, or retire an entry, call `retire-anchor `. -> That op flips `decisions_status` on the ledger and re-renders both `.md` files -> automatically. Numbers are never reused; retired entries are recoverable. - -This skill is loaded by the Dream agent after it has claimed the curation marker. -The agent has already done: claim (mv .json → .processing). Curation uses a single marker only. - -`assign-anchor` adds new entries; curation flips status only — never creates entries. - -## Procedure - -Touch the claimed `.devflow/dream/curation.{session}.processing` file. - -This task performs periodic housekeeping of the decisions ledger and rendered `.md` files. -Bounds: **≤5 changes per run**. **7-day protection window** — never touch any entry whose -`date` field in the ledger is within the past 7 days. The window key is the ledger row's -`date` field (YYYY-MM-DD), not anything in the `.md` file. - -Read all inputs: -```bash -# Active entry counts from the ledger (preferred) -node "$HOME/.devflow/scripts/hooks/json-helper.cjs" count-active \ - "decision" -node "$HOME/.devflow/scripts/hooks/json-helper.cjs" count-active \ - "pitfall" -``` - -Also read `.devflow/decisions/decisions-ledger.jsonl`, -`.devflow/decisions/decisions-log.jsonl`, and `.devflow/decisions/.decisions-usage.json`. - -Cite counts come from `.decisions-usage.json` — read it directly. Each entry is keyed by -anchor ID (`ADR-NNN` / `PF-NNN`) with `{ cites, last_cited, created }`. There is no separate -"scan" step here: `decisions-usage-scan.cjs` is a write-path tool that increments cite counts -from session text, not a reporter — do not call it from the curation task. - -**Rotate stale observations first** (before selecting curation candidates): - -Run under `.observations.lock` — never hold `.decisions.lock` and `.observations.lock` -simultaneously (ADR-017: if you need both, take `.decisions.lock` as the outer and complete -your observation reads before acquiring the inner — but in curation only rotation needs -`.observations.lock` and it is a self-contained step): - -```bash -node "$HOME/.devflow/scripts/hooks/json-helper.cjs" rotate-observations -``` - -This archives `observing` rows older than 30 days to `decisions-log.archive.jsonl` -(gitignored), keeping the writer's recurrence read bounded. It never touches anchored -(`anchor_id` set) or `created`/`ready` rows. After rotation, the live log is a clean -working set. - -**Staleness signal** (run once per curation task, before selecting candidates): - -```bash -node "$HOME/.devflow/scripts/hooks/lib/staleness.cjs" \ - ".devflow/decisions/decisions-log.jsonl" \ - "$(pwd)" -``` - -Entries flagged `mayBeStale: true` in the log (their referenced files no longer exist) are -**preferred retirement candidates**, WITHIN the existing 7-day protection window and ≤5-changes -bounds. This is a signal to prefer — not an automatic retirement. Apply normal LLM judgment: -a stale-referenced entry that is otherwise heavily cited should survive over one that is -uncited and stale. - -**LLM judgment — identify entries to retire or merge**: - -Retire an entry when it is: -- Superseded by a newer, more precise entry on the same topic -- Contradicted by evidence in recent sessions -- Never cited (0 cites) AND older than 30 days AND low-confidence in the log - -**ADR-XOR-PF awareness**: one incident yields exactly one of an ADR or a PF — never both. -If curation finds two entries covering the same incident (one ADR, one PF), consolidate to -the more accurate type and retire the other. Concrete failure mode → PF; forward-looking -architectural choice → ADR. - -**Dedup awareness**: before retiring, check whether two near-duplicate entries could be -consolidated. Retire the less specific one and update the surviving entry's `pattern` -description to absorb the key insight from the retired entry. - -**RETIRE BY STATUS — never hand-edit the .md** (rendered render invariant): - -To deprecate/supersede/retire an entry, call `retire-anchor` — this flips `decisions_status` -on the ledger row and re-renders both `.md` files atomically: - -```bash -# Single retirement — self-locking (acquires and releases .decisions.lock internally) -node "$HOME/.devflow/scripts/hooks/json-helper.cjs" \ - retire-anchor -# status ∈ Deprecated | Superseded | Retired -``` - -`retire-anchor` holds `.decisions.lock` across the full ledger-write + render critical -section. It is atomic and idempotent — calling it twice with the same status is safe. -The entry vanishes from the rendered `.md` but survives in the committed ledger; -numbers are never reused. - -**Batch retirement**: call `retire-anchor` once per entry — each call self-locks atomically. -Do NOT attempt to hold `.decisions.lock` across multiple `retire-anchor` invocations; that -would deadlock against `retire-anchor`'s own lock acquisition. - -**Recoverability**: to re-activate a retired entry (AC-F6), edit its row in -`decisions-ledger.jsonl` directly — set `decisions_status` back to `Accepted` (decisions) -or `Active` (pitfalls) — then re-render. (`retire-anchor` only accepts retiring statuses, -and `merge-observation` writes the raw observation log, not the ledger, so neither -re-activates an entry.) - -```bash -node "$HOME/.devflow/scripts/hooks/lib/render-decisions.cjs" render "$(pwd)" -``` - -**Citation preservation**: if an entry being retired has inbound `applies ADR-NNN` citations -in other entries, update those entries' `pattern` or `details` to reference the surviving -entry instead (edit those ledger rows directly, then re-render). - -**Cap enforcement**: stop after 5 changes regardless of remaining candidates. - -`.devflow/` is gitignored by default (ADR-021) — ledger and rendered `.md` files stay local. - -**Transparency**: after curation, emit a brief note in the agent output listing what was -retired/merged. If nothing was changed, stay silent. - -Delete the claimed `.processing` marker on success. - -**On any failure**: leave `.processing` files in place (dream-recover will retry them). diff --git a/shared/skills/dream-decisions/SKILL.md b/shared/skills/dream-decisions/SKILL.md deleted file mode 100644 index 2cca4dce..00000000 --- a/shared/skills/dream-decisions/SKILL.md +++ /dev/null @@ -1,116 +0,0 @@ ---- -name: dream-decisions -description: "Dream agent per-task procedure for the 'decisions' task. Loaded EXPLICITLY by the Dream agent via the Skill tool when the agent is spawned for a decisions task — not auto-activated. Handles decision/pitfall detection from dialog pairs and materialization via assign-anchor." -allowed-tools: Read, Bash, Write, Edit, Glob, Grep ---- - -# Dream Task: decisions - -## Iron Law - -> **assign-anchor OWNS NUMBERING; render OWNS THE .md; NEVER HAND-EDIT** -> -> ADR and PF numbers are assigned exclusively by `assign-anchor`. The `.md` files are -> written exclusively by `render-decisions.cjs`. Never write, edit, or infer an ADR-NNN -> or PF-NNN number directly into decisions.md or pitfalls.md. Never call `decisions-append`. -> One `assign-anchor` invocation claims one number and re-renders both files atomically. - -This skill is loaded by the Dream agent after it has claimed the decisions marker(s). -The agent has already done: claim (mv .json → .processing) and multi-marker merge -(concatenate dialogPairs strings; union existingObservationIds arrays). -Cap at the last 30 dialog-pairs before proceeding. - -## Procedure - -Touch all claimed `.devflow/dream/decisions.{session}.processing` files. - -Read the merged `dialogPairs`. Cap at last 30 pairs. -Read `.devflow/decisions/decisions-log.jsonl` in full (for dedup and recurrence patterns). - -**LLM judgment — creation bar (abstain-by-default)**: - -Most sessions produce nothing. If unsure, record nothing. Only capture what a future -contributor would need and could not reconstruct from the code. - -**NOT a decision**: bug fix, one-off UX tweak, routine refactor, applying an existing -pattern, dependency bump, or anything already covered by an existing ADR in the log. - -**NOT a pitfall**: typo, transient flake, mistake with no general lesson, or a problem -fully prevented by existing tooling. - -**Positive bar**: -- Decision = a deliberate architectural choice or trade-off with rationale that - constrains future work. It must be a real fork in the road, not an obvious choice. -- Pitfall = a non-obvious failure mode with a transferable lesson that the next - contributor cannot recover from the code alone. - -**ADR-XOR-PF (hard rule)**: one incident yields exactly one of an ADR or a PF — never -both. Concrete failure → PF; forward-looking architectural choice → ADR. - -**Dedup before creating**: read the log first. If an existing row (any status, including -Retired) already covers this concern, reinforce it (reuse its `obs_` id via -`merge-observation`) instead of creating a new entry. Duplication is worse than silence. - -For each pattern that clears the creation bar: -1. Scan the log for a matching existing entry. REUSE its `obs_` id if found. -2. Estimate `confidence` honestly — this is curation metadata only, NOT a gate. Estimate - what the evidence actually supports; do not inflate it. -3. Author full `details` string: `"context: X; decision: Y; rationale: Z"` (decision) or - `"area: X; issue: Y; impact: Z; resolution: W"` (pitfall). - -Write (or reinforce) each observation using bounded retry+backoff on `.observations.lock` -(explicit cap: 9 attempts, ~47s total backoff; on exhaustion leave `.processing` for retry): - -```bash -( - LOCK=".devflow/dream/.observations.lock" - _ACQUIRED=false - _BACKOFF=1 - for _ATTEMPT in 1 2 3 4 5 6 7 8 9; do - if mkdir "$LOCK" 2>/dev/null; then - _ACQUIRED=true - break - fi - sleep "$_BACKOFF" - _BACKOFF=$(( _BACKOFF < 8 ? _BACKOFF * 2 : 8 )) - done - if [ "$_ACQUIRED" != "true" ]; then - echo "dream-decisions: failed to acquire .observations.lock after 9 attempts — leaving .processing for retry" >&2 - exit 1 - fi - node "$HOME/.devflow/scripts/hooks/json-helper.cjs" merge-observation \ - ".devflow/decisions/decisions-log.jsonl" \ - '{"id":"obs_xxx","type":"decision","pattern":"...","evidence":["..."],"details":"context: ...; decision: ...; rationale: ...","confidence":0.8,"status":"observing","quality_ok":true}' - rmdir "$LOCK" 2>/dev/null || true -) -``` - -Replace the JSON with actual LLM-authored observation data (full fields shown above). - -**If promoting** (quality_ok=true, pattern recurs or is clearly significant after clearing -the creation bar above): promote via `assign-anchor`: - -```bash -node "$HOME/.devflow/scripts/hooks/json-helper.cjs" assign-anchor \ - "decision" \ - "obs_xxx" -``` - -For pitfalls: -```bash -node "$HOME/.devflow/scripts/hooks/json-helper.cjs" assign-anchor \ - "pitfall" \ - "obs_xxx" -``` - -`assign-anchor` scans the ledger for the current max anchor number (including Retired), -assigns max+1 as a zero-padded 3-digit ID, writes an anchored row to -`decisions-ledger.jsonl`, marks the log row as `created`, registers usage, and re-renders -both `decisions.md` and `pitfalls.md` — all atomically under `.decisions.lock`. - -NEVER call `decisions-append`. NEVER hand-edit `decisions.md` or `pitfalls.md`. - -`.devflow/` is gitignored by default (ADR-021) — materialized entries stay local. -Delete all claimed `.processing` markers on success. - -**On any failure**: leave `.processing` files in place (dream-recover will retry them). diff --git a/src/cli/commands/capture.ts b/src/cli/commands/capture.ts new file mode 100644 index 00000000..f5b32d4c --- /dev/null +++ b/src/cli/commands/capture.ts @@ -0,0 +1,161 @@ +import * as path from 'path'; +import type { Settings, HookMatcher } from '../utils/hooks.js'; + +// ─── Capture hook utilities ──────────────────────────────────────────────── +// +// The capture bundle (capture-prompt, capture-turn, capture-question) is +// always-on, like session-start-context (context.ts) — registered +// unconditionally by init, removed by uninstall. There is no per-feature +// toggle: capture hooks only append to the memory/dream queues, gated +// per-queue internally by each script's own dream-config read (see +// queue-append's queue_read_gates). Follows the context.ts add/remove/has +// pattern rather than memory.ts's toggle pattern. +// +// IMPORTANT — Stop-array ordering contract: capture-turn MUST be registered +// BEFORE memory-worker in the Stop hook array (see memory.ts). Settings.json +// hook arrays run in array order, and memory-worker's throttle/spawn decision +// assumes the current turn has already been appended to the queue by +// capture-turn earlier in the same Stop event (append-before-spawn). This +// module only ever pushes capture-turn; callers (init.ts) must register the +// capture bundle before the memory bundle to preserve this ordering. + +const CAPTURE_PROMPT_MARKER = 'capture-prompt'; +const CAPTURE_TURN_MARKER = 'capture-turn'; +const CAPTURE_QUESTION_MARKER = 'capture-question'; +const CAPTURE_QUESTION_MATCHER = 'AskUserQuestion'; + +/** + * Map of hook event type → filename marker for the capture hooks. + * Three hooks total: UserPromptSubmit, Stop, PostToolUse (matcher-scoped). + */ +const CAPTURE_HOOK_CONFIG: Record = { + UserPromptSubmit: CAPTURE_PROMPT_MARKER, + Stop: CAPTURE_TURN_MARKER, + PostToolUse: CAPTURE_QUESTION_MARKER, +}; + +/** + * Add all 3 capture hooks (UserPromptSubmit, Stop, PostToolUse) to settings JSON. + * Idempotent — skips hooks that already exist. Returns unchanged JSON if all 3 present. + * The PostToolUse entry is scoped with `matcher: "AskUserQuestion"` so it only fires + * for AskUserQuestion tool calls. + */ +export function addCaptureHooks(settingsJson: string, devflowDir: string): string { + const settings: Settings = JSON.parse(settingsJson); + + if (hasCaptureHooks(settings)) { + return settingsJson; + } + + if (!settings.hooks) { + settings.hooks = {}; + } + + for (const [hookType, marker] of Object.entries(CAPTURE_HOOK_CONFIG)) { + const existing = settings.hooks[hookType] ?? []; + const alreadyPresent = existing.some((matcher) => + matcher.hooks.some((h) => h.command.includes(marker)), + ); + + if (!alreadyPresent) { + const hookCommand = path.join(devflowDir, 'scripts', 'hooks', 'run-hook') + ` ${marker}`; + const newEntry: HookMatcher = { + hooks: [ + { + type: 'command', + command: hookCommand, + timeout: 10, + }, + ], + }; + + if (hookType === 'PostToolUse') { + newEntry.matcher = CAPTURE_QUESTION_MATCHER; + } + + if (!settings.hooks[hookType]) { + settings.hooks[hookType] = []; + } + + settings.hooks[hookType].push(newEntry); + } + } + + return JSON.stringify(settings, null, 2) + '\n'; +} + +/** + * Remove all capture hooks (UserPromptSubmit, Stop, PostToolUse) from settings JSON. + * Accepts either a JSON string or a parsed Settings object. + * Idempotent — returns unchanged JSON if no capture hooks present. + * Preserves non-capture hooks (e.g. ambient preamble on UserPromptSubmit, memory-worker on Stop). + */ +export function removeCaptureHooks(input: string | Settings): string { + const settingsJson = typeof input === 'string' ? input : JSON.stringify(input); + const settings: Settings = typeof input === 'string' ? JSON.parse(input) : structuredClone(input); + + if (!settings.hooks) { + return settingsJson; + } + + let changed = false; + + for (const [hookType, marker] of Object.entries(CAPTURE_HOOK_CONFIG)) { + if (!settings.hooks[hookType]) { + continue; + } + + const before = settings.hooks[hookType].length; + settings.hooks[hookType] = settings.hooks[hookType].filter( + (matcher) => !matcher.hooks.some((h) => h.command.includes(marker)), + ); + + if (settings.hooks[hookType].length !== before) { + changed = true; + } + + if (settings.hooks[hookType].length === 0) { + delete settings.hooks[hookType]; + } + } + + if (settings.hooks && Object.keys(settings.hooks).length === 0) { + delete settings.hooks; + } + + if (!changed) { + return settingsJson; + } + + return JSON.stringify(settings, null, 2) + '\n'; +} + +/** + * Check if ALL 3 capture hooks are registered in settings JSON or parsed Settings object. + */ +export function hasCaptureHooks(input: string | Settings): boolean { + return countCaptureHooks(input) === Object.keys(CAPTURE_HOOK_CONFIG).length; +} + +/** + * Count how many of the 3 capture hooks are present (0-3). + * Accepts either a JSON string or a parsed Settings object. + */ +export function countCaptureHooks(input: string | Settings): number { + const settings: Settings = typeof input === 'string' ? JSON.parse(input) : input; + + if (!settings.hooks) { + return 0; + } + + let count = 0; + + for (const [hookType, marker] of Object.entries(CAPTURE_HOOK_CONFIG)) { + const matchers = settings.hooks[hookType] ?? []; + if (matchers.some((matcher) => matcher.hooks.some((h) => h.command.includes(marker)))) { + count++; + } + } + + return count; +} diff --git a/src/cli/commands/decisions.ts b/src/cli/commands/decisions.ts index ec167f96..afce4002 100644 --- a/src/cli/commands/decisions.ts +++ b/src/cli/commands/decisions.ts @@ -12,9 +12,12 @@ import { getDecisionsManifestPath, getDecisionsLockDir, getDecisionsNotificationsPath, - getDecisionsRunsTodayPath, getDecisionsBatchIdsPath, getDecisionsDisabledSentinel, + getDreamPendingTurnsPath, + getDreamPendingTurnsProcessingPath, + getDreamLastOkPath, + getDreamWorkerLockDir, } from '../utils/project-paths.js'; import { updateFeature, isFeatureEnabled } from '../utils/dream-config.js'; import { syncManifestFeature } from '../utils/manifest.js'; @@ -161,42 +164,12 @@ export const decisionsCommand = new Command('decisions') if (options.configure) { p.intro(color.bgCyan(color.black(' Decisions Configuration '))); - const maxRuns = await p.text({ - message: 'Maximum background runs per day', - placeholder: '3', - defaultValue: '3', - validate: (v) => { - const n = Number(v); - if (isNaN(n) || n < 1 || n > 50) return 'Enter a number between 1 and 50'; - return undefined; - }, - }); - if (p.isCancel(maxRuns)) { - p.cancel('Configuration cancelled.'); - return; - } - - const throttle = await p.text({ - message: 'Throttle interval (minutes between runs)', - placeholder: '5', - defaultValue: '5', - validate: (v) => { - const n = Number(v); - if (isNaN(n) || n < 1 || n > 60) return 'Enter a number between 1 and 60'; - return undefined; - }, - }); - if (p.isCancel(throttle)) { - p.cancel('Configuration cancelled.'); - return; - } - const model = await p.select({ message: 'Model for decision detection', options: [ - { value: 'sonnet', label: 'Sonnet', hint: 'Recommended — good balance of quality and speed' }, + { value: 'opus', label: 'Opus', hint: 'Recommended — highest quality for detection + curation judgment' }, + { value: 'sonnet', label: 'Sonnet', hint: 'Good balance of quality and speed' }, { value: 'haiku', label: 'Haiku', hint: 'Fastest, lowest cost' }, - { value: 'opus', label: 'Opus', hint: 'Highest quality, highest cost' }, ], }); if (p.isCancel(model)) { @@ -226,8 +199,6 @@ export const decisionsCommand = new Command('decisions') } const config = { - max_daily_runs: Number(maxRuns), - throttle_minutes: Number(throttle), model, debug: debugMode, }; @@ -252,9 +223,25 @@ export const decisionsCommand = new Command('decisions') // --- --reset --- if (options.reset) { - const lockDir = getDecisionsLockDir(process.cwd()); + const gitRoot = await getGitRoot(); + if (!gitRoot) { + p.log.warn('Could not resolve git root — reset not performed'); + return; + } - // Acquire lock to prevent conflict with running background decisions agent. + // Never touch a project where a live background-dream-update worker holds + // .devflow/dream/.worker.lock — it is that worker's sole concurrency guard. + // This is a plain existence check, not a lock acquisition: we skip entirely + // rather than racing the worker. + try { + await fs.access(getDreamWorkerLockDir(gitRoot)); + p.log.error('A background dream worker is currently running. Try again in a moment.'); + return; + } catch { /* no live worker — safe to proceed */ } + + const lockDir = getDecisionsLockDir(gitRoot); + + // Acquire lock to prevent conflict with a concurrent `devflow decisions` invocation. try { await fs.mkdir(lockDir); } catch { @@ -264,12 +251,14 @@ export const decisionsCommand = new Command('decisions') try { const stateFilePaths = [ - getDecisionsLogPath(process.cwd()), - getDecisionsManifestPath(process.cwd()), - getDecisionsNotificationsPath(process.cwd()), - getDecisionsRunsTodayPath(process.cwd()), - getDecisionsBatchIdsPath(process.cwd()), - getDecisionsConfigPath(process.cwd()), + getDecisionsLogPath(gitRoot), + getDecisionsManifestPath(gitRoot), + getDecisionsNotificationsPath(gitRoot), + getDecisionsBatchIdsPath(gitRoot), + getDecisionsConfigPath(gitRoot), + getDreamPendingTurnsPath(gitRoot), + getDreamPendingTurnsProcessingPath(gitRoot), + getDreamLastOkPath(gitRoot), ]; if (process.stdin.isTTY) { @@ -293,19 +282,24 @@ export const decisionsCommand = new Command('decisions') // Remove decisions sentinel directory if present (may contain .disabled or other files). try { - await fs.rm(getDecisionsDir(process.cwd()), { recursive: true, force: true }); + await fs.rm(getDecisionsDir(gitRoot), { recursive: true, force: true }); } catch { /* best effort */ } - // Clean dream state files - const dreamDir = getDreamDir(process.cwd()); - for (const f of ['.decisions-runs-today']) { + // Clean legacy dream marker-pipeline stamps (pre-dream-system-simplification + // installs — config.json and the new .pending-turns.jsonl/.last-dream-ok are + // handled above/never touched here). + const dreamDir = getDreamDir(gitRoot); + for (const f of ['.decisions-runs-today', '.curation-last', '.processor-spawned-at']) { try { await fs.unlink(path.join(dreamDir, f)); } catch { /* may not exist */ } } - // Clean dream decisions markers + // Clean legacy per-session decisions/curation markers (.json/.processing/.retries/.failed) try { const dreamFiles = await fs.readdir(dreamDir); for (const f of dreamFiles) { - if (f.startsWith('decisions.') && f.endsWith('.json')) { + const isLegacyMarker = + (f.startsWith('decisions.') || f.startsWith('curation.')) && + (f.endsWith('.json') || f.endsWith('.processing') || f.endsWith('.retries') || f.endsWith('.failed')); + if (isLegacyMarker) { try { await fs.unlink(path.join(dreamDir, f)); } catch { /* ignore */ } } } @@ -320,8 +314,23 @@ export const decisionsCommand = new Command('decisions') // --- --clear --- if (options.clear) { + const gitRoot = await getGitRoot(); + if (!gitRoot) { + p.log.warn('Could not resolve git root — clear not performed'); + return; + } + + // Never touch a project where a live background-dream-update worker holds + // .devflow/dream/.worker.lock — it is that worker's sole concurrency guard. try { - await fs.access(logPath); + await fs.access(getDreamWorkerLockDir(gitRoot)); + p.log.error('A background dream worker is currently running. Try again in a moment.'); + return; + } catch { /* no live worker — safe to proceed */ } + + const decisionsLogPath = getDecisionsLogPath(gitRoot); + try { + await fs.access(decisionsLogPath); } catch { p.log.info('No decisions log to clear.'); return; @@ -338,7 +347,17 @@ export const decisionsCommand = new Command('decisions') } } - await fs.writeFile(logPath, '', 'utf-8'); + await fs.writeFile(decisionsLogPath, '', 'utf-8'); + + // Drain the dream (decisions-detection) queue and stamps so stale turns don't + // process on the next spawn — mirrors memory.ts's drain-on-disable behavior + // for the sibling memory queue. + await Promise.all([ + fs.unlink(getDreamPendingTurnsPath(gitRoot)).catch((e: NodeJS.ErrnoException) => { if (e.code !== 'ENOENT') throw e; }), + fs.unlink(getDreamPendingTurnsProcessingPath(gitRoot)).catch((e: NodeJS.ErrnoException) => { if (e.code !== 'ENOENT') throw e; }), + fs.unlink(getDreamLastOkPath(gitRoot)).catch((e: NodeJS.ErrnoException) => { if (e.code !== 'ENOENT') throw e; }), + ]); + p.log.success('Decisions log cleared.'); return; } diff --git a/src/cli/commands/dream.ts b/src/cli/commands/dream.ts new file mode 100644 index 00000000..c027b932 --- /dev/null +++ b/src/cli/commands/dream.ts @@ -0,0 +1,92 @@ +import * as path from 'path'; +import type { Settings, HookMatcher } from '../utils/hooks.js'; + +// ─── Dream worker hook utilities ─────────────────────────────────────────── +// +// spawn-dream-worker (SessionStart) is always-on, like session-start-context +// (context.ts) and the capture bundle (capture.ts) — registered unconditionally +// by init, removed by uninstall. There is no per-feature toggle at the hook +// registration level: the hook internally gates on the decisions dual-signal +// (dream/config.json's `decisions` field AND the `.devflow/decisions/.disabled` +// sentinel) before deciding whether to spawn background-dream-update. Follows +// the context.ts add/remove/has pattern. + +const SPAWN_DREAM_WORKER_MARKER = 'spawn-dream-worker'; + +/** + * Add the spawn-dream-worker hook to SessionStart in settings JSON. + * Idempotent — returns unchanged JSON if hook already present. + */ +export function addDreamHook(settingsJson: string, devflowDir: string): string { + if (hasDreamHook(settingsJson)) { + return settingsJson; + } + + const settings: Settings = JSON.parse(settingsJson); + + if (!settings.hooks) { + settings.hooks = {}; + } + + const hookCommand = path.join(devflowDir, 'scripts', 'hooks', 'run-hook') + ` ${SPAWN_DREAM_WORKER_MARKER}`; + const newEntry: HookMatcher = { + hooks: [ + { + type: 'command', + command: hookCommand, + timeout: 10, + }, + ], + }; + + if (!settings.hooks.SessionStart) { + settings.hooks.SessionStart = []; + } + + settings.hooks.SessionStart.push(newEntry); + + return JSON.stringify(settings, null, 2) + '\n'; +} + +/** + * Remove the spawn-dream-worker hook from settings JSON. + * Idempotent — returns unchanged JSON if hook not present. + * Preserves all other SessionStart hooks (session-start-memory, session-start-context). + */ +export function removeDreamHook(settingsJson: string): string { + const settings: Settings = JSON.parse(settingsJson); + + if (!settings.hooks?.SessionStart) { + return settingsJson; + } + + const before = settings.hooks.SessionStart.length; + settings.hooks.SessionStart = settings.hooks.SessionStart.filter( + (matcher) => !matcher.hooks.some((h) => h.command.includes(SPAWN_DREAM_WORKER_MARKER)), + ); + + if (settings.hooks.SessionStart.length === before) { + return settingsJson; + } + + if (settings.hooks.SessionStart.length === 0) { + delete settings.hooks.SessionStart; + } + + if (Object.keys(settings.hooks).length === 0) { + delete settings.hooks; + } + + return JSON.stringify(settings, null, 2) + '\n'; +} + +/** + * Check if the spawn-dream-worker hook is present in settings JSON. + * Accepts either a JSON string or a parsed Settings object. + */ +export function hasDreamHook(input: string | Settings): boolean { + const settings: Settings = typeof input === 'string' ? JSON.parse(input) : input; + return settings.hooks?.SessionStart?.some( + (matcher) => matcher.hooks.some((h) => h.command.includes(SPAWN_DREAM_WORKER_MARKER)), + ) ?? false; +} diff --git a/src/cli/commands/init.ts b/src/cli/commands/init.ts index fbaed2ab..0ebf2339 100644 --- a/src/cli/commands/init.ts +++ b/src/cli/commands/init.ts @@ -32,6 +32,8 @@ import { detectPlatform, detectShell, getProfilePath, getSafeDeleteInfo, hasSafe import { generateSafeDeleteBlock, installToProfile, removeFromProfile, getInstalledVersion, SAFE_DELETE_BLOCK_VERSION } from '../utils/safe-delete-install.js'; import { addAmbientHook, removeAmbientHook } from './ambient.js'; import { addMemoryHooks, removeMemoryHooks } from './memory.js'; +import { addCaptureHooks, removeCaptureHooks } from './capture.js'; +import { addDreamHook, removeDreamHook } from './dream.js'; // Settings/HookMatcher types used by hook utilities — each in their own module import { addHudStatusLine, removeHudStatusLine } from './hud.js'; import { loadConfig as loadHudConfig, saveConfig as saveHudConfig } from '../hud/config.js'; @@ -48,6 +50,8 @@ import * as os from 'os'; export { substituteSettingsTemplate, computeGitignoreAppend, mergeDenyList, discoverProjectGitRoots } from '../utils/post-install.js'; export { addAmbientHook, removeAmbientHook, hasAmbientHook } from './ambient.js'; export { addMemoryHooks, removeMemoryHooks, hasMemoryHooks } from './memory.js'; +export { addCaptureHooks, removeCaptureHooks, hasCaptureHooks } from './capture.js'; +export { addDreamHook, removeDreamHook, hasDreamHook } from './dream.js'; export { addHudStatusLine, removeHudStatusLine, hasHudStatusLine } from './hud.js'; // Re-export migrateShadowOverrides under its original name for backward compatibility export { migrateShadowOverridesRegistry as migrateShadowOverrides } from '../utils/shadow-overrides-migration.js'; @@ -1115,6 +1119,20 @@ export const initCommand = new Command('init') // Learning pipeline removed: eval-learning/eval-reinforce no longer sourced by dream-evaluate 'eval-learning', 'eval-reinforce', + // Dream system simplification: marker-pipeline scripts replaced by capture-prompt/ + // capture-turn/capture-question (queue-append) + spawn-dream-worker/background-dream-update + // (detached worker) — see capture.ts/dream.ts. copyDirectory is additive, so these must be + // actively swept on init or they linger in ~/.devflow/scripts/hooks/ after upgrade. + 'dream-capture', + 'dream-dispatch', + 'dream-evaluate', + 'eval-helpers', + 'eval-decisions', + 'eval-curation', + 'dream-collect-tasks', + 'dream-recover', + 'lib/transcript-filter.cjs', + 'lib/dream-ops.cjs', ]; const hooksDir = path.join(devflowDir, 'scripts', 'hooks'); for (const legacy of LEGACY_HOOK_FILES) { @@ -1138,9 +1156,18 @@ export const initCommand = new Command('init') const cleanedForAmbient = await removeAmbientHook(content); content = ambientEnabled ? await addAmbientHook(cleanedForAmbient, devflowDir) : cleanedForAmbient; - // Memory hooks — always remove-then-add to upgrade hook format (e.g., .sh → run-hook) - // Memory hooks include the unified dream hooks (dream-dispatch, dream-capture, - // dream-evaluate) which handle memory and decisions in the background. + // Capture hooks — always-on (like the context hook below), remove-then-add for + // upgrade safety. Queue-append only (capture-prompt/capture-turn/capture-question); + // each script gates its own per-queue write internally via dream config, so there + // is no CLI-level enable/disable toggle here. MUST run before addMemoryHooks below + // so capture-turn lands before memory-worker in the Stop array (AC-C2 ordering: + // append-before-spawn). + const cleanedForCapture = removeCaptureHooks(content); + content = addCaptureHooks(cleanedForCapture, devflowDir); + + // Memory hooks — always remove-then-add to upgrade hook format (e.g., .sh → run-hook). + // Three hooks: Stop (memory-worker), SessionStart (session-start-memory), PreCompact. + // Decisions detection/curation no longer live here — see the dream hook below. // Knowledge is handled in-command via write-through (knowledge_writeback MDS partial). const cleaned = removeMemoryHooks(content); content = memoryEnabled ? addMemoryHooks(cleaned, devflowDir) : cleaned; @@ -1154,6 +1181,15 @@ export const initCommand = new Command('init') const cleanedForContext = removeContextHook(content); content = addContextHook(cleanedForContext, devflowDir); + // Dream hook — always-on (SessionStart, like the context hook above), remove-then-add + // for upgrade safety. spawn-dream-worker internally gates on the decisions dual-signal + // (dream/config.json + .devflow/decisions/.disabled) before spawning anything — no + // CLI-level enable/disable toggle here. MUST run after addContextHook above so + // spawn-dream-worker lands last in the SessionStart array (AC-C2 ordering: + // session-start-memory, session-start-context, spawn-dream-worker). + const cleanedForDream = removeDreamHook(content); + content = addDreamHook(cleanedForDream, devflowDir); + // Claude Code flags — strip all managed keys, then re-apply selected flags content = stripFlags(content); content = applyFlags(content, enabledFlags); diff --git a/src/cli/commands/memory.ts b/src/cli/commands/memory.ts index 71480144..937f07d7 100644 --- a/src/cli/commands/memory.ts +++ b/src/cli/commands/memory.ts @@ -17,30 +17,41 @@ import type { HookMatcher, Settings } from '../utils/hooks.js'; import { updateFeature, isFeatureEnabled } from '../utils/dream-config.js'; /** - * Map of hook event type → filename marker for the dream hooks. - * Five hooks total: UserPromptSubmit, Stop, SessionEnd, SessionStart, PreCompact. + * Map of hook event type → filename marker for the memory hooks. + * Three hooks total: Stop, SessionStart, PreCompact. + * + * UserPromptSubmit and SessionEnd are NOT memory.ts's concern anymore: queue-append + * (formerly UserPromptSubmit/dream-dispatch, Stop/dream-capture) now lives in + * capture.ts (capture-prompt, capture-turn — always-on, not feature-gated at the + * hook-registration level), and SessionEnd (dream-evaluate) was removed entirely — + * decisions detection is a SessionStart-spawned detached worker now (see dream.ts). + * + * Stop-array ordering contract: memory-worker MUST be registered AFTER capture-turn + * in the Stop hook array (append-before-spawn — memory-worker's throttle/spawn + * decision assumes the current turn was already appended by capture-turn earlier + * in the same Stop event). Enforced by init.ts's registration order, not here. */ const MEMORY_HOOK_CONFIG: Record = { - UserPromptSubmit: 'dream-dispatch', - Stop: 'dream-capture', - SessionEnd: 'dream-evaluate', + Stop: 'memory-worker', SessionStart: 'session-start-memory', PreCompact: 'pre-compact-memory', }; /** - * Legacy hook filename markers from the pre-sidecar 8-hook system and the v3 sidecar→dream rename. - * Used by removeMemoryHooks to clean up hooks from upgrading users. + * Legacy hook filename markers from the pre-sidecar 8-hook system, the v3 + * sidecar→dream rename, and the dream-system-simplification cutover (dream-dispatch/ + * dream-capture/dream-evaluate marker pipeline retired in favor of capture.ts + + * dream.ts). Used by removeMemoryHooks to clean up hooks from upgrading users. */ const LEGACY_HOOK_MARKERS: Record = { - UserPromptSubmit: ['prompt-capture-memory', 'sidecar-dispatch'], - Stop: ['stop-update-memory', 'stop-update-learning', 'sidecar-capture'], - SessionEnd: ['session-end-learning', 'session-end-decisions', 'session-end-knowledge-refresh', 'sidecar-evaluate'], + UserPromptSubmit: ['prompt-capture-memory', 'sidecar-dispatch', 'dream-dispatch'], + Stop: ['stop-update-memory', 'stop-update-learning', 'sidecar-capture', 'dream-capture'], + SessionEnd: ['session-end-learning', 'session-end-decisions', 'session-end-knowledge-refresh', 'sidecar-evaluate', 'dream-evaluate'], }; /** - * Add all 5 memory hooks (UserPromptSubmit, Stop, SessionEnd, SessionStart, PreCompact) to settings JSON. - * Idempotent — skips hooks that already exist. Returns unchanged JSON if all 5 present. + * Add all 3 memory hooks (Stop, SessionStart, PreCompact) to settings JSON. + * Idempotent — skips hooks that already exist. Returns unchanged JSON if all 3 present. */ export function addMemoryHooks(settingsJson: string, devflowDir: string): string { const settings: Settings = JSON.parse(settingsJson); @@ -140,14 +151,14 @@ export function removeMemoryHooks(input: string | Settings): string { } /** - * Check if ALL 5 memory hooks are registered in settings JSON or parsed Settings object. + * Check if ALL 3 memory hooks are registered in settings JSON or parsed Settings object. */ export function hasMemoryHooks(input: string | Settings): boolean { return countMemoryHooks(input) === Object.keys(MEMORY_HOOK_CONFIG).length; } /** - * Count how many of the 5 memory hooks are present (0-5). + * Count how many of the 3 memory hooks are present (0-3). * Accepts either a JSON string or a parsed Settings object. */ export function countMemoryHooks(input: string | Settings): number { diff --git a/src/cli/commands/uninstall.ts b/src/cli/commands/uninstall.ts index 4c066106..4a6794a1 100644 --- a/src/cli/commands/uninstall.ts +++ b/src/cli/commands/uninstall.ts @@ -11,6 +11,8 @@ import { isClaudeCliAvailable } from '../utils/cli.js'; import { DEVFLOW_PLUGINS, getAllSkillNames, LEGACY_SKILL_NAMES, prefixSkillName, type PluginDefinition } from '../plugins.js'; import { removeAmbientHook } from './ambient.js'; import { removeMemoryHooks } from './memory.js'; +import { removeCaptureHooks } from './capture.js'; +import { removeDreamHook } from './dream.js'; import { removeHudStatusLine } from './hud.js'; import { removeContextHook } from './context.js'; import { listShadowed } from './skills.js'; @@ -399,6 +401,8 @@ export const uninstallCommand = new Command('uninstall') // Remove all Devflow hooks and flags in one pass (idempotent) let settingsContent = await removeAmbientHook(originalContent); settingsContent = removeMemoryHooks(settingsContent); + settingsContent = removeCaptureHooks(settingsContent); + settingsContent = removeDreamHook(settingsContent); settingsContent = removeHudStatusLine(settingsContent); settingsContent = removeContextHook(settingsContent); settingsContent = stripFlags(settingsContent); diff --git a/src/cli/plugins.ts b/src/cli/plugins.ts index 5eb2cd0e..e90bf8e3 100644 --- a/src/cli/plugins.ts +++ b/src/cli/plugins.ts @@ -48,15 +48,8 @@ export const DEVFLOW_PLUGINS: PluginDefinition[] = [ name: 'devflow-core-skills', description: 'Auto-activating quality enforcement skills - foundation layer for all Devflow plugins', commands: [], - // The Dream agent lives here (always-installed foundation plugin) because the - // session-start-context hook spawns Agent(subagent_type="Dream") unconditionally - // for the decisions/knowledge/curation subsystems — independent of whether - // the ambient plugin is installed. Declaring it only in devflow-ambient would break - // the dream subsystem under `devflow init --no-ambient` (agents install per selected - // plugin, unlike skills which install universally). Predecessor was the universally - // installed `devflow:sidecar` skill; core-skills preserves that guarantee. - agents: ['dream'], - skills: ['apply-decisions', 'apply-feature-knowledge', 'software-design', 'docs-framework', 'git', 'boundary-validation', 'test-driven-development', 'testing', 'dependency-research', 'dream-decisions', 'dream-curation'], + agents: [], + skills: ['apply-decisions', 'apply-feature-knowledge', 'software-design', 'docs-framework', 'git', 'boundary-validation', 'test-driven-development', 'testing', 'dependency-research'], rules: ['security', 'engineering', 'quality', 'reliability'], }, { @@ -152,7 +145,7 @@ export const DEVFLOW_PLUGINS: PluginDefinition[] = [ name: 'devflow-ambient', description: 'Keyword + plan auto-detection', commands: ['/ambient'], - agents: ['coder', 'validator', 'simplifier', 'scrutinizer', 'evaluator', 'tester', 'skimmer', 'reviewer', 'git', 'synthesizer', 'resolver', 'designer', 'knowledge', 'researcher', 'dream'], + agents: ['coder', 'validator', 'simplifier', 'scrutinizer', 'evaluator', 'tester', 'skimmer', 'reviewer', 'git', 'synthesizer', 'resolver', 'designer', 'knowledge', 'researcher'], skills: [ 'review-methodology', 'security', @@ -294,6 +287,11 @@ export const LEGACY_COMMAND_NAMES: string[] = [ */ export const LEGACY_AGENT_NAMES: string[] = [ 'shepherd', + // Dream system simplification: the Dream subagent (Agent(subagent_type="Dream")) + // is retired — decisions detection + curation now run via a detached `claude -p` + // worker (background-dream-update, spawned by spawn-dream-worker) that reads + // scripts/hooks/dream-procedure.md directly, not a Claude Code subagent. + 'dream', ]; /** @@ -536,21 +534,22 @@ const LEGACY_SKILLS_V2X: string[] = [ 'devflow:research:orch', 'devflow:release:orch', // v3.x dream per-task skills: bare names for pre-namespace installs. - // NOTE: dream-decisions and dream-curation are STILL-ACTIVE skills (declared in - // DEVFLOW_PLUGINS, installed at the namespaced path devflow:dream-*). - // These bare entries exist solely to clean up pre-namespace V2.x installs where - // skills were written without the devflow: prefix. On current installs the post-install - // fs.rm targets a bare path (e.g. ~/.claude/skills/dream-decisions) that does not exist - // — a harmless no-op. Do NOT remove these entries: upgrading users from V2.x need the - // stale bare-name dirs swept. (applies ADR-016; avoids PF-009) + // dream-decisions and dream-curation are now ALSO fully removed (dream system + // simplification: the detached background-dream-update worker reads + // scripts/hooks/dream-procedure.md directly instead of loading these as + // Claude Code skills), joining dream-memory and dream-knowledge below. These + // bare entries clean up pre-namespace V2.x installs where skills were written + // without the devflow: prefix — a harmless no-op on current installs. 'dream-memory', 'dream-knowledge', 'dream-decisions', 'dream-curation', - // v3.x dream-memory + dream-knowledge removal: namespaced names for cleanup of the - // installed devflow:dream-memory / devflow:dream-knowledge skills (both fully removed). + // Namespaced names for cleanup of the installed devflow:dream-* skills (all + // 4 dream per-task skills are now fully removed). 'devflow:dream-memory', 'devflow:dream-knowledge', + 'devflow:dream-decisions', + 'devflow:dream-curation', // v3.x agent-teams removal: namespaced name for cleanup of installed devflow:agent-teams skill 'devflow:agent-teams', // v2.x ambient refinements: devflow:-prefixed triage/guided/router names for cleanup diff --git a/src/cli/utils/decisions-config.ts b/src/cli/utils/decisions-config.ts index 6d09d860..5265d3da 100644 --- a/src/cli/utils/decisions-config.ts +++ b/src/cli/utils/decisions-config.ts @@ -6,30 +6,29 @@ import { getDecisionsConfigPath } from './project-paths.js'; /** * Merged decisions agent configuration from global and project-level config files. * - * Mirrors the shape of LearningConfig in learn.ts — decisions agent is a sibling - * pipeline with the same runtime knobs (model, throttle, daily cap, debug). + * The background dream worker (background-dream-update, spawned by spawn-dream-worker) + * has no daily-run cap or throttle of its own — it is gated by queue non-emptiness + * (spawn-dream-worker only spawns when the dream queue is non-empty or a leftover + * .processing batch exists) rather than a fixed daily/minute budget, so those knobs + * were dropped in the dream-system simplification. */ export interface DecisionsConfig { - /** Maximum number of background runs per day. Default: 3 */ - max_daily_runs: number; - /** Minimum minutes between consecutive runs. Default: 5 */ - throttle_minutes: number; - /** Model alias for the background Dream agent. Default: 'sonnet' */ + /** Model alias for the detached dream worker. Default: 'opus' */ model: string; /** Emit verbose logs when true. Default: false */ debug: boolean; } const DEFAULTS: DecisionsConfig = { - max_daily_runs: 3, - throttle_minutes: 5, - model: 'sonnet', + model: 'opus', debug: false, }; /** * Apply a single JSON config layer onto a DecisionsConfig, returning a new object. * Skips fields with wrong types. Swallows parse errors — callers see defaults. + * Unknown/dropped fields (e.g. a pre-simplification config still on disk with + * max_daily_runs/throttle_minutes) are silently ignored, not an error. */ export function applyDecisionsConfigLayer( config: DecisionsConfig, @@ -38,14 +37,6 @@ export function applyDecisionsConfigLayer( try { const raw = JSON.parse(json) as Record; return { - max_daily_runs: - typeof raw.max_daily_runs === 'number' - ? raw.max_daily_runs - : config.max_daily_runs, - throttle_minutes: - typeof raw.throttle_minutes === 'number' - ? raw.throttle_minutes - : config.throttle_minutes, model: typeof raw.model === 'string' ? raw.model : config.model, debug: diff --git a/src/cli/utils/project-paths.ts b/src/cli/utils/project-paths.ts index c9fa7c77..a67eeb0b 100644 --- a/src/cli/utils/project-paths.ts +++ b/src/cli/utils/project-paths.ts @@ -56,6 +56,26 @@ export function getDreamConfigPath(projectRoot: string): string { return path.join(projectRoot, '.devflow', 'dream', 'config.json'); } +/** .devflow/dream/.pending-turns.jsonl — decisions detection queue (dual-write with memory queue) */ +export function getDreamPendingTurnsPath(projectRoot: string): string { + return path.join(projectRoot, '.devflow', 'dream', '.pending-turns.jsonl'); +} + +/** .devflow/dream/.pending-turns.processing — atomic claim during background-dream-update processing */ +export function getDreamPendingTurnsProcessingPath(projectRoot: string): string { + return path.join(projectRoot, '.devflow', 'dream', '.pending-turns.processing'); +} + +/** .devflow/dream/.last-dream-ok — touched by the agent on successful dream worker completion */ +export function getDreamLastOkPath(projectRoot: string): string { + return path.join(projectRoot, '.devflow', 'dream', '.last-dream-ok'); +} + +/** .devflow/dream/.worker.lock — mkdir-based lock directory held by a live background-dream-update run */ +export function getDreamWorkerLockDir(projectRoot: string): string { + return path.join(projectRoot, '.devflow', 'dream', '.worker.lock'); +} + // --------------------------------------------------------------------------- // Decisions files // --------------------------------------------------------------------------- @@ -125,11 +145,6 @@ export function getDecisionsNotificationsPath(projectRoot: string): string { return path.join(projectRoot, '.devflow', 'decisions', '.decisions-notifications.json'); } -/** .devflow/decisions/.decisions-runs-today */ -export function getDecisionsRunsTodayPath(projectRoot: string): string { - return path.join(projectRoot, '.devflow', 'decisions', '.decisions-runs-today'); -} - /** .devflow/decisions/.decisions-batch-ids */ export function getDecisionsBatchIdsPath(projectRoot: string): string { return path.join(projectRoot, '.devflow', 'decisions', '.decisions-batch-ids'); diff --git a/src/templates/settings.json b/src/templates/settings.json index bbaf790d..155b1368 100644 --- a/src/templates/settings.json +++ b/src/templates/settings.json @@ -9,7 +9,16 @@ "hooks": [ { "type": "command", - "command": "${DEVFLOW_DIR}/scripts/hooks/run-hook stop-update-memory", + "command": "${DEVFLOW_DIR}/scripts/hooks/run-hook capture-turn", + "timeout": 10 + } + ] + }, + { + "hooks": [ + { + "type": "command", + "command": "${DEVFLOW_DIR}/scripts/hooks/run-hook memory-worker", "timeout": 10 } ] @@ -24,6 +33,24 @@ "timeout": 10 } ] + }, + { + "hooks": [ + { + "type": "command", + "command": "${DEVFLOW_DIR}/scripts/hooks/run-hook session-start-context", + "timeout": 10 + } + ] + }, + { + "hooks": [ + { + "type": "command", + "command": "${DEVFLOW_DIR}/scripts/hooks/run-hook spawn-dream-worker", + "timeout": 10 + } + ] } ], "PreCompact": [ diff --git a/tests/background-dream-update.test.ts b/tests/background-dream-update.test.ts index d3b7d20c..e9e22f6b 100644 --- a/tests/background-dream-update.test.ts +++ b/tests/background-dream-update.test.ts @@ -576,6 +576,76 @@ describe('DW8: CLI contract — allowedTools without skip-permissions, env flag, }); }); +// ============================================================================= +// DW9 — AC-C5: model resolution (project decisions.json -> global -> opus default) +// ============================================================================= +describe('DW9: AC-C5 — model resolution defaults to opus, project config wins', () => { + let projectDir: string; + let homeDir: string; + let shimDir: string; + + beforeEach(() => { + projectDir = fs.mkdtempSync(path.join(os.tmpdir(), 'bdu-dw9-')); + homeDir = fs.mkdtempSync(path.join(os.tmpdir(), 'bdu-dw9-home-')); + shimDir = fs.mkdtempSync(path.join(os.tmpdir(), 'bdu-dw9-shim-')); + fs.mkdirSync(path.join(projectDir, '.devflow', 'dream'), { recursive: true }); + fs.mkdirSync(path.join(projectDir, '.devflow', 'decisions'), { recursive: true }); + seedDreamQueue(projectDir); + }); + + afterEach(() => { + fs.rmSync(projectDir, { recursive: true, force: true }); + fs.rmSync(homeDir, { recursive: true, force: true }); + fs.rmSync(shimDir, { recursive: true, force: true }); + }); + + it('defaults to opus when neither project nor global decisions.json sets a model', () => { + const { argvCapture } = createRealAgentShim(shimDir, projectDir); + runWorker(projectDir, homeDir, shimDir); + const argv = fs.readFileSync(argvCapture, 'utf-8'); + expect(argv).toContain('--model opus'); + }); + + it('project decisions.json model field overrides the opus default', () => { + fs.writeFileSync( + path.join(projectDir, '.devflow', 'decisions', 'decisions.json'), + JSON.stringify({ model: 'haiku' }), + ); + const { argvCapture } = createRealAgentShim(shimDir, projectDir); + runWorker(projectDir, homeDir, shimDir); + const argv = fs.readFileSync(argvCapture, 'utf-8'); + expect(argv).toContain('--model haiku'); + }); + + it('global decisions.json model field is used when project config is absent', () => { + fs.mkdirSync(path.join(homeDir, '.devflow'), { recursive: true }); + fs.writeFileSync( + path.join(homeDir, '.devflow', 'decisions.json'), + JSON.stringify({ model: 'sonnet' }), + ); + const { argvCapture } = createRealAgentShim(shimDir, projectDir); + runWorker(projectDir, homeDir, shimDir); + const argv = fs.readFileSync(argvCapture, 'utf-8'); + expect(argv).toContain('--model sonnet'); + }); + + it('project decisions.json wins over global when both set a model', () => { + fs.mkdirSync(path.join(homeDir, '.devflow'), { recursive: true }); + fs.writeFileSync( + path.join(homeDir, '.devflow', 'decisions.json'), + JSON.stringify({ model: 'sonnet' }), + ); + fs.writeFileSync( + path.join(projectDir, '.devflow', 'decisions', 'decisions.json'), + JSON.stringify({ model: 'haiku' }), + ); + const { argvCapture } = createRealAgentShim(shimDir, projectDir); + runWorker(projectDir, homeDir, shimDir); + const argv = fs.readFileSync(argvCapture, 'utf-8'); + expect(argv).toContain('--model haiku'); + }); +}); + // ============================================================================= // AC-C3 — dream-procedure.md contract (pinned) // ============================================================================= diff --git a/tests/capture-hooks.test.ts b/tests/capture-hooks.test.ts index 9cbe5017..16831532 100644 --- a/tests/capture-hooks.test.ts +++ b/tests/capture-hooks.test.ts @@ -266,6 +266,38 @@ describe('capture-turn', () => { expect(exitCode).toBe(0); expect(fs.existsSync(path.join(projectDir, '.devflow'))).toBe(false); }); + + it('AC-F14: DEVFLOW_BG_UPDATER=1 -> exit 0, zero filesystem writes', () => { + const { exitCode } = runHook( + CAPTURE_TURN, + { cwd: projectDir, session_id: 't', last_assistant_message: 'hi' }, + homeDir, + { DEVFLOW_BG_UPDATER: '1' }, + ); + expect(exitCode).toBe(0); + expect(fs.existsSync(path.join(projectDir, '.devflow'))).toBe(false); + }); + + it('content truncated at 2000 chars', () => { + const longMessage = 'b'.repeat(5000); + runHook(CAPTURE_TURN, { cwd: projectDir, session_id: 't', last_assistant_message: longMessage }, homeDir); + const mem = readJsonl(path.join(projectDir, '.devflow', 'memory', '.pending-turns.jsonl')); + expect((mem[0].content as string).length).toBe(2000 + '... [truncated]'.length); + }); + + it('ignores legacy response_text field — only last_assistant_message gates capture', () => { + // Regression guard (PF-006 lineage): a payload carrying only the old field name + // (response_text) and not last_assistant_message must produce zero appends. + runHook(CAPTURE_TURN, { cwd: projectDir, session_id: 't', response_text: 'this should be ignored' }, homeDir); + expect(fs.existsSync(path.join(projectDir, '.devflow'))).toBe(false); + }); + + it('creates its own log file tagged [capture-turn] on successful capture', () => { + runHook(CAPTURE_TURN, { cwd: projectDir, session_id: 't', last_assistant_message: 'hello' }, homeDir); + const logFile = workerLogPath(projectDir, homeDir, 'capture-turn'); + expect(fs.existsSync(logFile)).toBe(true); + expect(fs.readFileSync(logFile, 'utf-8')).toContain('[capture-turn]'); + }); }); // ============================================================================= @@ -654,3 +686,34 @@ describe('spawn-dream-worker', () => { expect(elapsed).toBeLessThan(2000); }); }); + +// ============================================================================= +// capture-prompt + capture-turn integration +// ============================================================================= +describe('capture-prompt + capture-turn integration', () => { + let projectDir: string; + let homeDir: string; + + beforeEach(() => { + projectDir = fs.mkdtempSync(path.join(os.tmpdir(), 'cap-integ-')); + homeDir = fs.mkdtempSync(path.join(os.tmpdir(), 'cap-integ-home-')); + }); + + afterEach(() => { + fs.rmSync(projectDir, { recursive: true, force: true }); + fs.rmSync(homeDir, { recursive: true, force: true }); + }); + + it('user turn then assistant turn are appended in order to both queues', () => { + runHook(CAPTURE_PROMPT, { cwd: projectDir, session_id: 'integ', prompt: 'implement feature X' }, homeDir); + runHook(CAPTURE_TURN, { cwd: projectDir, session_id: 'integ', last_assistant_message: 'done' }, homeDir); + + for (const queue of ['memory', 'dream'] as const) { + const rows = readJsonl(path.join(projectDir, '.devflow', queue, '.pending-turns.jsonl')); + expect(rows).toEqual([ + { role: 'user', content: 'implement feature X', ts: expect.any(Number) }, + { role: 'assistant', content: 'done', ts: expect.any(Number) }, + ]); + } + }); +}); diff --git a/tests/capture.test.ts b/tests/capture.test.ts new file mode 100644 index 00000000..53361be8 --- /dev/null +++ b/tests/capture.test.ts @@ -0,0 +1,260 @@ +import { describe, it, expect } from 'vitest'; +import { + addCaptureHooks, + removeCaptureHooks, + hasCaptureHooks, + countCaptureHooks, +} from '../src/cli/commands/capture.js'; + +describe('addCaptureHooks', () => { + it('adds all 3 capture hook types to empty settings', () => { + const result = addCaptureHooks('{}', '/home/user/.devflow'); + const settings = JSON.parse(result); + + expect(settings.hooks.UserPromptSubmit).toHaveLength(1); + expect(settings.hooks.Stop).toHaveLength(1); + expect(settings.hooks.PostToolUse).toHaveLength(1); + expect(settings.hooks.UserPromptSubmit[0].hooks[0].command).toContain('capture-prompt'); + expect(settings.hooks.Stop[0].hooks[0].command).toContain('capture-turn'); + expect(settings.hooks.PostToolUse[0].hooks[0].command).toContain('capture-question'); + }); + + it('scopes the PostToolUse entry with matcher: "AskUserQuestion"', () => { + const result = addCaptureHooks('{}', '/home/user/.devflow'); + const settings = JSON.parse(result); + expect(settings.hooks.PostToolUse[0].matcher).toBe('AskUserQuestion'); + }); + + it('matcher persists through a JSON settings round-trip', () => { + const first = addCaptureHooks('{}', '/home/user/.devflow'); + // Simulate a settings.json read/write cycle (JSON.stringify -> disk -> JSON.parse) + const roundTripped = JSON.stringify(JSON.parse(first)); + const parsed = JSON.parse(roundTripped); + expect(parsed.hooks.PostToolUse[0].matcher).toBe('AskUserQuestion'); + }); + + it('is idempotent — calling twice returns identical JSON', () => { + const first = addCaptureHooks('{}', '/home/user/.devflow'); + const second = addCaptureHooks(first, '/home/user/.devflow'); + expect(second).toBe(first); + }); + + it('adds only missing hooks when partial state (1 hook missing)', () => { + const input = JSON.stringify({ + hooks: { + UserPromptSubmit: [{ hooks: [{ type: 'command', command: '/path/capture-prompt', timeout: 10 }] }], + Stop: [{ hooks: [{ type: 'command', command: '/path/capture-turn', timeout: 10 }] }], + }, + }); + const result = addCaptureHooks(input, '/home/user/.devflow'); + const settings = JSON.parse(result); + + expect(settings.hooks.UserPromptSubmit).toHaveLength(1); + expect(settings.hooks.Stop).toHaveLength(1); + expect(settings.hooks.PostToolUse).toHaveLength(1); + expect(settings.hooks.PostToolUse[0].hooks[0].command).toContain('capture-question'); + expect(settings.hooks.PostToolUse[0].matcher).toBe('AskUserQuestion'); + }); + + it('preserves existing ambient preamble hook on UserPromptSubmit when adding capture-prompt', () => { + const input = JSON.stringify({ + hooks: { + UserPromptSubmit: [{ hooks: [{ type: 'command', command: '/path/run-hook preamble' }] }], + }, + }); + const result = addCaptureHooks(input, '/home/user/.devflow'); + const settings = JSON.parse(result); + + expect(settings.hooks.UserPromptSubmit).toHaveLength(2); + const commands = settings.hooks.UserPromptSubmit.map((m: { hooks: { command: string }[] }) => m.hooks[0].command); + expect(commands.some((c: string) => c.includes('preamble'))).toBe(true); + expect(commands.some((c: string) => c.includes('capture-prompt'))).toBe(true); + }); + + it('creates hooks object if missing', () => { + const input = JSON.stringify({ statusLine: { type: 'command' } }); + const result = addCaptureHooks(input, '/home/user/.devflow'); + const settings = JSON.parse(result); + + expect(settings.hooks).toBeDefined(); + expect(settings.hooks.Stop).toHaveLength(1); + }); + + it('uses correct devflowDir path in command via run-hook wrapper', () => { + const result = addCaptureHooks('{}', '/custom/path/.devflow'); + const settings = JSON.parse(result); + + expect(settings.hooks.UserPromptSubmit[0].hooks[0].command).toContain('/custom/path/.devflow/scripts/hooks/run-hook'); + expect(settings.hooks.Stop[0].hooks[0].command).toContain('run-hook'); + expect(settings.hooks.PostToolUse[0].hooks[0].command).toContain('run-hook'); + }); + + it('preserves other settings (statusLine, env)', () => { + const input = JSON.stringify({ + statusLine: { type: 'command', command: 'statusline.sh' }, + env: { SOME_VAR: '1' }, + }); + const result = addCaptureHooks(input, '/home/user/.devflow'); + const settings = JSON.parse(result); + + expect(settings.statusLine.command).toBe('statusline.sh'); + expect(settings.env.SOME_VAR).toBe('1'); + }); + + it('sets timeout to 10 for all hooks', () => { + const result = addCaptureHooks('{}', '/home/user/.devflow'); + const settings = JSON.parse(result); + + expect(settings.hooks.UserPromptSubmit[0].hooks[0].timeout).toBe(10); + expect(settings.hooks.Stop[0].hooks[0].timeout).toBe(10); + expect(settings.hooks.PostToolUse[0].hooks[0].timeout).toBe(10); + }); +}); + +describe('removeCaptureHooks', () => { + it('removes all 3 capture hook types', () => { + const withHooks = addCaptureHooks('{}', '/home/user/.devflow'); + const result = removeCaptureHooks(withHooks); + const settings = JSON.parse(result); + + expect(settings.hooks).toBeUndefined(); + }); + + it('preserves ambient preamble on UserPromptSubmit when removing capture-prompt', () => { + const input = JSON.stringify({ + hooks: { + UserPromptSubmit: [ + { hooks: [{ type: 'command', command: 'preamble' }] }, + { hooks: [{ type: 'command', command: '/path/capture-prompt' }] }, + ], + }, + }); + const result = removeCaptureHooks(input); + const settings = JSON.parse(result); + + expect(settings.hooks.UserPromptSubmit).toHaveLength(1); + expect(settings.hooks.UserPromptSubmit[0].hooks[0].command).toBe('preamble'); + }); + + it('preserves memory-worker on Stop when removing capture-turn (adjacent hook, different module)', () => { + const input = JSON.stringify({ + hooks: { + Stop: [ + { hooks: [{ type: 'command', command: '/path/capture-turn' }] }, + { hooks: [{ type: 'command', command: '/path/memory-worker' }] }, + ], + }, + }); + const result = removeCaptureHooks(input); + const settings = JSON.parse(result); + + expect(settings.hooks.Stop).toHaveLength(1); + expect(settings.hooks.Stop[0].hooks[0].command).toContain('memory-worker'); + }); + + it('does not touch legacy dream-dispatch/dream-capture entries — that sweep belongs to memory.ts', () => { + // capture.ts is a pure always-on registrar (context.ts pattern) with no legacy + // awareness of its own; the dream-dispatch/dream-capture/dream-evaluate legacy + // sweep is memory.ts's LEGACY_HOOK_MARKERS responsibility (see memory.test.ts). + // This test proves removeCaptureHooks coexists safely: it removes only its own + // current markers and leaves legacy entries untouched for memory.ts to sweep. + const input = JSON.stringify({ + hooks: { + UserPromptSubmit: [ + { hooks: [{ type: 'command', command: '/path/run-hook dream-dispatch', timeout: 10 }] }, + { hooks: [{ type: 'command', command: '/path/run-hook capture-prompt', timeout: 10 }] }, + ], + Stop: [ + { hooks: [{ type: 'command', command: '/path/run-hook dream-capture', timeout: 10 }] }, + { hooks: [{ type: 'command', command: '/path/run-hook capture-turn', timeout: 10 }] }, + ], + }, + }); + const result = removeCaptureHooks(input); + const settings = JSON.parse(result); + + // Only the current capture-* markers are removed + expect(settings.hooks.UserPromptSubmit).toHaveLength(1); + expect(settings.hooks.UserPromptSubmit[0].hooks[0].command).toContain('dream-dispatch'); + expect(settings.hooks.Stop).toHaveLength(1); + expect(settings.hooks.Stop[0].hooks[0].command).toContain('dream-capture'); + }); + + it('is idempotent — safe to call when not present', () => { + const input = JSON.stringify({ + hooks: { UserPromptSubmit: [{ hooks: [{ type: 'command', command: 'other.sh' }] }] }, + }); + const result = removeCaptureHooks(input); + + expect(result).toBe(input); + }); + + it('cleans empty hooks object when all arrays removed', () => { + const withHooks = addCaptureHooks('{}', '/home/user/.devflow'); + const result = removeCaptureHooks(withHooks); + const settings = JSON.parse(result); + + expect(settings.hooks).toBeUndefined(); + }); + + it('accepts a parsed Settings object and does not mutate the original', () => { + const settings = { + hooks: { + Stop: [{ hooks: [{ type: 'command' as const, command: '/path/capture-turn', timeout: 10 }] }], + }, + }; + const result = removeCaptureHooks(settings); + const parsed = JSON.parse(result); + expect(parsed.hooks).toBeUndefined(); + // Original must be unchanged + expect(settings.hooks.Stop).toHaveLength(1); + }); + + it('toggle cycle: enable → disable → enable produces clean state', () => { + const enabled = addCaptureHooks('{}', '/home/user/.devflow'); + const disabled = removeCaptureHooks(enabled); + const reEnabled = addCaptureHooks(disabled, '/home/user/.devflow'); + const settings = JSON.parse(reEnabled); + + expect(settings.hooks.UserPromptSubmit).toHaveLength(1); + expect(settings.hooks.Stop).toHaveLength(1); + expect(settings.hooks.PostToolUse).toHaveLength(1); + expect(settings.hooks.PostToolUse[0].matcher).toBe('AskUserQuestion'); + }); +}); + +describe('hasCaptureHooks / countCaptureHooks', () => { + it('hasCaptureHooks returns true when all 3 present', () => { + const withHooks = addCaptureHooks('{}', '/home/user/.devflow'); + expect(hasCaptureHooks(withHooks)).toBe(true); + expect(countCaptureHooks(withHooks)).toBe(3); + }); + + it('hasCaptureHooks returns false when none present', () => { + expect(hasCaptureHooks('{}')).toBe(false); + expect(countCaptureHooks('{}')).toBe(0); + }); + + it('returns false/partial-count when only 2 of 3 present', () => { + const input = JSON.stringify({ + hooks: { + UserPromptSubmit: [{ hooks: [{ type: 'command', command: '/path/capture-prompt' }] }], + Stop: [{ hooks: [{ type: 'command', command: '/path/capture-turn' }] }], + }, + }); + expect(hasCaptureHooks(input)).toBe(false); + expect(countCaptureHooks(input)).toBe(2); + }); + + it('accepts a parsed Settings object (not just JSON string)', () => { + const settings = { + hooks: { + UserPromptSubmit: [{ hooks: [{ type: 'command' as const, command: '/path/capture-prompt', timeout: 10 }] }], + Stop: [{ hooks: [{ type: 'command' as const, command: '/path/capture-turn', timeout: 10 }] }], + PostToolUse: [{ matcher: 'AskUserQuestion', hooks: [{ type: 'command' as const, command: '/path/capture-question', timeout: 10 }] }], + }, + }; + expect(countCaptureHooks(settings)).toBe(3); + expect(hasCaptureHooks(settings)).toBe(true); + }); +}); diff --git a/tests/decisions/cli-subcommands.test.ts b/tests/decisions/cli-subcommands.test.ts index a25b17ce..3b01e3b9 100644 --- a/tests/decisions/cli-subcommands.test.ts +++ b/tests/decisions/cli-subcommands.test.ts @@ -16,9 +16,7 @@ import * as os from 'os'; vi.mock('../../src/cli/utils/decisions-config.js', () => ({ loadDecisionsConfig: vi.fn(() => ({ - max_daily_runs: 3, - throttle_minutes: 5, - model: 'sonnet', + model: 'opus', debug: false, })), })); @@ -274,12 +272,11 @@ describe('decisions --clear log truncation', () => { // --------------------------------------------------------------------------- describe('decisions --reset target files', () => { - it('reset targets decisions-specific state files only', () => { - const resetFiles = [ + it('reset targets decisions-specific state files (.devflow/decisions/)', () => { + const decisionsStateFiles = [ 'decisions-log.jsonl', '.decisions-manifest.json', '.decisions-notifications.json', - '.decisions-runs-today', '.decisions-batch-ids', 'decisions.json', ]; @@ -291,7 +288,7 @@ describe('decisions --reset target files', () => { 'WORKING-MEMORY.md', ]; - for (const f of resetFiles) { + for (const f of decisionsStateFiles) { expect( f.includes('decision') || f.includes('decisions'), `Expected "${f}" to contain "decision" or "decisions"`, @@ -299,34 +296,70 @@ describe('decisions --reset target files', () => { } for (const f of preservedFiles) { - expect(resetFiles).not.toContain(f); + expect(decisionsStateFiles).not.toContain(f); + } + }); + + it('reset also targets the dream (decisions-detection) queue and success stamp (.devflow/dream/)', () => { + // Not decision-prefixed by name — these live in .devflow/dream/, the queue the + // detached background-dream-update worker claims from. Reset must drain them + // too so a re-enable doesn't process stale pre-reset turns. + const dreamQueueFiles = [ + '.pending-turns.jsonl', + '.pending-turns.processing', + '.last-dream-ok', + ]; + + const preservedDreamFiles = [ + 'config.json', // shared multi-feature config — reset must never touch this + ]; + + for (const f of preservedDreamFiles) { + expect(dreamQueueFiles).not.toContain(f); } }); }); // --------------------------------------------------------------------------- -// --reset dream cleanup: verify dream state files are targeted +// --reset dream cleanup: verify legacy marker-pipeline state files are targeted // --------------------------------------------------------------------------- describe('decisions --reset dream state cleanup', () => { - it('dream cleanup targets .decisions-runs-today and decisions.*.json markers', () => { + it('dream cleanup targets legacy stamp files (.decisions-runs-today, .curation-last, .processor-spawned-at)', () => { const dreamFilesToClean = [ '.decisions-runs-today', + '.curation-last', + '.processor-spawned-at', ]; - const dreamMarkerPattern = /^decisions\..+\.json$/; for (const f of dreamFilesToClean) { - expect(f).toContain('decisions'); + expect(f.startsWith('.')).toBe(true); + } + }); + + it('dream cleanup targets legacy decisions.*/curation.* markers across all 4 suffixes', () => { + const dreamMarkerPattern = /^(decisions|curation)\..+\.(json|processing|retries|failed)$/; + + for (const f of [ + 'decisions.abc123.json', + 'decisions.session-xyz.processing', + 'decisions.abc123.retries', + 'decisions.abc123.failed', + 'curation.abc123.json', + 'curation.abc123.processing', + ]) { + expect(dreamMarkerPattern.test(f)).toBe(true); } - expect(dreamMarkerPattern.test('decisions.abc123.json')).toBe(true); - expect(dreamMarkerPattern.test('decisions.session-xyz.json')).toBe(true); - expect(dreamMarkerPattern.test('learning.abc123.json')).toBe(false); - expect(dreamMarkerPattern.test('decisions.json')).toBe(false); + // Never touches: learning markers (pipeline removed separately), the shared + // config.json, or the new .pending-turns.jsonl/.processing queue files. + for (const f of ['learning.abc123.json', 'decisions.json', 'config.json', '.pending-turns.jsonl', '.pending-turns.processing']) { + expect(dreamMarkerPattern.test(f)).toBe(false); + } }); it('dream cleanup does not target learning state files', () => { - const decisionsDreamFiles = ['.decisions-runs-today']; + const decisionsDreamFiles = ['.decisions-runs-today', '.curation-last', '.processor-spawned-at']; const learningFiles = ['.learning-runs-today', '.learning-sessions']; for (const lf of learningFiles) { diff --git a/tests/decisions/config.test.ts b/tests/decisions/config.test.ts index 3a9abc9a..76b64d4d 100644 --- a/tests/decisions/config.test.ts +++ b/tests/decisions/config.test.ts @@ -31,21 +31,10 @@ function ensureDir(dir: string): void { describe('applyDecisionsConfigLayer', () => { const base: DecisionsConfig = { - max_daily_runs: 3, - throttle_minutes: 5, - model: 'sonnet', + model: 'opus', debug: false, }; - it('overrides individual numeric fields', () => { - const result = applyDecisionsConfigLayer( - base, - JSON.stringify({ max_daily_runs: 10 }), - ); - expect(result.max_daily_runs).toBe(10); - expect(result.throttle_minutes).toBe(5); // unchanged - }); - it('overrides model string field', () => { const result = applyDecisionsConfigLayer(base, JSON.stringify({ model: 'haiku' })); expect(result.model).toBe('haiku'); @@ -56,14 +45,6 @@ describe('applyDecisionsConfigLayer', () => { expect(result.debug).toBe(true); }); - it('ignores non-numeric max_daily_runs', () => { - const result = applyDecisionsConfigLayer( - base, - JSON.stringify({ max_daily_runs: 'lots' }), - ); - expect(result.max_daily_runs).toBe(3); - }); - it('ignores non-boolean debug', () => { const result = applyDecisionsConfigLayer(base, JSON.stringify({ debug: 'yes' })); expect(result.debug).toBe(false); @@ -71,14 +52,14 @@ describe('applyDecisionsConfigLayer', () => { it('ignores non-string model', () => { const result = applyDecisionsConfigLayer(base, JSON.stringify({ model: 42 })); - expect(result.model).toBe('sonnet'); + expect(result.model).toBe('opus'); }); it('returns a new object — does not mutate input', () => { const input: DecisionsConfig = { ...base }; - const result = applyDecisionsConfigLayer(input, JSON.stringify({ max_daily_runs: 99 })); - expect(result.max_daily_runs).toBe(99); - expect(input.max_daily_runs).toBe(3); // not mutated + const result = applyDecisionsConfigLayer(input, JSON.stringify({ model: 'sonnet' })); + expect(result.model).toBe('sonnet'); + expect(input.model).toBe('opus'); // not mutated expect(result).not.toBe(input); }); @@ -92,6 +73,18 @@ describe('applyDecisionsConfigLayer', () => { const result = applyDecisionsConfigLayer(base, '{}'); expect(result).toEqual(base); }); + + it('ignores dropped legacy fields (max_daily_runs/throttle_minutes) without error', () => { + // On-disk configs from before the dream-system simplification may still carry + // these fields — they must load without error and be silently ignored. + const result = applyDecisionsConfigLayer( + base, + JSON.stringify({ max_daily_runs: 7, throttle_minutes: 15, model: 'haiku' }), + ); + expect(result).toEqual({ model: 'haiku', debug: false }); + expect((result as Record).max_daily_runs).toBeUndefined(); + expect((result as Record).throttle_minutes).toBeUndefined(); + }); }); // --------------------------------------------------------------------------- @@ -128,39 +121,37 @@ describe('loadDecisionsConfig', () => { it('returns all defaults when no config files exist', () => { const config = loadDecisionsConfig(projectCwd); - expect(config.max_daily_runs).toBe(3); - expect(config.throttle_minutes).toBe(5); - expect(config.model).toBe('sonnet'); + expect(config.model).toBe('opus'); expect(config.debug).toBe(false); }); it('global config overrides defaults', () => { - writeJson(devflowDir, 'decisions.json', { max_daily_runs: 7 }); + writeJson(devflowDir, 'decisions.json', { model: 'haiku' }); const config = loadDecisionsConfig(projectCwd); - expect(config.max_daily_runs).toBe(7); - expect(config.throttle_minutes).toBe(5); // default preserved + expect(config.model).toBe('haiku'); + expect(config.debug).toBe(false); // default preserved }); it('project config overrides global config', () => { writeJson(devflowDir, 'decisions.json', { - max_daily_runs: 7, model: 'haiku', + debug: true, }); writeJson(path.join(projectCwd, '.devflow', 'decisions'), 'decisions.json', { - max_daily_runs: 2, + model: 'sonnet', }); const config = loadDecisionsConfig(projectCwd); - expect(config.max_daily_runs).toBe(2); // project wins - expect(config.model).toBe('haiku'); // global preserved when project doesn't set + expect(config.model).toBe('sonnet'); // project wins + expect(config.debug).toBe(true); // global preserved when project doesn't set }); it('project config alone overrides defaults', () => { writeJson(path.join(projectCwd, '.devflow', 'decisions'), 'decisions.json', { - model: 'opus', + model: 'sonnet', }); const config = loadDecisionsConfig(projectCwd); - expect(config.model).toBe('opus'); - expect(config.max_daily_runs).toBe(3); // default + expect(config.model).toBe('sonnet'); + expect(config.debug).toBe(false); // default }); it('invalid JSON in global config returns defaults without crashing', () => { @@ -170,35 +161,46 @@ describe('loadDecisionsConfig', () => { 'utf-8', ); const config = loadDecisionsConfig(projectCwd); - expect(config.max_daily_runs).toBe(3); - expect(config.model).toBe('sonnet'); + expect(config.model).toBe('opus'); }); it('invalid JSON in project config falls back to global + defaults', () => { - writeJson(devflowDir, 'decisions.json', { max_daily_runs: 7 }); + writeJson(devflowDir, 'decisions.json', { model: 'haiku' }); fs.writeFileSync( path.join(projectCwd, '.devflow', 'decisions', 'decisions.json'), 'bad json', 'utf-8', ); const config = loadDecisionsConfig(projectCwd); - expect(config.max_daily_runs).toBe(7); // global applied - expect(config.model).toBe('sonnet'); // default + expect(config.model).toBe('haiku'); // global applied }); it('partial project override preserves global fields', () => { writeJson(devflowDir, 'decisions.json', { - throttle_minutes: 15, model: 'haiku', }); writeJson(path.join(projectCwd, '.devflow', 'decisions'), 'decisions.json', { debug: true, }); const config = loadDecisionsConfig(projectCwd); - expect(config.throttle_minutes).toBe(15); // from global expect(config.model).toBe('haiku'); // from global expect(config.debug).toBe(true); // from project - expect(config.max_daily_runs).toBe(3); // default }); + it('AC-C5: model defaults to opus (not sonnet) when nothing configures it', () => { + const config = loadDecisionsConfig(projectCwd); + expect(config.model).toBe('opus'); + }); + + it('on-disk config still containing dropped max_daily_runs/throttle_minutes loads without error', () => { + writeJson(path.join(projectCwd, '.devflow', 'decisions'), 'decisions.json', { + max_daily_runs: 3, + throttle_minutes: 5, + model: 'haiku', + }); + const config = loadDecisionsConfig(projectCwd); + expect(config.model).toBe('haiku'); + expect((config as Record).max_daily_runs).toBeUndefined(); + expect((config as Record).throttle_minutes).toBeUndefined(); + }); }); diff --git a/tests/decisions/decisions-format.test.ts b/tests/decisions/decisions-format.test.ts index 2bda1477..360221f2 100644 --- a/tests/decisions/decisions-format.test.ts +++ b/tests/decisions/decisions-format.test.ts @@ -360,15 +360,20 @@ describe('json-helper.cjs assign-anchor delegates to decisions-format', () => { }); // --------------------------------------------------------------------------- -// dream-decisions SKILL.md content-presence assertions (AC-F1, AC-F2) +// dream-procedure.md content-presence assertions (AC-F1, AC-F2) // --------------------------------------------------------------------------- -// These lightweight checks verify that the SKILL contains the creation-bar +// These lightweight checks verify that the procedure contains the creation-bar // elements required by the plan. They do not test LLM judgment — that is // validated by the Tester agent via scenarios. They lock the prose contract -// so that the SKILL cannot accidentally regress on the key phrases. +// so that the procedure cannot accidentally regress on the key phrases. +// +// The standalone dream-decisions SKILL.md was retired when the dream system +// was simplified to a detached `claude -p` worker (background-dream-update) +// that reads scripts/hooks/dream-procedure.md directly — it is not a Claude +// Code skill (skills do not load in `claude -p` sessions). -describe('dream-decisions SKILL.md creation-bar contract', () => { - const SKILL_PATH = path.join(ROOT, 'shared/skills/dream-decisions/SKILL.md'); +describe('dream-procedure.md creation-bar contract', () => { + const SKILL_PATH = path.join(ROOT, 'scripts/hooks/dream-procedure.md'); let skillContent: string; beforeAll(() => { @@ -376,7 +381,7 @@ describe('dream-decisions SKILL.md creation-bar contract', () => { }); it('contains abstain-by-default stance', () => { - expect(skillContent).toContain('Most sessions produce nothing'); + expect(skillContent).toContain('Most runs produce nothing'); expect(skillContent).toContain('If unsure, record nothing'); }); @@ -393,13 +398,14 @@ describe('dream-decisions SKILL.md creation-bar contract', () => { expect(skillContent).toContain('reinforce it'); }); - it('instructs agent to use assign-anchor and prohibits decisions-append', () => { - // The SKILL must instruct the agent to use assign-anchor for promotion + it('instructs agent to use assign-anchor for promotion, never invents numbers itself', () => { + // The procedure must instruct the agent to use assign-anchor for promotion expect(skillContent).toContain('assign-anchor'); - // The SKILL must prohibit decisions-append (mentioning it only to forbid it) - expect(skillContent).toContain('NEVER call `decisions-append`'); - // Must NOT contain a positive instruction to call decisions-append + // decisions-append is retired tooling and is not mentioned at all (nothing + // positively instructs calling it — there is no lingering reference to forbid). expect(skillContent).not.toMatch(/\bjson-helper\.cjs\b.*\bdecisions-append\b/); + expect(skillContent).not.toContain('decisions-append'); + expect(skillContent).toContain('NEVER invent an ADR-NNN/PF-NNN number'); }); it('has no numeric confidence gate (ADR-008)', () => { diff --git a/tests/decisions/dream-curation.test.ts b/tests/decisions/dream-curation.test.ts index 4ebde87b..19a8a30a 100644 --- a/tests/decisions/dream-curation.test.ts +++ b/tests/decisions/dream-curation.test.ts @@ -113,100 +113,89 @@ function readDecisionsMd(dir: string): string { } // --------------------------------------------------------------------------- -// dream-curation SKILL.md content-presence assertions +// dream-procedure.md content-presence assertions (AC-C3) +// +// The standalone dream-decisions/dream-curation SKILL.md files were retired +// when the dream system was simplified to a detached `claude -p` worker +// (background-dream-update) that reads scripts/hooks/dream-procedure.md +// directly — it is not a Claude Code skill (skills do not load in `claude -p` +// sessions). The Iron-Law strings this describe pins are the same contract +// the old SKILL.md files enforced, now consolidated into one procedure doc. // --------------------------------------------------------------------------- -describe('dream-curation SKILL.md curation contract (Phase 6)', () => { - const SKILL_PATH = path.join(ROOT, 'shared/skills/dream-curation/SKILL.md'); - let skillContent: string; +describe('dream-procedure.md curation contract (AC-C3)', () => { + const PROCEDURE_PATH = path.join(ROOT, 'scripts/hooks/dream-procedure.md'); + let procedureContent: string; beforeAll(() => { - skillContent = fs.readFileSync(SKILL_PATH, 'utf8'); - }); - - it('Iron Law says "RETIRE BY STATUS — THE LEDGER IS THE SOURCE OF TRUTH"', () => { - expect(skillContent).toContain('RETIRE BY STATUS'); - expect(skillContent).toContain('THE LEDGER IS THE SOURCE OF TRUTH'); + procedureContent = fs.readFileSync(PROCEDURE_PATH, 'utf8'); }); - it('states .md files are rendered views, never hand-edited', () => { - expect(skillContent).toContain('never hand-edited'); - // Or equivalent phrasing - expect(skillContent).toMatch(/rendered|pure render/i); + it('Iron Law says assign-anchor owns numbering, render owns the .md, never hand-edit', () => { + expect(procedureContent).toContain('assign-anchor OWNS NUMBERING'); + expect(procedureContent).toContain('render OWNS THE .md'); + expect(procedureContent).toContain('NEVER HAND-EDIT'); }); - it('instructs to call retire-anchor for deprecation/retirement', () => { - expect(skillContent).toContain('retire-anchor'); - expect(skillContent).toContain('decisions_status'); + it('instructs to call retire-anchor for deprecation/retirement, never hand-edit the .md', () => { + expect(procedureContent).toContain('retire-anchor'); + expect(procedureContent).toContain('RETIRE BY STATUS'); + expect(procedureContent).toContain('never hand-edit the .md'); }); - it('does NOT contain the old 3-call lock/Edit dance instruction', () => { - // The old SKILL had explicit bash acquire/release around an Edit tool call - expect(skillContent).not.toContain('_ACQUIRED=false'); - expect(skillContent).not.toContain('NEVER re-acquire it inside this window'); - // Old step: "Edit tool call: flip the Status line" - expect(skillContent).not.toMatch(/Edit tool call.*flip/); + it('states inputs are read-only (except where noted)', () => { + expect(procedureContent).toContain('Inputs (read-only, except where noted)'); }); - it('does NOT instruct direct .md editing for status changes', () => { - // The old instruction was to edit "- **Status**: Deprecated" directly - expect(skillContent).not.toContain('Flip its status to `- **Status**: Deprecated`'); - expect(skillContent).not.toContain('directly editing two lines'); + it('writes only via merge-observation/assign-anchor/retire-anchor/rotate-observations', () => { + expect(procedureContent).toContain('merge-observation'); + expect(procedureContent).toContain('assign-anchor'); + expect(procedureContent).toContain('retire-anchor'); + expect(procedureContent).toContain('rotate-observations'); }); - it('does NOT reference decisions-append positively', () => { - // decisions-append is removed; SKILL must not instruct to use it - // (it may mention it only in a prohibition context — but the old positive "decisions-append adds" is gone) - expect(skillContent).not.toContain('decisions-append adds'); - expect(skillContent).not.toContain('decisions-append` adds'); + it('touches .last-dream-ok as the final success signal', () => { + expect(procedureContent).toContain('.devflow/dream/.last-dream-ok'); + expect(procedureContent).toContain('LAST'); }); - it('references assign-anchor as the writer (for new entries)', () => { - // SKILL may reference assign-anchor in the context of how retire-anchor relates to assign-anchor - // OR in the count-active command description — either way the concept is present - expect(skillContent).toContain('assign-anchor'); + it('writes an optional last-run-summary only when the ledger changed', () => { + expect(procedureContent).toContain('dream/last-run-summary'); + expect(procedureContent).toMatch(/if nothing changed, do NOT create this file/i); }); - it('contains rotation step under .observations.lock', () => { - expect(skillContent).toContain('rotate-observations'); - expect(skillContent).toContain('.observations.lock'); - }); - - it('rotation step is for archiving stale observing rows (AC-F9)', () => { - expect(skillContent).toContain('observing'); - expect(skillContent).toMatch(/30 days|30-day/); - // never archives anchored rows - expect(skillContent).toMatch(/never touch.*anchor|never.*anchor.*archived/i); + it('contains the abstain-by-default creation bar', () => { + expect(procedureContent).toContain('abstain-by-default'); + expect(procedureContent).toMatch(/most runs produce nothing/i); }); it('contains ADR-XOR-PF awareness note', () => { - expect(skillContent).toContain('ADR-XOR-PF'); - // forward-looking / concrete failure distinction - expect(skillContent).toContain('forward-looking'); - expect(skillContent).toContain('Concrete failure'); + expect(procedureContent).toContain('ADR-XOR-PF'); + expect(procedureContent).toContain('forward-looking'); + expect(procedureContent).toContain('Concrete failure'); }); it('contains dedup awareness note', () => { - expect(skillContent).toMatch(/dedup|near-duplicate/i); + expect(procedureContent).toMatch(/dedup|near-duplicate/i); }); - it('7-day window is keyed off the ledger date field', () => { - // Not the .md file content - expect(skillContent).toContain("ledger row's"); - expect(skillContent).toContain('date` field'); + it('bounds curation to at most 5 changes per run', () => { + // \s+ tolerates an incidental mid-sentence line wrap in the markdown source + // (a literal newline between "curation" and "changes", not just a space). + expect(procedureContent).toMatch(/≤5\s+curation\s+changes/); + expect(procedureContent).toContain('stop after 5 changes'); }); - it('describes recoverability: re-activating a retired entry + render restores it (AC-F6)', () => { - expect(skillContent).toContain('Recoverability'); - expect(skillContent).toMatch(/re-activat|re.render/i); + it('7-day protection window is keyed off the ledger date field', () => { + expect(procedureContent).toContain('7-day protection window'); + expect(procedureContent).toContain("ledger row's"); + expect(procedureContent).toContain('date` field'); }); - it('never hold both .decisions.lock and .observations.lock simultaneously (ADR-017)', () => { - expect(skillContent).toContain('ADR-017'); - expect(skillContent).not.toContain('hold both'); - // The statement is actually "never hold both" — check the SKILL advises separation - expect(skillContent).toContain('.observations.lock'); - expect(skillContent).toContain('.decisions.lock'); + it('rotation step is for archiving stale observing rows (AC-F9)', () => { + expect(procedureContent).toContain('observing'); + expect(procedureContent).toMatch(/30 days|30-day/); + expect(procedureContent).toMatch(/never touches anchored|never touch.*anchor/i); }); }); @@ -434,39 +423,41 @@ describe('AC-F6: retired entry is recoverable — re-activate + render restores }); // --------------------------------------------------------------------------- -// AC-F9: rotation step — curation SKILL contract +// AC-F9: rotation step — dream-procedure.md contract // Already tested at the op level in ledger-ops.test.ts; here we verify -// the curation SKILL wires it correctly (contract-level check). +// the procedure wires it correctly (contract-level check). // --------------------------------------------------------------------------- describe('AC-F9: rotation step wired into curation (contract check)', () => { - const SKILL_PATH = path.join(ROOT, 'shared/skills/dream-curation/SKILL.md'); - let skillContent: string; + const PROCEDURE_PATH = path.join(ROOT, 'scripts/hooks/dream-procedure.md'); + let procedureContent: string; beforeAll(() => { - skillContent = fs.readFileSync(SKILL_PATH, 'utf8'); + procedureContent = fs.readFileSync(PROCEDURE_PATH, 'utf8'); }); - it('SKILL calls rotate-observations before selecting curation candidates', () => { - // The rotation step must appear BEFORE "LLM judgment" in the SKILL - const rotateIdx = skillContent.indexOf('rotate-observations'); - const judgmentIdx = skillContent.indexOf('LLM judgment'); + it('procedure calls rotate-observations before selecting curation retire/merge candidates', () => { + // The rotation step must appear BEFORE the Part 2 "LLM judgment" (retire/merge + // candidate selection) — use lastIndexOf since Part 1 has its own earlier + // "LLM judgment" heading (the creation-bar one) that rotation is NOT gated by. + const rotateIdx = procedureContent.indexOf('rotate-observations'); + const judgmentIdx = procedureContent.lastIndexOf('LLM judgment'); expect(rotateIdx).toBeGreaterThan(-1); expect(judgmentIdx).toBeGreaterThan(-1); expect(rotateIdx).toBeLessThan(judgmentIdx); }); - it('SKILL says rotation runs under .observations.lock', () => { + it('procedure says rotation runs under .observations.lock (self-locking)', () => { // The relevant paragraph must mention .observations.lock near rotate-observations - const rotateIdx = skillContent.indexOf('rotate-observations'); + const rotateIdx = procedureContent.indexOf('rotate-observations'); // Check within 400 chars of the rotate-observations mention - const context = skillContent.slice(Math.max(0, rotateIdx - 400), rotateIdx + 400); + const context = procedureContent.slice(Math.max(0, rotateIdx - 400), rotateIdx + 400); expect(context).toContain('.observations.lock'); }); - it('SKILL states rotation archives stale observing rows and never touches anchored rows', () => { - expect(skillContent).toContain('anchored'); - expect(skillContent).toContain('archive'); + it('procedure states rotation archives stale observing rows and never touches anchored rows', () => { + expect(procedureContent).toContain('anchored'); + expect(procedureContent).toContain('archive'); }); it('rotateObservations internal function: anchored rows never archived (AC-F9 contract)', () => { diff --git a/tests/eager-memory-refresh.test.ts b/tests/eager-memory-refresh.test.ts index 97cc117c..407ceb8e 100644 --- a/tests/eager-memory-refresh.test.ts +++ b/tests/eager-memory-refresh.test.ts @@ -5,11 +5,13 @@ * Covers AC-F1/F2/F3, AC-F4 (injection states), AC-F5/F6/F7, AC-C2/C3/C4, * AC-P3 (double-spawn), and no-regression scenarios. * - * Design constraint: tests that exercise dream-capture with a stale trigger MUST - * supply a fake claude shim on PATH (prepended before the system PATH). This - * prevents the nohup-spawned background-memory-update worker from invoking the - * real claude binary and hanging for 120s. Tests that only check queue-write - * behavior use a fresh trigger file so the 120s throttle prevents any spawn. + * The capture/spawn split: queue-append lives in capture-turn (never throttles, + * never spawns — see tests/capture-hooks.test.ts for its dedicated coverage), + * while the 120s throttle + detached background-memory-update spawn lives in + * memory-worker. Design constraint: tests that exercise memory-worker with a + * stale trigger MUST supply a fake claude shim on PATH (prepended before the + * system PATH). This prevents the nohup-spawned background-memory-update worker + * from invoking the real claude binary and hanging for 120s. */ import { describe, it, expect, beforeEach, afterEach } from 'vitest'; @@ -19,9 +21,9 @@ import * as path from 'path'; import * as os from 'os'; const HOOKS_DIR = path.resolve(__dirname, '..', 'scripts', 'hooks'); -const CAPTURE_HOOK = path.join(HOOKS_DIR, 'dream-capture'); +const CAPTURE_TURN_HOOK = path.join(HOOKS_DIR, 'capture-turn'); +const MEMORY_WORKER_HOOK = path.join(HOOKS_DIR, 'memory-worker'); const SESSION_START_MEMORY_HOOK = path.join(HOOKS_DIR, 'session-start-memory'); -const SESSION_START_CONTEXT_HOOK = path.join(HOOKS_DIR, 'session-start-context'); const BACKGROUND_UPDATER = path.join(HOOKS_DIR, 'background-memory-update'); // --------------------------------------------------------------------------- @@ -186,7 +188,7 @@ function initGitRepo(dir: string): void { // ============================================================================= // S1 — AC-F2/F3/C3: background-memory-update happy path // -// We run background-memory-update DIRECTLY (not via dream-capture) to avoid +// We run background-memory-update DIRECTLY (not via memory-worker) to avoid // the nohup-detach complexity. The fake claude shim writes a deterministic file. // ============================================================================= describe('S1: end-to-end happy path — background-memory-update worker (AC-F2/F3/C3)', () => { @@ -241,68 +243,6 @@ describe('S1: end-to-end happy path — background-memory-update worker (AC-F2/F }); }); -// ============================================================================= -// S1b — AC-F1: dream-capture touches trigger before spawning (throttle gate) -// -// We use a fake claude shim on PATH so the nohup-spawned worker does not -// invoke the real claude binary. -// ============================================================================= -describe('S1b: AC-F1 — dream-capture touches .working-memory-last-trigger', () => { - let projectDir: string; - let homeDir: string; - let shimDir: string; - - beforeEach(() => { - projectDir = fs.mkdtempSync(path.join(os.tmpdir(), 'emr-s1b-')); - homeDir = fs.mkdtempSync(path.join(os.tmpdir(), 'emr-s1b-home-')); - shimDir = fs.mkdtempSync(path.join(os.tmpdir(), 'emr-s1b-shim-')); - fs.mkdirSync(path.join(projectDir, '.devflow', 'memory'), { recursive: true }); - fs.mkdirSync(path.join(projectDir, '.devflow', 'dream'), { recursive: true }); - const memFile = path.join(projectDir, '.devflow', 'memory', 'WORKING-MEMORY.md'); - createFakeClaudeShim(shimDir, memFile); - seedQueue(projectDir); - }); - - afterEach(() => { - fs.rmSync(projectDir, { recursive: true, force: true }); - fs.rmSync(homeDir, { recursive: true, force: true }); - fs.rmSync(shimDir, { recursive: true, force: true }); - }); - - it('trigger file is touched (mtime refreshed) after throttle expires', () => { - const triggerFile = path.join(projectDir, '.devflow', 'memory', '.working-memory-last-trigger'); - fs.writeFileSync(triggerFile, ''); - backdateMtime(triggerFile, 600); // 10 minutes ago — throttle expired - - runHookWithFakeClaude( - CAPTURE_HOOK, - { cwd: projectDir, session_id: 'test', last_assistant_message: 'hello' }, - homeDir, - shimDir - ); - - const triggerStat = fs.statSync(triggerFile); - expect(triggerStat.mtimeMs).toBeGreaterThan(Date.now() - 15000); - }); - - it('no memory.json or memory.processing marker created by dream-capture (AC-C3)', () => { - const triggerFile = path.join(projectDir, '.devflow', 'memory', '.working-memory-last-trigger'); - fs.writeFileSync(triggerFile, ''); - backdateMtime(triggerFile, 600); - - runHookWithFakeClaude( - CAPTURE_HOOK, - { cwd: projectDir, session_id: 'test', last_assistant_message: 'hello' }, - homeDir, - shimDir - ); - - const dreamDir = path.join(projectDir, '.devflow', 'dream'); - const memMarkers = fs.readdirSync(dreamDir).filter((f) => f.startsWith('memory')); - expect(memMarkers).toHaveLength(0); - }); -}); - // ============================================================================= // S2 — AC-F4: Injection state rendering (session-start-memory) // ============================================================================= @@ -438,15 +378,14 @@ describe('S2: AC-F4 — session-start-memory injection states', () => { }); // ============================================================================= -// S3 — AC-F6: PATH without claude binary +// S3 — AC-F6: capture-turn is throttle-agnostic (queue-append only) // -// We keep trigger file FRESH (no backdating) so dream-capture throttles and -// does NOT try to spawn the worker. This lets us verify exit-0 and queue -// behavior without PATH manipulation worries. -// We test the capture-hook's "no claude found" exit path in S3b by explicitly -// using the fake shim approach with the trigger backdated. +// capture-turn never reads the trigger file at all — it only appends to the +// queue. These tests verify exit-0 and queue-append behavior regardless of +// trigger-file state. We test memory-worker's "no claude found" exit path +// separately in S3b. // ============================================================================= -describe('S3: AC-F6 — capture hook behavior when claude binary absent on PATH', () => { +describe('S3: AC-F6 — capture-turn queue-append is unaffected by trigger/throttle state', () => { let projectDir: string; let homeDir: string; @@ -456,7 +395,8 @@ describe('S3: AC-F6 — capture hook behavior when claude binary absent on PATH' fs.mkdirSync(path.join(projectDir, '.devflow', 'memory'), { recursive: true }); fs.mkdirSync(path.join(projectDir, '.devflow', 'dream'), { recursive: true }); seedQueue(projectDir); - // Fresh trigger (< 120s) — throttle will prevent spawn attempt + // Fresh trigger (< 120s) — irrelevant to capture-turn, present only to prove + // capture-turn ignores it entirely (that's memory-worker's concern). fs.writeFileSync( path.join(projectDir, '.devflow', 'memory', '.working-memory-last-trigger'), '' ); @@ -467,18 +407,18 @@ describe('S3: AC-F6 — capture hook behavior when claude binary absent on PATH' fs.rmSync(homeDir, { recursive: true, force: true }); }); - it('dream-capture exits 0 even when throttled (queue append only)', () => { + it('capture-turn exits 0 regardless of trigger-file state (queue append only)', () => { const { exitCode } = runHook( - CAPTURE_HOOK, + CAPTURE_TURN_HOOK, { cwd: projectDir, session_id: 'test', last_assistant_message: 'hello world' }, homeDir ); expect(exitCode).toBe(0); }); - it('queue still receives new turn (throttle does not block capture)', () => { + it('queue still receives new turn (capture-turn never consults the trigger file)', () => { runHook( - CAPTURE_HOOK, + CAPTURE_TURN_HOOK, { cwd: projectDir, session_id: 'test', last_assistant_message: 'hello world' }, homeDir ); @@ -488,9 +428,9 @@ describe('S3: AC-F6 — capture hook behavior when claude binary absent on PATH' expect(lines.length).toBeGreaterThan(0); }); - it('AC-F4/State C: after throttled capture, session-start-memory shows REFRESH FAILING', () => { + it('AC-F4/State C: after a capture with no worker run, session-start-memory shows REFRESH FAILING', () => { runHook( - CAPTURE_HOOK, + CAPTURE_TURN_HOOK, { cwd: projectDir, session_id: 'test', last_assistant_message: 'hello world' }, homeDir ); @@ -513,12 +453,12 @@ describe('S3: AC-F6 — capture hook behavior when claude binary absent on PATH' // S3b — AC-F6 source code: "no claude" path exits 0, logs skip message // ============================================================================= describe('S3b: AC-F6 source code — no-claude exit logged correctly', () => { - it('dream-capture code logs SKIP when claude binary not found', () => { - const captureSrc = fs.readFileSync(CAPTURE_HOOK, 'utf-8'); - expect(captureSrc).toContain('claude binary not found'); - expect(captureSrc).toContain('worker not spawned (queue intact)'); + it('memory-worker code logs SKIP when claude binary not found', () => { + const workerSrc = fs.readFileSync(MEMORY_WORKER_HOOK, 'utf-8'); + expect(workerSrc).toContain('claude binary not found'); + expect(workerSrc).toContain('worker not spawned (queue intact)'); // Exit must be 0 (not a hard failure) - expect(captureSrc).toContain('exit 0'); + expect(workerSrc).toContain('exit 0'); }); }); @@ -581,7 +521,7 @@ describe('S4: AC-F3/P3 — watchdog and failure path', () => { const okFile = path.join(projectDir, '.devflow', 'memory', '.last-refresh-ok'); expect(fs.existsSync(okFile)).toBe(false); - // .processing must be retained for dream-recover crash recovery. + // .processing must be retained for session-start-memory's own cold-path crash recovery (S19). const processingFile = path.join(projectDir, '.devflow', 'memory', '.pending-turns.processing'); expect(fs.existsSync(processingFile)).toBe(true); }, 20000); // 20s timeout: 2s watchdog sleep + 5s SIGTERM grace + margin @@ -702,41 +642,6 @@ describe('S6: AC-F5 — user-only queue skips LLM', () => { }); }); -// ============================================================================= -// S7 — AC-F7: memory:false in config -// ============================================================================= -describe('S7: AC-F7 — memory:false in dream config gates dream-capture', () => { - let projectDir: string; - let homeDir: string; - - beforeEach(() => { - projectDir = fs.mkdtempSync(path.join(os.tmpdir(), 'emr-s7-')); - homeDir = fs.mkdtempSync(path.join(os.tmpdir(), 'emr-s7-home-')); - writeDreamConfig(projectDir, { memory: false }); - fs.mkdirSync(path.join(projectDir, '.devflow', 'memory'), { recursive: true }); - }); - - afterEach(() => { - fs.rmSync(projectDir, { recursive: true, force: true }); - fs.rmSync(homeDir, { recursive: true, force: true }); - }); - - it('memory:false — no queue append, no trigger touch, no dream marker', () => { - runHook( - CAPTURE_HOOK, - { cwd: projectDir, session_id: 'test', last_assistant_message: 'hello' }, - homeDir - ); - - expect(fs.existsSync(path.join(projectDir, '.devflow', 'memory', '.pending-turns.jsonl'))).toBe(false); - expect(fs.existsSync(path.join(projectDir, '.devflow', 'memory', '.working-memory-last-trigger'))).toBe(false); - - const dreamDir = path.join(projectDir, '.devflow', 'dream'); - const markers = fs.readdirSync(dreamDir).filter((f) => f.startsWith('memory')); - expect(markers).toHaveLength(0); - }); -}); - // ============================================================================= // S8 — AC-C2/C4: Security — prompt via STDIN, DEVFLOW_BG_UPDATER, feedback loop // ============================================================================= @@ -791,24 +696,6 @@ describe('S8: AC-C2/C4 — security constraints', () => { fs.rmSync(shimDir, { recursive: true, force: true }); } }); - - it('dream-capture with DEVFLOW_BG_UPDATER=1 exits early — no queue write', () => { - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'emr-s8-')); - const homeDir = fs.mkdtempSync(path.join(os.tmpdir(), 'emr-s8-home-')); - fs.mkdirSync(path.join(tmpDir, '.devflow', 'memory'), { recursive: true }); - try { - runHook( - CAPTURE_HOOK, - { cwd: tmpDir, session_id: 'test', last_assistant_message: 'hello' }, - homeDir, - { DEVFLOW_BG_UPDATER: '1' } - ); - expect(fs.existsSync(path.join(tmpDir, '.devflow', 'memory', '.pending-turns.jsonl'))).toBe(false); - } finally { - fs.rmSync(tmpDir, { recursive: true, force: true }); - fs.rmSync(homeDir, { recursive: true, force: true }); - } - }); }); // ============================================================================= @@ -861,75 +748,11 @@ describe('S9: AC-C4 — .devflow/ local-by-default with the feature-knowledge ca }); // ============================================================================= -// S10 — No-regression: session-start-context decisions/knowledge agents -// ============================================================================= -describe('S10: No-regression — session-start-context decisions/knowledge unaffected', () => { - let projectDir: string; - let homeDir: string; - - beforeEach(() => { - projectDir = fs.mkdtempSync(path.join(os.tmpdir(), 'emr-s10-')); - homeDir = fs.mkdtempSync(path.join(os.tmpdir(), 'emr-s10-home-')); - fs.mkdirSync(path.join(projectDir, '.devflow', 'dream'), { recursive: true }); - fs.mkdirSync(path.join(projectDir, '.devflow', 'decisions'), { recursive: true }); - initGitRepo(projectDir); - }); - - afterEach(() => { - fs.rmSync(projectDir, { recursive: true, force: true }); - fs.rmSync(homeDir, { recursive: true, force: true }); - }); - - it('decisions.json marker → DREAM MAINTENANCE directive emitted, not memory', () => { - fs.writeFileSync( - path.join(projectDir, '.devflow', 'dream', 'decisions.test123.json'), - JSON.stringify({ type: 'decisions', session_id: 'test123' }) - ); - - const { stdout } = runHook(SESSION_START_CONTEXT_HOOK, { cwd: projectDir }, homeDir); - const parsed = JSON.parse(stdout.trim()) as { hookSpecificOutput?: { additionalContext?: string } }; - const ctx = parsed?.hookSpecificOutput?.additionalContext ?? ''; - - expect(ctx).toContain('DREAM MAINTENANCE'); - expect(ctx).toContain('decisions'); - expect(ctx).not.toMatch(/Agent\(.*memory.*\)/i); - }); - - it('stale memory.json marker deleted unconditionally (NOT in Dream directive)', () => { - const markerPath = path.join(projectDir, '.devflow', 'dream', 'memory.json'); - fs.writeFileSync(markerPath, JSON.stringify({ type: 'memory' })); - - runHook(SESSION_START_CONTEXT_HOOK, { cwd: projectDir }, homeDir); - - expect(fs.existsSync(markerPath)).toBe(false); - }); - - it('decisions usage scanner exits 0 when ADR-1/PF-1 appear in assistant message', () => { - // Use a FRESH trigger file (< 120s) so dream-capture throttles before spawning the worker - const memDir = path.join(projectDir, '.devflow', 'memory'); - fs.mkdirSync(memDir, { recursive: true }); - fs.writeFileSync(path.join(memDir, '.working-memory-last-trigger'), ''); - - fs.writeFileSync( - path.join(projectDir, '.devflow', 'decisions', 'decisions-log.jsonl'), '' - ); - - const { exitCode } = runHook( - CAPTURE_HOOK, - { cwd: projectDir, session_id: 'test', last_assistant_message: 'I applied ADR-1 and also PF-1 here' }, - homeDir - ); - - expect(exitCode).toBe(0); - }); -}); - -// ============================================================================= -// S11 — AC-C3: No memory.* marker in .devflow/dream/ after dream-capture +// S11 — AC-C3: No memory.* marker in .devflow/dream/ after a memory-worker spawn // // Uses fake claude on PATH so the nohup-spawned worker uses the shim, not real claude. // ============================================================================= -describe('S11: AC-C3 — no memory.* marker in .devflow/dream/ after dream-capture', () => { +describe('S11: AC-C3 — no memory.* marker in .devflow/dream/ after a memory-worker spawn', () => { let projectDir: string; let homeDir: string; let shimDir: string; @@ -956,10 +779,10 @@ describe('S11: AC-C3 — no memory.* marker in .devflow/dream/ after dream-captu fs.rmSync(shimDir, { recursive: true, force: true }); }); - it('no memory.* file in .devflow/dream/ after dream-capture (no marker created)', () => { + it('no memory.* file in .devflow/dream/ after memory-worker spawns the updater (no marker created)', () => { runHookWithFakeClaude( - CAPTURE_HOOK, - { cwd: projectDir, session_id: 'test', last_assistant_message: 'some response' }, + MEMORY_WORKER_HOOK, + { cwd: projectDir }, homeDir, shimDir ); @@ -1584,9 +1407,9 @@ exit 0 // ============================================================================= // S19 — session-start-memory cold-path recovery for orphaned .pending-turns.processing // -// Mirrors dream-recover's dream_recover_stale logic (duplicated, not sourced — -// session-start-memory has no dependency on dream-recover). Age >300s + no -// existing .jsonl -> recovered; .jsonl present -> left alone (non-clobber). +// session-start-memory owns this recovery itself (self-contained, no external +// helper dependency). Age >300s + no existing .jsonl -> recovered; .jsonl +// present -> left alone (non-clobber). // ============================================================================= describe('S19: session-start-memory cold-path .pending-turns.processing recovery', () => { let projectDir: string; diff --git a/tests/learning/filter.test.ts b/tests/learning/filter.test.ts deleted file mode 100644 index 4534e3f7..00000000 --- a/tests/learning/filter.test.ts +++ /dev/null @@ -1,268 +0,0 @@ -// tests/learning/filter.test.ts -// Tests for the channel-based transcript filter (D1, D2). -// Validates pollution rejection, channel population, and cap behaviour. - -import { describe, it, expect } from 'vitest'; -import { createRequire } from 'module'; -import * as path from 'path'; -import * as url from 'url'; - -const __filename = url.fileURLToPath(import.meta.url); -const __dirname = path.dirname(__filename); -const require = createRequire(import.meta.url); - -// Load the CJS module under test -const { extractChannels } = require( - path.resolve(__dirname, '../../scripts/hooks/lib/transcript-filter.cjs') -) as { extractChannels: (jsonl: string) => { userSignals: string[]; dialogPairs: { prior: string; user: string }[] } }; - -// Helper: build a JSONL line in the transcript envelope format used by Claude Code -function line(entry: Record): string { - return JSON.stringify(entry); -} -function userMsg(text: string, extra: Record = {}): string { - return line({ type: 'user', message: { role: 'user', content: text }, ...extra }); -} -function assistantMsg(text: string): string { - return line({ type: 'assistant', message: { role: 'assistant', content: text } }); -} -function userArrayMsg(items: unknown[]): string { - return line({ type: 'user', message: { role: 'user', content: items } }); -} - -describe('extractChannels — pollution rejection (D2)', () => { - it('rejects entries where isMeta is true', () => { - const input = [ - line({ type: 'user', isMeta: true, message: { role: 'user', content: 'some user text here' } }), - userMsg('keep this valid user message ok'), - ].join('\n'); - - const { userSignals } = extractChannels(input); - expect(userSignals).toHaveLength(1); - expect(userSignals[0]).toBe('keep this valid user message ok'); - }); - - it('rejects entries with sourceToolUseID present', () => { - const input = [ - line({ type: 'user', sourceToolUseID: 'tool-123', message: { role: 'user', content: 'hidden content here xx' } }), - userMsg('visible user message comes through ok'), - ].join('\n'); - - const { userSignals } = extractChannels(input); - expect(userSignals).toHaveLength(1); - expect(userSignals[0]).toBe('visible user message comes through ok'); - }); - - it('rejects entries with toolUseResult present', () => { - const input = [ - line({ type: 'user', toolUseResult: { output: 'foo' }, message: { role: 'user', content: 'tool result noise' } }), - userMsg('clean message after tool result ok'), - ].join('\n'); - - const { userSignals } = extractChannels(input); - expect(userSignals).toHaveLength(1); - expect(userSignals[0]).toBe('clean message after tool result ok'); - }); - - it('rejects string user content matching wrapper', () => { - const input = [ - userMsg('devflow:implement loaded context'), - userMsg('plain user message that is fine'), - ].join('\n'); - - const { userSignals } = extractChannels(input); - expect(userSignals).toHaveLength(1); - expect(userSignals[0]).toBe('plain user message that is fine'); - }); - - it('rejects string user content matching { - const input = [ - userMsg('bar baz content'), - userMsg('good user message here for signals'), - ].join('\n'); - - const { userSignals } = extractChannels(input); - expect(userSignals).toHaveLength(1); - expect(userSignals[0]).toBe('good user message here for signals'); - }); - - it('rejects string user content matching wrapper', () => { - const input = [ - userMsg('Do not use certain tools.'), - userMsg('actual user instruction that matters'), - ].join('\n'); - - const { userSignals } = extractChannels(input); - expect(userSignals).toHaveLength(1); - expect(userSignals[0]).toBe('actual user instruction that matters'); - }); - - it('rejects string user content matching wrapper', () => { - const input = [ - userMsg('here is an example content block'), - userMsg('real user request text goes here'), - ].join('\n'); - - const { userSignals } = extractChannels(input); - expect(userSignals).toHaveLength(1); - expect(userSignals[0]).toBe('real user request text goes here'); - }); - - it('rejects user array turn where any item is type tool_result', () => { - const input = [ - userArrayMsg([ - { type: 'tool_result', content: 'result output data here' }, - { type: 'text', text: 'this text should also be excluded' }, - ]), - userMsg('clean user message passes through ok'), - ].join('\n'); - - const { userSignals } = extractChannels(input); - expect(userSignals).toHaveLength(1); - expect(userSignals[0]).toBe('clean user message passes through ok'); - }); - - it('excludes noisy text items from array but keeps clean items', () => { - const input = [ - userArrayMsg([ - { type: 'text', text: 'injected context noise' }, - { type: 'text', text: 'actual user text that is clean and valid ok' }, - ]), - userMsg('another valid message here too'), - ].join('\n'); - - const { userSignals } = extractChannels(input); - // First message has clean text after filtering noisy item - expect(userSignals).toHaveLength(2); - expect(userSignals[0]).toBe('actual user text that is clean and valid ok'); - }); - - it('rejects empty user content (< 5 chars after trim)', () => { - const input = [ - userMsg(' ok '), // 2 chars after trim — rejected - userMsg('valid user text that is long enough'), - ].join('\n'); - - const { userSignals } = extractChannels(input); - expect(userSignals).toHaveLength(1); - expect(userSignals[0]).toBe('valid user text that is long enough'); - }); - - it('rejects invalid JSON lines gracefully', () => { - const input = [ - '{ invalid json line here }', - userMsg('valid message is kept after bad json'), - ].join('\n'); - - const { userSignals } = extractChannels(input); - expect(userSignals).toHaveLength(1); - }); -}); - -describe('extractChannels — channel population', () => { - it('populates USER_SIGNALS from plain user text', () => { - const input = [ - userMsg('implement the plan, then run /self-review, then commit'), - userMsg('squash merge the PR, pull main, delete the feature branch'), - ].join('\n'); - - const { userSignals, dialogPairs } = extractChannels(input); - expect(userSignals).toHaveLength(2); - expect(userSignals[0]).toContain('implement the plan'); - expect(dialogPairs).toHaveLength(0); // no assistant turns precede these - }); - - it('populates DIALOG_PAIRS when user turn directly follows assistant turn', () => { - const input = [ - assistantMsg("I'll add a try/catch around the Result parsing to be safe here"), - userMsg("no — we use Result types precisely to avoid try/catch. Do not wrap."), - ].join('\n'); - - const { userSignals, dialogPairs } = extractChannels(input); - expect(userSignals).toHaveLength(1); - expect(dialogPairs).toHaveLength(1); - expect(dialogPairs[0].prior).toContain("I'll add a try/catch"); - expect(dialogPairs[0].user).toContain("we use Result types"); - }); - - it('does NOT add to DIALOG_PAIRS when user follows another user (no assistant prior)', () => { - const input = [ - userMsg('first user message about workflow steps here'), - userMsg('second user message directly following first one'), - ].join('\n'); - - const { dialogPairs } = extractChannels(input); - expect(dialogPairs).toHaveLength(0); - }); - - it('does NOT include DIALOG_PAIR when assistant turn has only tool-use content (rejected)', () => { - // Assistant turn with only noisy content is filtered out — cannot be a "prior" - const input = [ - line({ - type: 'assistant', - message: { - role: 'assistant', - content: 'some-command', - }, - }), - userMsg('user message after rejected assistant turn here ok'), - ].join('\n'); - - const { userSignals, dialogPairs } = extractChannels(input); - // User message still appears in signals - expect(userSignals).toHaveLength(1); - // But no dialog pair because assistant turn was filtered - expect(dialogPairs).toHaveLength(0); - }); - - it('builds multiple dialog pairs correctly', () => { - const input = [ - assistantMsg("I'll update the file and amend the commit for you right now."), - userMsg("don't amend pushed commits. Create a new one."), - assistantMsg("Understood. I'll create a new commit with the changes needed."), - userMsg("correct — thank you for confirming that approach"), - ].join('\n'); - - const { dialogPairs } = extractChannels(input); - expect(dialogPairs).toHaveLength(2); - expect(dialogPairs[0].prior).toContain("amend the commit"); - expect(dialogPairs[0].user).toContain("don't amend pushed commits"); - expect(dialogPairs[1].prior).toContain("new commit"); - expect(dialogPairs[1].user).toContain("thank you"); - }); -}); - -describe('extractChannels — caps and limits', () => { - it('caps text to 1200 chars per turn', () => { - const longText = 'x'.repeat(2000); - const input = userMsg(longText); - - const { userSignals } = extractChannels(input); - expect(userSignals).toHaveLength(1); - expect(userSignals[0].length).toBe(1200); - }); - - it('caps to last 80 turns when more are present', () => { - // Create 90 user messages - const lines: string[] = []; - for (let i = 0; i < 90; i++) { - lines.push(userMsg(`user message number ${i} which is valid and long enough`)); - } - - const { userSignals } = extractChannels(lines.join('\n')); - // Should have at most 80 turns worth of signals - expect(userSignals.length).toBeLessThanOrEqual(80); - }); - - it('handles empty input gracefully', () => { - const { userSignals, dialogPairs } = extractChannels(''); - expect(userSignals).toHaveLength(0); - expect(dialogPairs).toHaveLength(0); - }); - - it('handles input with only blank lines', () => { - const { userSignals, dialogPairs } = extractChannels('\n\n\n'); - expect(userSignals).toHaveLength(0); - expect(dialogPairs).toHaveLength(0); - }); -}); diff --git a/tests/memory.test.ts b/tests/memory.test.ts index a9ed699e..18808417 100644 --- a/tests/memory.test.ts +++ b/tests/memory.test.ts @@ -6,37 +6,32 @@ import { exec } from 'child_process'; import { addMemoryHooks, removeMemoryHooks, hasMemoryHooks, countMemoryHooks, cleanQueueFiles, hasMemoryDir, filterProjectsWithMemory } from '../src/cli/commands/memory.js'; describe('addMemoryHooks', () => { - it('adds all 5 dream hook types to empty settings', () => { + it('adds all 3 memory hook types to empty settings', () => { const result = addMemoryHooks('{}', '/home/user/.devflow'); const settings = JSON.parse(result); - expect(settings.hooks.UserPromptSubmit).toHaveLength(1); expect(settings.hooks.Stop).toHaveLength(1); - expect(settings.hooks.SessionEnd).toHaveLength(1); expect(settings.hooks.SessionStart).toHaveLength(1); expect(settings.hooks.PreCompact).toHaveLength(1); - expect(settings.hooks.UserPromptSubmit[0].hooks[0].command).toContain('dream-dispatch'); - expect(settings.hooks.Stop[0].hooks[0].command).toContain('dream-capture'); - expect(settings.hooks.SessionEnd[0].hooks[0].command).toContain('dream-evaluate'); + expect(settings.hooks.UserPromptSubmit).toBeUndefined(); + expect(settings.hooks.SessionEnd).toBeUndefined(); + expect(settings.hooks.Stop[0].hooks[0].command).toContain('memory-worker'); expect(settings.hooks.SessionStart[0].hooks[0].command).toContain('session-start-memory'); expect(settings.hooks.PreCompact[0].hooks[0].command).toContain('pre-compact-memory'); }); - it('preserves existing ambient preamble hook when adding dream hooks', () => { + it('does not touch UserPromptSubmit — owned by capture.ts, not memory.ts', () => { const input = JSON.stringify({ hooks: { - UserPromptSubmit: [{ hooks: [{ type: 'command', command: 'preamble' }] }], + UserPromptSubmit: [{ hooks: [{ type: 'command', command: '/path/run-hook preamble' }] }], }, }); const result = addMemoryHooks(input, '/home/user/.devflow'); const settings = JSON.parse(result); - // Ambient preamble preserved alongside dream-dispatch - expect(settings.hooks.UserPromptSubmit).toHaveLength(2); - expect(settings.hooks.UserPromptSubmit[0].hooks[0].command).toBe('preamble'); - expect(settings.hooks.UserPromptSubmit[1].hooks[0].command).toContain('dream-dispatch'); + expect(settings.hooks.UserPromptSubmit).toHaveLength(1); + expect(settings.hooks.UserPromptSubmit[0].hooks[0].command).toContain('preamble'); expect(settings.hooks.Stop).toHaveLength(1); - expect(settings.hooks.SessionEnd).toHaveLength(1); expect(settings.hooks.SessionStart).toHaveLength(1); expect(settings.hooks.PreCompact).toHaveLength(1); }); @@ -51,9 +46,7 @@ describe('addMemoryHooks', () => { it('adds only missing hooks when partial state (1 hook missing)', () => { const input = JSON.stringify({ hooks: { - UserPromptSubmit: [{ hooks: [{ type: 'command', command: '/path/dream-dispatch', timeout: 10 }] }], - Stop: [{ hooks: [{ type: 'command', command: '/path/dream-capture', timeout: 10 }] }], - SessionEnd: [{ hooks: [{ type: 'command', command: '/path/dream-evaluate', timeout: 10 }] }], + Stop: [{ hooks: [{ type: 'command', command: '/path/memory-worker', timeout: 10 }] }], SessionStart: [{ hooks: [{ type: 'command', command: '/path/session-start-memory', timeout: 10 }] }], }, }); @@ -61,41 +54,13 @@ describe('addMemoryHooks', () => { const settings = JSON.parse(result); // Existing hooks preserved - expect(settings.hooks.UserPromptSubmit).toHaveLength(1); expect(settings.hooks.Stop).toHaveLength(1); - expect(settings.hooks.SessionEnd).toHaveLength(1); expect(settings.hooks.SessionStart).toHaveLength(1); // Missing hook added expect(settings.hooks.PreCompact).toHaveLength(1); expect(settings.hooks.PreCompact[0].hooks[0].command).toContain('pre-compact-memory'); }); - it('adds UserPromptSubmit dream-dispatch alongside existing preamble', () => { - // Simulate a partial install that already has ambient preamble - const input = JSON.stringify({ - hooks: { - UserPromptSubmit: [{ hooks: [{ type: 'command', command: '/path/preamble' }] }], - Stop: [{ hooks: [{ type: 'command', command: '/path/dream-capture', timeout: 10 }] }], - SessionEnd: [{ hooks: [{ type: 'command', command: '/path/dream-evaluate', timeout: 10 }] }], - SessionStart: [{ hooks: [{ type: 'command', command: '/path/session-start-memory', timeout: 10 }] }], - PreCompact: [{ hooks: [{ type: 'command', command: '/path/pre-compact-memory', timeout: 10 }] }], - }, - }); - const result = addMemoryHooks(input, '/home/user/.devflow'); - const settings = JSON.parse(result); - - // dream-dispatch added; preamble kept - expect(settings.hooks.UserPromptSubmit).toHaveLength(2); - const commands = settings.hooks.UserPromptSubmit.map((m: { hooks: { command: string }[] }) => m.hooks[0].command); - expect(commands.some((c: string) => c.includes('preamble'))).toBe(true); - expect(commands.some((c: string) => c.includes('dream-dispatch'))).toBe(true); - // Other hooks unchanged - expect(settings.hooks.Stop).toHaveLength(1); - expect(settings.hooks.SessionEnd).toHaveLength(1); - expect(settings.hooks.SessionStart).toHaveLength(1); - expect(settings.hooks.PreCompact).toHaveLength(1); - }); - it('creates hooks object if missing', () => { const input = JSON.stringify({ statusLine: { type: 'command' } }); const result = addMemoryHooks(input, '/home/user/.devflow'); @@ -109,12 +74,8 @@ describe('addMemoryHooks', () => { const result = addMemoryHooks('{}', '/custom/path/.devflow'); const settings = JSON.parse(result); - expect(settings.hooks.UserPromptSubmit[0].hooks[0].command).toContain('/custom/path/.devflow/scripts/hooks/run-hook'); - expect(settings.hooks.UserPromptSubmit[0].hooks[0].command).toContain('dream-dispatch'); expect(settings.hooks.Stop[0].hooks[0].command).toContain('/custom/path/.devflow/scripts/hooks/run-hook'); - expect(settings.hooks.Stop[0].hooks[0].command).toContain('dream-capture'); - expect(settings.hooks.SessionEnd[0].hooks[0].command).toContain('run-hook'); - expect(settings.hooks.SessionEnd[0].hooks[0].command).toContain('dream-evaluate'); + expect(settings.hooks.Stop[0].hooks[0].command).toContain('memory-worker'); expect(settings.hooks.SessionStart[0].hooks[0].command).toContain('run-hook'); expect(settings.hooks.SessionStart[0].hooks[0].command).toContain('session-start-memory'); expect(settings.hooks.PreCompact[0].hooks[0].command).toContain('run-hook'); @@ -138,16 +99,14 @@ describe('addMemoryHooks', () => { const result = addMemoryHooks('{}', '/home/user/.devflow'); const settings = JSON.parse(result); - expect(settings.hooks.UserPromptSubmit[0].hooks[0].timeout).toBe(10); expect(settings.hooks.Stop[0].hooks[0].timeout).toBe(10); - expect(settings.hooks.SessionEnd[0].hooks[0].timeout).toBe(10); expect(settings.hooks.SessionStart[0].hooks[0].timeout).toBe(10); expect(settings.hooks.PreCompact[0].hooks[0].timeout).toBe(10); }); }); describe('removeMemoryHooks', () => { - it('removes all 5 dream hook types', () => { + it('removes all 3 memory hook types', () => { const withHooks = addMemoryHooks('{}', '/home/user/.devflow'); const result = removeMemoryHooks(withHooks); const settings = JSON.parse(result); @@ -155,29 +114,22 @@ describe('removeMemoryHooks', () => { expect(settings.hooks).toBeUndefined(); }); - it('preserves ambient preamble when removing dream hooks', () => { + it('does not touch UserPromptSubmit — owned by capture.ts, not memory.ts', () => { const input = JSON.stringify({ hooks: { UserPromptSubmit: [ - { hooks: [{ type: 'command', command: 'preamble' }] }, - { hooks: [{ type: 'command', command: '/path/dream-dispatch' }] }, + { hooks: [{ type: 'command', command: '/path/run-hook preamble' }] }, + { hooks: [{ type: 'command', command: '/path/run-hook capture-prompt' }] }, ], - Stop: [{ hooks: [{ type: 'command', command: '/path/dream-capture' }] }], - SessionEnd: [{ hooks: [{ type: 'command', command: '/path/dream-evaluate' }] }], - SessionStart: [{ hooks: [{ type: 'command', command: '/path/session-start-memory' }] }], - PreCompact: [{ hooks: [{ type: 'command', command: '/path/pre-compact-memory' }] }], + Stop: [{ hooks: [{ type: 'command', command: '/path/memory-worker' }] }], }, }); const result = removeMemoryHooks(input); const settings = JSON.parse(result); - // Ambient preamble preserved; dream-dispatch removed - expect(settings.hooks.UserPromptSubmit).toHaveLength(1); - expect(settings.hooks.UserPromptSubmit[0].hooks[0].command).toBe('preamble'); + // UserPromptSubmit is untouched — both entries survive; only Stop is cleared + expect(settings.hooks.UserPromptSubmit).toHaveLength(2); expect(settings.hooks.Stop).toBeUndefined(); - expect(settings.hooks.SessionEnd).toBeUndefined(); - expect(settings.hooks.SessionStart).toBeUndefined(); - expect(settings.hooks.PreCompact).toBeUndefined(); }); it('is idempotent — safe to call when not present', () => { @@ -192,7 +144,7 @@ describe('removeMemoryHooks', () => { it('cleans empty hook type arrays', () => { const input = JSON.stringify({ hooks: { - Stop: [{ hooks: [{ type: 'command', command: '/path/dream-capture' }] }], + Stop: [{ hooks: [{ type: 'command', command: '/path/memory-worker' }] }], }, }); const result = removeMemoryHooks(input); @@ -212,8 +164,8 @@ describe('removeMemoryHooks', () => { it('removes only the hooks that exist (partial)', () => { const input = JSON.stringify({ hooks: { - Stop: [{ hooks: [{ type: 'command', command: '/path/dream-capture' }] }], - // UserPromptSubmit, SessionEnd, SessionStart, PreCompact already missing + Stop: [{ hooks: [{ type: 'command', command: '/path/memory-worker' }] }], + // SessionStart, PreCompact already missing }, }); const result = removeMemoryHooks(input); @@ -226,9 +178,7 @@ describe('removeMemoryHooks', () => { const input = JSON.stringify({ statusLine: { type: 'command' }, hooks: { - UserPromptSubmit: [{ hooks: [{ type: 'command', command: '/path/dream-dispatch' }] }], - Stop: [{ hooks: [{ type: 'command', command: '/path/dream-capture' }] }], - SessionEnd: [{ hooks: [{ type: 'command', command: '/path/dream-evaluate' }] }], + Stop: [{ hooks: [{ type: 'command', command: '/path/memory-worker' }] }], SessionStart: [{ hooks: [{ type: 'command', command: '/path/session-start-memory' }] }], PreCompact: [{ hooks: [{ type: 'command', command: '/path/pre-compact-memory' }] }], }, @@ -245,16 +195,14 @@ describe('removeMemoryHooks', () => { const reEnabled = addMemoryHooks(disabled, '/home/user/.devflow'); const settings = JSON.parse(reEnabled); - expect(settings.hooks.UserPromptSubmit).toHaveLength(1); expect(settings.hooks.Stop).toHaveLength(1); - expect(settings.hooks.SessionEnd).toHaveLength(1); expect(settings.hooks.SessionStart).toHaveLength(1); expect(settings.hooks.PreCompact).toHaveLength(1); }); }); describe('hasMemoryHooks', () => { - it('returns true when all 5 dream hooks present', () => { + it('returns true when all 3 memory hooks present', () => { const withHooks = addMemoryHooks('{}', '/home/user/.devflow'); expect(hasMemoryHooks(withHooks)).toBe(true); }); @@ -263,28 +211,26 @@ describe('hasMemoryHooks', () => { expect(hasMemoryHooks('{}')).toBe(false); }); - it('returns false when partial (1 of 5)', () => { + it('returns false when partial (1 of 3)', () => { const input = JSON.stringify({ hooks: { - Stop: [{ hooks: [{ type: 'command', command: '/path/dream-capture' }] }], + Stop: [{ hooks: [{ type: 'command', command: '/path/memory-worker' }] }], }, }); expect(hasMemoryHooks(input)).toBe(false); }); - it('returns false when partial (4 of 5 — missing SessionEnd)', () => { + it('returns false when partial (2 of 3 — missing PreCompact)', () => { const input = JSON.stringify({ hooks: { - UserPromptSubmit: [{ hooks: [{ type: 'command', command: '/path/dream-dispatch' }] }], - Stop: [{ hooks: [{ type: 'command', command: '/path/dream-capture' }] }], + Stop: [{ hooks: [{ type: 'command', command: '/path/memory-worker' }] }], SessionStart: [{ hooks: [{ type: 'command', command: '/path/session-start-memory' }] }], - PreCompact: [{ hooks: [{ type: 'command', command: '/path/pre-compact-memory' }] }], }, }); expect(hasMemoryHooks(input)).toBe(false); }); - it('returns false for ambient preamble only (not a dream hook)', () => { + it('returns false when only unrelated hooks (e.g. ambient preamble) are present', () => { const input = JSON.stringify({ hooks: { UserPromptSubmit: [{ hooks: [{ type: 'command', command: 'preamble' }] }], @@ -295,37 +241,35 @@ describe('hasMemoryHooks', () => { }); describe('countMemoryHooks', () => { - it('returns 5 when all dream hooks present', () => { + it('returns 3 when all memory hooks present', () => { const withHooks = addMemoryHooks('{}', '/home/user/.devflow'); - expect(countMemoryHooks(withHooks)).toBe(5); + expect(countMemoryHooks(withHooks)).toBe(3); }); it('returns 0 when none present', () => { expect(countMemoryHooks('{}')).toBe(0); }); - it('returns correct partial count (2 of 5)', () => { + it('returns correct partial count (2 of 3)', () => { const input = JSON.stringify({ hooks: { - Stop: [{ hooks: [{ type: 'command', command: '/path/dream-capture' }] }], + Stop: [{ hooks: [{ type: 'command', command: '/path/memory-worker' }] }], SessionStart: [{ hooks: [{ type: 'command', command: '/path/session-start-memory' }] }], }, }); expect(countMemoryHooks(input)).toBe(2); }); - it('does not count ambient preamble as dream-dispatch', () => { + it('does not count unrelated UserPromptSubmit hooks toward the memory count', () => { const input = JSON.stringify({ hooks: { UserPromptSubmit: [{ hooks: [{ type: 'command', command: '/path/preamble' }] }], - Stop: [{ hooks: [{ type: 'command', command: '/path/dream-capture' }] }], - SessionEnd: [{ hooks: [{ type: 'command', command: '/path/dream-evaluate' }] }], + Stop: [{ hooks: [{ type: 'command', command: '/path/memory-worker' }] }], SessionStart: [{ hooks: [{ type: 'command', command: '/path/session-start-memory' }] }], PreCompact: [{ hooks: [{ type: 'command', command: '/path/pre-compact-memory' }] }], }, }); - // preamble does not match 'dream-dispatch' marker - expect(countMemoryHooks(input)).toBe(4); + expect(countMemoryHooks(input)).toBe(3); }); }); @@ -333,14 +277,12 @@ describe('countMemoryHooks accepts parsed Settings', () => { it('accepts a parsed Settings object (not just JSON string)', () => { const settings = { hooks: { - UserPromptSubmit: [{ hooks: [{ type: 'command' as const, command: '/path/dream-dispatch', timeout: 10 }] }], - Stop: [{ hooks: [{ type: 'command' as const, command: '/path/dream-capture', timeout: 10 }] }], - SessionEnd: [{ hooks: [{ type: 'command' as const, command: '/path/dream-evaluate', timeout: 10 }] }], + Stop: [{ hooks: [{ type: 'command' as const, command: '/path/memory-worker', timeout: 10 }] }], SessionStart: [{ hooks: [{ type: 'command' as const, command: '/path/session-start-memory', timeout: 10 }] }], PreCompact: [{ hooks: [{ type: 'command' as const, command: '/path/pre-compact-memory', timeout: 10 }] }], }, }; - expect(countMemoryHooks(settings)).toBe(5); + expect(countMemoryHooks(settings)).toBe(3); expect(hasMemoryHooks(settings)).toBe(true); }); @@ -353,7 +295,7 @@ describe('countMemoryHooks accepts parsed Settings', () => { it('accepts parsed Settings with partial hooks', () => { const settings = { hooks: { - Stop: [{ hooks: [{ type: 'command' as const, command: '/path/dream-capture', timeout: 10 }] }], + Stop: [{ hooks: [{ type: 'command' as const, command: '/path/memory-worker', timeout: 10 }] }], SessionStart: [{ hooks: [{ type: 'command' as const, command: '/path/session-start-memory', timeout: 10 }] }], }, }; @@ -518,9 +460,7 @@ describe('removeMemoryHooks accepts parsed Settings', () => { it('accepts a parsed Settings object and returns JSON string', () => { const settings = { hooks: { - UserPromptSubmit: [{ hooks: [{ type: 'command' as const, command: '/path/dream-dispatch', timeout: 10 }] }], - Stop: [{ hooks: [{ type: 'command' as const, command: '/path/dream-capture', timeout: 10 }] }], - SessionEnd: [{ hooks: [{ type: 'command' as const, command: '/path/dream-evaluate', timeout: 10 }] }], + Stop: [{ hooks: [{ type: 'command' as const, command: '/path/memory-worker', timeout: 10 }] }], SessionStart: [{ hooks: [{ type: 'command' as const, command: '/path/session-start-memory', timeout: 10 }] }], PreCompact: [{ hooks: [{ type: 'command' as const, command: '/path/pre-compact-memory', timeout: 10 }] }], }, @@ -533,7 +473,7 @@ describe('removeMemoryHooks accepts parsed Settings', () => { it('does not mutate the original Settings object when passed by reference', () => { const settings = { hooks: { - Stop: [{ hooks: [{ type: 'command' as const, command: '/path/dream-capture', timeout: 10 }] }], + Stop: [{ hooks: [{ type: 'command' as const, command: '/path/memory-worker', timeout: 10 }] }], }, }; removeMemoryHooks(settings); @@ -544,7 +484,7 @@ describe('removeMemoryHooks accepts parsed Settings', () => { it('consistent API: string and Settings produce same result', () => { const settingsObj = { hooks: { - Stop: [{ hooks: [{ type: 'command' as const, command: '/path/dream-capture', timeout: 10 }] }], + Stop: [{ hooks: [{ type: 'command' as const, command: '/path/memory-worker', timeout: 10 }] }], }, }; const resultFromObj = removeMemoryHooks(settingsObj); @@ -667,8 +607,8 @@ describe('session-start-context hook integration', () => { }); }); -describe('removeMemoryHooks removes legacy pre-sidecar hooks', () => { - it('removes sidecar-dispatch from UserPromptSubmit (v3 sidecar→dream rename)', () => { +describe('removeMemoryHooks removes legacy hook registrations', () => { + it('removes sidecar-dispatch from UserPromptSubmit (v3 sidecar→dream rename) — legacy sweep only, UserPromptSubmit untouched otherwise', () => { const input = JSON.stringify({ hooks: { UserPromptSubmit: [ @@ -750,18 +690,46 @@ describe('removeMemoryHooks removes legacy pre-sidecar hooks', () => { { hooks: [{ type: 'command', command: '/path/run-hook session-end-learning', timeout: 10 }] }, { hooks: [{ type: 'command', command: '/path/run-hook session-end-decisions', timeout: 10 }] }, { hooks: [{ type: 'command', command: '/path/run-hook session-end-knowledge-refresh', timeout: 10 }] }, - { hooks: [{ type: 'command', command: '/path/run-hook dream-evaluate', timeout: 10 }] }, ], }, }); const result = removeMemoryHooks(input); const settings = JSON.parse(result); - // Legacy hooks removed; dream-evaluate (new hook) removed by MEMORY_HOOK_CONFIG + // Legacy pre-sidecar SessionEnd hooks removed via LEGACY_HOOK_MARKERS expect(settings.hooks).toBeUndefined(); }); - it('handles mix of old and new hooks — removes all', () => { + it('removes the dream-dispatch/dream-capture/dream-evaluate marker pipeline (dream system simplification)', () => { + // Upgrading users still carry the pre-cutover dream-* hook registrations — + // these are swept via LEGACY_HOOK_MARKERS, not MEMORY_HOOK_CONFIG (which no + // longer includes UserPromptSubmit or SessionEnd at all). + const input = JSON.stringify({ + hooks: { + UserPromptSubmit: [ + { hooks: [{ type: 'command', command: '/path/run-hook dream-dispatch', timeout: 10 }] }, + { hooks: [{ type: 'command', command: '/path/run-hook preamble', timeout: 10 }] }, + ], + Stop: [ + { hooks: [{ type: 'command', command: '/path/run-hook dream-capture', timeout: 10 }] }, + ], + SessionEnd: [ + { hooks: [{ type: 'command', command: '/path/run-hook dream-evaluate', timeout: 10 }] }, + ], + }, + }); + const result = removeMemoryHooks(input); + const settings = JSON.parse(result); + + // dream-dispatch removed; preamble preserved + expect(settings.hooks.UserPromptSubmit).toHaveLength(1); + expect(settings.hooks.UserPromptSubmit[0].hooks[0].command).toContain('preamble'); + // Stop and SessionEnd cleared entirely (only legacy entries were present) + expect(settings.hooks.Stop).toBeUndefined(); + expect(settings.hooks.SessionEnd).toBeUndefined(); + }); + + it('handles mix of old and new hooks — removes all legacy, preserves current registrations', () => { // Simulate an upgrading user with both old and new hooks installed const input = JSON.stringify({ hooks: { @@ -772,6 +740,7 @@ describe('removeMemoryHooks removes legacy pre-sidecar hooks', () => { Stop: [ { hooks: [{ type: 'command', command: '/path/run-hook stop-update-memory', timeout: 10 }] }, { hooks: [{ type: 'command', command: '/path/run-hook dream-capture', timeout: 10 }] }, + { hooks: [{ type: 'command', command: '/path/run-hook memory-worker', timeout: 10 }] }, ], SessionEnd: [ { hooks: [{ type: 'command', command: '/path/run-hook session-end-learning', timeout: 10 }] }, @@ -788,6 +757,9 @@ describe('removeMemoryHooks removes legacy pre-sidecar hooks', () => { const result = removeMemoryHooks(input); const settings = JSON.parse(result); + // Every entry present (across both legacy-swept and current MEMORY_HOOK_CONFIG + // arrays) was either a legacy marker or a current memory hook, so nothing + // survives — the whole `hooks` object is cleaned up entirely. expect(settings.hooks).toBeUndefined(); }); }); diff --git a/tests/project-paths.test.ts b/tests/project-paths.test.ts index 36172868..4ce22f73 100644 --- a/tests/project-paths.test.ts +++ b/tests/project-paths.test.ts @@ -21,6 +21,10 @@ import { getFeaturesDir, getDocsDir, getDreamConfigPath, + getDreamPendingTurnsPath, + getDreamPendingTurnsProcessingPath, + getDreamLastOkPath, + getDreamWorkerLockDir, getDecisionsFilePath, getPitfallsFilePath, getDecisionsDisabledSentinel, @@ -34,7 +38,6 @@ import { getDecisionsUsageLockDir, getObservationsLockDir, getDecisionsNotificationsPath, - getDecisionsRunsTodayPath, getDecisionsBatchIdsPath, getWorkingMemoryPath, getBackupPath, @@ -84,6 +87,22 @@ describe('project-paths TypeScript module', () => { it('getDreamConfigPath returns .devflow/dream/config.json', () => { expect(getDreamConfigPath(ROOT)).toBe('/some/project/.devflow/dream/config.json'); }); + + it('getDreamPendingTurnsPath returns .devflow/dream/.pending-turns.jsonl', () => { + expect(getDreamPendingTurnsPath(ROOT)).toBe('/some/project/.devflow/dream/.pending-turns.jsonl'); + }); + + it('getDreamPendingTurnsProcessingPath returns .devflow/dream/.pending-turns.processing', () => { + expect(getDreamPendingTurnsProcessingPath(ROOT)).toBe('/some/project/.devflow/dream/.pending-turns.processing'); + }); + + it('getDreamLastOkPath returns .devflow/dream/.last-dream-ok', () => { + expect(getDreamLastOkPath(ROOT)).toBe('/some/project/.devflow/dream/.last-dream-ok'); + }); + + it('getDreamWorkerLockDir returns .devflow/dream/.worker.lock', () => { + expect(getDreamWorkerLockDir(ROOT)).toBe('/some/project/.devflow/dream/.worker.lock'); + }); }); describe('decisions files', () => { @@ -127,10 +146,6 @@ describe('project-paths TypeScript module', () => { expect(getDecisionsNotificationsPath(ROOT)).toBe('/some/project/.devflow/decisions/.decisions-notifications.json'); }); - it('getDecisionsRunsTodayPath returns .devflow/decisions/.decisions-runs-today', () => { - expect(getDecisionsRunsTodayPath(ROOT)).toBe('/some/project/.devflow/decisions/.decisions-runs-today'); - }); - it('getDecisionsBatchIdsPath returns .devflow/decisions/.decisions-batch-ids', () => { expect(getDecisionsBatchIdsPath(ROOT)).toBe('/some/project/.devflow/decisions/.decisions-batch-ids'); }); @@ -231,6 +246,10 @@ describe('CJS project-paths parity', () => { { name: 'getFeaturesDir', ts: getFeaturesDir, cjs: cjsPaths.getFeaturesDir }, { name: 'getDocsDir', ts: getDocsDir, cjs: cjsPaths.getDocsDir }, { name: 'getDreamConfigPath', ts: getDreamConfigPath, cjs: cjsPaths.getDreamConfigPath }, + { name: 'getDreamPendingTurnsPath', ts: getDreamPendingTurnsPath, cjs: cjsPaths.getDreamPendingTurnsPath }, + { name: 'getDreamPendingTurnsProcessingPath', ts: getDreamPendingTurnsProcessingPath, cjs: cjsPaths.getDreamPendingTurnsProcessingPath }, + { name: 'getDreamLastOkPath', ts: getDreamLastOkPath, cjs: cjsPaths.getDreamLastOkPath }, + { name: 'getDreamWorkerLockDir', ts: getDreamWorkerLockDir, cjs: cjsPaths.getDreamWorkerLockDir }, { name: 'getDecisionsFilePath', ts: getDecisionsFilePath, cjs: cjsPaths.getDecisionsFilePath }, { name: 'getPitfallsFilePath', ts: getPitfallsFilePath, cjs: cjsPaths.getPitfallsFilePath }, { name: 'getDecisionsDisabledSentinel', ts: getDecisionsDisabledSentinel, cjs: cjsPaths.getDecisionsDisabledSentinel }, @@ -244,7 +263,6 @@ describe('CJS project-paths parity', () => { { name: 'getDecisionsUsageLockDir', ts: getDecisionsUsageLockDir, cjs: cjsPaths.getDecisionsUsageLockDir }, { name: 'getObservationsLockDir', ts: getObservationsLockDir, cjs: cjsPaths.getObservationsLockDir }, { name: 'getDecisionsNotificationsPath', ts: getDecisionsNotificationsPath, cjs: cjsPaths.getDecisionsNotificationsPath }, - { name: 'getDecisionsRunsTodayPath', ts: getDecisionsRunsTodayPath, cjs: cjsPaths.getDecisionsRunsTodayPath }, { name: 'getDecisionsBatchIdsPath', ts: getDecisionsBatchIdsPath, cjs: cjsPaths.getDecisionsBatchIdsPath }, { name: 'getWorkingMemoryPath', ts: getWorkingMemoryPath, cjs: cjsPaths.getWorkingMemoryPath }, { name: 'getBackupPath', ts: getBackupPath, cjs: cjsPaths.getBackupPath }, diff --git a/tests/sentinel.test.ts b/tests/sentinel.test.ts index 71a758b8..be0b8fbf 100644 --- a/tests/sentinel.test.ts +++ b/tests/sentinel.test.ts @@ -52,124 +52,6 @@ function parseHookOutput(rawOutput: string): string { return hookOutput.additionalContext as string; } -// ─── Part B: Sentinel guards for dream capture hooks ────────────────────── - -describe('sentinel guard: dream-dispatch', () => { - const HOOK = path.join(HOOKS_DIR, 'dream-dispatch'); - let tmpDir: string; - - beforeEach(() => { tmpDir = mkTmpDir(); }); - afterEach(() => { fs.rmSync(tmpDir, { recursive: true, force: true }); }); - - it('captures user prompt normally when memory enabled (no config)', () => { - mkMemoryDir(tmpDir); - const input = sessionInput(tmpDir, { prompt: 'test prompt' }); - execSync(`bash "${HOOK}"`, { input, stdio: ['pipe', 'pipe', 'pipe'] }); - expect(fs.existsSync(path.join(tmpDir, '.devflow', 'memory', '.pending-turns.jsonl'))).toBe(true); - }); - - it('exits early and writes nothing when config has memory: false', () => { - mkMemoryDir(tmpDir); - const dreamDir = path.join(tmpDir, '.devflow', 'dream'); - fs.mkdirSync(dreamDir, { recursive: true }); - fs.writeFileSync(path.join(dreamDir, 'config.json'), JSON.stringify({ memory: false })); - const input = sessionInput(tmpDir, { prompt: 'test prompt' }); - execSync(`bash "${HOOK}"`, { input, stdio: ['pipe', 'pipe', 'pipe'] }); - expect(fs.existsSync(path.join(tmpDir, '.devflow', 'memory', '.pending-turns.jsonl'))).toBe(false); - }); -}); - -describe('sentinel guard: dream-capture', () => { - const HOOK = path.join(HOOKS_DIR, 'dream-capture'); - let tmpDir: string; - - beforeEach(() => { tmpDir = mkTmpDir(); }); - afterEach(() => { fs.rmSync(tmpDir, { recursive: true, force: true }); }); - - it('appends assistant turn normally when memory enabled (no config)', () => { - mkMemoryDir(tmpDir); - // Write stale WORKING-MEMORY.md so throttle passes - const memFile = path.join(tmpDir, '.devflow', 'memory', 'WORKING-MEMORY.md'); - fs.writeFileSync(memFile, '## Now\n- testing'); - const tenMinutesAgo = new Date(Date.now() - 600 * 1000); - fs.utimesSync(memFile, tenMinutesAgo, tenMinutesAgo); - const input = sessionInput(tmpDir, { last_assistant_message: 'hello' }); - execSync(`bash "${HOOK}"`, { input, stdio: ['pipe', 'pipe', 'pipe'] }); - expect(fs.existsSync(path.join(tmpDir, '.devflow', 'memory', '.pending-turns.jsonl'))).toBe(true); - }); - - it('exits early when config has memory: false', () => { - mkMemoryDir(tmpDir); - const dreamDir = path.join(tmpDir, '.devflow', 'dream'); - fs.mkdirSync(dreamDir, { recursive: true }); - fs.writeFileSync(path.join(dreamDir, 'config.json'), JSON.stringify({ memory: false })); - const input = sessionInput(tmpDir, { last_assistant_message: 'hello' }); - execSync(`bash "${HOOK}"`, { input, stdio: ['pipe', 'pipe', 'pipe'] }); - expect(fs.existsSync(path.join(tmpDir, '.devflow', 'memory', '.pending-turns.jsonl'))).toBe(false); - }); - - it('captures when stop_reason is "tool_use" and last_assistant_message is present', () => { - // Regression: the old code filtered on stop_reason=end_turn, which meant tool_use - // stops with a valid last_assistant_message were silently dropped. After the rename, - // stop_reason is ignored — only last_assistant_message presence gates capture. - mkMemoryDir(tmpDir); - const memFile = path.join(tmpDir, '.devflow', 'memory', 'WORKING-MEMORY.md'); - fs.writeFileSync(memFile, '## Now\n- testing'); - const tenMinutesAgo = new Date(Date.now() - 600 * 1000); - fs.utimesSync(memFile, tenMinutesAgo, tenMinutesAgo); - const input = sessionInput(tmpDir, { - stop_reason: 'tool_use', - last_assistant_message: 'response with tool call', - }); - execSync(`bash "${HOOK}"`, { input, stdio: ['pipe', 'pipe', 'pipe'] }); - // Capture must proceed regardless of stop_reason value - expect(fs.existsSync(path.join(tmpDir, '.devflow', 'memory', '.pending-turns.jsonl'))).toBe(true); - }); - - it('ignores legacy response_text field (avoids PF-006)', () => { - // Regression guard: when input carries only the old field name (response_text) - // and NOT the new field (last_assistant_message), ASSISTANT_MSG is empty and - // the hook must exit without creating the queue file. - mkMemoryDir(tmpDir); - const memFile = path.join(tmpDir, '.devflow', 'memory', 'WORKING-MEMORY.md'); - fs.writeFileSync(memFile, '## Now\n- testing'); - const tenMinutesAgo = new Date(Date.now() - 600 * 1000); - fs.utimesSync(memFile, tenMinutesAgo, tenMinutesAgo); - const input = sessionInput(tmpDir, { - stop_reason: 'end_turn', - response_text: 'this should be ignored', - }); - execSync(`bash "${HOOK}"`, { input, stdio: ['pipe', 'pipe', 'pipe'] }); - // Queue must NOT be created — old field name carries no capture value - expect(fs.existsSync(path.join(tmpDir, '.devflow', 'memory', '.pending-turns.jsonl'))).toBe(false); - }); - - it('creates log file on successful capture', () => { - mkMemoryDir(tmpDir); - const memFile = path.join(tmpDir, '.devflow', 'memory', 'WORKING-MEMORY.md'); - fs.writeFileSync(memFile, '## Now\n- testing'); - const tenMinutesAgo = new Date(Date.now() - 600 * 1000); - fs.utimesSync(memFile, tenMinutesAgo, tenMinutesAgo); - const input = sessionInput(tmpDir, { last_assistant_message: 'hello' }); - // Create a temp home so log writes land in our temp dir - const tmpHome = fs.mkdtempSync(os.tmpdir() + '/devflow-log-home-'); - try { - execSync(`bash "${HOOK}"`, { - input, - stdio: ['pipe', 'pipe', 'pipe'], - env: { ...process.env, HOME: tmpHome }, - }); - const slug = tmpDir.replace(/^\//, '').replace(/\//g, '-'); - const logFile = path.join(tmpHome, '.devflow', 'logs', slug, '.dream-capture.log'); - expect(fs.existsSync(logFile)).toBe(true); - const content = fs.readFileSync(logFile, 'utf-8'); - expect(content).toContain('[dream-capture]'); - } finally { - fs.rmSync(tmpHome, { recursive: true, force: true }); - } - }); -}); - describe('sentinel guard: pre-compact-memory', () => { const HOOK = path.join(HOOKS_DIR, 'pre-compact-memory'); let tmpDir: string; @@ -231,39 +113,6 @@ describe('sentinel guard: session-start-memory', () => { }); }); -// ─── Part C: Sentinel guard for dream-evaluate hook ──────────────────────── - -describe('sentinel guard: dream-evaluate', () => { - const HOOK = path.join(HOOKS_DIR, 'dream-evaluate'); - let tmpDir: string; - - beforeEach(() => { tmpDir = mkTmpDir(); }); - afterEach(() => { fs.rmSync(tmpDir, { recursive: true, force: true }); }); - - it('exits cleanly when no session_id is provided', () => { - mkMemoryDir(tmpDir); - const input = JSON.stringify({ cwd: tmpDir }); - expect(() => { - execSync(`bash "${HOOK}"`, { input, stdio: ['pipe', 'pipe', 'pipe'] }); - }).not.toThrow(); - }); - - it('exits cleanly for DEVFLOW_BG_LEARNER=1 (feedback loop guard)', () => { - expect(() => { - execSync(`DEVFLOW_BG_LEARNER=1 bash "${HOOK}"`, { stdio: 'ignore' }); - }).not.toThrow(); - }); - - it('exits cleanly when .devflow/memory/ does not exist (no markers written)', () => { - const input = sessionInput(tmpDir); - expect(() => { - execSync(`bash "${HOOK}"`, { input, stdio: ['pipe', 'pipe', 'pipe'] }); - }).not.toThrow(); - // Should not create any dream marker files - expect(fs.existsSync(path.join(tmpDir, '.devflow', 'dream'))).toBe(false); - }); -}); - // ─── Part D: Decisions scanner fix ────────────────────────────────────────── describe('sentinel guard: decisions-usage-scan.cjs', () => { @@ -298,23 +147,15 @@ describe('sentinel guard: decisions-usage-scan.cjs', () => { }); }); -describe('sentinel guard: dream-capture decisions scanner gating', () => { - const HOOK = path.join(HOOKS_DIR, 'dream-capture'); +describe('sentinel guard: capture-turn decisions scanner gating', () => { + const HOOK = path.join(HOOKS_DIR, 'capture-turn'); let tmpDir: string; beforeEach(() => { tmpDir = mkTmpDir(); }); afterEach(() => { fs.rmSync(tmpDir, { recursive: true, force: true }); }); - function writeStaleWorkingMemory(dir: string): void { - const memFile = path.join(dir, '.devflow', 'memory', 'WORKING-MEMORY.md'); - fs.writeFileSync(memFile, '## Now\n- stale memory'); - const tenMinutesAgo = new Date(Date.now() - 600 * 1000); - fs.utimesSync(memFile, tenMinutesAgo, tenMinutesAgo); - } - it('does NOT run scanner when decisions/.disabled exists', () => { mkMemoryDir(tmpDir); - writeStaleWorkingMemory(tmpDir); writeDisabledSentinel(path.join(tmpDir, '.devflow', 'decisions', '.disabled')); // Create usage file to detect if scanner would have run const usagePath = path.join(tmpDir, '.devflow', 'decisions', '.decisions-usage.json'); @@ -331,7 +172,6 @@ describe('sentinel guard: dream-capture decisions scanner gating', () => { it('runs scanner when decisions/.disabled absent', () => { mkMemoryDir(tmpDir); - writeStaleWorkingMemory(tmpDir); // Create usage file to detect scanner run const usagePath = path.join(tmpDir, '.devflow', 'decisions', '.decisions-usage.json'); fs.writeFileSync(usagePath, JSON.stringify({ @@ -495,6 +335,75 @@ describe('context hook registration', () => { }); }); +// ─── Part A: dream hook registration (spawn-dream-worker) ────────────────── +// +// spawn-dream-worker is always-on (SessionStart), like session-start-context — +// registered unconditionally by init, removed by uninstall. Follows the same +// add/remove/has contract as context hook registration above. + +describe('dream hook registration', () => { + it('addDreamHook adds spawn-dream-worker to SessionStart', async () => { + const { addDreamHook } = await import('../src/cli/commands/init.js'); + const result = addDreamHook('{}', '/home/user/.devflow'); + const settings = JSON.parse(result); + expect(settings.hooks?.SessionStart).toBeDefined(); + const hookPresent = settings.hooks.SessionStart.some( + (m: { hooks: { command: string }[] }) => + m.hooks.some((h: { command: string }) => h.command.includes('spawn-dream-worker')), + ); + expect(hookPresent).toBe(true); + }); + + it('removeDreamHook removes spawn-dream-worker from settings', async () => { + const { addDreamHook, removeDreamHook } = await import('../src/cli/commands/init.js'); + const withHook = addDreamHook('{}', '/home/user/.devflow'); + const removed = removeDreamHook(withHook); + const settings = JSON.parse(removed); + const hookPresent = settings.hooks?.SessionStart?.some( + (m: { hooks: { command: string }[] }) => + m.hooks.some((h: { command: string }) => h.command.includes('spawn-dream-worker')), + ) ?? false; + expect(hookPresent).toBe(false); + }); + + it('hasDreamHook returns true when hook registered', async () => { + const { addDreamHook, hasDreamHook } = await import('../src/cli/commands/init.js'); + const withHook = addDreamHook('{}', '/home/user/.devflow'); + expect(hasDreamHook(withHook)).toBe(true); + }); + + it('hasDreamHook returns false when hook absent', async () => { + const { hasDreamHook } = await import('../src/cli/commands/init.js'); + expect(hasDreamHook('{}')).toBe(false); + }); + + it('addDreamHook is idempotent', async () => { + const { addDreamHook } = await import('../src/cli/commands/init.js'); + const first = addDreamHook('{}', '/home/user/.devflow'); + const second = addDreamHook(first, '/home/user/.devflow'); + expect(second).toBe(first); + }); + + it('removeDreamHook preserves other SessionStart hooks (session-start-memory, session-start-context)', async () => { + const { addDreamHook, removeDreamHook } = await import('../src/cli/commands/init.js'); + const input = JSON.stringify({ + hooks: { + SessionStart: [ + { hooks: [{ type: 'command', command: '/path/run-hook session-start-memory' }] }, + { hooks: [{ type: 'command', command: '/path/run-hook session-start-context' }] }, + ], + }, + }); + const withDream = addDreamHook(input, '/home/user/.devflow'); + const removed = removeDreamHook(withDream); + const settings = JSON.parse(removed); + expect(settings.hooks?.SessionStart).toHaveLength(2); + const commands = settings.hooks.SessionStart.map((m: { hooks: { command: string }[] }) => m.hooks[0].command); + expect(commands.some((c: string) => c.includes('session-start-memory'))).toBe(true); + expect(commands.some((c: string) => c.includes('session-start-context'))).toBe(true); + }); +}); + // ─── Part E: manageSentinel utility ───────────────────────────────────────── describe('manageSentinel utility', () => { @@ -550,9 +459,9 @@ describe('manageSentinel utility', () => { // session-start-context, session-start-memory, and pre-compact-memory had no // re-entrancy guard at all (a latent pre-existing gap): the memory or dream // worker's own nested claude -p session fires these SessionStart/PreCompact -// hooks too. Guards were added additively — these are NEW test cases; the -// existing guard describes above (dream-dispatch, dream-capture, -// dream-evaluate) are untouched. +// hooks too. capture-prompt, capture-turn, and capture-question carry the +// equivalent DEVFLOW_BG_UPDATER/DEVFLOW_BG_DREAM guards and are covered in +// tests/capture-hooks.test.ts. describe('sentinel guard: session-start-context DEVFLOW_BG_* re-entrancy', () => { const HOOK = path.join(HOOKS_DIR, 'session-start-context'); diff --git a/tests/shell-hooks.test.ts b/tests/shell-hooks.test.ts index 6dad1839..79f22fad 100644 --- a/tests/shell-hooks.test.ts +++ b/tests/shell-hooks.test.ts @@ -26,11 +26,6 @@ const HOOK_SCRIPTS = [ 'ensure-devflow-init', 'ensure-root-gitignore', 'resolve-project-root', - 'dream-capture', - 'dream-evaluate', - 'dream-dispatch', - 'eval-helpers', - 'eval-decisions', 'queue-append', 'capture-prompt', 'capture-turn', @@ -184,190 +179,6 @@ describe('debug-trace helper behaviors', () => { }); }); -const EVAL_HELPERS = path.join(HOOKS_DIR, 'eval-helpers'); -const DREAM_LOCK = path.join(HOOKS_DIR, 'dream-lock'); - -describe('eval-helpers: read_daily_cap', () => { - it('returns 0 when counter file absent', () => { - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'devflow-test-')); - try { - const counterFile = path.join(tmpDir, '.runs-today'); - const result = execSync(`bash -c ' - TODAY=$(date +%Y-%m-%d) - source "${EVAL_HELPERS}" - read_daily_cap "${counterFile}" 0 - '`, { stdio: 'pipe' }).toString().trim(); - expect(result).toBe('0'); - } finally { - fs.rmSync(tmpDir, { recursive: true, force: true }); - } - }); - - it('returns count when date matches today', () => { - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'devflow-test-')); - const counterFile = path.join(tmpDir, '.runs-today'); - const today = execSync('date +%Y-%m-%d', { stdio: 'pipe' }).toString().trim(); - try { - fs.writeFileSync(counterFile, `${today}\t5\n`); - const result = execSync(`bash -c ' - TODAY=$(date +%Y-%m-%d) - source "${EVAL_HELPERS}" - read_daily_cap "${counterFile}" 0 - '`, { stdio: 'pipe' }).toString().trim(); - expect(result).toBe('5'); - } finally { - fs.rmSync(tmpDir, { recursive: true, force: true }); - } - }); - - it('returns 0 when counter file has a stale date', () => { - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'devflow-test-')); - const counterFile = path.join(tmpDir, '.runs-today'); - try { - fs.writeFileSync(counterFile, `2020-01-01\t99\n`); - const result = execSync(`bash -c ' - TODAY=$(date +%Y-%m-%d) - source "${EVAL_HELPERS}" - read_daily_cap "${counterFile}" 0 - '`, { stdio: 'pipe' }).toString().trim(); - expect(result).toBe('0'); - } finally { - fs.rmSync(tmpDir, { recursive: true, force: true }); - } - }); -}); - -describe('eval-helpers: atomic_increment_daily', () => { - it('creates counter file with count 1 on first call', () => { - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'devflow-test-')); - const counterFile = path.join(tmpDir, '.runs-today'); - const today = execSync('date +%Y-%m-%d', { stdio: 'pipe' }).toString().trim(); - try { - execSync(`bash -c ' - TODAY=$(date +%Y-%m-%d) - source "${DREAM_LOCK}" - source "${EVAL_HELPERS}" - atomic_increment_daily "${counterFile}" "$TODAY" - '`, { stdio: 'pipe' }); - const content = fs.readFileSync(counterFile, 'utf-8').trim(); - expect(content).toBe(`${today}\t1`); - } finally { - fs.rmSync(tmpDir, { recursive: true, force: true }); - } - }); - - it('increments existing count on subsequent call', () => { - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'devflow-test-')); - const counterFile = path.join(tmpDir, '.runs-today'); - const today = execSync('date +%Y-%m-%d', { stdio: 'pipe' }).toString().trim(); - try { - fs.writeFileSync(counterFile, `${today}\t3\n`); - execSync(`bash -c ' - TODAY=$(date +%Y-%m-%d) - source "${DREAM_LOCK}" - source "${EVAL_HELPERS}" - atomic_increment_daily "${counterFile}" "$TODAY" - '`, { stdio: 'pipe' }); - const content = fs.readFileSync(counterFile, 'utf-8').trim(); - expect(content).toBe(`${today}\t4`); - } finally { - fs.rmSync(tmpDir, { recursive: true, force: true }); - } - }); -}); - -describe('eval-helpers: load_existing_ids', () => { - it('returns empty array when log file is absent', () => { - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'devflow-test-')); - try { - const missingFile = path.join(tmpDir, 'nonexistent.jsonl'); - const result = execSync(`bash -c ' - source "${EVAL_HELPERS}" - load_existing_ids "${missingFile}" - '`, { stdio: 'pipe' }).toString().trim(); - expect(JSON.parse(result)).toEqual([]); - } finally { - fs.rmSync(tmpDir, { recursive: true, force: true }); - } - }); - - it('returns empty array when log file is empty', () => { - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'devflow-test-')); - const logFile = path.join(tmpDir, 'empty.jsonl'); - try { - fs.writeFileSync(logFile, ''); - const result = execSync(`bash -c ' - source "${EVAL_HELPERS}" - load_existing_ids "${logFile}" - '`, { stdio: 'pipe' }).toString().trim(); - expect(JSON.parse(result)).toEqual([]); - } finally { - fs.rmSync(tmpDir, { recursive: true, force: true }); - } - }); - - it('returns ids from valid JSONL as JSON array', () => { - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'devflow-test-')); - const logFile = path.join(tmpDir, 'learning.jsonl'); - try { - fs.writeFileSync(logFile, [ - JSON.stringify({ id: 'obs_aaa', type: 'workflow', confidence: 0.9 }), - JSON.stringify({ id: 'obs_bbb', type: 'procedural', confidence: 0.7 }), - JSON.stringify({ id: 'obs_ccc', type: 'decision', confidence: 0.8 }), - ].join('\n') + '\n'); - const result = execSync(`bash -c ' - source "${EVAL_HELPERS}" - load_existing_ids "${logFile}" - '`, { stdio: 'pipe' }).toString().trim(); - const ids = JSON.parse(result); - expect(ids).toContain('obs_aaa'); - expect(ids).toContain('obs_bbb'); - expect(ids).toContain('obs_ccc'); - expect(ids).toHaveLength(3); - } finally { - fs.rmSync(tmpDir, { recursive: true, force: true }); - } - }); -}); - -describe('eval-helpers: _eval_release_lock', () => { - it('releases a held lock dir by removing it', () => { - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'devflow-test-')); - const lockDir = path.join(tmpDir, 'test.lock'); - try { - // Create the lock directory (simulating an acquired lock) - fs.mkdirSync(lockDir); - expect(fs.existsSync(lockDir)).toBe(true); - - execSync(`bash -c ' - source "${DREAM_LOCK}" - source "${EVAL_HELPERS}" - _eval_release_lock "${lockDir}" - '`, { stdio: 'pipe' }); - - expect(fs.existsSync(lockDir)).toBe(false); - } finally { - fs.rmSync(tmpDir, { recursive: true, force: true }); - } - }); - - it('is a no-op when lock dir does not exist', () => { - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'devflow-test-')); - const lockDir = path.join(tmpDir, 'nonexistent.lock'); - try { - expect(() => { - execSync(`bash -c ' - source "${DREAM_LOCK}" - source "${EVAL_HELPERS}" - _eval_release_lock "${lockDir}" - '`, { stdio: 'pipe' }); - }).not.toThrow(); - } finally { - fs.rmSync(tmpDir, { recursive: true, force: true }); - } - }); -}); - describe('json-helper.js operations', () => { it('get-field extracts a field with default', () => { const result = execSync( @@ -536,445 +347,6 @@ describe('json-parse wrapper', () => { }); }); -describe('working memory queue behavior', () => { - const STOP_HOOK = path.join(HOOKS_DIR, 'dream-capture'); - const PREAMBLE_HOOK = path.join(HOOKS_DIR, 'preamble'); - const PROMPT_CAPTURE_HOOK = path.join(HOOKS_DIR, 'dream-dispatch'); - - let tmpDir: string; - - beforeEach(() => { - tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'devflow-queue-test-')); - }); - - afterEach(() => { - fs.rmSync(tmpDir, { recursive: true, force: true }); - }); - - // Helper: write a fresh WORKING-MEMORY.md with an old mtime so the throttle passes. - // dream-capture throttles if WORKING-MEMORY.md was updated <120s ago. - // We write the file then backdate its mtime to 10 minutes ago. - function writeStaleWorkingMemory(tmpDir: string): void { - const memFile = path.join(tmpDir, '.devflow', 'memory', 'WORKING-MEMORY.md'); - fs.writeFileSync(memFile, '## Now\n- stale memory'); - // Backdate 10 minutes (600 seconds) - const tenMinutesAgo = new Date(Date.now() - 600 * 1000); - fs.utimesSync(memFile, tenMinutesAgo, tenMinutesAgo); - } - - it('empty last_assistant_message — no queue append', () => { - // Hook exits early when last_assistant_message is absent/empty - fs.mkdirSync(path.join(tmpDir, '.devflow', 'memory'), { recursive: true }); - - const input = JSON.stringify({ - cwd: tmpDir, - session_id: 'test-session-001', - last_assistant_message: '', - }); - - execSync(`bash "${STOP_HOOK}"`, { input, stdio: ['pipe', 'pipe', 'pipe'] }); - - const queueFile = path.join(tmpDir, '.devflow', 'memory', '.pending-turns.jsonl'); - expect(fs.existsSync(queueFile)).toBe(false); - }); - - it('last_assistant_message present — appends assistant turn to queue', () => { - // Create .devflow/memory/ directory - fs.mkdirSync(path.join(tmpDir, '.devflow', 'memory'), { recursive: true }); - // Write stale WORKING-MEMORY.md so throttle check passes (no memory.processing marker needed) - writeStaleWorkingMemory(tmpDir); - - const input = JSON.stringify({ - cwd: tmpDir, - session_id: 'test-session-002', - last_assistant_message: 'test response', - }); - - execSync(`bash "${STOP_HOOK}"`, { input, stdio: ['pipe', 'pipe', 'pipe'] }); - - const queueFile = path.join(tmpDir, '.devflow', 'memory', '.pending-turns.jsonl'); - expect(fs.existsSync(queueFile)).toBe(true); - - const lines = fs.readFileSync(queueFile, 'utf-8').trim().split('\n').filter(Boolean); - expect(lines).toHaveLength(1); - - const entry = JSON.parse(lines[0]) as { role: string; content: string; ts: number }; - expect(entry.role).toBe('assistant'); - expect(entry.content).toBe('test response'); - expect(typeof entry.ts).toBe('number'); - }); - - it('dream-dispatch captures user prompt to queue', () => { - // Create .devflow/memory/ directory so capture is triggered - fs.mkdirSync(path.join(tmpDir, '.devflow', 'memory'), { recursive: true }); - - const input = JSON.stringify({ - cwd: tmpDir, - session_id: 'test-session-003', - prompt: 'implement the cache', - }); - - execSync(`bash "${PROMPT_CAPTURE_HOOK}"`, { input, stdio: ['pipe', 'pipe', 'pipe'] }); - - const queueFile = path.join(tmpDir, '.devflow', 'memory', '.pending-turns.jsonl'); - expect(fs.existsSync(queueFile)).toBe(true); - - const lines = fs.readFileSync(queueFile, 'utf-8').trim().split('\n').filter(Boolean); - expect(lines).toHaveLength(1); - - const entry = JSON.parse(lines[0]) as { role: string; content: string; ts: number }; - expect(entry.role).toBe('user'); - expect(entry.content).toBe('implement the cache'); - expect(typeof entry.ts).toBe('number'); - }); - - it('dream-dispatch with missing .devflow/ — creates it via ensure-devflow-init, exit 0', () => { - // tmpDir exists but has no .devflow/ subdirectory — ensure-devflow-init creates it - const input = JSON.stringify({ - cwd: tmpDir, - session_id: 'test-session-004a', - prompt: 'implement the cache', - }); - - expect(() => { - execSync(`bash "${PROMPT_CAPTURE_HOOK}"`, { input, stdio: ['pipe', 'pipe', 'pipe'] }); - }).not.toThrow(); - - // Hook creates .devflow/memory/ and writes to queue - const queueFile = path.join(tmpDir, '.devflow', 'memory', '.pending-turns.jsonl'); - expect(fs.existsSync(queueFile)).toBe(true); - }); - - it('preamble does NOT write to queue — zero file I/O', () => { - // Create .devflow/memory/ to confirm preamble doesn't touch the queue even when it exists - fs.mkdirSync(path.join(tmpDir, '.devflow', 'memory'), { recursive: true }); - - const input = JSON.stringify({ - cwd: tmpDir, - session_id: 'test-session-004', - prompt: 'implement the cache', - }); - - // Should not throw (exit 0) - expect(() => { - execSync(`bash "${PREAMBLE_HOOK}"`, { input, stdio: ['pipe', 'pipe', 'pipe'] }); - }).not.toThrow(); - - const queueFile = path.join(tmpDir, '.devflow', 'memory', '.pending-turns.jsonl'); - expect(fs.existsSync(queueFile)).toBe(false); - }); - - it('preamble with slash command — exits 0, no queue write', () => { - fs.mkdirSync(path.join(tmpDir, '.devflow', 'memory'), { recursive: true }); - - const input = JSON.stringify({ - cwd: tmpDir, - session_id: 'test-session-004b', - prompt: '/code-review', - }); - - expect(() => { - execSync(`bash "${PREAMBLE_HOOK}"`, { input, stdio: ['pipe', 'pipe', 'pipe'] }); - }).not.toThrow(); - - const queueFile = path.join(tmpDir, '.devflow', 'memory', '.pending-turns.jsonl'); - expect(fs.existsSync(queueFile)).toBe(false); - }); - - it('queue JSONL format — each line is valid JSON with role, content, ts', () => { - fs.mkdirSync(path.join(tmpDir, '.devflow', 'memory'), { recursive: true }); - const queueFile = path.join(tmpDir, '.devflow', 'memory', '.pending-turns.jsonl'); - - const now = Math.floor(Date.now() / 1000); - const entries = [ - { role: 'user', content: 'hello world', ts: now }, - { role: 'assistant', content: 'I will help you', ts: now + 1 }, - { role: 'user', content: 'thanks', ts: now + 2 }, - ]; - - fs.writeFileSync(queueFile, entries.map(e => JSON.stringify(e)).join('\n') + '\n'); - - const lines = fs.readFileSync(queueFile, 'utf-8').trim().split('\n').filter(Boolean); - expect(lines).toHaveLength(3); - - for (const line of lines) { - const parsed = JSON.parse(line); - expect(['user', 'assistant']).toContain(parsed.role); - expect(typeof parsed.content).toBe('string'); - expect(typeof parsed.ts).toBe('number'); - } - }); - - it('preserves existing user entries when appending assistant turn', () => { - fs.mkdirSync(path.join(tmpDir, '.devflow', 'memory'), { recursive: true }); - writeStaleWorkingMemory(tmpDir); - - const queueFile = path.join(tmpDir, '.devflow', 'memory', '.pending-turns.jsonl'); - const now = Math.floor(Date.now() / 1000); - // Pre-populate with user-only entries (from dream-dispatch) - const userLines = Array.from({ length: 5 }, (_, i) => - JSON.stringify({ role: 'user', content: `user prompt ${i}`, ts: now + i }), - ); - fs.writeFileSync(queueFile, userLines.join('\n') + '\n'); - - const input = JSON.stringify({ - cwd: tmpDir, - session_id: 'test-session-autoclean', - last_assistant_message: 'first real assistant response', - }); - - execSync(`bash "${STOP_HOOK}"`, { input, stdio: ['pipe', 'pipe', 'pipe'] }); - - const lines = fs.readFileSync(queueFile, 'utf-8').trim().split('\n').filter(Boolean); - // All 5 user entries preserved + 1 new assistant entry - expect(lines).toHaveLength(6); - const lastEntry = JSON.parse(lines[5]) as { role: string; content: string; ts: number }; - expect(lastEntry.role).toBe('assistant'); - expect(lastEntry.content).toBe('first real assistant response'); - }); - - it('empty queue file — assistant entry appended normally', () => { - fs.mkdirSync(path.join(tmpDir, '.devflow', 'memory'), { recursive: true }); - writeStaleWorkingMemory(tmpDir); - - const queueFile = path.join(tmpDir, '.devflow', 'memory', '.pending-turns.jsonl'); - fs.writeFileSync(queueFile, ''); - - const input = JSON.stringify({ - cwd: tmpDir, - session_id: 'test-session-autoclean-empty', - last_assistant_message: 'response after empty queue', - }); - - execSync(`bash "${STOP_HOOK}"`, { input, stdio: ['pipe', 'pipe', 'pipe'] }); - - const lines = fs.readFileSync(queueFile, 'utf-8').trim().split('\n').filter(Boolean); - expect(lines).toHaveLength(1); - const entry = JSON.parse(lines[0]) as { role: string; content: string; ts: number }; - expect(entry.role).toBe('assistant'); - expect(entry.content).toBe('response after empty queue'); - }); - - it('single user entry preserved when assistant turn appends', () => { - fs.mkdirSync(path.join(tmpDir, '.devflow', 'memory'), { recursive: true }); - writeStaleWorkingMemory(tmpDir); - - const queueFile = path.join(tmpDir, '.devflow', 'memory', '.pending-turns.jsonl'); - const now = Math.floor(Date.now() / 1000); - const userEntry = JSON.stringify({ role: 'user', content: 'user prompt', ts: now }); - fs.writeFileSync(queueFile, userEntry + '\n'); - - const input = JSON.stringify({ - cwd: tmpDir, - session_id: 'test-session-autoclean-single', - last_assistant_message: 'response after user prompt', - }); - - execSync(`bash "${STOP_HOOK}"`, { input, stdio: ['pipe', 'pipe', 'pipe'] }); - - const lines = fs.readFileSync(queueFile, 'utf-8').trim().split('\n').filter(Boolean); - // User entry preserved + new assistant entry = 2 lines - expect(lines).toHaveLength(2); - const firstEntry = JSON.parse(lines[0]) as { role: string; content: string; ts: number }; - expect(firstEntry.role).toBe('user'); - expect(firstEntry.content).toBe('user prompt'); - const lastEntry = JSON.parse(lines[1]) as { role: string; content: string; ts: number }; - expect(lastEntry.role).toBe('assistant'); - expect(lastEntry.content).toBe('response after user prompt'); - }); - - it('queue with mixed entries preserved when assistant turn appends', () => { - fs.mkdirSync(path.join(tmpDir, '.devflow', 'memory'), { recursive: true }); - writeStaleWorkingMemory(tmpDir); - - const queueFile = path.join(tmpDir, '.devflow', 'memory', '.pending-turns.jsonl'); - const now = Math.floor(Date.now() / 1000); - // Pre-populate with mixed entries (healthy queue) - const mixedLines = [ - JSON.stringify({ role: 'user', content: 'hello', ts: now }), - JSON.stringify({ role: 'assistant', content: 'hi there', ts: now + 1 }), - JSON.stringify({ role: 'user', content: 'next prompt', ts: now + 2 }), - ]; - fs.writeFileSync(queueFile, mixedLines.join('\n') + '\n'); - - const input = JSON.stringify({ - cwd: tmpDir, - session_id: 'test-session-no-autoclean', - last_assistant_message: 'another response', - }); - - execSync(`bash "${STOP_HOOK}"`, { input, stdio: ['pipe', 'pipe', 'pipe'] }); - - const lines = fs.readFileSync(queueFile, 'utf-8').trim().split('\n').filter(Boolean); - // 3 existing + 1 new = 4 - expect(lines).toHaveLength(4); - }); - - it('queue with only assistant entries preserved when new entry appends', () => { - fs.mkdirSync(path.join(tmpDir, '.devflow', 'memory'), { recursive: true }); - writeStaleWorkingMemory(tmpDir); - - const queueFile = path.join(tmpDir, '.devflow', 'memory', '.pending-turns.jsonl'); - const now = Math.floor(Date.now() / 1000); - // Pre-populate with assistant-only entries - const assistantLines = Array.from({ length: 3 }, (_, i) => - JSON.stringify({ role: 'assistant', content: `response ${i}`, ts: now + i }), - ); - fs.writeFileSync(queueFile, assistantLines.join('\n') + '\n'); - - const input = JSON.stringify({ - cwd: tmpDir, - session_id: 'test-session-autoclean-assistant-only', - last_assistant_message: 'new assistant response', - }); - - execSync(`bash "${STOP_HOOK}"`, { input, stdio: ['pipe', 'pipe', 'pipe'] }); - - const resultLines = fs.readFileSync(queueFile, 'utf-8').trim().split('\n').filter(Boolean); - // 3 existing assistant entries preserved + 1 new = 4 - expect(resultLines).toHaveLength(4); - const lastEntry = JSON.parse(resultLines[resultLines.length - 1]) as { role: string; content: string; ts: number }; - expect(lastEntry.role).toBe('assistant'); - expect(lastEntry.content).toBe('new assistant response'); - }); - - it('queue overflow — >200 lines truncated to last 100', () => { - fs.mkdirSync(path.join(tmpDir, '.devflow', 'memory'), { recursive: true }); - // Write stale WORKING-MEMORY.md so throttle passes - writeStaleWorkingMemory(tmpDir); - - const queueFile = path.join(tmpDir, '.devflow', 'memory', '.pending-turns.jsonl'); - const now = Math.floor(Date.now() / 1000); - - // Pre-populate queue with 201 entries (mixed roles to avoid auto-clean) - const existingLines = Array.from({ length: 201 }, (_, i) => - JSON.stringify({ role: i % 2 === 0 ? 'user' : 'assistant', content: `entry ${i}`, ts: now + i }), - ); - fs.writeFileSync(queueFile, existingLines.join('\n') + '\n'); - - // Trigger stop hook — appends 1 more entry, then overflow check fires - const input = JSON.stringify({ - cwd: tmpDir, - session_id: 'test-session-006', - last_assistant_message: 'overflow trigger response', - }); - - execSync(`bash "${STOP_HOOK}"`, { input, stdio: ['pipe', 'pipe', 'pipe'] }); - - // After overflow: 201 pre-existing + 1 new = 202 lines → truncated to last 100 - const resultLines = fs.readFileSync(queueFile, 'utf-8').trim().split('\n').filter(Boolean); - expect(resultLines).toHaveLength(100); - - // The first preserved entry must be from the tail of the original data - // 201 pre-existing + 1 new = 202 lines → tail -100 starts at index 102 - const firstEntry = JSON.parse(resultLines[0]) as { role: string; content: string; ts: number }; - expect(firstEntry.content).toBe('entry 102'); - - // The new entry (the assistant turn) must be present as the last line - const lastEntry = JSON.parse(resultLines[resultLines.length - 1]) as { role: string; content: string; ts: number }; - expect(lastEntry.role).toBe('assistant'); - expect(lastEntry.content).toBe('overflow trigger response'); - }); - - it('dispatch then capture preserves user turn in fresh queue', () => { - fs.mkdirSync(path.join(tmpDir, '.devflow', 'memory'), { recursive: true }); - writeStaleWorkingMemory(tmpDir); - - const queueFile = path.join(tmpDir, '.devflow', 'memory', '.pending-turns.jsonl'); - - // Step 1: dream-dispatch writes user turn - execSync(`bash "${PROMPT_CAPTURE_HOOK}"`, { - input: JSON.stringify({ cwd: tmpDir, session_id: 'test-fresh', prompt: 'implement feature X' }), - stdio: ['pipe', 'pipe', 'pipe'], - }); - - // Step 2: dream-capture writes assistant turn - execSync(`bash "${STOP_HOOK}"`, { - input: JSON.stringify({ cwd: tmpDir, session_id: 'test-fresh', last_assistant_message: 'done' }), - stdio: ['pipe', 'pipe', 'pipe'], - }); - - const lines = fs.readFileSync(queueFile, 'utf-8').trim().split('\n').filter(Boolean); - expect(lines).toHaveLength(2); - const first = JSON.parse(lines[0]) as { role: string; content: string }; - const second = JSON.parse(lines[1]) as { role: string; content: string }; - expect(first.role).toBe('user'); - expect(first.content).toBe('implement feature X'); - expect(second.role).toBe('assistant'); - expect(second.content).toBe('done'); - }); - - it('dream-dispatch truncates prompts longer than 2000 chars', () => { - fs.mkdirSync(path.join(tmpDir, '.devflow', 'memory'), { recursive: true }); - - const longPrompt = 'a'.repeat(3000); - const input = JSON.stringify({ - cwd: tmpDir, - session_id: 'test-trunc-001', - prompt: longPrompt, - }); - - execSync(`bash "${PROMPT_CAPTURE_HOOK}"`, { input, stdio: ['pipe', 'pipe', 'pipe'] }); - - const queueFile = path.join(tmpDir, '.devflow', 'memory', '.pending-turns.jsonl'); - const lines = fs.readFileSync(queueFile, 'utf-8').trim().split('\n').filter(Boolean); - expect(lines).toHaveLength(1); - - const entry = JSON.parse(lines[0]) as { role: string; content: string; ts: number }; - // Truncated at 2000 chars + '... [truncated]' suffix (15 chars) = 2015 - expect(entry.content.length).toBe(2015); - expect(entry.content).toContain('[truncated]'); - }); - - it('dream-capture truncates assistant content longer than 2000 chars', () => { - fs.mkdirSync(path.join(tmpDir, '.devflow', 'memory'), { recursive: true }); - // Write stale WORKING-MEMORY.md so throttle passes - writeStaleWorkingMemory(tmpDir); - - const longMessage = 'b'.repeat(5000); - const input = JSON.stringify({ - cwd: tmpDir, - session_id: 'test-trunc-002', - last_assistant_message: longMessage, - }); - - execSync(`bash "${STOP_HOOK}"`, { input, stdio: ['pipe', 'pipe', 'pipe'] }); - - const queueFile = path.join(tmpDir, '.devflow', 'memory', '.pending-turns.jsonl'); - const lines = fs.readFileSync(queueFile, 'utf-8').trim().split('\n').filter(Boolean); - expect(lines).toHaveLength(1); - - const entry = JSON.parse(lines[0]) as { role: string; content: string; ts: number }; - // Truncated at 2000 chars + '... [truncated]' suffix (15 chars) = 2015 - expect(entry.content.length).toBe(2015); - expect(entry.content).toContain('[truncated]'); - }); - - it('dream-capture exits cleanly when DEVFLOW_BG_UPDATER=1', () => { - fs.mkdirSync(path.join(tmpDir, '.devflow', 'memory'), { recursive: true }); - - // Hook exits at line 11 before reading stdin, so don't pipe input — would race - // and EPIPE on Node 20 when bash closes the pipe before execSync flushes. - expect(() => { - execSync(`DEVFLOW_BG_UPDATER=1 bash "${STOP_HOOK}"`, { stdio: 'ignore' }); - }).not.toThrow(); - - const queueFile = path.join(tmpDir, '.devflow', 'memory', '.pending-turns.jsonl'); - expect(fs.existsSync(queueFile)).toBe(false); - }); - - it('dream-dispatch exits cleanly when DEVFLOW_BG_UPDATER=1', () => { - fs.mkdirSync(path.join(tmpDir, '.devflow', 'memory'), { recursive: true }); - - expect(() => { - execSync(`DEVFLOW_BG_UPDATER=1 bash "${PROMPT_CAPTURE_HOOK}"`, { stdio: 'ignore' }); - }).not.toThrow(); - - const queueFile = path.join(tmpDir, '.devflow', 'memory', '.pending-turns.jsonl'); - expect(fs.existsSync(queueFile)).toBe(false); - }); -}); - // ============================================================================= // resolve-project-root — anchor .devflow/ to the project root (Fix 6) // ============================================================================= @@ -1041,20 +413,14 @@ describe('resolve-project-root: df_resolve_root', () => { }); describe('hooks anchor .devflow/ to the project root (no stray nested .devflow/)', () => { - const STOP_HOOK = path.join(HOOKS_DIR, 'dream-capture'); + const STOP_HOOK = path.join(HOOKS_DIR, 'capture-turn'); - it('dream-capture run with a CWD inside .devflow/ writes the queue at the repo root, not a nested .devflow/', () => { + it('capture-turn run with a CWD inside .devflow/ writes the queue at the repo root, not a nested .devflow/', () => { const repo = fs.mkdtempSync(path.join(os.tmpdir(), 'devflow-anchor-')); try { execSync(`git init -q "${repo}"`, { stdio: 'pipe' }); const real = fs.realpathSync(repo); - // Pre-scaffold the real .devflow/memory and a fresh trigger so the hook stays - // throttled and never spawns a background worker during the test. - const memDir = path.join(real, '.devflow', 'memory'); - fs.mkdirSync(memDir, { recursive: true }); - fs.writeFileSync(path.join(memDir, '.working-memory-last-trigger'), ''); - // The hook runs with a CWD deep inside .devflow/ — the stray-nesting scenario. const nestedCwd = path.join(real, '.devflow', 'docs', 'waves', 'w', 'tickets'); fs.mkdirSync(nestedCwd, { recursive: true }); @@ -1595,121 +961,38 @@ describe('get-mtime behavioral', () => { }); }); +describe('run-hook behavioral', () => { + const RUN_HOOK = path.join(HOOKS_DIR, 'run-hook'); + + it('exits 0 with a stderr warning when the named script is absent', () => { + // Covers the init upgrade-swap window and hand-edited settings.json entries + // that still point at a hook name that was renamed or removed (e.g. a + // pre-cutover dream-* registration lingering until the next `devflow init`). + // Exit is 0 (no throw), so the warning is captured via 2>&1 redirection + // rather than relying on execSync's catch-path stderr capture. + const output = execSync(`bash "${RUN_HOOK}" definitely-not-a-real-hook-name 2>&1`, { + stdio: ['ignore', 'pipe', 'pipe'], + }).toString(); + expect(output).toContain('definitely-not-a-real-hook-name'); + expect(output).toContain('not found'); + }); + + it('still execs a real hook script normally (get-mtime sourced via a no-op wrapper check)', () => { + // run-hook execs `bash