From c261ee709433eeedc5f768561c49426185f1159e Mon Sep 17 00:00:00 2001 From: David Cramer Date: Tue, 14 Jul 2026 21:00:55 -0700 Subject: [PATCH 01/58] ref(conversations): Cut over to canonical event history Make ConversationEvent the sole application history contract and derive Pi context through an adapter. Rename the SQL table in place, version rows, and rewrite legacy Pi messages through a bounded upgrade migration. Keep only rollout-scoped database and Redis import compatibility while removing the legacy application store and projection paths. Co-Authored-By: GPT-5 Codex --- .agents/skills/junior-qa/SKILL.md | 13 +- .../conversation-event-log/proposal.md | 82 ++ .../changes/conversation-event-log/tasks.md | 52 + .../specs/agent-step-history/spec.md | 81 -- .../specs/conversation-messages/spec.md | 44 - .../specs/conversation-retention/spec.md | 68 -- packages/docs/src/content/docs/cli/upgrade.md | 7 +- .../evals/core/coding-file-tools.eval.ts | 60 +- .../evals/core/conversation-storage.eval.ts | 57 +- packages/junior-evals/src/helpers.ts | 47 +- .../migrations/0005_conversation_events.sql | 66 ++ packages/junior/migrations/README.md | 6 + .../junior/migrations/meta/0005_snapshot.json | 1048 +++++++++++++++++ packages/junior/migrations/meta/_journal.json | 7 + .../junior/src/api/conversations/activity.ts | 100 +- .../api/conversations/detail-projection.ts | 110 +- .../junior/src/chat/agent-dispatch/runner.ts | 2 +- packages/junior/src/chat/agent/index.ts | 6 +- packages/junior/src/chat/agent/prompt.ts | 2 +- packages/junior/src/chat/agent/request.ts | 4 +- packages/junior/src/chat/agent/resume.ts | 6 +- .../junior/src/chat/conversations/README.md | 29 +- .../junior/src/chat/conversations/history.ts | 260 ++-- .../conversations/legacy-advisor-session.ts | 2 +- .../src/chat/conversations/legacy-import.ts | 44 +- .../src/chat/conversations/projection.ts | 283 ++--- .../src/chat/conversations/provenance.ts | 75 ++ .../conversations/sql/conversation-row.ts | 2 +- .../src/chat/conversations/sql/history.ts | 140 +-- .../sql/legacy-history-import.ts | 104 +- .../src/chat/conversations/sql/purge.ts | 12 +- .../junior/src/chat/conversations/store.ts | 2 +- .../chat/conversations/visible-compactions.ts | 30 +- .../chat/conversations/visible-messages.ts | 2 +- packages/junior/src/chat/db.ts | 16 +- packages/junior/src/chat/local/runner.ts | 4 +- .../junior/src/chat/pi/conversation-events.ts | 102 ++ .../junior/src/chat/plugins/task-runner.ts | 14 +- packages/junior/src/chat/runtime/README.md | 2 +- .../junior/src/chat/runtime/reply-executor.ts | 31 +- .../src/chat/services/context-compaction.ts | 18 +- .../services/plugin-auth-orchestration.ts | 2 +- .../src/chat/services/turn-session-record.ts | 4 +- .../junior/src/chat/state/conversation.ts | 4 +- packages/junior/src/chat/state/session-log.ts | 1039 +--------------- .../junior/src/chat/state/turn-session.ts | 28 +- .../junior/src/chat/task-execution/state.ts | 2 +- .../junior/src/chat/task-execution/store.ts | 2 +- packages/junior/src/cli/upgrade.ts | 2 + .../migrations/conversation-event-data.ts | 114 ++ .../upgrade/migrations/conversation-usage.ts | 24 +- .../migrations/conversations-history-sql.ts | 4 +- packages/junior/src/db/schema.ts | 6 +- packages/junior/src/db/schema/agent-steps.ts | 39 - .../src/db/schema/conversation-events.ts | 46 + .../junior/src/handlers/mcp-oauth-callback.ts | 2 +- .../junior/src/handlers/oauth-callback.ts | 2 +- .../cli/conversation-event-migration.test.ts | 374 ++++++ .../tests/component/cli/upgrade-cli.test.ts | 44 +- .../conversation-transcripts-sql.test.ts | 297 +++-- .../conversations/legacy-import.test.ts | 97 +- .../component/conversations/retention.test.ts | 43 +- .../services/context-compaction.test.ts | 12 +- .../services/turn-session-record.test.ts | 2 +- .../integration/agent-continue-slack.test.ts | 21 +- .../integration/dashboard-reporting.test.ts | 303 +++-- .../runtime/agent-run-model-handoff.test.ts | 16 +- .../integration/slack/bot-handlers.test.ts | 6 +- .../unit/chat/pi/conversation-events.test.ts | 108 ++ .../legacy-history-import.test.ts | 72 +- .../unit/conversations/provenance.test.ts | 68 ++ .../unit/state/conversation-state.test.ts | 2 +- .../tests/unit/state/session-log.test.ts | 1019 ++-------------- policies/data-redaction.md | 2 +- policies/interface-design.md | 4 +- 75 files changed, 3558 insertions(+), 3292 deletions(-) create mode 100644 openspec/changes/conversation-event-log/proposal.md create mode 100644 openspec/changes/conversation-event-log/tasks.md delete mode 100644 openspec/changes/sql-conversation-storage/specs/agent-step-history/spec.md delete mode 100644 openspec/changes/sql-conversation-storage/specs/conversation-messages/spec.md delete mode 100644 openspec/changes/sql-conversation-storage/specs/conversation-retention/spec.md create mode 100644 packages/junior/migrations/0005_conversation_events.sql create mode 100644 packages/junior/migrations/meta/0005_snapshot.json create mode 100644 packages/junior/src/chat/conversations/provenance.ts create mode 100644 packages/junior/src/chat/pi/conversation-events.ts create mode 100644 packages/junior/src/cli/upgrade/migrations/conversation-event-data.ts delete mode 100644 packages/junior/src/db/schema/agent-steps.ts create mode 100644 packages/junior/src/db/schema/conversation-events.ts create mode 100644 packages/junior/tests/component/cli/conversation-event-migration.test.ts create mode 100644 packages/junior/tests/unit/chat/pi/conversation-events.test.ts create mode 100644 packages/junior/tests/unit/conversations/provenance.test.ts diff --git a/.agents/skills/junior-qa/SKILL.md b/.agents/skills/junior-qa/SKILL.md index 8743258a0..c95082a91 100644 --- a/.agents/skills/junior-qa/SKILL.md +++ b/.agents/skills/junior-qa/SKILL.md @@ -123,18 +123,19 @@ pnpm --filter @sentry/junior exec vitest run tests/integration/mcp-oauth-callbac For SQL conversation storage changes, verify the resumed turn rebuilds context from SQL, not `thread-state` mirrors: conversation context must hydrate from `junior_conversation_messages` (`hydrateConversationMessages`) and pi history -from `junior_agent_steps` (`loadProjection`). In those tests, transcripts -seeded only into `thread-state` must also be persisted to SQL -(`persistConversationMessages`) before the callback runs, and the resumed -agent-run input `conversationContext` must contain the SQL-seeded messages. +from `junior_conversation_events` through the Pi adapter (`loadProjection`). In +those tests, transcripts seeded only into `thread-state` must also be persisted +to SQL (`persistConversationMessages`) before the callback runs, and the +resumed agent-run input `conversationContext` must contain the SQL-seeded +messages. ## Failure Handling If local chat fails because credentials are missing or expired, refresh the environment when appropriate with `pnpm dev:env`, then rerun the same command. If local chat fails with a `junior_conversation_messages` or -`junior_agent_steps` query error, the local Postgres schema predates the SQL -conversation storage cutover; run `pnpm cli -- upgrade`, then rerun. +`junior_conversation_events` query error, the local Postgres schema predates the +SQL conversation storage cutover; run `pnpm cli -- upgrade`, then rerun. If Redis errors appear during ordinary local QA, check whether `JUNIOR_STATE_ADAPTER=redis` was set; local chat normally defaults to memory state. diff --git a/openspec/changes/conversation-event-log/proposal.md b/openspec/changes/conversation-event-log/proposal.md new file mode 100644 index 000000000..0e70066d4 --- /dev/null +++ b/openspec/changes/conversation-event-log/proposal.md @@ -0,0 +1,82 @@ +# Canonical Conversation Event Log + +## Summary + +Make one ordered, append-only Junior conversation event log the canonical +history for model context, operator history, delivery facts, and child-agent +lineage. Pi and the dashboard consume separate projections of the same events. + +## Motivation + +Before this change, Junior reconstructed related views from agent steps, delivered +messages, turn-session records, activity records, and context events. Pi needs +an exact model context, while the dashboard needs chronological operational +history. Treating both as variants of one transcript makes failures difficult +to correlate and encourages duplicate message representations. + +The pre-cutover store persisted ordered agent steps and filtered host-only steps +out of Pi context. This change makes that event boundary explicit and extends it +rather than creating another transcript or lifecycle sidecar. + +## Design + +`ConversationEvent` is the canonical persisted history contract. Every event +has stable conversation ordering, a timestamp, and a versioned payload. The +first slice formalizes only the durable event kinds Junior already stores; turn +lifecycle, delivery, correlation, and lineage variants are added with their +own write boundaries in later slices. + +The storage cutover rewrites existing `pi_message` rows into Junior-owned +`message` events. Their opaque continuity payload is interpreted as a Pi +message only by the Pi adapter; Pi no longer owns the persisted event contract. + +Two primary adapters consume the log: + +- the Pi adapter selects model-relevant events and deterministically produces + `PiMessage[]`; +- the conversation detail API projects persistence events into a separate + Junior-owned, authorized, redacted reporting-event contract, and the + dashboard builds its own timeline presentation from that safe contract. + +The raw persistence union is never an API or dashboard contract. In particular, +legacy Pi roles, provider fields, and permissive message payloads do not cross +the reporting boundary merely because they are readable during migration. + +Conversation list/search tables may remain materialized read models. Queue +wakeups, leases, resumability cursors, and credentials remain mutable control +state rather than conversation history. + +Subagents use independent conversation streams in the same event system. Child +conversation metadata records parent/root lineage and a shared-context fork +point when needed. Parent events reference child execution without copying the +child event stream. + +## Rollout + +1. Formalize the event contract, isolate Pi projection, rename the physical + table, persist schema versions, and cut every application reader and writer + over to canonical Junior events. +2. Rewrite legacy SQL message rows in bounded batches. A database-only view + translates old worker reads and writes during the rolling-deploy window; it + stores no duplicate rows and is removed in the next release. +3. Expose privacy-safe events from the detail API and move timeline shaping to + the dashboard. +4. Add child-conversation lineage and context-fork projection. +5. Remove obsolete transcript reconstruction and demote remaining message + stores to explicit read models. + +## Non-Goals + +- Event-sourcing queue leases, credentials, or other mutable control state. +- Sending delivery, retry, failure, or provider diagnostics into Pi context. +- Copying child conversation events into parent conversations. +- Duplicating physical event rows or retaining parallel application stores. +- Persisting raw provider errors or private payloads as operational metadata. + +## Compatibility + +The in-place table rename preserves sequence identity and payload bytes. Old +workers remain compatible through the temporary SQL view while the application +uses only `junior_conversation_events`; the view and triggers are dropped after +the rolling-deploy window. Internal runtime and dashboard contracts may use +hard cutovers because their producers and consumers ship together. diff --git a/openspec/changes/conversation-event-log/tasks.md b/openspec/changes/conversation-event-log/tasks.md new file mode 100644 index 000000000..6cecab296 --- /dev/null +++ b/openspec/changes/conversation-event-log/tasks.md @@ -0,0 +1,52 @@ +# Tasks + +## 1. Event Contract And Pi Adapter + +- [x] Define the runtime-schema-owned `ConversationEvent` envelope and event + data union over only the existing ordered history kinds. +- [x] Isolate deterministic `ConversationEvent[] -> PiMessage[]` projection in + the Pi-owned module. +- [x] Preserve message boundaries, provenance, context epochs, model binding, + authorization observations, and bounded sequence projection. +- [x] Hard-cut storage writes and SQL rows to canonical events without changing + API payloads or runtime behavior. +- [x] Restrict legacy `pi_message` to the bounded Redis/import and rolling-worker + compatibility seams; never expose raw persistence data through the API. +- [x] Add focused adapter parity tests and update owning module documentation. + +## 2. Canonical Event Writes + +- [x] Use `(conversationId, seq)` as stable event identity and persist schema + versions on every physical event row. +- [x] Persist Junior-owned message, tool, authorization, subagent, and context + events through `ConversationEventStore`. +- [x] Rewrite legacy SQL message rows in bounded, retry-safe batches while a + database-only compatibility view supports rolling old workers. +- [x] Keep Pi messages derived rather than separately persisted. +- [ ] Add turn outcome, delivery, and explicit turn-correlation event variants. + +## 3. Event API And Dashboard + +- [ ] Return one ordered, authorized, privacy-safe event array from conversation + detail reporting. +- [ ] Move turn grouping, tool activity, failures, compactions, and delivery + presentation into the dashboard client. +- [ ] Remove the parallel transcript, activity, and context-event API views once + all consumers use events. + +## 4. Child Conversations + +- [ ] Add parent/root conversation lineage and parent turn/event correlation. +- [ ] Record subagent start and finish references without copying child events. +- [ ] Represent shared context with an immutable parent sequence fork point. +- [ ] Verify isolated and shared child Pi projections and inherited privacy. + +## 5. Cleanup And Verification + +- [ ] Make remaining visible-message and aggregate stores explicit read models. +- [ ] Remove obsolete transcript reconstruction and legacy compatibility after + migration completion. +- [ ] Verify retention, purge, redaction, idempotency, replay parity, and event + tree behavior. +- [ ] Move durable invariants into owning code and documentation, then delete + this completed plan. diff --git a/openspec/changes/sql-conversation-storage/specs/agent-step-history/spec.md b/openspec/changes/sql-conversation-storage/specs/agent-step-history/spec.md deleted file mode 100644 index 35b43c5c4..000000000 --- a/openspec/changes/sql-conversation-storage/specs/agent-step-history/spec.md +++ /dev/null @@ -1,81 +0,0 @@ -# agent-step-history - -## ADDED Requirements - -### Requirement: Execution history is stored as one row per agent step - -The system SHALL store the durable model execution history in `junior_agent_steps` with one row per step: `conversation_id` (FK to `junior_conversations`), `seq` (per-conversation order), `context_epoch`, `type` (discriminant), optional `role` (denormalized for `pi_message` rows), `payload` JSON, and `created_at`. The primary key SHALL be `(conversation_id, seq)`. The table SHALL replace the Redis list at `junior:agent-session-log:` as the execution-history authority. Rows are append-only; deletion happens only through retention purge or erasure. - -#### Scenario: Steps appended at a safe boundary - -- **WHEN** a worker commits model and tool activity at a safe boundary -- **THEN** each step is one row and reading the conversation's steps returns them in `seq` order - -### Requirement: Sequence assignment is lease-fenced and fails loudly - -`seq` SHALL be assigned transactionally per conversation by a writer holding the conversation lease. A conflicting write SHALL fail with a primary-key violation rather than silently interleaving or overwriting. - -#### Scenario: Fencing violation surfaces as an error - -- **WHEN** a writer that lost its lease attempts to append with a `seq` already used -- **THEN** the insert fails with a constraint error and no stored row is modified - -### Requirement: The current context epoch is the model context - -The model-visible context for a conversation SHALL be exactly the `pi_message` steps in the conversation's highest `context_epoch`, ordered by `seq`. Epochs start at 0. Steps in older epochs SHALL remain readable as audit history and SHALL NOT contribute to model context. Host-only step types (activity, auth, provider-connection facts) SHALL be excluded from the Pi projection as today. - -#### Scenario: Resume restores context from the current epoch - -- **WHEN** a queue-driven worker resumes a conversation -- **THEN** it restores `agent.state.messages` from the `pi_message` rows of the highest epoch in `seq` order and calls `continue()` - -### Requirement: Compaction and rollback start a new epoch atomically - -Context rebuilds SHALL NOT store embedded transcript arrays. Compaction and safe-boundary rollback SHALL, in one transaction: append a `context_epoch_started` marker step carrying `reason` (`compaction` | `rollback`), then append the replacement context as ordinary `pi_message` rows in the new epoch, preserving each message's original timestamp so replay is byte-stable. - -#### Scenario: Compaction writes a new epoch - -- **WHEN** context compaction runs for a conversation in epoch N -- **THEN** epoch N+1 contains a `context_epoch_started {reason: "compaction"}` marker followed by the retained messages and the synthetic compaction summary as individual rows, and epoch N remains as audit history - -#### Scenario: Provider-retry rollback - -- **WHEN** a transient provider failure requires trimming trailing failed assistant output -- **THEN** a new epoch is written with `reason: "rollback"` containing the trimmed history, and `continue()` resumes from it - -### Requirement: Subagent histories are child conversations - -A subagent (advisor) execution SHALL be recorded as its own conversation row with `parent_conversation_id` set, and its steps stored in `junior_agent_steps` under its own `conversation_id`. The `subagent_started` step SHALL reference the child by `childConversationId`. The polymorphic `transcriptRef {type, key}` reference and the ad-hoc `junior::advisor_session` Redis key SHALL be removed. Reading a subagent transcript SHALL use the same query path as any conversation. - -#### Scenario: Advisor invocation creates a child conversation - -- **WHEN** the advisor tool runs inside a parent conversation -- **THEN** a child conversation row exists with `parent_conversation_id` pointing at the parent, the parent's `subagent_started` step carries the child's `conversationId`, and the child's transcript renders through the standard conversation read path - -#### Scenario: Top-level listings exclude children - -- **WHEN** the dashboard lists recent conversations -- **THEN** conversations with a non-null `parent_conversation_id` are excluded - -### Requirement: Strict envelope, permissive payload - -Row envelope fields (`conversation_id`, `seq`, `context_epoch`, `type`, `created_at`) SHALL be strictly validated with the existing Zod step-type union at the store boundary; an unknown `type` or invalid envelope SHALL fail loudly as corrupt state. `pi_message` payload content SHALL remain permissive (Pi SDK owns the message shape) and SHALL carry its payload `schemaVersion` per row. - -#### Scenario: Corrupt row fails loudly - -- **WHEN** a read encounters a row whose envelope fails validation -- **THEN** the read fails with an explicit error instead of guessing behavior for the unknown shape - -### Requirement: One-time migration from Redis - -`junior upgrade` SHALL bulk-import legacy Redis session logs (bounded, newest-first), translating `sessionId` markers to integer epochs, exploding `projection_reset` entries into `context_epoch_started` markers plus per-message rows, converting advisor session keys into child conversations, and normalizing legacy v1 entry shapes. Import SHALL be idempotent per conversation (skip when step rows already exist). For conversations touched by the old deployment during promotion, the first read that finds no SQL rows while a Redis log exists SHALL perform a one-time lazy import under the conversation lease. Backfilled `pi_message` rows SHALL take `created_at` from message-internal timestamps when present, else fall back to the conversation's timestamps; fabricated import-time timestamps SHALL NOT be used. The lazy-import path SHALL be removed after the legacy Redis TTL horizon passes. - -#### Scenario: Bulk backfill during upgrade - -- **WHEN** `junior upgrade` runs against a database with legacy Redis session logs present -- **THEN** each imported conversation's steps exist in SQL with correct epochs and ordering, and re-running the upgrade imports nothing twice - -#### Scenario: Straggler conversation lazily imported - -- **WHEN** a worker resumes a conversation that has a Redis session log but no SQL step rows -- **THEN** the worker imports the log once under the conversation lease before continuing execution diff --git a/openspec/changes/sql-conversation-storage/specs/conversation-messages/spec.md b/openspec/changes/sql-conversation-storage/specs/conversation-messages/spec.md deleted file mode 100644 index be937f825..000000000 --- a/openspec/changes/sql-conversation-storage/specs/conversation-messages/spec.md +++ /dev/null @@ -1,44 +0,0 @@ -# conversation-messages - -## ADDED Requirements - -### Requirement: Visible messages are stored as SQL rows - -The system SHALL store the visible conversation transcript in `junior_conversation_messages` with one row per message, keyed by `(conversation_id, message_id)` where `conversation_id` is the globally unique conversation key and `message_id` is the source-scoped message identity (Slack `ts`-derived, local sequence). Each row SHALL carry `role` (`user` | `assistant` | `system`), `text`, an optional `author_identity_id` FK to `junior_identities`, optional `meta` JSON for bounded source facts, and `created_at`. - -#### Scenario: Message recorded and queryable - -- **WHEN** an inbound user message is accepted for a conversation -- **THEN** one message row exists for that `(conversation_id, message_id)` and listing the conversation's messages returns it in `created_at` order - -#### Scenario: Duplicate recording is idempotent - -- **WHEN** the same source message is recorded twice (source retry or redelivery) -- **THEN** exactly one row exists for that `(conversation_id, message_id)` - -### Requirement: Source facts are immutable; delivery marks are explicitly updatable - -Message rows SHALL treat `role`, `text`, `author_identity_id`, and `created_at` as immutable after insert. Mutable bookkeeping SHALL be limited to the `replied_at` delivery mark and wholesale refresh of the bounded `meta` JSON when the same message is idempotently re-recorded (late vision hydration, routing/skip marks); no other field of a stored message may be updated in place. - -#### Scenario: Reply mark set without content mutation - -- **WHEN** delivery finalizes an assistant reply that answers a stored user message -- **THEN** the user message's `replied_at` is set and its `text`, `role`, and `created_at` are unchanged - -### Requirement: SQL is the single visible-transcript authority - -The `conversation.messages` and `conversation.piMessages` mirrors in Redis `thread-state:` SHALL be removed. Reply policy, channel-context assembly, and reporting SHALL read visible messages through the `ConversationMessageStore` port. `thread-state` SHALL retain only runtime scratch (artifacts, sandbox identity, processing state) with a single Junior-owned TTL constant and a single writer. - -#### Scenario: Reply policy reads from the store - -- **WHEN** the runtime evaluates channel context or reply policy for a conversation -- **THEN** message history is read through `ConversationMessageStore` and no transcript data is read from `thread-state` - -### Requirement: Store port boundary - -Runtime, services, ingress, and dashboard modules SHALL depend on the `ConversationMessageStore` port. Drizzle client, table, and ORM types SHALL NOT leak outside `chat/conversations/sql/`. - -#### Scenario: Consumer imports the port - -- **WHEN** a runtime module needs visible message history -- **THEN** it imports the store interface, not Drizzle schema or client types diff --git a/openspec/changes/sql-conversation-storage/specs/conversation-retention/spec.md b/openspec/changes/sql-conversation-storage/specs/conversation-retention/spec.md deleted file mode 100644 index 927af903d..000000000 --- a/openspec/changes/sql-conversation-storage/specs/conversation-retention/spec.md +++ /dev/null @@ -1,68 +0,0 @@ -# conversation-retention - -## ADDED Requirements - -### Requirement: Content retention follows conversation visibility from last activity - -Conversation content (message rows, step rows, and descendant conversations' content) SHALL be retained for `window(visibility)` after the conversation's `last_activity_at`: 90 days when the root conversation's destination has persisted visibility `public`, 14 days otherwise. Any visibility other than persisted `public` — including `private`, `direct`, `unknown`, and a missing destination — SHALL resolve to the private window (fail closed). Windows SHALL be owned by named policy constants; storage write paths SHALL NOT accept or apply per-write TTLs. - -#### Scenario: Private conversation expires at 14 days - -- **WHEN** a private conversation's `last_activity_at` is more than 14 days old at purge time -- **THEN** its content is deleted - -#### Scenario: Public conversation retained past 14 days - -- **WHEN** a public conversation's `last_activity_at` is 15 days old -- **THEN** its content is retained until 90 days after last activity - -#### Scenario: New activity restarts the clock - -- **WHEN** a conversation receives an accepted inbound message or a finalized assistant delivery -- **THEN** `last_activity_at` advances and the retention window is measured from the new value - -### Requirement: Visibility is resolved at purge time through the root conversation - -The purge job SHALL compute each conversation's window at purge time by resolving the parent chain to the root conversation and reading its destination's current persisted visibility. No `expires_at` SHALL be stored. Descendant (subagent) conversations SHALL NOT have independent retention clocks; they purge with their root. - -#### Scenario: Visibility flip shortens retention - -- **WHEN** a channel's destination visibility changes from `public` to `private` -- **THEN** the next purge pass applies the 14-day window to conversations in that destination - -#### Scenario: Child rides the root's window - -- **WHEN** an advisor child conversation belongs to a public root conversation -- **THEN** the child's content is retained on the root's 90-day clock and deleted when the root is purged - -### Requirement: Purge deletes content wholesale and scrubs private metadata - -When a conversation expires, the purge job SHALL, in bounded work: delete all of its message rows, step rows, and descendants' content; stamp `transcript_purged_at` on the conversation row; and, for non-public conversations, null the raw-payload metadata fields (`title`, `channel_name`, actor JSON) so purged private conversations retain only safe metadata. The conversation metadata row itself SHALL survive purge. Reporting SHALL present purged content as expired, distinct from redacted. - -#### Scenario: Private conversation purged - -- **WHEN** the purge job processes an expired private conversation -- **THEN** its messages and steps are deleted, `transcript_purged_at` is set, its title/channel name/actor JSON are nulled, and the conversation still appears in listings with a generic label - -#### Scenario: Expired is distinct from redacted - -- **WHEN** the dashboard requests a purged conversation's transcript -- **THEN** the response indicates the content expired under retention, not that it was redacted for privacy - -### Requirement: Purge runs as a dedicated bounded cron - -Retention SHALL be enforced by a dedicated scheduled job (daily Vercel cron at `/api/internal/retention`), not by the heartbeat repair loop. Each run SHALL process a bounded batch ordered by `last_activity_at` and leave remaining work for later runs. A purge failure SHALL NOT affect task execution, heartbeat recovery, or delivery paths. - -#### Scenario: Bounded batch under backlog - -- **WHEN** more conversations are expired than one run's batch limit -- **THEN** the run purges up to the limit and the next run continues from the remaining backlog - -### Requirement: Single-conversation erasure uses the purge primitive - -The system SHALL expose `purgeConversation(conversationId)` deleting one conversation's content and descendants immediately, regardless of age, applying the same metadata scrubbing. - -#### Scenario: Erasure request honored - -- **WHEN** an operator invokes erasure for one conversation id -- **THEN** its messages, steps, and descendant content are deleted and its private metadata fields are scrubbed without waiting for the retention window diff --git a/packages/docs/src/content/docs/cli/upgrade.md b/packages/docs/src/content/docs/cli/upgrade.md index f18765d37..ad4fec09d 100644 --- a/packages/docs/src/content/docs/cli/upgrade.md +++ b/packages/docs/src/content/docs/cli/upgrade.md @@ -29,9 +29,10 @@ The command takes no extra arguments. - Move legacy `junior:conversation-work:*` Redis state into the newer conversation record and index state used by the durable worker and dashboard feed. - Backfill retained conversation records into the shared Junior SQL database. The upgrade requires `DATABASE_URL`. -- Repair legacy token and estimated-cost rollups from durable SQL agent steps in bounded batches. Conversations that are active during the repair are left unchanged and can be repaired by rerunning the command after they become idle. +- Rewrite legacy SQL Pi-message rows into canonical conversation events in bounded batches. +- Repair legacy token and estimated-cost rollups from durable SQL conversation events in bounded batches. Conversations that are active during the repair are left unchanged and can be repaired by rerunning the command after they become idle. -The migrations are idempotent: rerunning them skips records that were already moved, removes stale legacy index entries that no longer have a record, and upserts SQL conversation rows. The SQL conversation backfill copies a bounded legacy slice of Redis conversation metadata; after cutover, durable conversation metadata is written to SQL while Redis remains the transcript and execution/cache store. +The migrations are idempotent: rerunning them skips records that were already moved, removes stale legacy index entries that no longer have a record, and upserts SQL conversation rows. After cutover, SQL owns durable conversation metadata and event history; retained Redis session data is only a bounded import source. ## Vercel deploys @@ -61,6 +62,8 @@ Running migration migrate-redis-conversation-state... Finished migration migrate-redis-conversation-state: scanned=2 migrated=1 existing=0 missing=1 Running migration backfill-conversations-sql... Finished migration backfill-conversations-sql: scanned=2 migrated=2 existing=0 missing=0 +Running migration rewrite-conversation-event-data... +Finished migration rewrite-conversation-event-data: scanned=3 migrated=3 existing=0 missing=0 Running migration repair-conversation-usage... Finished migration repair-conversation-usage: scanned=2 migrated=1 existing=1 missing=0 Junior upgrade complete. diff --git a/packages/junior-evals/evals/core/coding-file-tools.eval.ts b/packages/junior-evals/evals/core/coding-file-tools.eval.ts index 5549d0ea2..d89400fa0 100644 --- a/packages/junior-evals/evals/core/coding-file-tools.eval.ts +++ b/packages/junior-evals/evals/core/coding-file-tools.eval.ts @@ -1,7 +1,7 @@ import { assistantMessages, describeEval, toolCalls } from "vitest-evals"; import { expect } from "vitest"; import { - agentSteps, + conversationEvents, mention, rubric, slackEvals, @@ -124,14 +124,14 @@ describeEval("Coding File Tools", slackEvals, (it) => { }); expect(calls.filter((call) => call.name === "handoff")).toHaveLength(1); - const steps = await agentSteps(result.session); - const markers = steps.filter( - (step) => - step.entry.type === "context_epoch_started" && - step.entry.reason === "handoff", + const events = await conversationEvents(result.session); + const markers = events.filter( + (event) => + event.data.type === "context_epoch_started" && + event.data.reason === "handoff", ); expect(markers).toHaveLength(1); - expect(markers[0]?.entry).toMatchObject({ + expect(markers[0]?.data).toMatchObject({ type: "context_epoch_started", reason: "handoff", modelProfile: "coding", @@ -147,43 +147,41 @@ describeEval("Coding File Tools", slackEvals, (it) => { return args.includes("sha256sum") && args.includes("handoff-proof.txt"); }), ).toBe(true); - const followUp = steps.find( - (step) => - step.entry.type === "pi_message" && - step.role === "user" && - JSON.stringify(step.entry.message).includes( + const followUp = events.find( + (event) => + event.data.type === "message" && + event.data.message.role === "user" && + JSON.stringify(event.data.message).includes( "run sha256sum on skills/coding-workspace-fixture/project/handoff-proof.txt", ), ); expect(followUp).toBeDefined(); - const firstHandoffModels = steps + const firstHandoffModels = events .filter( - (step) => - step.seq > markers[0]!.seq && - step.seq < followUp!.seq && - step.entry.type === "pi_message" && - step.role === "assistant", + (event) => + event.seq > markers[0]!.seq && + event.seq < followUp!.seq && + event.data.type === "message" && + event.data.message.role === "assistant", ) - .map((step) => - step.entry.type === "pi_message" && - step.entry.message.role === "assistant" - ? step.entry.message.model + .map((event) => + event.data.type === "message" && event.data.message.role === "assistant" + ? (event.data.message as { model?: string }).model : undefined, ); const handoffModel = firstHandoffModels.at(-1); expect(handoffModel).toBeDefined(); expect(handoffModel).not.toBe(process.env.AI_MODEL); - const followUpModels = steps + const followUpModels = events .filter( - (step) => - step.seq > followUp!.seq && - step.entry.type === "pi_message" && - step.role === "assistant", + (event) => + event.seq > followUp!.seq && + event.data.type === "message" && + event.data.message.role === "assistant", ) - .map((step) => - step.entry.type === "pi_message" && - step.entry.message.role === "assistant" - ? step.entry.message.model + .map((event) => + event.data.type === "message" && event.data.message.role === "assistant" + ? (event.data.message as { model?: string }).model : undefined, ); expect(followUpModels.length).toBeGreaterThan(0); diff --git a/packages/junior-evals/evals/core/conversation-storage.eval.ts b/packages/junior-evals/evals/core/conversation-storage.eval.ts index 3bfc4ed1b..447e585be 100644 --- a/packages/junior-evals/evals/core/conversation-storage.eval.ts +++ b/packages/junior-evals/evals/core/conversation-storage.eval.ts @@ -1,7 +1,7 @@ import { describeEval, toolCalls } from "vitest-evals"; import { expect } from "vitest"; import { - agentSteps, + conversationEvents, conversationMessages, mention, rubric, @@ -81,24 +81,29 @@ describeEval("Conversation Storage", slackEvals, (it) => { }), }); - // (a) The durable step history holds the turn's user and assistant - // pi_message rows in the current (highest) epoch, in seq order. - const steps = await agentSteps(result.session); - const currentEpoch = Math.max(...steps.map((step) => step.contextEpoch)); - const currentPiMessages = steps.filter( - (step) => - step.type === "pi_message" && step.contextEpoch === currentEpoch, + // (a) The durable event history holds the turn's user and assistant + // Message events in the current (highest) epoch, in seq order. + const events = await conversationEvents(result.session); + const currentEpoch = Math.max(...events.map((event) => event.contextEpoch)); + const currentMessages = events.filter( + (event) => + event.data.type === "message" && event.contextEpoch === currentEpoch, ); - const firstUser = currentPiMessages.find((step) => step.role === "user"); - const firstAssistant = currentPiMessages.find( - (step) => step.role === "assistant", + const firstUser = currentMessages.find( + (event) => + event.data.type === "message" && event.data.message.role === "user", + ); + const firstAssistant = currentMessages.find( + (event) => + event.data.type === "message" && + event.data.message.role === "assistant", ); expect(firstUser).toBeDefined(); expect(firstAssistant).toBeDefined(); expect(firstUser!.seq).toBeLessThan(firstAssistant!.seq); // seq order is preserved by loadHistory; the filtered slice stays ascending. - const seqs = currentPiMessages.map((step) => step.seq); + const seqs = currentMessages.map((event) => event.seq); expect(seqs).toEqual([...seqs].sort((left, right) => left - right)); // (b) The visible message transcript holds the user message and the @@ -115,7 +120,7 @@ describeEval("Conversation Storage", slackEvals, (it) => { }); // Regression guard for lost MCP provider-connection facts between turns. A - // durable `mcp_provider_connected` step recorded on the first turn must be + // durable `mcp_provider_connected` event recorded on the first turn must be // visible to the follow-up turn so an already-connected provider is reused // instead of re-authorized. (The concrete bug: a projection reader that // skipped the lazy legacy import missed a prior connection and re-prompted.) @@ -164,24 +169,24 @@ describeEval("Conversation Storage", slackEvals, (it) => { }), }); - // (1) The durable step history records the provider connection exactly once + // (1) The durable event history records the provider connection exactly once // for the whole conversation. A lost turn-1 fact forces a second connection // on turn 2; a duplicated fact signals a re-connect. - const steps = await agentSteps(result.session); - const connectedSteps = steps.filter( - (step) => - step.entry.type === "mcp_provider_connected" && - step.entry.provider === EVAL_MCP_PROVIDER, + const events = await conversationEvents(result.session); + const connectedEvents = events.filter( + (event) => + event.data.type === "mcp_provider_connected" && + event.data.provider === EVAL_MCP_PROVIDER, ); - expect(connectedSteps).toHaveLength(1); + expect(connectedEvents).toHaveLength(1); // (2) No re-authorization after the connection: any `authorization_requested` - // step ordered after the first connection means the follow-up re-prompted. - const firstConnectSeq = connectedSteps[0]!.seq; - const authAfterConnect = steps.filter( - (step) => - step.entry.type === "authorization_requested" && - step.seq > firstConnectSeq, + // event ordered after the first connection means the follow-up re-prompted. + const firstConnectSeq = connectedEvents[0]!.seq; + const authAfterConnect = events.filter( + (event) => + event.data.type === "authorization_requested" && + event.seq > firstConnectSeq, ); expect(authAfterConnect).toEqual([]); diff --git a/packages/junior-evals/src/helpers.ts b/packages/junior-evals/src/helpers.ts index 8a9887054..8fb653ca1 100644 --- a/packages/junior-evals/src/helpers.ts +++ b/packages/junior-evals/src/helpers.ts @@ -20,8 +20,11 @@ import { type ToolCallRecord, } from "vitest-evals/harness"; import { registerLogRecordSink, type EmittedLogRecord } from "@/chat/logging"; -import { getAgentStepStore, getConversationMessageStore } from "@/chat/db"; -import type { AgentStepEntry } from "@/chat/conversations/history"; +import { + getConversationEventStore, + getConversationMessageStore, +} from "@/chat/db"; +import type { ConversationEvent } from "@/chat/conversations/history"; import type { ConversationMessage } from "@/chat/conversations/messages"; import { renderResourceEventNotificationText } from "@/chat/resource-events/notification"; import { @@ -1037,17 +1040,6 @@ export function threadStart(opts?: { const CONVERSATION_IDS_METADATA_KEY = "conversation_ids"; -/** One durable execution step, flattened for ordering/presence assertions. */ -export interface AgentStepView { - seq: number; - contextEpoch: number; - type: AgentStepEntry["type"]; - /** Message role for `pi_message` steps; absent for host-only step types. */ - role?: string; - /** The strictly validated step payload, for deeper assertions when needed. */ - entry: AgentStepEntry; -} - function conversationIdsFromSession(session: NormalizedSession): string[] { const raw = session.metadata?.[CONVERSATION_IDS_METADATA_KEY]; if (!Array.isArray(raw) || raw.some((id) => typeof id !== "string")) { @@ -1066,40 +1058,23 @@ export function conversationId(session: NormalizedSession): string { const ids = conversationIdsFromSession(session); if (ids.length !== 1) { throw new Error( - `Expected exactly one conversation for this run but found ${ids.length} (${ids.join(", ")}). Pass an explicit conversationId to agentSteps/conversationMessages.`, + `Expected exactly one conversation for this run but found ${ids.length} (${ids.join(", ")}). Pass an explicit conversationId to conversationEvents/conversationMessages.`, ); } return ids[0]!; } -function toStepRole(entry: AgentStepEntry): string | undefined { - if (entry.type === "pi_message") { - const role = (entry.message as { role?: unknown }).role; - return typeof role === "string" ? role : undefined; - } - return undefined; -} - /** - * The run's durable execution step rows, in `seq` order across every epoch, - * read via `AgentStepStore.loadHistory`. Defaults to the run's sole + * The run's durable conversation events, in `seq` order across every epoch, + * read via `ConversationEventStore.loadHistory`. Defaults to the run's sole * conversation; pass a conversation id to inspect a child conversation. */ -export async function agentSteps( +export async function conversationEvents( session: NormalizedSession, conversationIdOverride?: string, -): Promise { +): Promise { const id = conversationIdOverride ?? conversationId(session); - const steps = await getAgentStepStore().loadHistory(id); - return steps.map((step) => ({ - seq: step.seq, - contextEpoch: step.contextEpoch, - type: step.entry.type, - ...(toStepRole(step.entry) !== undefined - ? { role: toStepRole(step.entry) } - : {}), - entry: step.entry, - })); + return await getConversationEventStore().loadHistory(id); } /** diff --git a/packages/junior/migrations/0005_conversation_events.sql b/packages/junior/migrations/0005_conversation_events.sql new file mode 100644 index 000000000..2c817dd2d --- /dev/null +++ b/packages/junior/migrations/0005_conversation_events.sql @@ -0,0 +1,66 @@ +ALTER TABLE "junior_agent_steps" RENAME TO "junior_conversation_events";--> statement-breakpoint +ALTER TABLE "junior_conversation_events" RENAME CONSTRAINT "junior_agent_steps_conversation_id_seq_pk" TO "junior_conversation_events_conversation_id_seq_pk";--> statement-breakpoint +ALTER TABLE "junior_conversation_events" RENAME CONSTRAINT "junior_agent_steps_conversation_id_junior_conversations_conversation_id_fk" TO "junior_conversation_events_conversation_id_junior_conversations_conversation_id_fk";--> statement-breakpoint +ALTER INDEX "junior_agent_steps_epoch_idx" RENAME TO "junior_conversation_events_epoch_idx";--> statement-breakpoint +ALTER TABLE "junior_conversation_events" ADD COLUMN "schema_version" integer DEFAULT 1 NOT NULL;--> statement-breakpoint +-- TODO(v0.104.0): Remove the junior_agent_steps compatibility view and trigger +-- functions after the 0.103.x rolling-deploy window. +CREATE VIEW "junior_agent_steps" AS +SELECT + "conversation_id", + "seq", + "context_epoch", + CASE WHEN "type" = 'message' THEN 'pi_message' ELSE "type" END AS "type", + "role", + "payload", + "created_at" +FROM "junior_conversation_events";--> statement-breakpoint +CREATE FUNCTION "junior_agent_steps_insert_compat"() +RETURNS trigger +LANGUAGE plpgsql +AS $$ +BEGIN + INSERT INTO "junior_conversation_events" ( + "conversation_id", + "seq", + "context_epoch", + "schema_version", + "type", + "role", + "payload", + "created_at" + ) VALUES ( + NEW."conversation_id", + NEW."seq", + NEW."context_epoch", + 1, + CASE WHEN NEW."type" = 'pi_message' THEN 'message' ELSE NEW."type" END, + NEW."role", + CASE + WHEN NEW."type" = 'pi_message' + AND jsonb_typeof(NEW."payload") = 'object' + THEN NEW."payload" - 'schemaVersion' + ELSE NEW."payload" + END, + NEW."created_at" + ); + RETURN NEW; +END; +$$;--> statement-breakpoint +CREATE TRIGGER "junior_agent_steps_insert_compat_trigger" +INSTEAD OF INSERT ON "junior_agent_steps" +FOR EACH ROW EXECUTE FUNCTION "junior_agent_steps_insert_compat"();--> statement-breakpoint +CREATE FUNCTION "junior_agent_steps_delete_compat"() +RETURNS trigger +LANGUAGE plpgsql +AS $$ +BEGIN + DELETE FROM "junior_conversation_events" + WHERE "conversation_id" = OLD."conversation_id" + AND "seq" = OLD."seq"; + RETURN OLD; +END; +$$;--> statement-breakpoint +CREATE TRIGGER "junior_agent_steps_delete_compat_trigger" +INSTEAD OF DELETE ON "junior_agent_steps" +FOR EACH ROW EXECUTE FUNCTION "junior_agent_steps_delete_compat"(); diff --git a/packages/junior/migrations/README.md b/packages/junior/migrations/README.md index a0b5c2aea..4974f66da 100644 --- a/packages/junior/migrations/README.md +++ b/packages/junior/migrations/README.md @@ -12,3 +12,9 @@ The `0000_initial.sql` baseline represents the schema already deployed by the pre-Drizzle Junior migration runner. During upgrade, existing installations adopt that baseline once; new installations execute it normally. All later migrations are applied by Drizzle in journal order. + +`0004_conversation_events.sql` keeps `junior_agent_steps` as an updatable +compatibility view for the 0.103.x rolling-deploy window. It maps legacy +`pi_message` reads and writes to canonical `message` events while the CLI +upgrade backfills existing rows in bounded batches. Remove the view and its +trigger functions in 0.104.0 after 0.103.x workers are no longer supported. diff --git a/packages/junior/migrations/meta/0005_snapshot.json b/packages/junior/migrations/meta/0005_snapshot.json new file mode 100644 index 000000000..f50845eb1 --- /dev/null +++ b/packages/junior/migrations/meta/0005_snapshot.json @@ -0,0 +1,1048 @@ +{ + "id": "5bcf5afe-2cc5-497a-a873-08c54ab1dfb0", + "prevId": "f4f247d8-7b7d-420b-a78f-7a36ba8c1472", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.junior_conversation_events": { + "name": "junior_conversation_events", + "schema": "", + "columns": { + "conversation_id": { + "name": "conversation_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "seq": { + "name": "seq", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "context_epoch": { + "name": "context_epoch", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "schema_version": { + "name": "schema_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "junior_conversation_events_epoch_idx": { + "name": "junior_conversation_events_epoch_idx", + "columns": [ + { + "expression": "conversation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "context_epoch", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "seq", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "junior_conversation_events_conversation_id_junior_conversations_conversation_id_fk": { + "name": "junior_conversation_events_conversation_id_junior_conversations_conversation_id_fk", + "tableFrom": "junior_conversation_events", + "tableTo": "junior_conversations", + "columnsFrom": [ + "conversation_id" + ], + "columnsTo": [ + "conversation_id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "junior_conversation_events_conversation_id_seq_pk": { + "name": "junior_conversation_events_conversation_id_seq_pk", + "columns": [ + "conversation_id", + "seq" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.junior_conversation_messages": { + "name": "junior_conversation_messages", + "schema": "", + "columns": { + "conversation_id": { + "name": "conversation_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "message_id": { + "name": "message_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "author_identity_id": { + "name": "author_identity_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "text": { + "name": "text", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "meta": { + "name": "meta", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "replied_at": { + "name": "replied_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "junior_conversation_messages_activity_idx": { + "name": "junior_conversation_messages_activity_idx", + "columns": [ + { + "expression": "conversation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "junior_conversation_messages_search_idx": { + "name": "junior_conversation_messages_search_idx", + "columns": [ + { + "expression": "to_tsvector('english', \"text\")", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gin", + "with": {} + } + }, + "foreignKeys": { + "junior_conversation_messages_conversation_id_junior_conversations_conversation_id_fk": { + "name": "junior_conversation_messages_conversation_id_junior_conversations_conversation_id_fk", + "tableFrom": "junior_conversation_messages", + "tableTo": "junior_conversations", + "columnsFrom": [ + "conversation_id" + ], + "columnsTo": [ + "conversation_id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "junior_conversation_messages_author_identity_id_junior_identities_id_fk": { + "name": "junior_conversation_messages_author_identity_id_junior_identities_id_fk", + "tableFrom": "junior_conversation_messages", + "tableTo": "junior_identities", + "columnsFrom": [ + "author_identity_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "junior_conversation_messages_conversation_id_message_id_pk": { + "name": "junior_conversation_messages_conversation_id_message_id_pk", + "columns": [ + "conversation_id", + "message_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.junior_conversations": { + "name": "junior_conversations", + "schema": "", + "columns": { + "conversation_id": { + "name": "conversation_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "schema_version": { + "name": "schema_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "origin_type": { + "name": "origin_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "origin_id": { + "name": "origin_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "origin_run_id": { + "name": "origin_run_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "destination_id": { + "name": "destination_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "destination_json": { + "name": "destination_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "actor_identity_id": { + "name": "actor_identity_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "creator_identity_id": { + "name": "creator_identity_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "credential_subject_identity_id": { + "name": "credential_subject_identity_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "actor_json": { + "name": "actor_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "channel_name": { + "name": "channel_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "last_activity_at": { + "name": "last_activity_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "execution_updated_at": { + "name": "execution_updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "execution_status": { + "name": "execution_status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_checkpoint_at": { + "name": "last_checkpoint_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_enqueued_at": { + "name": "last_enqueued_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "parent_conversation_id": { + "name": "parent_conversation_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "transcript_purged_at": { + "name": "transcript_purged_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "duration_ms": { + "name": "duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "usage_json": { + "name": "usage_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "execution_duration_ms": { + "name": "execution_duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "execution_usage_json": { + "name": "execution_usage_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "metric_run_id": { + "name": "metric_run_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "archived_at": { + "name": "archived_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "junior_conversations_last_activity_idx": { + "name": "junior_conversations_last_activity_idx", + "columns": [ + { + "expression": "last_activity_at", + "isExpression": false, + "asc": false, + "nulls": "last" + }, + { + "expression": "conversation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "junior_conversations_active_idx": { + "name": "junior_conversations_active_idx", + "columns": [ + { + "expression": "coalesce(\"execution_updated_at\", \"updated_at\")", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "conversation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"junior_conversations\".\"execution_status\" <> 'idle'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "junior_conversations_destination_activity_idx": { + "name": "junior_conversations_destination_activity_idx", + "columns": [ + { + "expression": "destination_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "last_activity_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "junior_conversations_actor_activity_idx": { + "name": "junior_conversations_actor_activity_idx", + "columns": [ + { + "expression": "actor_identity_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "last_activity_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "junior_conversations_origin_idx": { + "name": "junior_conversations_origin_idx", + "columns": [ + { + "expression": "origin_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "origin_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "last_activity_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "junior_conversations_parent_idx": { + "name": "junior_conversations_parent_idx", + "columns": [ + { + "expression": "parent_conversation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "junior_conversations_destination_id_junior_destinations_id_fk": { + "name": "junior_conversations_destination_id_junior_destinations_id_fk", + "tableFrom": "junior_conversations", + "tableTo": "junior_destinations", + "columnsFrom": [ + "destination_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "junior_conversations_actor_identity_id_junior_identities_id_fk": { + "name": "junior_conversations_actor_identity_id_junior_identities_id_fk", + "tableFrom": "junior_conversations", + "tableTo": "junior_identities", + "columnsFrom": [ + "actor_identity_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "junior_conversations_creator_identity_id_junior_identities_id_fk": { + "name": "junior_conversations_creator_identity_id_junior_identities_id_fk", + "tableFrom": "junior_conversations", + "tableTo": "junior_identities", + "columnsFrom": [ + "creator_identity_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "junior_conversations_credential_subject_identity_id_junior_identities_id_fk": { + "name": "junior_conversations_credential_subject_identity_id_junior_identities_id_fk", + "tableFrom": "junior_conversations", + "tableTo": "junior_identities", + "columnsFrom": [ + "credential_subject_identity_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "junior_conversations_parent_conversation_id_junior_conversations_conversation_id_fk": { + "name": "junior_conversations_parent_conversation_id_junior_conversations_conversation_id_fk", + "tableFrom": "junior_conversations", + "tableTo": "junior_conversations", + "columnsFrom": [ + "parent_conversation_id" + ], + "columnsTo": [ + "conversation_id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.junior_destinations": { + "name": "junior_destinations", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_tenant_id": { + "name": "provider_tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "provider_destination_id": { + "name": "provider_destination_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "parent_destination_id": { + "name": "parent_destination_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "visibility": { + "name": "visibility", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'unknown'" + }, + "metadata_json": { + "name": "metadata_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "junior_destinations_provider_destination_uidx": { + "name": "junior_destinations_provider_destination_uidx", + "columns": [ + { + "expression": "provider", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_tenant_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_destination_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "junior_destinations_provider_kind_idx": { + "name": "junior_destinations_provider_kind_idx", + "columns": [ + { + "expression": "provider", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.junior_identities": { + "name": "junior_identities", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_tenant_id": { + "name": "provider_tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "provider_subject_id": { + "name": "provider_subject_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "handle": { + "name": "handle", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "avatar_url": { + "name": "avatar_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata_json": { + "name": "metadata_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "email_normalized": { + "name": "email_normalized", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "email_verified": { + "name": "email_verified", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + } + }, + "indexes": { + "junior_identities_provider_subject_uidx": { + "name": "junior_identities_provider_subject_uidx", + "columns": [ + { + "expression": "provider", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_tenant_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_subject_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "junior_identities_user_idx": { + "name": "junior_identities_user_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "junior_identities_verified_email_idx": { + "name": "junior_identities_verified_email_idx", + "columns": [ + { + "expression": "email_normalized", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"junior_identities\".\"email_verified\" = true AND \"junior_identities\".\"email_normalized\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "junior_identities_kind_provider_idx": { + "name": "junior_identities_kind_provider_idx", + "columns": [ + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "junior_identities_user_id_junior_users_id_fk": { + "name": "junior_identities_user_id_junior_users_id_fk", + "tableFrom": "junior_identities", + "tableTo": "junior_users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.junior_users": { + "name": "junior_users", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "primary_email": { + "name": "primary_email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "primary_email_normalized": { + "name": "primary_email_normalized", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "junior_users_primary_email_normalized_uidx": { + "name": "junior_users_primary_email_normalized_uidx", + "columns": [ + { + "expression": "primary_email_normalized", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} diff --git a/packages/junior/migrations/meta/_journal.json b/packages/junior/migrations/meta/_journal.json index e97022daa..7f4c61ea1 100644 --- a/packages/junior/migrations/meta/_journal.json +++ b/packages/junior/migrations/meta/_journal.json @@ -36,6 +36,13 @@ "when": 1784137322420, "tag": "0004_useful_magus", "breakpoints": true + }, + { + "idx": 5, + "version": "7", + "when": 1784137322421, + "tag": "0005_conversation_events", + "breakpoints": true } ] } diff --git a/packages/junior/src/api/conversations/activity.ts b/packages/junior/src/api/conversations/activity.ts index d0bc0bb89..8c2c65624 100644 --- a/packages/junior/src/api/conversations/activity.ts +++ b/packages/junior/src/api/conversations/activity.ts @@ -1,6 +1,6 @@ import type { - AgentStepEntry, - StoredAgentStep, + ConversationEvent, + ConversationEventData, } from "@/chat/conversations/history"; import type { PiMessage } from "@/chat/pi/messages"; import { redactedPayloadFields } from "./transcript"; @@ -43,51 +43,51 @@ function activityPayloadFields( } /** - * Build the current-run activity timeline from durable agent steps. + * Build the current-run activity timeline from durable conversation events. * * Tool executions, subagent starts/ends, and their nesting are derived from the - * conversation's `junior_agent_steps` rows instead of the legacy Redis session - * log; tool statuses come from the aligned `pi_message` tool results. Redaction + * conversation's durable events instead of the legacy Redis session log; tool + * statuses come from aligned model-message tool results. Redaction * stays byte-compatible with the prior session-log path. */ -export function buildConversationActivityFromSteps(args: { +export function buildConversationActivityFromEvents(args: { canExposePayload: boolean; - steps: StoredAgentStep[]; + events: ConversationEvent[]; messages: PiMessage[]; }): ConversationActivityReport[] { const toolStatuses = toolResultStatuses(args.messages); - const subagentEnds = new Map(); + const subagentEnds = new Map(); const subagentsByToolCallId = new Map< string, ConversationSubagentActivityReport[] >(); const orphanSubagents: ConversationSubagentActivityReport[] = []; - for (const step of args.steps) { - if (step.entry.type === "subagent_ended") { + for (const event of args.events) { + if (event.data.type === "subagent_ended") { subagentEnds.set( - step.entry.subagentInvocationId, - step as SubagentEndedStep, + event.data.subagentInvocationId, + event as SubagentEndedEvent, ); } } - for (const step of args.steps) { - if (step.entry.type !== "subagent_started") { + for (const event of args.events) { + if (event.data.type !== "subagent_started") { continue; } - const start = step as SubagentStartedStep; - const parentStatus = start.entry.parentToolCallId - ? toolStatuses.get(start.entry.parentToolCallId) + const start = event as SubagentStartedEvent; + const parentStatus = start.data.parentToolCallId + ? toolStatuses.get(start.data.parentToolCallId) : undefined; - const activity = subagentActivityFromSteps( + const activity = subagentActivityFromEvents( start, - subagentEnds.get(start.entry.subagentInvocationId), + subagentEnds.get(start.data.subagentInvocationId), { canExposeTranscript: args.canExposePayload, parentStatus }, ); - if (start.entry.parentToolCallId) { - subagentsByToolCallId.set(start.entry.parentToolCallId, [ - ...(subagentsByToolCallId.get(start.entry.parentToolCallId) ?? []), + if (start.data.parentToolCallId) { + subagentsByToolCallId.set(start.data.parentToolCallId, [ + ...(subagentsByToolCallId.get(start.data.parentToolCallId) ?? []), activity, ]); continue; @@ -96,19 +96,19 @@ export function buildConversationActivityFromSteps(args: { } const rows: ConversationActivityReport[] = []; - for (const step of args.steps) { - if (step.entry.type !== "tool_execution_started") { + for (const event of args.events) { + if (event.data.type !== "tool_execution_started") { continue; } rows.push({ type: "tool_execution", - id: step.entry.toolCallId, - toolCallId: step.entry.toolCallId, - toolName: step.entry.toolName, - createdAt: new Date(step.createdAtMs).toISOString(), - status: toolStatuses.get(step.entry.toolCallId) ?? "running", - subagents: subagentsByToolCallId.get(step.entry.toolCallId) ?? [], - ...activityPayloadFields(step.entry.args, args.canExposePayload), + id: event.data.toolCallId, + toolCallId: event.data.toolCallId, + toolName: event.data.toolName, + createdAt: new Date(event.createdAtMs).toISOString(), + status: toolStatuses.get(event.data.toolCallId) ?? "running", + subagents: subagentsByToolCallId.get(event.data.toolCallId) ?? [], + ...activityPayloadFields(event.data.args, args.canExposePayload), }); } @@ -119,17 +119,17 @@ export function buildConversationActivityFromSteps(args: { ); } -export type SubagentStartedStep = StoredAgentStep & { - entry: Extract; +export type SubagentStartedEvent = ConversationEvent & { + data: Extract; }; -export type SubagentEndedStep = StoredAgentStep & { - entry: Extract; +export type SubagentEndedEvent = ConversationEvent & { + data: Extract; }; -/** Pair durable subagent start and end steps into one activity report. */ -export function subagentActivityFromSteps( - start: SubagentStartedStep, - end: SubagentEndedStep | undefined, +/** Pair durable subagent start and end events into one activity report. */ +export function subagentActivityFromEvents( + start: SubagentStartedEvent, + end: SubagentEndedEvent | undefined, options: { canExposeTranscript?: boolean; parentStatus?: ConversationActivityStatus; @@ -137,21 +137,21 @@ export function subagentActivityFromSteps( ): ConversationSubagentActivityReport { return { type: "subagent", - id: start.entry.subagentInvocationId, - subagentKind: start.entry.subagentKind, - ...(start.entry.modelId ? { modelId: start.entry.modelId } : {}), - ...(start.entry.parentToolCallId - ? { parentToolCallId: start.entry.parentToolCallId } + id: start.data.subagentInvocationId, + subagentKind: start.data.subagentKind, + ...(start.data.modelId ? { modelId: start.data.modelId } : {}), + ...(start.data.parentToolCallId + ? { parentToolCallId: start.data.parentToolCallId } : {}), - ...(start.entry.reasoningLevel - ? { reasoningLevel: start.entry.reasoningLevel } + ...(start.data.reasoningLevel + ? { reasoningLevel: start.data.reasoningLevel } : {}), createdAt: new Date(start.createdAtMs).toISOString(), ...(end ? { endedAt: new Date(end.createdAtMs).toISOString(), - outcome: end.entry.outcome, - status: end.entry.outcome, + outcome: end.data.outcome, + status: end.data.outcome, // Every subagent is a child conversation whose transcript loads on // demand; expose the affordance only when the parent is public. ...(options.canExposeTranscript ? { transcriptAvailable: true } : {}), @@ -163,9 +163,9 @@ export function subagentActivityFromSteps( /** * Read one child-agent transcript through its parent conversation. * - * The parent records `subagent_started`/`subagent_ended` as durable steps that + * The parent records `subagent_started`/`subagent_ended` as durable events that * name the child by `childConversationId`; the transcript is the child * conversation's own projected Pi messages. `runId` is retained for the route - * signature but no longer scopes the lookup — subagent steps live on the parent + * signature but no longer scopes the lookup — subagent events live on the parent * conversation regardless of the run that produced them. */ diff --git a/packages/junior/src/api/conversations/detail-projection.ts b/packages/junior/src/api/conversations/detail-projection.ts index 427c36931..036c5d3ef 100644 --- a/packages/junior/src/api/conversations/detail-projection.ts +++ b/packages/junior/src/api/conversations/detail-projection.ts @@ -5,12 +5,16 @@ import type { ConversationMessageStore, } from "@/chat/conversations/messages"; import type { - AgentStepStore, - StoredAgentStep, + ConversationEventStore, + ConversationEvent, } from "@/chat/conversations/history"; import type { Conversation } from "@/chat/conversations/store"; -import { loadProjection, projectSteps } from "@/chat/conversations/projection"; -import { getAgentStepStore, getConversationMessageStore } from "@/chat/db"; +import { loadProjection } from "@/chat/conversations/projection"; +import { projectConversationEvents } from "@/chat/pi/conversation-events"; +import { + getConversationEventStore, + getConversationMessageStore, +} from "@/chat/db"; import type { PiMessage } from "@/chat/pi/messages"; import { stripRuntimeTurnContext } from "@/chat/pi/transcript"; import { @@ -18,10 +22,10 @@ import { buildSentryTraceUrl, } from "@/chat/sentry-links"; import { - buildConversationActivityFromSteps, - subagentActivityFromSteps, - type SubagentEndedStep, - type SubagentStartedStep, + buildConversationActivityFromEvents, + subagentActivityFromEvents, + type SubagentEndedEvent, + type SubagentStartedEvent, } from "./activity"; import { conversationSummaryFromStoredConversation } from "./projection"; import { @@ -47,8 +51,8 @@ const COMPACTION_SUMMARY_PREFIXES = [ const MODEL_HANDOFF_SUMMARY_PREFIX = "Model handoff checkpoint. Continue the outstanding request now using this summary as the complete prior context:"; -type EpochStartedStep = StoredAgentStep & { - entry: Extract; +type EpochStartedEvent = ConversationEvent & { + data: Extract; }; function messageText(message: PiMessage): string { @@ -100,28 +104,28 @@ function matchingPrefix(left: PiMessage[], right: PiMessage[]): number { */ function historyContent(args: { canExposePayload: boolean; - steps: StoredAgentStep[]; + events: ConversationEvent[]; }): { contextEvents: ConversationContextEvent[]; messages: PiMessage[]; } { const contextEvents: ConversationContextEvent[] = []; const messages: PiMessage[] = []; - const epochs = new Map(); - for (const step of args.steps) { - const epoch = epochs.get(step.contextEpoch); - if (epoch) epoch.push(step); - else epochs.set(step.contextEpoch, [step]); + const epochs = new Map(); + for (const event of args.events) { + const epoch = epochs.get(event.contextEpoch); + if (epoch) epoch.push(event); + else epochs.set(event.contextEpoch, [event]); } let previousModelId: string | undefined; let previousProjection: PiMessage[] = []; - for (const steps of epochs.values()) { - const marker = steps.find( - (step): step is EpochStartedStep => - step.entry.type === "context_epoch_started", + for (const events of epochs.values()) { + const marker = events.find( + (event): event is EpochStartedEvent => + event.data.type === "context_epoch_started", ); - const projection = projectSteps(steps); + const projection = projectConversationEvents(events); const projected: PiMessage[] = []; const projectedProvenance: typeof projection.provenance = []; projection.messages.forEach((message, index) => { @@ -131,43 +135,43 @@ function historyContent(args: { } }); const replacementSummaryIndex = - marker?.entry.reason === "compaction" + marker?.data.reason === "compaction" ? summaryIndex( projected, projectedProvenance, COMPACTION_SUMMARY_PREFIXES, ) - : marker?.entry.reason === "handoff" + : marker?.data.reason === "handoff" ? summaryIndex(projected, projectedProvenance, [ MODEL_HANDOFF_SUMMARY_PREFIX, ]) : -1; const summary = - marker?.entry.reason === "compaction" && replacementSummaryIndex >= 0 + marker?.data.reason === "compaction" && replacementSummaryIndex >= 0 ? summaryAfterPrefix( projected[replacementSummaryIndex]!, COMPACTION_SUMMARY_PREFIXES, ) : undefined; const handoffMessage = - marker?.entry.reason === "handoff" && replacementSummaryIndex >= 0 + marker?.data.reason === "handoff" && replacementSummaryIndex >= 0 ? messageText(projected[replacementSummaryIndex]!) || undefined : undefined; - if (marker?.entry.reason === "compaction") { + if (marker?.data.reason === "compaction") { contextEvents.push({ type: "context_compacted", createdAt: new Date(marker.createdAtMs).toISOString(), - ...(marker.entry.modelId ? { modelId: marker.entry.modelId } : {}), + ...(marker.data.modelId ? { modelId: marker.data.modelId } : {}), ...(args.canExposePayload && summary ? { summary } : {}), transcriptIndex: messages.length, }); - } else if (marker?.entry.reason === "handoff") { + } else if (marker?.data.reason === "handoff") { contextEvents.push({ type: "model_handoff", createdAt: new Date(marker.createdAtMs).toISOString(), ...(previousModelId ? { fromModelId: previousModelId } : {}), - toModelId: marker.entry.modelId, + toModelId: marker.data.modelId, ...(args.canExposePayload && handoffMessage ? { message: handoffMessage } : {}), @@ -175,7 +179,7 @@ function historyContent(args: { }); } - if (marker?.entry.reason === "rollback") { + if (marker?.data.reason === "rollback") { messages.push( ...projected.slice(matchingPrefix(previousProjection, projected)), ); @@ -185,7 +189,7 @@ function historyContent(args: { if (index === replacementSummaryIndex) return; let copiedCompactionMessage = false; if ( - marker?.entry.reason === "compaction" && + marker?.data.reason === "compaction" && replacementSummaryIndex >= 0 && index < replacementSummaryIndex ) { @@ -200,7 +204,7 @@ function historyContent(args: { if (!copiedCompactionMessage) messages.push(message); }); } - previousModelId = marker?.entry.modelId ?? previousModelId; + previousModelId = marker?.data.modelId ?? previousModelId; previousProjection = projected; } @@ -210,17 +214,17 @@ function historyContent(args: { async function conversationContent(args: { conversationId: string; messageStore: ConversationMessageStore; - stepStore: AgentStepStore; + eventStore: ConversationEventStore; canExposePayload: boolean; }): Promise<{ activity: ConversationActivityReport[]; contextEvents: ConversationContextEvent[]; transcript: TranscriptMessage[]; }> { - const steps = await args.stepStore.loadHistory(args.conversationId); + const events = await args.eventStore.loadHistory(args.conversationId); const history = historyContent({ canExposePayload: args.canExposePayload, - steps, + events, }); const messages = history.messages; const transcript = @@ -230,9 +234,9 @@ async function conversationContent(args: { visibleMessageTranscript, ); return { - activity: buildConversationActivityFromSteps({ + activity: buildConversationActivityFromEvents({ canExposePayload: args.canExposePayload, - steps, + events, messages, }), contextEvents: history.contextEvents, @@ -260,7 +264,7 @@ export async function buildConversationDetail(args: { const { conversation } = args; const conversationId = conversation.conversationId; const nowMs = Date.now(); - const stepStore = getAgentStepStore(); + const eventStore = getConversationEventStore(); const messageStore = getConversationMessageStore(); const transcriptPurgedAtMs = conversation.transcriptPurgedAtMs; const transcriptExpiredAt = @@ -270,7 +274,7 @@ export async function buildConversationDetail(args: { // Reporting reads the complete durable execution history. Context rebuilds // become explicit events while copied replacement messages are de-duplicated. - // Purged conversations have no steps to read. + // Purged conversations have no events to read. const canExposeSqlContent = canExposeConversationPayload({ conversationId, visibility: conversation.visibility, @@ -280,7 +284,7 @@ export async function buildConversationDetail(args: { ? await conversationContent({ conversationId, messageStore, - stepStore, + eventStore, canExposePayload: canExposeSqlContent, }) : { activity: [], contextEvents: [], transcript: [] }; @@ -341,10 +345,10 @@ export async function buildConversationSubagent( subagentId: string, ): Promise { const conversationId = conversation.conversationId; - const stepStore = getAgentStepStore(); - const parentSteps = await stepStore.loadHistory(conversationId); + const eventStore = getConversationEventStore(); + const parentEvents = await eventStore.loadHistory(conversationId); - // Retention purge deletes the parent tree's steps wholesale; present the + // Retention purge deletes the parent tree's events wholesale; present the // subagent as expired rather than "not found" (data-redaction.md). if (conversation?.transcriptPurgedAtMs !== undefined) { return { @@ -362,10 +366,10 @@ export async function buildConversationSubagent( }; } - const start = parentSteps.find( - (step): step is SubagentStartedStep => - step.entry.type === "subagent_started" && - step.entry.subagentInvocationId === subagentId, + const start = parentEvents.find( + (event): event is SubagentStartedEvent => + event.data.type === "subagent_started" && + event.data.subagentInvocationId === subagentId, ); if (!start) { return { @@ -379,14 +383,14 @@ export async function buildConversationSubagent( unavailableReason: "not_found", }; } - const end = parentSteps.find( - (step): step is SubagentEndedStep => - step.entry.type === "subagent_ended" && - step.entry.subagentInvocationId === subagentId, + const end = parentEvents.find( + (event): event is SubagentEndedEvent => + event.data.type === "subagent_ended" && + event.data.subagentInvocationId === subagentId, ); - const childConversationId = start.entry.childConversationId; - const activity = subagentActivityFromSteps(start, end); + const childConversationId = start.data.childConversationId; + const activity = subagentActivityFromEvents(start, end); const subagentSentryConversationUrl = buildSentryConversationUrl(childConversationId); const conversationFields = { diff --git a/packages/junior/src/chat/agent-dispatch/runner.ts b/packages/junior/src/chat/agent-dispatch/runner.ts index 46a811c8a..93931643d 100644 --- a/packages/junior/src/chat/agent-dispatch/runner.ts +++ b/packages/junior/src/chat/agent-dispatch/runner.ts @@ -311,7 +311,7 @@ export async function runAgentDispatchSlice( messageText: dispatch.input, conversationContext, // Pi history for redelivered dispatch slices comes from the SQL - // step-store projection, not a thread-state mirror. + // event-store projection, not a thread-state mirror. piMessages: await loadProjection({ conversationId }), }, routing: { diff --git a/packages/junior/src/chat/agent/index.ts b/packages/junior/src/chat/agent/index.ts index cfe1dc545..4bb72e6ca 100644 --- a/packages/junior/src/chat/agent/index.ts +++ b/packages/junior/src/chat/agent/index.ts @@ -44,8 +44,8 @@ import { import { instructionActors, instructionProvenanceFor, - type PiMessageProvenance, -} from "@/chat/state/session-log"; + type ConversationMessageProvenance, +} from "@/chat/conversations/provenance"; import type { Actor } from "@/chat/actor"; import { GEN_AI_PROVIDER_NAME, @@ -341,7 +341,7 @@ async function executeAgentRunInPrivacyContext( // agent starts, then adds the current actor's turn-start instruction. // Steering appends to this array as it drains, so `run.actors` stays a // pure, live projection of committed instruction provenance. - const committedInstructionProvenance: PiMessageProvenance[] = [ + const committedInstructionProvenance: ConversationMessageProvenance[] = [ ...(existingSessionRecord?.piMessageProvenance ?? []), ...(existingSessionRecord?.actors ?? []).map(instructionProvenanceFor), ...(resumedFromSessionRecord ? [] : [instructionProvenanceFor(actor)]), diff --git a/packages/junior/src/chat/agent/prompt.ts b/packages/junior/src/chat/agent/prompt.ts index 95f336b57..d8685e5be 100644 --- a/packages/junior/src/chat/agent/prompt.ts +++ b/packages/junior/src/chat/agent/prompt.ts @@ -286,7 +286,7 @@ export function buildPromptInput(input: AgentRunInput): PromptInput { /** * Convert a mid-run user message into the Pi user message shape used for - * steering injection and parked-conversation session-log appends, so both + * steering injection and parked-conversation event-log appends, so both * paths store identical durable history. */ export function buildSteeringPiMessage( diff --git a/packages/junior/src/chat/agent/request.ts b/packages/junior/src/chat/agent/request.ts index 03b0f3346..b4fe5226f 100644 --- a/packages/junior/src/chat/agent/request.ts +++ b/packages/junior/src/chat/agent/request.ts @@ -25,7 +25,7 @@ import type { SlackConversationContext } from "@/chat/slack/conversation-context import { parseSlackThreadId } from "@/chat/slack/context"; import type { ThreadArtifactsState } from "@/chat/state/artifacts"; import type { ConversationPendingAuthState } from "@/chat/state/conversation"; -import type { PiMessageProvenance } from "@/chat/state/session-log"; +import type { ConversationMessageProvenance } from "@/chat/conversations/provenance"; import type { AgentTurnSurface } from "@/chat/state/turn-session"; import type { ToolExecutionReport } from "@/chat/tool-support/tool-execution-report"; import type { SlackActionToken } from "@/chat/slack/action-token"; @@ -52,7 +52,7 @@ export interface AgentRunInstructionActor { export interface AgentRunSteeringMessage { actor?: AgentRunInstructionActor; /** Provenance of this queued/steered message, carrying its original author. */ - provenance: PiMessageProvenance; + provenance: ConversationMessageProvenance; omittedImageAttachmentCount?: number; text: string; timestampMs?: number; diff --git a/packages/junior/src/chat/agent/resume.ts b/packages/junior/src/chat/agent/resume.ts index e020389cb..ce67c675e 100644 --- a/packages/junior/src/chat/agent/resume.ts +++ b/packages/junior/src/chat/agent/resume.ts @@ -10,7 +10,7 @@ import type { Destination, Source } from "@sentry/junior-plugin-api"; import { botConfig } from "@/chat/config"; import type { PiMessage } from "@/chat/pi/messages"; -import type { PiMessageProvenance } from "@/chat/state/session-log"; +import type { ConversationMessageProvenance } from "@/chat/conversations/provenance"; import { CooperativeTurnYieldError, TurnInputCommitLostError, @@ -148,7 +148,7 @@ export function createResumeState(args: ResumeStateArgs) { }, async persistSafeBoundary( messages: PiMessage[], - trailingMessageProvenance?: PiMessageProvenance[], + trailingMessageProvenance?: ConversationMessageProvenance[], ): Promise { if (!canPersistSession) { return false; @@ -172,7 +172,7 @@ export function createResumeState(args: ResumeStateArgs) { }, async requireDurableInputCheckpoint( messages: PiMessage[], - trailingMessageProvenance?: PiMessageProvenance[], + trailingMessageProvenance?: ConversationMessageProvenance[], ): Promise { const persisted = await this.persistSafeBoundary( messages, diff --git a/packages/junior/src/chat/conversations/README.md b/packages/junior/src/chat/conversations/README.md index 65694bf78..b32c2b47b 100644 --- a/packages/junior/src/chat/conversations/README.md +++ b/packages/junior/src/chat/conversations/README.md @@ -1,14 +1,16 @@ # Conversation Storage This module owns the durable product record for conversations, visible messages, -agent steps, compaction boundaries, search, retention, and legacy import. +conversation events, compaction boundaries, search, retention, and legacy import. ## Records - Conversation rows identify the source, destination, participants, visibility, and lifecycle metadata. - Visible messages are the destination-facing user and assistant history. -- Agent steps are append-only execution history used to restore Pi state. +- Conversation events are the versioned, append-only execution history used to + restore Pi state. Pi messages and context are projections of this log, not a + second authority. - Context epochs identify replacement boundaries created by compaction or model handoff. - Provider payloads and old state-store mirrors are migration inputs, not @@ -16,18 +18,32 @@ agent steps, compaction boundaries, search, retention, and legacy import. The schemas and migrations under `sql/` are authoritative. +`junior_conversation_events` is the single physical event table. Every row +persists its schema version, sequence, context epoch, type, payload, and +timestamp. The `(conversation_id, seq)` key is both stable event identity and +the lease-fencing tripwire. + +The Junior-owned `message` event retains an opaque model-continuity payload and +canonical provenance. The Pi adapter is the only module that interprets that +payload as a Pi message; the legacy Redis `pi_message` shape exists only as a +bounded import input. + +Reporting APIs must project events into an authorized, redacted product +contract. Raw `ConversationEventData` is an internal persistence boundary and +must not become a dashboard or external API payload. + ## Write Rules - Persist user input before agent execution. - Persist assistant text only after successful destination delivery. -- Append agent steps in monotonic sequence order. -- Restore state from durable steps rather than a duplicate transcript cache. +- Append conversation events in monotonic sequence order. +- Restore state from durable events rather than a duplicate transcript cache. - Compaction replaces prior model context without rewriting visible history. - Imports and migrations are idempotent and preserve stable conversation IDs. ## Visibility And Retention -Destination visibility is the privacy authority. Messages, steps, child +Destination visibility is the privacy authority. Messages, conversation events, child conversations, and plugin projections inherit it. Retention is enforced by the conversation purge paths and must distinguish expired content from redacted content. @@ -42,6 +58,9 @@ Follow `../../../../../policies/data-redaction.md` and - Data rewrites use explicit migrations or resumable import code. - Legacy fields remain readable only for the migration window and are removed after the new authority is verified. +- The database-only `junior_agent_steps` compatibility view supports old workers + during the 0.103.x rollout and is removed in 0.104.0; it stores no duplicate + rows and is never an application read or write path. - Purge and migration jobs operate in bounded batches and are safe to retry. Representative coverage lives in diff --git a/packages/junior/src/chat/conversations/history.ts b/packages/junior/src/chat/conversations/history.ts index a60f986d7..d4a923954 100644 --- a/packages/junior/src/chat/conversations/history.ts +++ b/packages/junior/src/chat/conversations/history.ts @@ -1,35 +1,39 @@ /** - * Durable agent execution history port. + * Durable conversation event history port. * - * Steps are appended one row at a time under the conversation lease; context + * Events append under the conversation lease in stable sequence order. Context * rebuilds (compaction, handoff, rollback) open a new context epoch instead of - * rewriting history. Each new epoch binds the model profile that owns it and - * records the exact resolved model id for audit. - * The step envelope is strictly validated — an unknown type or malformed shape - * is corrupt state and fails loudly — while `pi_message` content stays - * permissive because the Pi SDK owns the message shape. + * rewriting history. Unknown event types or malformed shapes fail loudly; + * nested model-message fields remain opaque continuity data and are interpreted + * only by the Pi adapter. */ import { z } from "zod"; -import { piMessageSchema } from "@/chat/pi/messages"; import type { ConversationCompaction } from "@/chat/state/conversation"; -import { piMessageProvenanceSchema } from "@/chat/state/session-log"; import { modelProfileSchema } from "@/chat/model-profile"; +import { conversationMessageProvenanceSchema } from "./provenance"; const handoffModelProfileSchema = modelProfileSchema.refine( (profile) => profile !== "standard", "handoff profile must not be standard", ); -const piMessageStepEntrySchema = z.object({ - type: z.literal("pi_message"), - schemaVersion: z.number().int().optional(), - message: piMessageSchema, - provenance: piMessageProvenanceSchema.optional(), -}); +/** Junior-owned durable message shape; Pi-specific validation belongs to its adapter. */ +export const conversationModelMessageSchema = z + .object({ role: z.string() }) + .passthrough() + .transform((value) => value as { role: string }); + +const messageEventDataSchema = z + .object({ + type: z.literal("message"), + message: conversationModelMessageSchema, + provenance: conversationMessageProvenanceSchema.optional(), + }) + .strict(); // Replaces the legacy `projection_reset` payload at the SQL layer: a marker plus -// ordinary pi_message rows in the new epoch, not an embedded transcript array. -const contextEpochStartedEntrySchema = z.union([ +// ordinary message events in the new epoch, not an embedded transcript array. +const contextEpochStartedEventDataSchema = z.union([ z .object({ type: z.literal("context_epoch_started"), @@ -50,7 +54,7 @@ const contextEpochStartedEntrySchema = z.union([ .object({ type: z.literal("context_epoch_started"), reason: z.union([z.literal("compaction"), z.literal("rollback")]), - // TODO(v0.97.0): Remove support for deployed compaction/rollback markers + // TODO(v0.104.0): Remove support for deployed compaction/rollback markers // without model bindings after those rows pass the retention horizon. modelProfile: z.undefined().optional(), modelId: z.undefined().optional(), @@ -66,11 +70,11 @@ const contextEpochStartedEntrySchema = z.union([ .strict(), ]); -const piMessageStepSchema = z +const contextEpochMessageSchema = z .object({ - message: piMessageSchema, + message: conversationModelMessageSchema, createdAtMs: z.number().finite(), - provenance: piMessageProvenanceSchema.optional(), + provenance: conversationMessageProvenanceSchema.optional(), }) .strict(); @@ -81,7 +85,7 @@ export const contextEpochStartSchema = z.discriminatedUnion("reason", [ reason: z.literal("initial"), modelProfile: z.literal("standard"), modelId: z.string().min(1), - messages: z.array(piMessageStepSchema), + messages: z.array(contextEpochMessageSchema), }) .strict(), z @@ -89,7 +93,7 @@ export const contextEpochStartSchema = z.discriminatedUnion("reason", [ reason: z.literal("handoff"), modelProfile: handoffModelProfileSchema, modelId: z.string().min(1), - messages: z.array(piMessageStepSchema), + messages: z.array(contextEpochMessageSchema), }) .strict(), z @@ -97,7 +101,7 @@ export const contextEpochStartSchema = z.discriminatedUnion("reason", [ reason: z.union([z.literal("compaction"), z.literal("rollback")]), modelProfile: modelProfileSchema, modelId: z.string().min(1), - messages: z.array(piMessageStepSchema), + messages: z.array(contextEpochMessageSchema), }) .strict(), ]); @@ -105,58 +109,73 @@ export const contextEpochStartSchema = z.discriminatedUnion("reason", [ /** One atomically persisted context epoch and its model binding. */ export type ContextEpochStart = z.output; -const mcpProviderConnectedEntrySchema = z.object({ - type: z.literal("mcp_provider_connected"), - provider: z.string().min(1), -}); +const mcpProviderConnectedEventDataSchema = z + .object({ + type: z.literal("mcp_provider_connected"), + provider: z.string().min(1), + }) + .strict(); -const authorizationKindSchema = z.union([ +export const authorizationKindSchema = z.union([ z.literal("plugin"), z.literal("mcp"), ]); -const authorizationRequestedEntrySchema = z.object({ - type: z.literal("authorization_requested"), - kind: authorizationKindSchema, - provider: z.string().min(1), - actorId: z.string().min(1), - authorizationId: z.string().min(1), - delivery: z.union([ - z.literal("private_link_sent"), - z.literal("private_link_reused"), - ]), -}); - -const authorizationCompletedEntrySchema = z.object({ - type: z.literal("authorization_completed"), - kind: authorizationKindSchema, - provider: z.string().min(1), - actorId: z.string().min(1), - authorizationId: z.string().min(1), -}); - -const toolExecutionStartedEntrySchema = z.object({ - type: z.literal("tool_execution_started"), - toolCallId: z.string().min(1), - toolName: z.string().min(1), - args: z.unknown().optional(), -}); - -const visibleContextCompactedEntrySchema = z.object({ - type: z.literal("visible_context_compacted"), - compactions: z.array( - z.object({ - coveredMessageIds: z.array(z.string()), - createdAtMs: z.number(), - id: z.string().min(1), - summary: z.string(), - }), - ) satisfies z.ZodType, -}); +/** Provider authorization family recorded by conversation events. */ +export type AuthorizationKind = z.output; + +const authorizationRequestedEventDataSchema = z + .object({ + type: z.literal("authorization_requested"), + kind: authorizationKindSchema, + provider: z.string().min(1), + actorId: z.string().min(1), + authorizationId: z.string().min(1), + delivery: z.union([ + z.literal("private_link_sent"), + z.literal("private_link_reused"), + ]), + }) + .strict(); + +const authorizationCompletedEventDataSchema = z + .object({ + type: z.literal("authorization_completed"), + kind: authorizationKindSchema, + provider: z.string().min(1), + actorId: z.string().min(1), + authorizationId: z.string().min(1), + }) + .strict(); + +const toolExecutionStartedEventDataSchema = z + .object({ + type: z.literal("tool_execution_started"), + toolCallId: z.string().min(1), + toolName: z.string().min(1), + args: z.unknown().optional(), + }) + .strict(); + +const visibleContextCompactedEventDataSchema = z + .object({ + type: z.literal("visible_context_compacted"), + compactions: z.array( + z + .object({ + coveredMessageIds: z.array(z.string()), + createdAtMs: z.number(), + id: z.string().min(1), + summary: z.string(), + }) + .strict(), + ) satisfies z.ZodType, + }) + .strict(); // Subagent histories are child conversations; the marker references the child by // its own conversation id rather than a polymorphic transcript locator. -const subagentStartedEntrySchema = z +const subagentStartedEventDataSchema = z .object({ type: z.literal("subagent_started"), subagentInvocationId: z.string().min(1), @@ -169,70 +188,83 @@ const subagentStartedEntrySchema = z }) .strict(); -const subagentEndedEntrySchema = z.object({ - type: z.literal("subagent_ended"), - subagentInvocationId: z.string().min(1), - outcome: z.union([ - z.literal("success"), - z.literal("error"), - z.literal("aborted"), - ]), - errorCode: z.string().min(1).optional(), -}); +const subagentEndedEventDataSchema = z + .object({ + type: z.literal("subagent_ended"), + subagentInvocationId: z.string().min(1), + outcome: z.union([ + z.literal("success"), + z.literal("error"), + z.literal("aborted"), + ]), + errorCode: z.string().min(1).optional(), + }) + .strict(); /** Prevent ordinary appends from bypassing context-epoch lifecycle validation. */ -const appendableAgentStepEntrySchema = z.union([ - piMessageStepEntrySchema, - mcpProviderConnectedEntrySchema, - authorizationRequestedEntrySchema, - authorizationCompletedEntrySchema, - toolExecutionStartedEntrySchema, - visibleContextCompactedEntrySchema, - subagentStartedEntrySchema, - subagentEndedEntrySchema, +const appendableConversationEventDataSchema = z.union([ + messageEventDataSchema, + mcpProviderConnectedEventDataSchema, + authorizationRequestedEventDataSchema, + authorizationCompletedEventDataSchema, + toolExecutionStartedEventDataSchema, + visibleContextCompactedEventDataSchema, + subagentStartedEventDataSchema, + subagentEndedEventDataSchema, ]); -/** Strict step envelope reused by the SQL row codec; unknown types fail loudly. */ -export const agentStepEntrySchema = z.union([ - appendableAgentStepEntrySchema, - contextEpochStartedEntrySchema, +/** Strict event-data contract reused by the SQL row codec. */ +export const conversationEventDataSchema = z.union([ + appendableConversationEventDataSchema, + contextEpochStartedEventDataSchema, ]); -/** One durable execution step's validated payload (sessionId lifted to epoch). */ -export type AgentStepEntry = z.infer; +/** One durable conversation event's validated data. */ +export type ConversationEventData = z.output< + typeof conversationEventDataSchema +>; -/** A step read back from storage with its assigned order and epoch. */ -export interface StoredAgentStep { - seq: number; - contextEpoch: number; - createdAtMs: number; - entry: AgentStepEntry; -} +/** + * Canonical ordered envelope for the durable conversation log. + * `schemaVersion` is persisted with every physical event row. + */ +export const conversationEventSchema = z + .object({ + schemaVersion: z.literal(1), + seq: z.number().int().nonnegative(), + contextEpoch: z.number().int().nonnegative(), + createdAtMs: z.number().finite(), + data: conversationEventDataSchema, + }) + .strict(); + +/** One versioned event read from the durable conversation log. */ +export type ConversationEvent = z.output; /** Validate a current-epoch append without permitting epoch markers. */ -export const newAgentStepSchema = z +export const newConversationEventSchema = z .object({ - entry: appendableAgentStepEntrySchema, + data: appendableConversationEventDataSchema, createdAtMs: z.number().finite(), }) .strict(); -/** A step to append; the store assigns `seq` and the current `context_epoch`. */ -export type NewAgentStep = z.output; +/** An event to append; the store assigns `seq` and the current context epoch. */ +export type NewConversationEvent = z.output; -/** A replacement Pi message written into a freshly opened epoch. */ -export type PiMessageStep = z.output; +/** A model message written into a freshly opened context epoch. */ +export type ContextEpochMessage = z.output; -/** Persist and read the durable per-conversation agent execution history. */ -export interface AgentStepStore { - /** Append steps in one transaction, assigning `seq = max+1` under the lease. */ - append(conversationId: string, steps: NewAgentStep[]): Promise; +/** Persist and read the canonical per-conversation event log. */ +export interface ConversationEventStore { + /** Append events atomically, assigning `seq = max+1` under the lease. */ + append(conversationId: string, events: NewConversationEvent[]): Promise; /** * Open initial epoch 0 or the next replacement epoch in one transaction. */ startEpoch(conversationId: string, opts: ContextEpochStart): Promise; - /** Steps of the highest epoch in `seq` order (all types; caller filters). */ - loadCurrentEpoch(conversationId: string): Promise; - /** All steps across all epochs in `seq` order, for audit and reporting. */ - loadHistory(conversationId: string): Promise; + /** Events of the highest epoch in `seq` order (all types; caller filters). */ + loadCurrentEpoch(conversationId: string): Promise; + /** All events across every epoch in `seq` order, for audit and reporting. */ + loadHistory(conversationId: string): Promise; } diff --git a/packages/junior/src/chat/conversations/legacy-advisor-session.ts b/packages/junior/src/chat/conversations/legacy-advisor-session.ts index 9489a15ec..87f1b7187 100644 --- a/packages/junior/src/chat/conversations/legacy-advisor-session.ts +++ b/packages/junior/src/chat/conversations/legacy-advisor-session.ts @@ -2,7 +2,7 @@ import { z } from "zod"; import { piMessageSchema, type PiMessage } from "@/chat/pi/messages"; import { getStateAdapter } from "@/chat/state/adapter"; -// TODO(v0.95.0): Remove this reader with the bounded Redis-to-SQL import path +// TODO(v0.104.0): Remove this reader with the bounded Redis-to-SQL import path // after the legacy import horizon. const legacyAdvisorMessagesSchema = z.array(piMessageSchema); diff --git a/packages/junior/src/chat/conversations/legacy-import.ts b/packages/junior/src/chat/conversations/legacy-import.ts index 812c7adbc..d5c63ea9c 100644 --- a/packages/junior/src/chat/conversations/legacy-import.ts +++ b/packages/junior/src/chat/conversations/legacy-import.ts @@ -3,16 +3,16 @@ * * A single per-conversation import unit shared by `junior upgrade` (bulk, * bounded newest-first) and the lazy first-read straggler path. It converts the - * legacy session log into `junior_agent_steps`, imports the advisor session blob - * as a child conversation, and copies the `thread-state` visible messages into - * `junior_conversation_messages`. Import is idempotent per conversation: step + * legacy session log into `junior_conversation_events`, imports the advisor + * session blob as a child conversation, and copies the `thread-state` visible messages into + * `junior_conversation_messages`. Import is idempotent per conversation: event * rows seal normal imports, while message-only imports verify their complete * SQL projection before skipping. It never fabricates import-time timestamps. * * This module and its lazy hook are removed wholesale after the legacy Redis TTL * horizon passes; keeping it separate keeps that deletion mechanical. */ -// TODO(v0.95.0): Remove this module and its advisor-session reader after the +// TODO(v0.104.0): Remove this module and its advisor-session reader after the // legacy Redis-to-SQL import horizon. import { isDeepStrictEqual } from "node:util"; import { eq } from "drizzle-orm"; @@ -36,11 +36,11 @@ import { import type { JuniorSqlDatabase } from "@/db/db"; import { juniorConversations } from "@/db/schema"; import { - getAgentStepStore, + getConversationEventStore, getConversationMessageStore, getSqlExecutor, } from "@/chat/db"; -import { createSqlAgentStepStore } from "./sql/history"; +import { createSqlConversationEventStore } from "./sql/history"; import type { ConversationMessageStore } from "./messages"; import type { Conversation } from "./store"; import { @@ -150,15 +150,15 @@ function intrinsicTimestamps( /** * Import one conversation's legacy Redis history into SQL, idempotently. * - * Returns whether an import ran (false when step rows already exist or there is + * Returns whether an import ran (false when event rows already exist or there is * nothing legacy to import). */ export async function importConversationFromLegacy( conversationId: string, deps: LegacyImportDeps, ): Promise<{ imported: boolean }> { - const stepStore = createSqlAgentStepStore(deps.executor); - const existing = await stepStore.loadCurrentEpoch(conversationId); + const eventStore = createSqlConversationEventStore(deps.executor); + const existing = await eventStore.loadCurrentEpoch(conversationId); if (existing.length > 0) { return { imported: false }; } @@ -212,15 +212,15 @@ export async function importConversationFromLegacy( fallbackCreatedAtMs, }); if (compactions.length > 0) { - converted.steps.push({ - seq: converted.steps.length, - contextEpoch: converted.steps.at(-1)?.contextEpoch ?? 0, - entry: { type: "visible_context_compacted", compactions }, + converted.events.push({ + seq: converted.events.length, + contextEpoch: converted.events.at(-1)?.contextEpoch ?? 0, + data: { type: "visible_context_compacted", compactions }, createdAtMs: compactions.at(-1)?.createdAtMs ?? fallbackCreatedAtMs, }); } - if (converted.steps.length === 0 && visible.length > 0) { + if (converted.events.length === 0 && visible.length > 0) { const existingMessages = new Map( (await deps.messageStore.list(conversationId)).map((message) => [ message.messageId, @@ -244,7 +244,7 @@ export async function importConversationFromLegacy( conversationId, fallbackCreatedAtMs, lastActivityAtMs, - steps: [], + events: [], }); return { imported: false }; } @@ -253,13 +253,13 @@ export async function importConversationFromLegacy( let child: | { conversationId: string; - steps: ReturnType; + events: ReturnType; } | undefined; if (converted.advisorChildConversationId) { child = { conversationId: converted.advisorChildConversationId, - steps: convertAdvisorMessages(advisorMessages, fallbackCreatedAtMs), + events: convertAdvisorMessages(advisorMessages, fallbackCreatedAtMs), }; } @@ -268,14 +268,14 @@ export async function importConversationFromLegacy( ...(message.meta?.replied ? { repliedAtMs: message.createdAtMs } : {}), })); - // Messages and steps share one locked SQL transaction so retention can never + // Messages and events share one locked SQL transaction so retention can never // purge between the legacy-source check and the import commit. const imported = await writeLegacyImport(deps.executor, { conversationId, fallbackCreatedAtMs, lastActivityAtMs, ...(messages.length > 0 ? { messages } : {}), - steps: converted.steps, + events: converted.events, ...(child ? { child } : {}), }); @@ -292,8 +292,8 @@ export async function importConversationFromLegacy( export async function ensureLegacyConversationImport(args: { conversationId: string; }): Promise { - const stepStore = getAgentStepStore(); - if ((await stepStore.loadCurrentEpoch(args.conversationId)).length > 0) { + const eventStore = getConversationEventStore(); + if ((await eventStore.loadCurrentEpoch(args.conversationId)).length > 0) { return; } const entries = await readSessionLogEntries({ @@ -320,7 +320,7 @@ export async function ensureLegacyConversationImport(args: { await importConversationFromLegacy(args.conversationId, { executor, messageStore: getConversationMessageStore(), - sessionLogStore: { read: async () => entries, append: async () => {} }, + sessionLogStore: { read: async () => entries }, loadVisibleMessages: async () => visible, legacyCompactions: snapshot.compactions, ...(snapshot.lastActivityAtMs === undefined diff --git a/packages/junior/src/chat/conversations/projection.ts b/packages/junior/src/chat/conversations/projection.ts index 2b821c9ab..a28de60f7 100644 --- a/packages/junior/src/chat/conversations/projection.ts +++ b/packages/junior/src/chat/conversations/projection.ts @@ -1,124 +1,36 @@ /** - * Agent-step projection. + * Conversation projection orchestration. * - * Materializes the model-visible Pi context and derived run facts (connected - * MCP providers, latest instruction actor) from the durable `AgentStepStore`. - * The current model context is exactly the highest context epoch's `pi_message` - * steps in `seq` order; host-only step types (activity, provider connection, - * authorization requests, epoch markers) never reach `agent.state.messages`. - * Compaction, handoff, and rollback open a new epoch instead of rewriting - * history, so the context reducer only walks one epoch. The epoch marker binds - * the projection's model profile, which later replacements inherit, and the - * exact resolved model id used when the epoch was opened for audit. + * Materializes model-visible Pi context and derived run facts such as connected + * providers. Storage and commit lifecycle stay in the conversations domain; + * Pi-specific event reduction lives in `pi/conversation-events`. */ import { isDeepStrictEqual } from "node:util"; import type { PiMessage } from "@/chat/pi/messages"; import { contextProvenance, - type AuthorizationKind, - type PiMessageProvenance, - type SessionProjection, -} from "@/chat/state/session-log"; + type ConversationMessageProvenance, +} from "@/chat/conversations/provenance"; import type { - AgentStepEntry, - StoredAgentStep, + AuthorizationKind, + ConversationEvent, } from "@/chat/conversations/history"; -import { getAgentStepStore } from "@/chat/db"; +import { getConversationEventStore } from "@/chat/db"; import { ensureLegacyConversationImport } from "@/chat/conversations/legacy-import"; -import type { ModelProfile } from "@/chat/model-profile"; - -type PiMessageStepEntry = Extract; -type AuthorizationCompletedEntry = Extract< - AgentStepEntry, - { type: "authorization_completed" } ->; - -/** Current conversation context with its authoritative model binding. */ -export interface ConversationProjection extends SessionProjection { - /** Model profile bound to this projection. */ - modelProfile: ModelProfile; - /** Audit snapshot; runtime model selection remains profile-driven. */ - modelId: string | undefined; -} - -/** Aligned step projection: `provenance[i]` and `seqs[i]` describe `messages[i]`. */ -export interface StepProjection extends ConversationProjection { - /** The `seq` of the step each projected message came from. */ - seqs: number[]; -} - -/** - * Synthesize the host observation a completed authorization contributes to Pi. - * Mirrors the legacy session-log projection so a resumed run still learns the - * provider unblocked and should retry the blocked operation. - */ -function authorizationObservationMessage( - entry: AuthorizationCompletedEntry, - createdAtMs: number, -): PiMessage { - const label = entry.kind === "mcp" ? "MCP authorization" : "Authorization"; - return { - role: "user", - content: [ - { - type: "text", - text: `${label} completed for provider "${entry.provider}". Continue the blocked request and retry the provider operation if needed.`, - }, - ], - timestamp: createdAtMs, - } as PiMessage; -} - -function piEntryProvenance(entry: PiMessageStepEntry): PiMessageProvenance { - return entry.provenance ?? contextProvenance; -} - -/** - * Materialize Pi messages with aligned provenance and source seqs from one - * epoch's steps. Steps above `maxSeq` (when given) are excluded so a terminal - * turn record reproduces exactly the boundary it committed. - */ -export function projectSteps( - steps: StoredAgentStep[], - opts?: { maxSeq?: number }, -): StepProjection { - const messages: PiMessage[] = []; - const provenance: PiMessageProvenance[] = []; - const seqs: number[] = []; - let modelProfile: ModelProfile = "standard"; - let modelId: string | undefined; - for (const step of steps) { - if (opts?.maxSeq !== undefined && step.seq > opts.maxSeq) { - break; - } - if (step.entry.type === "context_epoch_started") { - modelProfile = step.entry.modelProfile ?? "standard"; - modelId = step.entry.modelId; - continue; - } - if (step.entry.type === "pi_message") { - messages.push(step.entry.message); - provenance.push(piEntryProvenance(step.entry)); - seqs.push(step.seq); - continue; - } - if (step.entry.type === "authorization_completed") { - messages.push( - authorizationObservationMessage(step.entry, step.createdAtMs), - ); - provenance.push(contextProvenance); - seqs.push(step.seq); - } - } - return { messages, provenance, seqs, modelProfile, modelId }; -} +import { + projectConversationEvents, + type PiConversationEventProjection, + type PiConversationProjection, +} from "@/chat/pi/conversation-events"; -/** Distinct MCP providers durably connected in the given steps, sorted. */ -function connectedMcpProvidersFromSteps(steps: StoredAgentStep[]): string[] { +/** Distinct MCP providers durably connected in the given events, sorted. */ +function connectedMcpProvidersFromEvents( + events: ConversationEvent[], +): string[] { const providers = new Set(); - for (const step of steps) { - if (step.entry.type === "mcp_provider_connected") { - providers.add(step.entry.provider); + for (const event of events) { + if (event.data.type === "mcp_provider_connected") { + providers.add(event.data.provider); } } return [...providers].sort((left, right) => left.localeCompare(right)); @@ -147,13 +59,13 @@ function countMatchingPrefix(left: PiMessage[], right: PiMessage[]): number { * the last new user message — the current turn's input. */ function resolveCommitProvenance(args: { - existing: SessionProjection; + existing: Pick; nextMessages: PiMessage[]; matchingPrefix: number; - explicitProvenance?: PiMessageProvenance[]; - trailingMessageProvenance?: PiMessageProvenance[]; - newMessageProvenance?: PiMessageProvenance; -}): PiMessageProvenance[] { + explicitProvenance?: ConversationMessageProvenance[]; + trailingMessageProvenance?: ConversationMessageProvenance[]; + newMessageProvenance?: ConversationMessageProvenance; +}): ConversationMessageProvenance[] { if (args.explicitProvenance) { if (args.explicitProvenance.length !== args.nextMessages.length) { throw new Error("commit provenance must align one-to-one with messages"); @@ -218,40 +130,45 @@ export async function loadProjection( args: ScopedConversation, ): Promise { await importLegacyIfNeeded(args); - const steps = await getAgentStepStore().loadCurrentEpoch(args.conversationId); - return projectSteps(steps).messages; + const events = await getConversationEventStore().loadCurrentEpoch( + args.conversationId, + ); + return projectConversationEvents(events).messages; } /** Load the current-epoch Pi projection with aligned per-message provenance. */ export async function loadConversationProjection( args: ScopedConversation, -): Promise { +): Promise { await importLegacyIfNeeded(args); - const steps = await getAgentStepStore().loadCurrentEpoch(args.conversationId); - const { messages, provenance, modelProfile, modelId } = projectSteps(steps); + const events = await getConversationEventStore().loadCurrentEpoch( + args.conversationId, + ); + const { messages, provenance, modelProfile, modelId } = + projectConversationEvents(events); return { messages, provenance, modelProfile, modelId }; } /** Open a standard initial epoch before a conversation's first model request. */ export async function openConversationProjection( args: ScopedConversation & { modelId: string }, -): Promise { +): Promise { await importLegacyIfNeeded(args); - const stepStore = getAgentStepStore(); - const steps = await stepStore.loadCurrentEpoch(args.conversationId); - const projection = projectSteps(steps); + const eventStore = getConversationEventStore(); + const events = await eventStore.loadCurrentEpoch(args.conversationId); + const projection = projectConversationEvents(events); if ( - steps.some( - (step) => - step.entry.type === "context_epoch_started" || - step.entry.type === "pi_message", + events.some( + (event) => + event.data.type === "context_epoch_started" || + event.data.type === "message", ) ) { return projection; } // Host facts may predate the first model request. Keep them in epoch 0 and // make that formerly implicit epoch explicit before model execution. - await stepStore.startEpoch(args.conversationId, { + await eventStore.startEpoch(args.conversationId, { reason: "initial", modelProfile: "standard", modelId: args.modelId, @@ -266,7 +183,7 @@ export async function openConversationProjection( } /** - * Load a turn's committed Pi projection from the durable step store. + * Load a turn's committed Pi projection from the durable event store. * * The record stays pinned to the epoch containing its committed boundary, so a * later rollback or compaction cannot silently rewrite what a stale record @@ -281,25 +198,29 @@ export async function loadTurnProjection(args: { conversationId: string; committedSeq: number; includeTail: boolean; -}): Promise { +}): Promise { await importLegacyIfNeeded(args); - const stepStore = getAgentStepStore(); + const eventStore = getConversationEventStore(); // A record that committed no messages materializes the live projection, the // same way count-based records with a zero cursor did. if (args.committedSeq < 0) { - return projectSteps(await stepStore.loadCurrentEpoch(args.conversationId)); + return projectConversationEvents( + await eventStore.loadCurrentEpoch(args.conversationId), + ); } - const history = await stepStore.loadHistory(args.conversationId); - const committedStep = history.find((step) => step.seq === args.committedSeq); - if (!committedStep) { + const history = await eventStore.loadHistory(args.conversationId); + const committedEvent = history.find( + (event) => event.seq === args.committedSeq, + ); + if (!committedEvent) { return undefined; } - const epochSteps = history.filter( - (step) => step.contextEpoch === committedStep.contextEpoch, + const epochEvents = history.filter( + (event) => event.contextEpoch === committedEvent.contextEpoch, ); return args.includeTail - ? projectSteps(epochSteps) - : projectSteps(epochSteps, { maxSeq: args.committedSeq }); + ? projectConversationEvents(epochEvents) + : projectConversationEvents(epochEvents, { maxSeq: args.committedSeq }); } /** Load MCP providers durably connected in this conversation's current epoch. */ @@ -307,8 +228,10 @@ export async function loadConnectedMcpProviders( args: ScopedConversation, ): Promise { await importLegacyIfNeeded(args); - const steps = await getAgentStepStore().loadCurrentEpoch(args.conversationId); - return connectedMcpProvidersFromSteps(steps); + const events = await getConversationEventStore().loadCurrentEpoch( + args.conversationId, + ); + return connectedMcpProvidersFromEvents(events); } function messageTimestamp(message: PiMessage): number { @@ -320,7 +243,7 @@ function messageTimestamp(message: PiMessage): number { * Commit the turn's Pi history: append when it advanced the committed * projection normally, or open a `rollback` epoch when it diverged (a * provider-retry trim regenerated trailing assistant output). Returns the - * resolved provenance, the per-message step seqs, and the `seq` boundary that + * resolved provenance, the per-message event seqs, and the `seq` boundary that * reproduces exactly the committed messages. A first commit atomically opens * the standard initial epoch; the run boundary opens it earlier when a model * may act before any session checkpoint, such as recordless handoff. @@ -331,19 +254,19 @@ export async function commitMessages(args: { modelId: string; messages: PiMessage[]; /** Explicit per-message provenance aligned one-to-one with `messages`. */ - provenance?: PiMessageProvenance[]; + provenance?: ConversationMessageProvenance[]; /** Explicit provenance for the trailing newly committed messages. */ - trailingMessageProvenance?: PiMessageProvenance[]; + trailingMessageProvenance?: ConversationMessageProvenance[]; /** Default applied to the last new user message when no explicit array. */ - newMessageProvenance?: PiMessageProvenance; + newMessageProvenance?: ConversationMessageProvenance; }): Promise<{ committedSeq: number; messageSeqs: number[]; - provenance: PiMessageProvenance[]; + provenance: ConversationMessageProvenance[]; }> { - const stepStore = getAgentStepStore(); - const currentSteps = await stepStore.loadCurrentEpoch(args.conversationId); - const existing = projectSteps(currentSteps); + const eventStore = getConversationEventStore(); + const currentEvents = await eventStore.loadCurrentEpoch(args.conversationId); + const existing = projectConversationEvents(currentEvents); const matchingPrefix = countMatchingPrefix(existing.messages, args.messages); const nextProvenance = resolveCommitProvenance({ existing, @@ -357,8 +280,8 @@ export async function commitMessages(args: { ? { newMessageProvenance: args.newMessageProvenance } : {}), }); - if (currentSteps.length === 0) { - await stepStore.startEpoch(args.conversationId, { + if (currentEvents.length === 0) { + await eventStore.startEpoch(args.conversationId, { reason: "initial", modelProfile: "standard", modelId: args.modelId, @@ -370,11 +293,11 @@ export async function commitMessages(args: { }); } else if (matchingPrefix === existing.messages.length) { const newMessages = args.messages.slice(matchingPrefix); - await stepStore.append( + await eventStore.append( args.conversationId, newMessages.map((message, index) => ({ - entry: { - type: "pi_message" as const, + data: { + type: "message" as const, message, provenance: nextProvenance[matchingPrefix + index]!, }, @@ -382,7 +305,7 @@ export async function commitMessages(args: { })), ); } else { - await stepStore.startEpoch(args.conversationId, { + await eventStore.startEpoch(args.conversationId, { reason: "rollback", modelProfile: existing.modelProfile, modelId: args.modelId, @@ -393,8 +316,8 @@ export async function commitMessages(args: { })), }); } - const committed = projectSteps( - await stepStore.loadCurrentEpoch(args.conversationId), + const committed = projectConversationEvents( + await eventStore.loadCurrentEpoch(args.conversationId), ); return { committedSeq: committed.seqs.at(-1) ?? -1, @@ -408,14 +331,14 @@ export async function recordMcpProviderConnected(args: { conversationId: string; provider: string; }): Promise { - const stepStore = getAgentStepStore(); - const steps = await stepStore.loadCurrentEpoch(args.conversationId); - if (connectedMcpProvidersFromSteps(steps).includes(args.provider)) { + const eventStore = getConversationEventStore(); + const events = await eventStore.loadCurrentEpoch(args.conversationId); + if (connectedMcpProvidersFromEvents(events).includes(args.provider)) { return; } - await stepStore.append(args.conversationId, [ + await eventStore.append(args.conversationId, [ { - entry: { type: "mcp_provider_connected", provider: args.provider }, + data: { type: "mcp_provider_connected", provider: args.provider }, createdAtMs: Date.now(), }, ]); @@ -430,20 +353,20 @@ export async function recordAuthorizationRequested(args: { authorizationId: string; delivery: "private_link_sent" | "private_link_reused"; }): Promise { - const stepStore = getAgentStepStore(); - const steps = await stepStore.loadCurrentEpoch(args.conversationId); + const eventStore = getConversationEventStore(); + const events = await eventStore.loadCurrentEpoch(args.conversationId); if ( - steps.some( - (step) => - step.entry.type === "authorization_requested" && - step.entry.authorizationId === args.authorizationId, + events.some( + (event) => + event.data.type === "authorization_requested" && + event.data.authorizationId === args.authorizationId, ) ) { return; } - await stepStore.append(args.conversationId, [ + await eventStore.append(args.conversationId, [ { - entry: { + data: { type: "authorization_requested", kind: args.kind, provider: args.provider, @@ -464,20 +387,20 @@ export async function recordAuthorizationCompleted(args: { actorId: string; authorizationId: string; }): Promise { - const stepStore = getAgentStepStore(); - const steps = await stepStore.loadCurrentEpoch(args.conversationId); + const eventStore = getConversationEventStore(); + const events = await eventStore.loadCurrentEpoch(args.conversationId); if ( - steps.some( - (step) => - step.entry.type === "authorization_completed" && - step.entry.authorizationId === args.authorizationId, + events.some( + (event) => + event.data.type === "authorization_completed" && + event.data.authorizationId === args.authorizationId, ) ) { return; } - await stepStore.append(args.conversationId, [ + await eventStore.append(args.conversationId, [ { - entry: { + data: { type: "authorization_completed", kind: args.kind, provider: args.provider, @@ -497,9 +420,9 @@ export async function recordToolExecutionStarted(args: { toolCallId: string; toolName: string; }): Promise { - await getAgentStepStore().append(args.conversationId, [ + await getConversationEventStore().append(args.conversationId, [ { - entry: { + data: { type: "tool_execution_started", toolCallId: args.toolCallId, toolName: args.toolName, diff --git a/packages/junior/src/chat/conversations/provenance.ts b/packages/junior/src/chat/conversations/provenance.ts new file mode 100644 index 000000000..af2c51720 --- /dev/null +++ b/packages/junior/src/chat/conversations/provenance.ts @@ -0,0 +1,75 @@ +/** + * Durable message attribution shared by storage and projections. + * Attribution never grants credential authority. + */ +import { actorSchema, type Actor } from "@sentry/junior-plugin-api"; +import { z } from "zod"; + +const conversationMessageAuthoritySchema = z.union([ + z.literal("instruction"), + z.literal("context"), +]); + +/** Canonical authority and actor metadata for one durable message event. */ +export const conversationMessageProvenanceSchema = z + .object({ + authority: conversationMessageAuthoritySchema, + actor: actorSchema.optional(), + }) + .strict(); + +/** Whether a message is a durable instruction or ambient conversation context. */ +export type ConversationMessageAuthority = z.output< + typeof conversationMessageAuthoritySchema +>; + +/** Actor and authority metadata aligned with one durable message event. */ +export type ConversationMessageProvenance = z.output< + typeof conversationMessageProvenanceSchema +>; + +/** Build instruction provenance for an optional actor. */ +export function instructionProvenanceFor( + actor: Actor | undefined, +): ConversationMessageProvenance { + return actor + ? { authority: "instruction", actor } + : { authority: "instruction" }; +} + +/** Unattributed ambient-context provenance for non-instruction messages. */ +export const contextProvenance: ConversationMessageProvenance = { + authority: "context", +}; + +function actorIdentityKey(actor: Actor): string { + if (actor.platform === "system") { + return `system ${actor.name}`; + } + return actor.platform === "slack" + ? `slack\u0000${actor.teamId}\u0000${actor.userId}` + : `${actor.platform}\u0000${actor.userId}`; +} + +/** + * Return distinct instruction actors in first-seen order for attribution only. + * Never use this list as credential authority. + */ +export function instructionActors( + provenance: ConversationMessageProvenance[], +): Actor[] { + const seen = new Set(); + const actors: Actor[] = []; + for (const entry of provenance) { + if (entry.authority !== "instruction" || !entry.actor) { + continue; + } + const identity = actorIdentityKey(entry.actor); + if (seen.has(identity)) { + continue; + } + seen.add(identity); + actors.push(entry.actor); + } + return actors; +} diff --git a/packages/junior/src/chat/conversations/sql/conversation-row.ts b/packages/junior/src/chat/conversations/sql/conversation-row.ts index 92aaa736c..4c6b70651 100644 --- a/packages/junior/src/chat/conversations/sql/conversation-row.ts +++ b/packages/junior/src/chat/conversations/sql/conversation-row.ts @@ -4,7 +4,7 @@ import { juniorConversations } from "@/db/schema"; /** * Establish the conversation metadata row on first contact for content writes - * (agent steps, visible transcript) that can land before activity recording has + * (conversation events, visible transcript) that can land before activity has * created the row, and whose tables FK to it. * * First contact creates the row; every later content write advances the diff --git a/packages/junior/src/chat/conversations/sql/history.ts b/packages/junior/src/chat/conversations/sql/history.ts index a2a24d1ed..0d8b525e9 100644 --- a/packages/junior/src/chat/conversations/sql/history.ts +++ b/packages/junior/src/chat/conversations/sql/history.ts @@ -1,86 +1,92 @@ import { and, asc, eq, isNotNull, sql } from "drizzle-orm"; import type { JuniorSqlDatabase } from "@/db/db"; import { - agentStepEntrySchema, + conversationEventDataSchema, + conversationEventSchema, contextEpochStartSchema, - newAgentStepSchema, - type AgentStepEntry, - type AgentStepStore, + newConversationEventSchema, + type ConversationEvent, + type ConversationEventStore, + type ConversationEventData, type ContextEpochStart, - type NewAgentStep, - type PiMessageStep, - type StoredAgentStep, + type ContextEpochMessage, + type NewConversationEvent, } from "../history"; import { ensureConversationRow } from "./conversation-row"; -import { juniorAgentSteps, juniorConversations } from "@/db/schema"; +import { juniorConversationEvents, juniorConversations } from "@/db/schema"; import { sanitizePostgresJson } from "@/db/postgres-json"; -type AgentStepRow = typeof juniorAgentSteps.$inferSelect; -type AgentStepInsert = typeof juniorAgentSteps.$inferInsert; -type PersistedAgentStep = { - entry: AgentStepEntry; +type ConversationEventRow = typeof juniorConversationEvents.$inferSelect; +type ConversationEventInsert = typeof juniorConversationEvents.$inferInsert; +type PersistedConversationEvent = { + data: ConversationEventData; createdAtMs: number; }; -function messageRole(entry: AgentStepEntry): string | null { - if (entry.type !== "pi_message") { +function messageRole(data: ConversationEventData): string | null { + if (data.type !== "message") { return null; } - const role = (entry.message as { role?: unknown }).role; - return typeof role === "string" ? role : null; + return data.message.role; } -/** Split the validated entry into its column-lifted envelope and jsonb payload. */ -function insertFromStep( +/** Split validated event data into column-lifted and JSON payload fields. */ +function insertFromEvent( conversationId: string, seq: number, contextEpoch: number, - step: PersistedAgentStep, -): AgentStepInsert { - const { type, ...payload } = agentStepEntrySchema.parse(step.entry); + event: PersistedConversationEvent, +): ConversationEventInsert { + const { type, ...payload } = conversationEventDataSchema.parse(event.data); return { conversationId, seq, contextEpoch, + schemaVersion: 1, type, - role: messageRole(step.entry), + role: messageRole(event.data), payload: sanitizePostgresJson(payload), - createdAt: new Date(step.createdAtMs), + createdAt: new Date(event.createdAtMs), }; } -/** Reconstruct the domain entry from a row; corrupt envelopes fail loudly. */ -function stepFromRow(row: AgentStepRow): StoredAgentStep { - const entry = agentStepEntrySchema.parse({ type: row.type, ...row.payload }); - return { +/** Parse one physical event row into the canonical domain envelope. */ +function eventFromRow(row: ConversationEventRow): ConversationEvent { + return conversationEventSchema.parse({ + schemaVersion: row.schemaVersion, seq: row.seq, contextEpoch: row.contextEpoch, createdAtMs: row.createdAt.getTime(), - entry, - }; + data: { ...row.payload, type: row.type }, + }); } -function piMessageStep(step: PiMessageStep): NewAgentStep { +function epochMessageEvent(message: ContextEpochMessage): NewConversationEvent { return { - entry: { - type: "pi_message", - message: step.message, - ...(step.provenance ? { provenance: step.provenance } : {}), + data: { + type: "message", + message: message.message, + ...(message.provenance ? { provenance: message.provenance } : {}), }, - createdAtMs: step.createdAtMs, + createdAtMs: message.createdAtMs, }; } -class SqlAgentStepStore implements AgentStepStore { +class SqlConversationEventStore implements ConversationEventStore { constructor(private readonly executor: JuniorSqlDatabase) {} - async append(conversationId: string, steps: NewAgentStep[]): Promise { - const parsed = steps.map((step) => newAgentStepSchema.parse(step)); + async append( + conversationId: string, + events: NewConversationEvent[], + ): Promise { + const parsed = events.map((event) => + newConversationEventSchema.parse(event), + ); if (parsed.length === 0) { return; } const newestCreatedAtMs = Math.max( - ...parsed.map((step) => step.createdAtMs), + ...parsed.map((event) => event.createdAtMs), ); await this.executor.transaction(async () => { await ensureConversationRow( @@ -101,10 +107,10 @@ class SqlAgentStepStore implements AgentStepStore { const cursor = await this.readCursor(conversationId); const contextEpoch = cursor.maxEpoch ?? 0; let seq = cursor.nextSeq; - const rows = parsed.map((step) => - insertFromStep(conversationId, seq++, contextEpoch, step), + const rows = parsed.map((event) => + insertFromEvent(conversationId, seq++, contextEpoch, event), ); - await this.executor.db().insert(juniorAgentSteps).values(rows); + await this.executor.db().insert(juniorConversationEvents).values(rows); }); } @@ -132,18 +138,18 @@ class SqlAgentStepStore implements AgentStepStore { : (cursor.maxEpoch ?? -1) + 1; let seq = cursor.nextSeq; const { messages, ...binding } = parsed; - const marker: PersistedAgentStep = { - entry: { type: "context_epoch_started", ...binding }, + const marker: PersistedConversationEvent = { + data: { type: "context_epoch_started", ...binding }, createdAtMs: Date.now(), }; - const rows = [marker, ...messages.map(piMessageStep)].map((step) => - insertFromStep(conversationId, seq++, contextEpoch, step), + const rows = [marker, ...messages.map(epochMessageEvent)].map((event) => + insertFromEvent(conversationId, seq++, contextEpoch, event), ); - await this.executor.db().insert(juniorAgentSteps).values(rows); + await this.executor.db().insert(juniorConversationEvents).values(rows); }); } - async loadCurrentEpoch(conversationId: string): Promise { + async loadCurrentEpoch(conversationId: string): Promise { const cursor = await this.readCursor(conversationId); if (cursor.maxEpoch === null) { return []; @@ -151,25 +157,25 @@ class SqlAgentStepStore implements AgentStepStore { const rows = await this.executor .db() .select() - .from(juniorAgentSteps) + .from(juniorConversationEvents) .where( and( - eq(juniorAgentSteps.conversationId, conversationId), - eq(juniorAgentSteps.contextEpoch, cursor.maxEpoch), + eq(juniorConversationEvents.conversationId, conversationId), + eq(juniorConversationEvents.contextEpoch, cursor.maxEpoch), ), ) - .orderBy(asc(juniorAgentSteps.seq)); - return rows.map(stepFromRow); + .orderBy(asc(juniorConversationEvents.seq)); + return rows.map(eventFromRow); } - async loadHistory(conversationId: string): Promise { + async loadHistory(conversationId: string): Promise { const rows = await this.executor .db() .select() - .from(juniorAgentSteps) - .where(eq(juniorAgentSteps.conversationId, conversationId)) - .orderBy(asc(juniorAgentSteps.seq)); - return rows.map(stepFromRow); + .from(juniorConversationEvents) + .where(eq(juniorConversationEvents.conversationId, conversationId)) + .orderBy(asc(juniorConversationEvents.seq)); + return rows.map(eventFromRow); } /** Read the next `seq` and current highest epoch for one conversation. */ @@ -179,11 +185,13 @@ class SqlAgentStepStore implements AgentStepStore { const rows = await this.executor .db() .select({ - maxSeq: sql`max(${juniorAgentSteps.seq})`, - maxEpoch: sql`max(${juniorAgentSteps.contextEpoch})`, + maxSeq: sql`max(${juniorConversationEvents.seq})`, + maxEpoch: sql< + number | null + >`max(${juniorConversationEvents.contextEpoch})`, }) - .from(juniorAgentSteps) - .where(eq(juniorAgentSteps.conversationId, conversationId)); + .from(juniorConversationEvents) + .where(eq(juniorConversationEvents.conversationId, conversationId)); const maxSeq = rows[0]?.maxSeq; const maxEpoch = rows[0]?.maxEpoch; return { @@ -194,9 +202,9 @@ class SqlAgentStepStore implements AgentStepStore { } } -/** Create a SQL-backed agent step store. */ -export function createSqlAgentStepStore( +/** Create a SQL-backed canonical conversation event store. */ +export function createSqlConversationEventStore( executor: JuniorSqlDatabase, -): AgentStepStore { - return new SqlAgentStepStore(executor); +): ConversationEventStore { + return new SqlConversationEventStore(executor); } diff --git a/packages/junior/src/chat/conversations/sql/legacy-history-import.ts b/packages/junior/src/chat/conversations/sql/legacy-history-import.ts index bb56f9483..c2f95f74a 100644 --- a/packages/junior/src/chat/conversations/sql/legacy-history-import.ts +++ b/packages/junior/src/chat/conversations/sql/legacy-history-import.ts @@ -2,7 +2,7 @@ * One-time Redis→SQL legacy history importer (import-only, deletion-scoped). * * Translates the legacy `junior:agent-session-log:` list shape into - * `junior_agent_steps` rows: `sessionId` markers become integer context epochs, + * `junior_conversation_events` rows: `sessionId` markers become context epochs, * `projection_reset` entries explode into a `context_epoch_started` marker plus * per-message rows, advisor `transcriptRef` links become `childConversationId`, * and legacy v1 provenance normalizes exactly as the legacy reducer does. This @@ -17,14 +17,19 @@ import type { PiMessage } from "@/chat/pi/messages"; import { unescapeXml } from "@/chat/xml"; import type { NewConversationMessage } from "../messages"; import { - contextProvenance, legacyActorProvenance, - type PiMessageProvenance, type SessionLogEntry, } from "@/chat/state/session-log"; -import { agentStepEntrySchema, type AgentStepEntry } from "../history"; import { - juniorAgentSteps, + contextProvenance, + type ConversationMessageProvenance, +} from "../provenance"; +import { + conversationEventDataSchema, + type ConversationEventData, +} from "../history"; +import { + juniorConversationEvents, juniorConversationMessages, juniorConversations, } from "@/db/schema"; @@ -35,17 +40,17 @@ const ADVISOR_TASK_CLOSE = "\n"; const ADVISOR_CONTEXT_OPEN = "\n"; const ADVISOR_CONTEXT_CLOSE = "\n"; -/** A converted legacy step with its explicit order and epoch pinned. */ -export interface ImportedStep { +/** A converted legacy event with its explicit order and epoch pinned. */ +export interface ImportedEvent { seq: number; contextEpoch: number; - entry: AgentStepEntry; + data: ConversationEventData; createdAtMs: number; } /** Result of converting one conversation's legacy log. */ export interface ConvertedLegacyLog { - steps: ImportedStep[]; + events: ImportedEvent[]; /** Deterministic advisor child conversation id, when a subagent was recorded. */ advisorChildConversationId?: string; } @@ -119,7 +124,7 @@ function normalizeAdvisorMessage(message: PiMessage): PiMessage { /** Decode a legacy pi_message entry's provenance, tolerating v1 actor shapes. */ function piEntryProvenance( entry: Extract, -): PiMessageProvenance { +): ConversationMessageProvenance { if (entry.provenance) { return entry.provenance; } @@ -130,7 +135,7 @@ function piEntryProvenance( } /** - * Convert a legacy session log into ordered step rows with explicit epochs. + * Convert a legacy session log into ordered events with explicit epochs. * * `fallbackCreatedAtMs` supplies `created_at` for rows without an intrinsic * timestamp (epoch markers, provider facts, timestamp-less messages); a real @@ -141,16 +146,16 @@ export function convertLegacySessionLog(args: { entries: SessionLogEntry[]; fallbackCreatedAtMs: number; }): ConvertedLegacyLog { - const steps: ImportedStep[] = []; + const events: ImportedEvent[] = []; const fallback = args.fallbackCreatedAtMs; let advisorChildConversationId: string | undefined; let seq = 0; const push = ( contextEpoch: number, - entry: AgentStepEntry, + data: ConversationEventData, createdAtMs: number, ): void => { - steps.push({ seq, contextEpoch, entry, createdAtMs }); + events.push({ seq, contextEpoch, data, createdAtMs }); seq += 1; }; @@ -161,7 +166,7 @@ export function convertLegacySessionLog(args: { push( epoch, { - type: "pi_message", + type: "message", message: entry.message, provenance: piEntryProvenance(entry), }, @@ -188,7 +193,7 @@ export function convertLegacySessionLog(args: { push( epoch, { - type: "pi_message", + type: "message", message, provenance: provenance[index]!, }, @@ -292,7 +297,7 @@ export function convertLegacySessionLog(args: { } return { - steps, + events, ...(advisorChildConversationId ? { advisorChildConversationId } : {}), }; } @@ -301,41 +306,42 @@ export function convertLegacySessionLog(args: { export function convertAdvisorMessages( messages: PiMessage[], fallbackCreatedAtMs: number, -): ImportedStep[] { +): ImportedEvent[] { return messages.map((sourceMessage, seq) => { const message = normalizeAdvisorMessage(sourceMessage); return { seq, contextEpoch: 0, - entry: { type: "pi_message", message, provenance: contextProvenance }, + data: { type: "message", message, provenance: contextProvenance }, createdAtMs: messageTimestampMs(message) ?? fallbackCreatedAtMs, }; }); } -type AgentStepInsert = typeof juniorAgentSteps.$inferInsert; +type ConversationEventInsert = typeof juniorConversationEvents.$inferInsert; -function messageRole(entry: AgentStepEntry): string | null { - if (entry.type !== "pi_message") { +function messageRole(entry: ConversationEventData): string | null { + if (entry.type !== "message") { return null; } - const role = (entry.message as { role?: unknown }).role; - return typeof role === "string" ? role : null; + return entry.message.role; } +/** Encode one validated imported event as a physical SQL row. */ function insertRow( conversationId: string, - step: ImportedStep, -): AgentStepInsert { - const { type, ...payload } = agentStepEntrySchema.parse(step.entry); + event: ImportedEvent, +): ConversationEventInsert { + const { type, ...payload } = conversationEventDataSchema.parse(event.data); return { conversationId, - seq: step.seq, - contextEpoch: step.contextEpoch, + seq: event.seq, + contextEpoch: event.contextEpoch, + schemaVersion: 1, type, - role: messageRole(step.entry), + role: messageRole(event.data), payload: sanitizePostgresJson(payload), - createdAt: new Date(step.createdAtMs), + createdAt: new Date(event.createdAtMs), }; } @@ -345,14 +351,14 @@ export interface LegacyImportWrite { fallbackCreatedAtMs: number; lastActivityAtMs: number; messages?: Array; - steps: ImportedStep[]; - child?: { conversationId: string; steps: ImportedStep[] }; + events: ImportedEvent[]; + child?: { conversationId: string; events: ImportedEvent[] }; } /** * Write a converted legacy history for one conversation, all-or-nothing. * - * Serialized by a per-conversation advisory lock and skipped when step rows + * Serialized by a per-conversation advisory lock and skipped when event rows * already exist, so a re-run (bulk or lazy) never double-imports. Parent and * advisor-child rows land in one transaction; explicit `seq`/`context_epoch` * are what make this need a dedicated writer rather than the narrow port. @@ -377,9 +383,11 @@ export async function writeLegacyImport( return false; } const existing = await db - .select({ seq: juniorAgentSteps.seq }) - .from(juniorAgentSteps) - .where(eq(juniorAgentSteps.conversationId, args.conversationId)) + .select({ seq: juniorConversationEvents.seq }) + .from(juniorConversationEvents) + .where( + eq(juniorConversationEvents.conversationId, args.conversationId), + ) .limit(1); if (existing.length > 0) { return false; @@ -420,21 +428,21 @@ export async function writeLegacyImport( }, }); } - if (args.steps.length > 0) { + if (args.events.length > 0) { await db - .insert(juniorAgentSteps) + .insert(juniorConversationEvents) .values( - args.steps.map((step) => insertRow(args.conversationId, step)), + args.events.map((event) => insertRow(args.conversationId, event)), ); } if (args.child) { const childCreatedAtMs = - args.child.steps.length > 0 - ? Math.min(...args.child.steps.map((step) => step.createdAtMs)) + args.child.events.length > 0 + ? Math.min(...args.child.events.map((event) => event.createdAtMs)) : args.fallbackCreatedAtMs; const childLastActivityAtMs = - args.child.steps.length > 0 - ? Math.max(...args.child.steps.map((step) => step.createdAtMs)) + args.child.events.length > 0 + ? Math.max(...args.child.events.map((event) => event.createdAtMs)) : childCreatedAtMs; await ensureChildConversationRow( executor, @@ -443,12 +451,12 @@ export async function writeLegacyImport( new Date(childCreatedAtMs), new Date(childLastActivityAtMs), ); - if (args.child.steps.length > 0) { + if (args.child.events.length > 0) { await db - .insert(juniorAgentSteps) + .insert(juniorConversationEvents) .values( - args.child.steps.map((step) => - insertRow(args.child!.conversationId, step), + args.child.events.map((event) => + insertRow(args.child!.conversationId, event), ), ); } diff --git a/packages/junior/src/chat/conversations/sql/purge.ts b/packages/junior/src/chat/conversations/sql/purge.ts index 494219109..a58b52ce3 100644 --- a/packages/junior/src/chat/conversations/sql/purge.ts +++ b/packages/junior/src/chat/conversations/sql/purge.ts @@ -1,7 +1,7 @@ import { and, asc, eq, inArray, isNull, sql } from "drizzle-orm"; import type { JuniorSqlDatabase } from "@/db/db"; import { - juniorAgentSteps, + juniorConversationEvents, juniorConversationMessages, juniorConversations, juniorDestinations, @@ -79,8 +79,8 @@ export async function selectExpiredRoots( select 1 from conversation_tree tree where exists ( - select 1 from junior_agent_steps steps - where steps.conversation_id = tree.conversation_id + select 1 from junior_conversation_events events + where events.conversation_id = tree.conversation_id ) or exists ( select 1 from junior_conversation_messages messages @@ -130,7 +130,7 @@ export async function selectExpiredRoots( /** * Purge one conversation tree in a single transaction: delete all message and - * step rows for the given conversation and every descendant, stamp + * event rows for the given conversation and every descendant, stamp * `transcript_purged_at`, and — for non-public content — null the raw-payload * metadata (`title`, `channel_name`, legacy actor JSON) so purged private * conversations keep only safe metadata. The metadata rows themselves survive. @@ -178,8 +178,8 @@ export async function purgeConversationTree( const ids = await conversationTreeIds(executor, args.rootConversationId); await executor .db() - .delete(juniorAgentSteps) - .where(inArray(juniorAgentSteps.conversationId, ids)); + .delete(juniorConversationEvents) + .where(inArray(juniorConversationEvents.conversationId, ids)); await executor .db() .delete(juniorConversationMessages) diff --git a/packages/junior/src/chat/conversations/store.ts b/packages/junior/src/chat/conversations/store.ts index c2217fe51..10686403a 100644 --- a/packages/junior/src/chat/conversations/store.ts +++ b/packages/junior/src/chat/conversations/store.ts @@ -42,7 +42,7 @@ export interface Conversation { updatedAtMs: number; /** * When retention purged this conversation's content. Set means messages and - * steps were deleted wholesale; reporting presents the transcript as expired + * events were deleted wholesale; reporting presents the transcript as expired * rather than privacy-redacted (`../../../../../policies/data-redaction.md`). */ transcriptPurgedAtMs?: number; diff --git a/packages/junior/src/chat/conversations/visible-compactions.ts b/packages/junior/src/chat/conversations/visible-compactions.ts index e4a8226e2..501b6c443 100644 --- a/packages/junior/src/chat/conversations/visible-compactions.ts +++ b/packages/junior/src/chat/conversations/visible-compactions.ts @@ -1,16 +1,16 @@ import { isDeepStrictEqual } from "node:util"; -import { getAgentStepStore } from "@/chat/db"; -import type { StoredAgentStep } from "./history"; +import { getConversationEventStore } from "@/chat/db"; +import type { ConversationEvent } from "./history"; import type { ConversationCompaction, ThreadConversationState, } from "@/chat/state/conversation"; -function latestSnapshot(steps: StoredAgentStep[]): ConversationCompaction[] { - for (let index = steps.length - 1; index >= 0; index -= 1) { - const entry = steps[index]?.entry; - if (entry?.type === "visible_context_compacted") { - return entry.compactions; +function latestSnapshot(events: ConversationEvent[]): ConversationCompaction[] { + for (let index = events.length - 1; index >= 0; index -= 1) { + const data = events[index]?.data; + if (data?.type === "visible_context_compacted") { + return data.compactions; } } return []; @@ -21,25 +21,27 @@ export async function hydrateConversationCompactions(args: { conversation: ThreadConversationState; conversationId: string; }): Promise { - const steps = await getAgentStepStore().loadHistory(args.conversationId); - args.conversation.compactions = latestSnapshot(steps); + const events = await getConversationEventStore().loadHistory( + args.conversationId, + ); + args.conversation.compactions = latestSnapshot(events); } -/** Persist a changed visible-context compaction snapshot in SQL agent history. */ +/** Persist a changed visible-context compaction snapshot in event history. */ export async function persistConversationCompactions(args: { conversation: ThreadConversationState; conversationId: string; }): Promise { - const stepStore = getAgentStepStore(); + const eventStore = getConversationEventStore(); const existing = latestSnapshot( - await stepStore.loadHistory(args.conversationId), + await eventStore.loadHistory(args.conversationId), ); if (isDeepStrictEqual(existing, args.conversation.compactions)) { return; } - await stepStore.append(args.conversationId, [ + await eventStore.append(args.conversationId, [ { - entry: { + data: { type: "visible_context_compacted", compactions: args.conversation.compactions, }, diff --git a/packages/junior/src/chat/conversations/visible-messages.ts b/packages/junior/src/chat/conversations/visible-messages.ts index c89cd5be6..395cbc03b 100644 --- a/packages/junior/src/chat/conversations/visible-messages.ts +++ b/packages/junior/src/chat/conversations/visible-messages.ts @@ -80,7 +80,7 @@ function fromStoredMessage( * projection read (turn-dedupe, delivered-message redelivery guards, * channel-context assembly) would otherwise make correctness decisions on an * empty transcript for promotion-window stragglers whose history is still only - * in legacy Redis. The import is idempotent (skips when SQL step rows exist) + * in legacy Redis. The import is idempotent (skips when SQL event rows exist) * and no-ops cheaply when there is nothing legacy to read. */ export async function hydrateConversationMessages(args: { diff --git a/packages/junior/src/chat/db.ts b/packages/junior/src/chat/db.ts index ea1f73f9f..2d11c9618 100644 --- a/packages/junior/src/chat/db.ts +++ b/packages/junior/src/chat/db.ts @@ -1,8 +1,8 @@ import { getChatConfig, type SqlDriver } from "@/chat/config"; import { createSqlStore } from "@/chat/conversations/sql/store"; import type { ConversationStore } from "@/chat/conversations/store"; -import { createSqlAgentStepStore } from "@/chat/conversations/sql/history"; -import type { AgentStepStore } from "@/chat/conversations/history"; +import { createSqlConversationEventStore } from "@/chat/conversations/sql/history"; +import type { ConversationEventStore } from "@/chat/conversations/history"; import { createSqlConversationMessageStore } from "@/chat/conversations/sql/messages"; import type { ConversationMessageStore } from "@/chat/conversations/messages"; import { createSqlConversationSearchStore } from "@/chat/conversations/sql/search"; @@ -16,7 +16,7 @@ let current: db: JuniorSqlExecutor; driver: SqlDriver; store: ConversationStore; - stepStore: AgentStepStore; + eventStore: ConversationEventStore; messageStore: ConversationMessageStore; searchStore: ConversationSearchStore; } @@ -34,7 +34,7 @@ function createDb(args: { /** * Return the process SQL executor. Exposed for the one-time legacy import - * writer, which needs explicit-`seq`/epoch inserts the step-store port omits. + * writer, which needs explicit-`seq`/epoch inserts the event-store port omits. */ export function getSqlExecutor(): JuniorSqlExecutor { const { sql } = getChatConfig(); @@ -56,7 +56,7 @@ export function getSqlExecutor(): JuniorSqlExecutor { driver: sql.driver, db, store: createSqlStore(db), - stepStore: createSqlAgentStepStore(db), + eventStore: createSqlConversationEventStore(db), messageStore: createSqlConversationMessageStore(db), searchStore: createSqlConversationSearchStore(db), }; @@ -75,10 +75,10 @@ export function getConversationStore(): ConversationStore { return current!.store; } -/** Return the SQL-backed durable agent step store. */ -export function getAgentStepStore(): AgentStepStore { +/** Return the canonical SQL-backed conversation event store. */ +export function getConversationEventStore(): ConversationEventStore { getSqlExecutor(); - return current!.stepStore; + return current!.eventStore; } /** Return the SQL-backed visible conversation message store. */ diff --git a/packages/junior/src/chat/local/runner.ts b/packages/junior/src/chat/local/runner.ts index ad1b4bf58..b7c880bb9 100644 --- a/packages/junior/src/chat/local/runner.ts +++ b/packages/junior/src/chat/local/runner.ts @@ -112,7 +112,7 @@ function nextUserMessageSequence( ); } -/** Load the durable local Pi history from the SQL step-store projection. */ +/** Load durable local Pi history from the conversation-event projection. */ async function loadLocalPiMessages(args: { conversationId: string; }): Promise { @@ -341,7 +341,7 @@ export async function runLocalAgentTurn( }); if (reply.piMessages?.length) { // Destination acceptance is the completion boundary: this first commits - // the final assistant messages to the session log and marks the session + // the final assistant messages to the event log and marks the session // record completed only after the CLI sink accepted the reply. await completeDeliveredTurn({ conversationId: input.conversationId, diff --git a/packages/junior/src/chat/pi/conversation-events.ts b/packages/junior/src/chat/pi/conversation-events.ts new file mode 100644 index 000000000..8f889b6fa --- /dev/null +++ b/packages/junior/src/chat/pi/conversation-events.ts @@ -0,0 +1,102 @@ +/** + * Pi adapter for Junior conversation events. + * + * Conversation storage owns the canonical ordered event log. This module is + * the only reducer that translates those Junior events into Pi messages, + * aligned provenance, source sequence numbers, and epoch model binding. + */ +import type { ModelProfile } from "@/chat/model-profile"; +import type { + ConversationEvent, + ConversationEventData, +} from "@/chat/conversations/history"; +import { piMessageSchema, type PiMessage } from "@/chat/pi/messages"; +import { + contextProvenance, + type ConversationMessageProvenance, +} from "@/chat/conversations/provenance"; + +type MessageEventData = Extract; +type AuthorizationCompletedEventData = Extract< + ConversationEventData, + { type: "authorization_completed" } +>; + +/** Pi context projected from one Junior conversation epoch. */ +export interface PiConversationProjection { + messages: PiMessage[]; + provenance: ConversationMessageProvenance[]; + modelProfile: ModelProfile; + modelId: string | undefined; +} + +/** Pi context with the source event sequence for every projected message. */ +export interface PiConversationEventProjection extends PiConversationProjection { + seqs: number[]; +} + +function authorizationObservationMessage( + data: AuthorizationCompletedEventData, + createdAtMs: number, +): PiMessage { + const label = data.kind === "mcp" ? "MCP authorization" : "Authorization"; + return { + role: "user", + content: [ + { + type: "text", + text: `${label} completed for provider "${data.provider}". Continue the blocked request and retry the provider operation if needed.`, + }, + ], + timestamp: createdAtMs, + } as PiMessage; +} + +function messageEventProvenance( + data: MessageEventData, +): ConversationMessageProvenance { + return data.provenance ?? contextProvenance; +} + +/** + * Project ordered Junior events into Pi context. + * + * Host-only events are filtered, completed authorization becomes a synthetic + * observation, and `maxSeq` reproduces an exact committed boundary. + */ +export function projectConversationEvents( + events: ConversationEvent[], + options?: { maxSeq?: number }, +): PiConversationEventProjection { + const messages: PiMessage[] = []; + const provenance: ConversationMessageProvenance[] = []; + const seqs: number[] = []; + let modelProfile: ModelProfile = "standard"; + let modelId: string | undefined; + + for (const event of events) { + if (options?.maxSeq !== undefined && event.seq > options.maxSeq) { + break; + } + if (event.data.type === "context_epoch_started") { + modelProfile = event.data.modelProfile ?? "standard"; + modelId = event.data.modelId; + continue; + } + if (event.data.type === "message") { + messages.push(piMessageSchema.parse(event.data.message)); + provenance.push(messageEventProvenance(event.data)); + seqs.push(event.seq); + continue; + } + if (event.data.type === "authorization_completed") { + messages.push( + authorizationObservationMessage(event.data, event.createdAtMs), + ); + provenance.push(contextProvenance); + seqs.push(event.seq); + } + } + + return { messages, provenance, seqs, modelProfile, modelId }; +} diff --git a/packages/junior/src/chat/plugins/task-runner.ts b/packages/junior/src/chat/plugins/task-runner.ts index 75e828911..4e90ae1a8 100644 --- a/packages/junior/src/chat/plugins/task-runner.ts +++ b/packages/junior/src/chat/plugins/task-runner.ts @@ -35,7 +35,7 @@ import { coerceThreadConversationState } from "@/chat/state/conversation"; import { hydrateConversationMessages } from "@/chat/conversations/visible-messages"; import type { ConversationMessage } from "@/chat/state/conversation"; import { parseSlackMessageTs } from "@/chat/slack/timestamp"; -import type { PiMessageProvenance } from "@/chat/state/session-log"; +import type { ConversationMessageProvenance } from "@/chat/conversations/provenance"; import { getAgentTurnSessionRecord, type AgentTurnSessionRecord, @@ -139,7 +139,7 @@ function sameActorIdentity( /** Build the transcript provenance for a user message from its Pi provenance. */ function messageProvenance( - provenance: PiMessageProvenance, + provenance: ConversationMessageProvenance, ): PluginRunTranscriptProvenance { return { authority: provenance.authority, @@ -149,7 +149,7 @@ function messageProvenance( function runTranscriptEntry( message: PiMessage, - provenance: PiMessageProvenance, + provenance: ConversationMessageProvenance, runActor: Actor | undefined, ): PluginRunTranscriptEntry | undefined { const role = getPiMessageRole(message); @@ -200,12 +200,14 @@ function runTranscriptEntry( */ function turnMessagesWithProvenance( record: AgentTurnSessionRecord, -): Array<{ message: PiMessage; provenance: PiMessageProvenance }> { +): Array<{ message: PiMessage; provenance: ConversationMessageProvenance }> { const startIndex = record.turnStartMessageIndex ?? 0; const messages = record.piMessages.slice(startIndex); const provenance = record.piMessageProvenance.slice(startIndex); - const paired: Array<{ message: PiMessage; provenance: PiMessageProvenance }> = - []; + const paired: Array<{ + message: PiMessage; + provenance: ConversationMessageProvenance; + }> = []; for (const [index, message] of messages.entries()) { for (const stripped of stripRuntimeTurnContext([message])) { paired.push({ diff --git a/packages/junior/src/chat/runtime/README.md b/packages/junior/src/chat/runtime/README.md index 7b304a171..93773a2eb 100644 --- a/packages/junior/src/chat/runtime/README.md +++ b/packages/junior/src/chat/runtime/README.md @@ -18,7 +18,7 @@ this directory owns product orchestration around it. ## Durable Continuation -- Agent steps are appended at safe boundaries with monotonic sequence numbers. +- Conversation events append at safe boundaries with monotonic sequence numbers. - Restoration reduces the current context epoch into Pi messages and derived runtime state. - A timeout or soft execution limit yields only at a boundary where tool results diff --git a/packages/junior/src/chat/runtime/reply-executor.ts b/packages/junior/src/chat/runtime/reply-executor.ts index 1f754578b..d5d723dec 100644 --- a/packages/junior/src/chat/runtime/reply-executor.ts +++ b/packages/junior/src/chat/runtime/reply-executor.ts @@ -116,8 +116,8 @@ import { getConversationStore } from "@/chat/db"; import { contextProvenance, instructionProvenanceFor, - type PiMessageProvenance, -} from "@/chat/state/session-log"; + type ConversationMessageProvenance, +} from "@/chat/conversations/provenance"; import { commitMessages, loadProjection, @@ -250,7 +250,7 @@ function queuedInstructionActor( function queuedInstructionProvenance( queued: QueuedTurnMessage, teamId: string, -): PiMessageProvenance { +): ConversationMessageProvenance { if (isResourceEventMessage(queued.message)) { return contextProvenance; } @@ -641,7 +641,7 @@ export function createReplyToThread(deps: ReplyExecutorDeps) { }; /** * Commit drained parked/batched input pairs to the conversation - * session log, deduping by `parkedInputKey` so a redelivery never + * event log, deduping by `parkedInputKey` so a redelivery never * double-appends. This is the Membership-Rule commit point * Membership rule: each drained message is written with its * own author's instruction provenance while that author is still known, @@ -650,11 +650,14 @@ export function createReplyToThread(deps: ReplyExecutorDeps) { * The read-compute-append races a concurrently-resumed slice, which * runs under the thread resume lock; take the same lock so the two * writers never interleave. Returns false when the lock is busy (a live - * resume owns the session log): the caller must leave the mailbox + * resume owns the event log): the caller must leave the mailbox * message pending for the next drain instead of consuming it. */ - const drainParkedInputToSessionLog = async ( - pairs: Array<{ message: PiMessage; provenance: PiMessageProvenance }>, + const drainParkedInputToEventLog = async ( + pairs: Array<{ + message: PiMessage; + provenance: ConversationMessageProvenance; + }>, ): Promise => { if (!conversationId || pairs.length === 0) { return true; @@ -703,7 +706,7 @@ export function createReplyToThread(deps: ReplyExecutorDeps) { } }; /** - * Durably append this turn's parked user input to the session log at + * Durably append this turn's parked user input to the event log at * the parked safe boundary so the resumed `continue()` sees it. The * awaiting record pins the log session and materializes the projection * tail, so the append needs no record mutation. Must complete before @@ -740,7 +743,7 @@ export function createReplyToThread(deps: ReplyExecutorDeps) { message: buildSteeringPiMessage(steering), provenance: steering.provenance, })); - return drainParkedInputToSessionLog(parkedPairs); + return drainParkedInputToEventLog(parkedPairs); }; if (preparedState.userMessageAlreadyReplied) { await persistThreadState(thread, { @@ -758,7 +761,7 @@ export function createReplyToThread(deps: ReplyExecutorDeps) { sessionId: activeTurnId, }); if (resumeRequest) { - // Durable session-log append first: rescheduling a continuation + // Durable event-log append first: rescheduling a continuation // does not consume the message, and `ack` may only // fire after the input is model-visible. if (!(await appendParkedTurnInput(resumeRequest.sessionId))) { @@ -802,7 +805,7 @@ export function createReplyToThread(deps: ReplyExecutorDeps) { // A user follow-up supersedes the auth-parked run: answer it // now as a fresh turn instead of consuming it into a pause that // may never resume. The parked prompt stays model-visible via - // the session-log projection, pendingAuth state keeps the + // the event-log projection, pendingAuth state keeps the // authorization link reusable, and the abandoned record turns a // late OAuth callback into a stale no-op instead of a competing // run. @@ -1051,14 +1054,14 @@ export function createReplyToThread(deps: ReplyExecutorDeps) { message: buildSteeringPiMessage(steering), provenance: steering.provenance, })); - // Commit the batch to the session log before the run starts — the + // Commit the batch to the event log before the run starts — the // Membership-rule commit point — so its // authors' instruction provenance is durable while they are known. // The fresh prompt checkpoint then matches these merged messages as // an already-committed prefix and reuses that provenance instead of // collapsing them to the live actor. - if (!(await drainParkedInputToSessionLog(batchedInstructions))) { - // A live resume owns the session-log read-modify-write. Defer the + if (!(await drainParkedInputToEventLog(batchedInstructions))) { + // A live resume owns the event-log read-modify-write. Defer the // turn (as appendParkedTurnInput does) so the worker releases the // lease and the next drain commits provenance before running; // never run with the batch uncommitted. diff --git a/packages/junior/src/chat/services/context-compaction.ts b/packages/junior/src/chat/services/context-compaction.ts index 22dad28a9..e242a405a 100644 --- a/packages/junior/src/chat/services/context-compaction.ts +++ b/packages/junior/src/chat/services/context-compaction.ts @@ -3,7 +3,7 @@ * * This module bounds visible Pi history for long conversations. It strips * runtime-only turn context before summarizing and opens replacement epochs in - * the durable step store. Capacity compaction retains recent user intent; + * the durable event store. Capacity compaction retains recent user intent; * handoff starts a profile-bound epoch with only its summary. Normal checkpoints * may later append the current bootstrap; future replacement strips it again. */ @@ -24,10 +24,10 @@ import { } from "@/chat/services/context-budget"; import { contextProvenance, - type PiMessageProvenance, -} from "@/chat/state/session-log"; + type ConversationMessageProvenance, +} from "@/chat/conversations/provenance"; import { loadConversationProjection } from "@/chat/conversations/projection"; -import { getAgentStepStore } from "@/chat/db"; +import { getConversationEventStore } from "@/chat/db"; import type { ThreadConversationState } from "@/chat/state/conversation"; import { logWarn, setSpanAttributes } from "@/chat/logging"; import { @@ -354,8 +354,8 @@ function estimateHistoryTokens(messages: PiMessage[]): number { */ function buildReplacementProvenance(args: { retained: RetainedUserMessage[]; - sourceProvenance: PiMessageProvenance[]; -}): PiMessageProvenance[] { + sourceProvenance: ConversationMessageProvenance[]; +}): ConversationMessageProvenance[] { return [ ...args.retained.map( (entry) => args.sourceProvenance[entry.sourceIndex] ?? contextProvenance, @@ -451,7 +451,7 @@ async function writeCompactedThreadContext( triggerTokens?: number; }, ): Promise { - const stepStore = getAgentStepStore(); + const eventStore = getConversationEventStore(); const sourceProjection = await loadConversationProjection({ conversationId: args.conversationId, }); @@ -468,7 +468,7 @@ async function writeCompactedThreadContext( retained, sourceProvenance: sourceProjection.provenance, }); - await stepStore.startEpoch(args.conversationId, { + await eventStore.startEpoch(args.conversationId, { reason: "compaction", modelProfile: sourceProjection.modelProfile, modelId: modelIdForProfile(botConfig, sourceProjection.modelProfile), @@ -526,7 +526,7 @@ export async function compactContextForHandoff( } as PiMessage; const messages = [message]; args.signal?.throwIfAborted(); - await getAgentStepStore().startEpoch(args.conversationId, { + await getConversationEventStore().startEpoch(args.conversationId, { reason: "handoff", modelProfile: args.target.modelProfile, modelId: args.target.modelId, diff --git a/packages/junior/src/chat/services/plugin-auth-orchestration.ts b/packages/junior/src/chat/services/plugin-auth-orchestration.ts index dd6c09892..cce12ffa3 100644 --- a/packages/junior/src/chat/services/plugin-auth-orchestration.ts +++ b/packages/junior/src/chat/services/plugin-auth-orchestration.ts @@ -3,7 +3,7 @@ * * This module detects plugin credential failures from the sandbox egress layer * and maps them onto the same paused-run contract used by MCP auth. It owns - * provider attribution, private-link delivery/reuse, session-log recording, + * provider attribution, private-link delivery/reuse, event-log recording, * and credential cleanup. * * Auth failures are detected exclusively through the structured `auth_required` diff --git a/packages/junior/src/chat/services/turn-session-record.ts b/packages/junior/src/chat/services/turn-session-record.ts index 73cd81c28..7e1c31132 100644 --- a/packages/junior/src/chat/services/turn-session-record.ts +++ b/packages/junior/src/chat/services/turn-session-record.ts @@ -8,7 +8,7 @@ import type { ConversationPrivacy } from "@/chat/conversation-privacy"; import type { Destination, Actor, Source } from "@sentry/junior-plugin-api"; import { getActiveTraceId, logException } from "@/chat/logging"; import type { PiMessage } from "@/chat/pi/messages"; -import type { PiMessageProvenance } from "@/chat/state/session-log"; +import type { ConversationMessageProvenance } from "@/chat/conversations/provenance"; import { getPiMessageRole, trimTrailingAssistantMessages, @@ -136,7 +136,7 @@ export async function persistRunningSessionRecord(args: { sliceId: number; messages: PiMessage[]; /** Provenance for trailing newly committed messages, such as steering. */ - trailingMessageProvenance?: PiMessageProvenance[]; + trailingMessageProvenance?: ConversationMessageProvenance[]; loadedSkillNames?: string[]; logContext: SessionRecordLogContext; modelId: string; diff --git a/packages/junior/src/chat/state/conversation.ts b/packages/junior/src/chat/state/conversation.ts index bf0dba8b0..49430bf37 100644 --- a/packages/junior/src/chat/state/conversation.ts +++ b/packages/junior/src/chat/state/conversation.ts @@ -163,7 +163,7 @@ export function coerceThreadConversationState( const base = defaultConversationState(); // The visible transcript lives in the ConversationMessageStore and Pi - // history lives in the AgentStepStore (both SQL). Legacy `messages` / + // history lives in the ConversationEventStore (both SQL). Legacy `messages` / // `piMessages` mirrors left in old thread-state payloads are ignored on // read; consumers hydrate from SQL instead. const messages: ConversationMessage[] = []; @@ -262,7 +262,7 @@ export function coerceThreadConversationState( * Wrap a conversation state into the storage envelope for persistence. The * visible transcript (`messages`) is not written to `thread-state`; it lives * in SQL (`ConversationMessageStore`), and Pi history lives in the SQL - * `AgentStepStore`. Only runtime scratch is persisted here. + * `ConversationEventStore`. Only runtime scratch is persisted here. */ export function buildConversationStatePatch( conversation: ThreadConversationState, diff --git a/packages/junior/src/chat/state/session-log.ts b/packages/junior/src/chat/state/session-log.ts index cb1be1908..49e3d0e6e 100644 --- a/packages/junior/src/chat/state/session-log.ts +++ b/packages/junior/src/chat/state/session-log.ts @@ -1,66 +1,34 @@ /** - * Conversation-scoped Pi session log. + * Legacy Redis Pi session-log read port. * - * This append-only log is the durable source for reusable Pi history across - * turns. Projection resets mark internal session boundaries after compaction; - * readers normally load the current projection so older sessions do not make - * active context grow without bound. + * SQL conversation events are the sole durable history authority. This module + * exists only to decode and read pre-cutover Redis entries during the bounded + * Redis-to-SQL import window. */ -import { isDeepStrictEqual } from "node:util"; import type { RedisStateAdapter } from "@chat-adapter/state-redis"; import { z } from "zod"; -import { actorSchema, type Actor } from "@sentry/junior-plugin-api"; import { getChatConfig } from "@/chat/config"; -import { piMessageSchema, type PiMessage } from "@/chat/pi/messages"; +import { piMessageSchema } from "@/chat/pi/messages"; import { storedSlackActorSchema, type StoredSlackActor } from "@/chat/actor"; import { getConnectedStateContext, getStateAdapter, } from "@/chat/state/adapter"; +import { + contextProvenance, + conversationMessageProvenanceSchema, + instructionProvenanceFor, + type ConversationMessageProvenance, +} from "@/chat/conversations/provenance"; const AGENT_SESSION_LOG_PREFIX = "junior:agent-session-log"; const AGENT_SESSION_LOG_SCHEMA_VERSION = 2; const INITIAL_SESSION_ID = "session_0"; -const SESSION_ID_PREFIX = "session_"; -const STATE_STORE_LOCK_TTL_MS = 5_000; -// Decode both the current (v2, per-entry provenance) and legacy (v1, -// latest-wins actor) session-log shapes; new writes always emit v2. +// Decode both deployed v2 (per-entry provenance) and legacy v1 (latest-wins +// actor) session-log shapes from before SQL conversation events became primary. const schemaVersionSchema = z.union([z.literal(1), z.literal(2)]); -const piMessageAuthoritySchema = z.union([ - z.literal("instruction"), - z.literal("context"), -]); - -/** Per-message provenance payload reused by the SQL agent-step envelope. */ -export const piMessageProvenanceSchema = z - .object({ - authority: piMessageAuthoritySchema, - actor: actorSchema.optional(), - }) - .strict(); - -/** Whether a user-role Pi message is a durable instruction or ambient context. */ -export type PiMessageAuthority = z.output; -/** Per-message record of the actor a Pi message came from and its authority weight. */ -export type PiMessageProvenance = z.output; - -const unattributedContextProvenance: PiMessageProvenance = { - authority: "context", -}; - -function instructionProvenance(actor?: Actor): PiMessageProvenance { - return actor - ? { authority: "instruction", actor } - : { authority: "instruction" }; -} - -/** A provenance entry carries no signal when it is unattributed ambient context. */ -function isDefaultContextProvenance(provenance: PiMessageProvenance): boolean { - return provenance.authority === "context" && !provenance.actor; -} - /** * Recover per-message provenance from a legacy v1 pi_message. A stored Slack * actor on the entry meant that user message was the turn instruction, so it @@ -69,9 +37,9 @@ function isDefaultContextProvenance(provenance: PiMessageProvenance): boolean { */ export function legacyActorProvenance( actor: StoredSlackActor, -): PiMessageProvenance { +): ConversationMessageProvenance { if (actor.teamId && actor.slackUserId && actor.platform) { - return instructionProvenance({ + return instructionProvenanceFor({ platform: "slack", teamId: actor.teamId, userId: actor.slackUserId, @@ -80,7 +48,7 @@ export function legacyActorProvenance( ...(actor.email ? { email: actor.email } : {}), }); } - return unattributedContextProvenance; + return contextProvenance; } const piMessageEntrySchema = z.object({ @@ -88,7 +56,7 @@ const piMessageEntrySchema = z.object({ type: z.literal("pi_message"), sessionId: z.string().min(1).default(INITIAL_SESSION_ID), message: piMessageSchema, - provenance: piMessageProvenanceSchema.optional(), + provenance: conversationMessageProvenanceSchema.optional(), // Legacy v1 latest-wins actor, decoded into provenance on read. actor: storedSlackActorSchema.optional(), }); @@ -98,7 +66,7 @@ const projectionResetEntrySchema = z.object({ type: z.literal("projection_reset"), sessionId: z.string().min(1).default(INITIAL_SESSION_ID), messages: z.array(piMessageSchema), - provenance: z.array(piMessageProvenanceSchema).optional(), + provenance: z.array(conversationMessageProvenanceSchema).optional(), // Legacy v1 latest-wins actor; v1 resets carry no per-message provenance. actor: storedSlackActorSchema.optional(), }); @@ -210,27 +178,15 @@ const sessionLogEntrySchema = z.discriminatedUnion("type", [ subagentEndedEntrySchema, ]); -/** Actor identity stored with turn-start messages for durable continuation. */ +/** One decoded pre-cutover Redis entry accepted by the SQL history importer. */ export type SessionLogEntry = z.infer; -export type AuthorizationKind = z.infer; -export type TranscriptRef = z.infer; -export type SessionActivityEntry = - | Extract - | Extract - | Extract; interface Scope { conversationId: string; } -interface AppendArgs { - entries: SessionLogEntry[]; - scope: Scope; - ttlMs: number; -} - +/** Read-only port for the bounded legacy Redis-to-SQL import. */ export interface SessionLogStore { - append(args: AppendArgs): Promise; read(scope: Scope): Promise; } @@ -253,6 +209,7 @@ function storedRecord(value: unknown): Record | undefined { : undefined; } +/** Normalize deployed requester-era fields before legacy schema validation. */ function migrateStoredEntry(value: unknown): unknown { const record = storedRecord(value); if (!record) { @@ -260,7 +217,7 @@ function migrateStoredEntry(value: unknown): unknown { } const migrated = { ...record }; - // TODO(v0.91.0): Remove legacy requester session-log entry migration. + // TODO(v0.104.0): Remove legacy requester session-log entry migration. if ("requester" in migrated && !("actor" in migrated)) { migrated.actor = migrated.requester; } @@ -277,283 +234,7 @@ function migrateStoredEntry(value: unknown): unknown { return migrated; } -function normalizeMessageCount(value: number): number { - return Number.isFinite(value) ? Math.max(0, Math.floor(value)) : 0; -} - -function countMatchingPrefix(left: PiMessage[], right: PiMessage[]): number { - const limit = Math.min(left.length, right.length); - for (let index = 0; index < limit; index += 1) { - if (!isDeepStrictEqual(left[index], right[index])) { - return index; - } - } - return limit; -} - -function entrySessionId(entry: SessionLogEntry): string { - return entry.sessionId ?? INITIAL_SESSION_ID; -} - -function isActivityEntry( - entry: SessionLogEntry, -): entry is SessionActivityEntry { - return ( - entry.type === "tool_execution_started" || - entry.type === "subagent_started" || - entry.type === "subagent_ended" - ); -} - -function latestProjectionResetIndex(entries: SessionLogEntry[]): number { - for (let index = entries.length - 1; index >= 0; index -= 1) { - if (entries[index]?.type === "projection_reset") { - return index; - } - } - return -1; -} - -/** Return the active projection session that new entries should join. */ -function currentSessionId(entries: SessionLogEntry[]): string { - const resetIndex = latestProjectionResetIndex(entries); - if (resetIndex < 0) { - return INITIAL_SESSION_ID; - } - return entrySessionId(entries[resetIndex]!); -} - -/** Allocate the next projection session after a reset changes visible history. */ -function nextSessionId(entries: SessionLogEntry[]): string { - const resetCount = entries.filter( - (entry) => entry.type === "projection_reset", - ).length; - return `${SESSION_ID_PREFIX}${resetCount + 1}`; -} - -/** - * Select the visible log segment for a session. Without an explicit session, - * readers see only the latest projection reset and entries after it. - */ -function projectionEntries( - entries: SessionLogEntry[], - sessionId?: string, -): SessionLogEntry[] { - if (sessionId) { - const sessionEntries: SessionLogEntry[] = []; - let started = false; - for (const entry of entries) { - const entryId = entrySessionId(entry); - if (!started) { - if (entryId !== sessionId) { - continue; - } - started = true; - } else if (entry.type === "projection_reset" && entryId !== sessionId) { - break; - } - if (entryId === sessionId) { - sessionEntries.push(entry); - } - } - return sessionEntries; - } - - const resetIndex = latestProjectionResetIndex(entries); - const startIndex = resetIndex < 0 ? 0 : resetIndex; - const currentId = - resetIndex < 0 ? INITIAL_SESSION_ID : entrySessionId(entries[resetIndex]!); - - return entries - .slice(startIndex) - .filter((entry) => entrySessionId(entry) === currentId); -} - -function piEntry( - message: PiMessage, - sessionId: string, - provenance?: PiMessageProvenance, -): SessionLogEntry { - return { - schemaVersion: AGENT_SESSION_LOG_SCHEMA_VERSION, - type: "pi_message", - sessionId, - message, - // Ambient context is the decode default, so only attributed/instruction - // provenance needs to be persisted on the entry. - ...(provenance && !isDefaultContextProvenance(provenance) - ? { provenance } - : {}), - }; -} - -function resetEntry( - messages: PiMessage[], - sessionId: string, - provenance: PiMessageProvenance[], -): SessionLogEntry { - if (provenance.length !== messages.length) { - throw new Error( - "projection_reset provenance must align one-to-one with messages", - ); - } - return { - schemaVersion: AGENT_SESSION_LOG_SCHEMA_VERSION, - type: "projection_reset", - sessionId, - messages, - provenance, - }; -} - -function mcpProviderConnectedEntry( - provider: string, - sessionId: string, -): SessionLogEntry { - return { - schemaVersion: AGENT_SESSION_LOG_SCHEMA_VERSION, - type: "mcp_provider_connected", - sessionId, - provider, - }; -} - -function authorizationObservationMessage(entry: { - createdAtMs: number; - kind: AuthorizationKind; - provider: string; -}): PiMessage { - const label = entry.kind === "mcp" ? "MCP authorization" : "Authorization"; - return { - role: "user", - content: [ - { - type: "text", - text: `${label} completed for provider "${entry.provider}". Continue the blocked request and retry the provider operation if needed.`, - }, - ], - timestamp: entry.createdAtMs, - } as PiMessage; -} - -function authorizationRequestedEntry(args: { - createdAtMs: number; - kind: AuthorizationKind; - sessionId: string; - provider: string; - actorId: string; - authorizationId: string; - delivery: "private_link_sent" | "private_link_reused"; -}): SessionLogEntry { - return { - schemaVersion: AGENT_SESSION_LOG_SCHEMA_VERSION, - type: "authorization_requested", - sessionId: args.sessionId, - createdAtMs: args.createdAtMs, - kind: args.kind, - provider: args.provider, - actorId: args.actorId, - authorizationId: args.authorizationId, - delivery: args.delivery, - }; -} - -function authorizationCompletedEntry(args: { - createdAtMs: number; - kind: AuthorizationKind; - sessionId: string; - provider: string; - actorId: string; - authorizationId: string; -}): SessionLogEntry { - return { - schemaVersion: AGENT_SESSION_LOG_SCHEMA_VERSION, - type: "authorization_completed", - sessionId: args.sessionId, - createdAtMs: args.createdAtMs, - kind: args.kind, - provider: args.provider, - actorId: args.actorId, - authorizationId: args.authorizationId, - }; -} - -function toolExecutionStartedEntry(args: { - args?: unknown; - createdAtMs: number; - sessionId: string; - toolCallId: string; - toolName: string; -}): SessionLogEntry { - return { - schemaVersion: AGENT_SESSION_LOG_SCHEMA_VERSION, - type: "tool_execution_started", - sessionId: args.sessionId, - createdAtMs: args.createdAtMs, - toolCallId: args.toolCallId, - toolName: args.toolName, - ...(args.args !== undefined ? { args: args.args } : {}), - }; -} - -function subagentStartedEntry(args: { - createdAtMs: number; - historyMode: "shared"; - modelId?: string; - parentConversationId: string; - parentSessionId?: string; - parentToolCallId?: string; - reasoningLevel?: string; - sessionId: string; - subagentInvocationId: string; - subagentKind: string; - transcriptRef: TranscriptRef; -}): SessionLogEntry { - return { - schemaVersion: AGENT_SESSION_LOG_SCHEMA_VERSION, - type: "subagent_started", - sessionId: args.sessionId, - subagentInvocationId: args.subagentInvocationId, - subagentKind: args.subagentKind, - ...(args.modelId ? { modelId: args.modelId } : {}), - ...(args.parentToolCallId - ? { parentToolCallId: args.parentToolCallId } - : {}), - parentConversationId: args.parentConversationId, - ...(args.parentSessionId ? { parentSessionId: args.parentSessionId } : {}), - ...(args.reasoningLevel ? { reasoningLevel: args.reasoningLevel } : {}), - transcriptRef: args.transcriptRef, - historyMode: args.historyMode, - createdAtMs: args.createdAtMs, - }; -} - -function subagentEndedEntry(args: { - createdAtMs: number; - errorCode?: string; - outcome: "success" | "error" | "aborted"; - sessionId: string; - subagentInvocationId: string; - transcriptEndMessageIndex?: number; - transcriptStartMessageIndex?: number; -}): SessionLogEntry { - return { - schemaVersion: AGENT_SESSION_LOG_SCHEMA_VERSION, - type: "subagent_ended", - sessionId: args.sessionId, - subagentInvocationId: args.subagentInvocationId, - outcome: args.outcome, - ...(args.errorCode ? { errorCode: args.errorCode } : {}), - ...(args.transcriptEndMessageIndex !== undefined - ? { transcriptEndMessageIndex: args.transcriptEndMessageIndex } - : {}), - ...(args.transcriptStartMessageIndex !== undefined - ? { transcriptStartMessageIndex: args.transcriptStartMessageIndex } - : {}), - createdAtMs: args.createdAtMs, - }; -} - +/** Decode an enveloped legacy entry, falling back to a raw Pi message shape. */ function decode(value: unknown): SessionLogEntry { if (typeof value === "string") { return decode(JSON.parse(value) as unknown); @@ -564,235 +245,11 @@ function decode(value: unknown): SessionLogEntry { return parsed.data; } - return piEntry(piMessageSchema.parse(value), INITIAL_SESSION_ID); -} - -/** Aligned Pi-message projection: `provenance[i]` describes `messages[i]`. */ -export interface SessionProjection { - messages: PiMessage[]; - provenance: PiMessageProvenance[]; -} - -/** Decode the provenance a projected pi_message carries, tolerating v1 shapes. */ -function piEntryProvenance( - entry: Extract, -): PiMessageProvenance { - if (entry.provenance) { - return entry.provenance; - } - if (entry.actor) { - return legacyActorProvenance(entry.actor); - } - return unattributedContextProvenance; -} - -/** - * Materialize Pi messages and per-message provenance from log entries. - * - * Each projected message carries its own provenance instead of a single - * latest-wins actor; legacy entries without provenance decode as - * unauthored context, and misaligned reset provenance fails closed. - */ -function project( - entries: SessionLogEntry[], - sessionId?: string, -): SessionProjection { - let messages: PiMessage[] = []; - let provenance: PiMessageProvenance[] = []; - for (const entry of projectionEntries(entries, sessionId)) { - if (entry.type === "pi_message") { - messages.push(entry.message); - provenance.push(piEntryProvenance(entry)); - continue; - } - if (entry.type === "authorization_completed") { - messages.push(authorizationObservationMessage(entry)); - provenance.push(unattributedContextProvenance); - continue; - } - if (entry.type === "projection_reset") { - const resetProvenance = - entry.provenance ?? - entry.messages.map(() => unattributedContextProvenance); - if (resetProvenance.length !== entry.messages.length) { - throw new Error( - "projection_reset provenance must align one-to-one with messages", - ); - } - messages = [...entry.messages]; - provenance = [...resetProvenance]; - continue; - } - } - return { messages, provenance }; -} - -function projectMessages( - entries: SessionLogEntry[], - sessionId?: string, -): PiMessage[] { - return project(entries, sessionId).messages; -} - -/** Find the newest instruction actor, used for latest-actor compatibility. */ -function latestInstructionActor( - provenance: PiMessageProvenance[], -): Actor | undefined { - for (let index = provenance.length - 1; index >= 0; index -= 1) { - const entry = provenance[index]!; - if (entry.authority === "instruction" && entry.actor) { - return entry.actor; - } - } - return undefined; -} - -/** - * Stable identity key for an actor: platform + name for system actors, - * platform + team + user for Slack, platform + user otherwise. Never uses - * display fields, so the same human under two profiles collapses to one - * identity. - */ -function actorIdentityKey(actor: Actor): string { - if (actor.platform === "system") { - return `system ${actor.name}`; - } - return actor.platform === "slack" - ? `slack${actor.teamId}${actor.userId}` - : `${actor.platform}${actor.userId}`; -} - -/** - * All distinct actors annotated on instruction-authority messages, in - * first-seen order — the run's actors. This is attribution, never - * authority: it exists so provenance consumers know which actors - * contributed instructions to a run. It must never feed credential - * issuance, credential-subject selection, or scope ownership. - * Unattributable instructions (no resolvable actor) never join; - * distinctness is by identity ids only, never display fields. - */ -export function instructionActors(provenance: PiMessageProvenance[]): Actor[] { - const seen = new Set(); - const actors: Actor[] = []; - for (const entry of provenance) { - if (entry.authority !== "instruction" || !entry.actor) { - continue; - } - const identity = actorIdentityKey(entry.actor); - if (seen.has(identity)) { - continue; - } - seen.add(identity); - actors.push(entry.actor); - } - return actors; -} - -function connectedMcpProviders( - entries: SessionLogEntry[], - sessionId?: string, -): string[] { - const providers = new Set(); - for (const entry of projectionEntries(entries, sessionId)) { - if (entry.type === "mcp_provider_connected") { - providers.add(entry.provider); - } - } - return [...providers].sort((left, right) => left.localeCompare(right)); -} - -function isUserMessage(message: PiMessage): boolean { - return (message as { role?: unknown }).role === "user"; -} - -/** - * Resolve the aligned provenance to persist for `nextMessages`. - * - * Explicit per-message provenance always wins; otherwise the unchanged prefix - * reuses its committed provenance, new messages default to unauthored context, - * and any new-user-message default (the turn author's instruction) is attached - * to the last new user message — the current turn's input. - */ -function resolveCommitProvenance(args: { - existing: SessionProjection; - nextMessages: PiMessage[]; - explicitProvenance?: PiMessageProvenance[]; - trailingMessageProvenance?: PiMessageProvenance[]; - newMessageProvenance?: PiMessageProvenance; -}): PiMessageProvenance[] { - if (args.explicitProvenance) { - if (args.explicitProvenance.length !== args.nextMessages.length) { - throw new Error("commit provenance must align one-to-one with messages"); - } - return args.explicitProvenance; - } - const matchingPrefix = countMatchingPrefix( - args.existing.messages, - args.nextMessages, - ); - const provenance = args.nextMessages.map((_, index) => - index < matchingPrefix - ? (args.existing.provenance[index] ?? unattributedContextProvenance) - : unattributedContextProvenance, - ); - if (args.newMessageProvenance) { - for ( - let index = args.nextMessages.length - 1; - index >= matchingPrefix; - index -= 1 - ) { - if (isUserMessage(args.nextMessages[index]!)) { - provenance[index] = args.newMessageProvenance; - break; - } - } - } - if (args.trailingMessageProvenance) { - if (args.trailingMessageProvenance.length > provenance.length) { - throw new Error( - "trailing commit provenance cannot exceed committed messages", - ); - } - const newMessageCount = args.nextMessages.length - matchingPrefix; - if (args.trailingMessageProvenance.length > newMessageCount) { - throw new Error( - "trailing commit provenance must align to newly committed messages", - ); - } - const start = provenance.length - args.trailingMessageProvenance.length; - args.trailingMessageProvenance.forEach((entry, offset) => { - provenance[start + offset] = entry; - }); - } - return provenance; -} - -/** - * Commit by appending when history advanced normally, or by writing an explicit - * projection reset when the runtime intentionally replaces visible history. - */ -function commitEntries( - existing: SessionProjection, - nextMessages: PiMessage[], - nextProvenance: PiMessageProvenance[], - sessionId: string, - entries: SessionLogEntry[], -): { entries: SessionLogEntry[]; sessionId: string } { - const matchingPrefix = countMatchingPrefix(existing.messages, nextMessages); - if (matchingPrefix === existing.messages.length) { - const newMessages = nextMessages.slice(matchingPrefix); - const newProvenance = nextProvenance.slice(matchingPrefix); - return { - entries: newMessages.map((message, index) => - piEntry(message, sessionId, newProvenance[index]), - ), - sessionId, - }; - } - const resetSessionId = nextSessionId(entries); return { - entries: [resetEntry(nextMessages, resetSessionId, nextProvenance)], - sessionId: resetSessionId, + schemaVersion: AGENT_SESSION_LOG_SCHEMA_VERSION, + type: "pi_message", + sessionId: INITIAL_SESSION_ID, + message: piMessageSchema.parse(value), }; } @@ -800,16 +257,6 @@ function redisStore(redisStateAdapter: RedisStateAdapter): SessionLogStore { const client = redisStateAdapter.getClient(); return { - async append({ entries, scope, ttlMs }) { - const listKey = key(scope); - if (entries.length > 0) { - await client.rPush( - listKey, - entries.map((entry) => JSON.stringify(entry)), - ); - } - await client.pExpire(listKey, Math.max(1, ttlMs)); - }, async read(scope) { const values = await client.lRange(key(scope), 0, -1); return values.map(decode); @@ -821,29 +268,6 @@ function stateStore(): SessionLogStore { const stateAdapter = getStateAdapter(); return { - async append({ entries, scope, ttlMs }) { - const listKey = rawKey(scope); - const lock = await stateAdapter.acquireLock( - `${listKey}:commit`, - STATE_STORE_LOCK_TTL_MS, - ); - if (!lock) { - throw new Error("Could not acquire session log commit lock"); - } - try { - const existingValue = await stateAdapter.get(listKey); - const existingEntries = Array.isArray(existingValue) - ? existingValue.map(decode) - : (await stateAdapter.getList(listKey)).map(decode); - await stateAdapter.set( - listKey, - [...existingEntries, ...entries], - Math.max(1, ttlMs), - ); - } finally { - await stateAdapter.releaseLock(lock); - } - }, async read(scope) { const listKey = rawKey(scope); const value = await stateAdapter.get(listKey); @@ -889,412 +313,3 @@ export async function readSessionLogEntries( ): Promise { return loadEntries(args); } - -/** Load chronological host-only runtime activity entries for reporting. */ -export async function loadActivityEntries( - args: Scope & { - store?: SessionLogStore; - sessionId?: string; - }, -): Promise { - const entries = await loadEntries(args); - return projectionEntries(entries, args.sessionId).filter(isActivityEntry); -} - -/** Load the committed Pi-message projection for a conversation. */ -export async function loadMessages( - args: Scope & { - store?: SessionLogStore; - messageCount: number; - sessionId?: string; - }, -): Promise { - const messageCount = normalizeMessageCount(args.messageCount); - if (messageCount === 0) { - return []; - } - - const messages = projectMessages(await loadEntries(args), args.sessionId); - return messages.length >= messageCount - ? messages.slice(0, messageCount) - : undefined; -} - -/** Load the committed Pi-message projection with aligned per-message provenance. */ -export async function loadMessagesWithProvenance( - args: Scope & { - store?: SessionLogStore; - messageCount: number; - sessionId?: string; - }, -): Promise { - const messageCount = normalizeMessageCount(args.messageCount); - if (messageCount === 0) { - return { messages: [], provenance: [] }; - } - - const projection = project(await loadEntries(args), args.sessionId); - return projection.messages.length >= messageCount - ? { - messages: projection.messages.slice(0, messageCount), - provenance: projection.provenance.slice(0, messageCount), - } - : undefined; -} - -/** Load the full current Pi-message projection for a conversation. */ -export async function loadProjection( - args: Scope & { - store?: SessionLogStore; - sessionId?: string; - }, -): Promise { - return project(await loadEntries(args), args.sessionId).messages; -} - -/** - * Load the Pi-message projection with aligned per-message provenance in one - * read. Used at continuation boundaries to avoid a second log scan. - */ -export async function loadProjectionWithProvenance( - args: Scope & { - store?: SessionLogStore; - sessionId?: string; - }, -): Promise { - return project(await loadEntries(args), args.sessionId); -} - -/** - * Load the projection with the latest instruction actor as a stored Slack - * actor. Retained for callers that still key on a single latest actor; it - * derives the actor from per-message provenance rather than a latest-wins - * field. - */ -export async function loadProjectionWithActor( - args: Scope & { - store?: SessionLogStore; - sessionId?: string; - }, -): Promise<{ messages: PiMessage[]; actor?: StoredSlackActor }> { - const projection = project(await loadEntries(args), args.sessionId); - const actor = latestInstructionActor(projection.provenance); - if (actor?.platform === "slack") { - return { - messages: projection.messages, - actor: { - platform: "slack", - slackUserId: actor.userId, - teamId: actor.teamId, - ...(actor.userName ? { slackUserName: actor.userName } : {}), - ...(actor.fullName ? { fullName: actor.fullName } : {}), - ...(actor.email ? { email: actor.email } : {}), - }, - }; - } - return { messages: projection.messages }; -} - -/** Load MCP providers that were durably connected in this conversation. */ -export async function loadConnectedMcpProviders( - args: Scope & { - store?: SessionLogStore; - }, -): Promise { - return connectedMcpProviders(await loadEntries(args)); -} - -/** Record a successful MCP provider connection without duplicating the fact. */ -export async function recordMcpProviderConnected( - args: Scope & { - store?: SessionLogStore; - provider: string; - ttlMs: number; - }, -): Promise { - const store = args.store ?? (await defaultStore()); - const entries = await store.read(args); - const sessionId = currentSessionId(entries); - if (connectedMcpProviders(entries).includes(args.provider)) { - return; - } - await store.append({ - scope: args, - entries: [mcpProviderConnectedEntry(args.provider, sessionId)], - ttlMs: args.ttlMs, - }); -} - -/** Record that an OAuth/MCP authorization link was delivered or reused. */ -export async function recordAuthorizationRequested( - args: Scope & { - store?: SessionLogStore; - kind: AuthorizationKind; - provider: string; - actorId: string; - authorizationId: string; - delivery: "private_link_sent" | "private_link_reused"; - ttlMs: number; - }, -): Promise { - const store = args.store ?? (await defaultStore()); - const entries = await store.read(args); - const sessionId = currentSessionId(entries); - if ( - projectionEntries(entries).some( - (entry) => - entry.type === "authorization_requested" && - entry.authorizationId === args.authorizationId, - ) - ) { - return; - } - await store.append({ - scope: args, - entries: [ - authorizationRequestedEntry({ - createdAtMs: Date.now(), - kind: args.kind, - sessionId, - provider: args.provider, - actorId: args.actorId, - authorizationId: args.authorizationId, - delivery: args.delivery, - }), - ], - ttlMs: args.ttlMs, - }); -} - -/** Record completed authorization as a chronological host observation for Pi. */ -export async function recordAuthorizationCompleted( - args: Scope & { - store?: SessionLogStore; - kind: AuthorizationKind; - provider: string; - actorId: string; - authorizationId: string; - ttlMs: number; - }, -): Promise { - const store = args.store ?? (await defaultStore()); - const entries = await store.read(args); - const sessionId = currentSessionId(entries); - if ( - projectionEntries(entries).some( - (entry) => - entry.type === "authorization_completed" && - entry.authorizationId === args.authorizationId, - ) - ) { - return; - } - await store.append({ - scope: args, - entries: [ - authorizationCompletedEntry({ - createdAtMs: Date.now(), - kind: args.kind, - sessionId, - provider: args.provider, - actorId: args.actorId, - authorizationId: args.authorizationId, - }), - ], - ttlMs: args.ttlMs, - }); -} - -/** Record a host-observed parent tool start without adding it to Pi replay. */ -export async function recordToolExecutionStarted( - args: Scope & { - args?: unknown; - createdAtMs?: number; - sessionId?: string; - store?: SessionLogStore; - toolCallId: string; - toolName: string; - ttlMs: number; - }, -): Promise { - const store = args.store ?? (await defaultStore()); - const entries = await store.read(args); - const sessionId = args.sessionId ?? currentSessionId(entries); - await store.append({ - scope: args, - entries: [ - toolExecutionStartedEntry({ - args: args.args, - createdAtMs: args.createdAtMs ?? Date.now(), - sessionId, - toolCallId: args.toolCallId, - toolName: args.toolName, - }), - ], - ttlMs: args.ttlMs, - }); -} - -/** Record that a child agent execution became visible from this parent run. */ -export async function recordSubagentStarted( - args: Scope & { - createdAtMs?: number; - historyMode: "shared"; - modelId?: string; - parentConversationId: string; - parentSessionId?: string; - parentToolCallId?: string; - reasoningLevel?: string; - sessionId?: string; - store?: SessionLogStore; - subagentInvocationId: string; - subagentKind: string; - transcriptRef: TranscriptRef; - ttlMs: number; - }, -): Promise { - const store = args.store ?? (await defaultStore()); - const entries = await store.read(args); - const sessionId = args.sessionId ?? currentSessionId(entries); - await store.append({ - scope: args, - entries: [ - subagentStartedEntry({ - createdAtMs: args.createdAtMs ?? Date.now(), - historyMode: args.historyMode, - modelId: args.modelId, - parentConversationId: args.parentConversationId, - parentSessionId: args.parentSessionId, - parentToolCallId: args.parentToolCallId, - reasoningLevel: args.reasoningLevel, - sessionId, - subagentInvocationId: args.subagentInvocationId, - subagentKind: args.subagentKind, - transcriptRef: args.transcriptRef, - }), - ], - ttlMs: args.ttlMs, - }); -} - -/** Record the terminal state for a previously-started child agent execution. */ -export async function recordSubagentEnded( - args: Scope & { - createdAtMs?: number; - errorCode?: string; - outcome: "success" | "error" | "aborted"; - sessionId?: string; - store?: SessionLogStore; - subagentInvocationId: string; - transcriptEndMessageIndex?: number; - transcriptStartMessageIndex?: number; - ttlMs: number; - }, -): Promise { - const store = args.store ?? (await defaultStore()); - const entries = await store.read(args); - const sessionId = args.sessionId ?? currentSessionId(entries); - await store.append({ - scope: args, - entries: [ - subagentEndedEntry({ - createdAtMs: args.createdAtMs ?? Date.now(), - errorCode: args.errorCode, - outcome: args.outcome, - sessionId, - subagentInvocationId: args.subagentInvocationId, - transcriptEndMessageIndex: args.transcriptEndMessageIndex, - transcriptStartMessageIndex: args.transcriptStartMessageIndex, - }), - ], - ttlMs: args.ttlMs, - }); -} - -/** - * Append conversation-log entries and advance the current Pi projection. - * - * Normal commits append new Pi messages. If the runtime rolls back to an - * earlier safe boundary, the log records that projection reset as an explicit - * event instead of rewriting prior history. - */ -export async function commitMessages( - args: Scope & { - store?: SessionLogStore; - messages: PiMessage[]; - ttlMs: number; - /** Explicit per-message provenance aligned one-to-one with `messages`. */ - provenance?: PiMessageProvenance[]; - /** Explicit provenance for the trailing newly committed messages. */ - trailingMessageProvenance?: PiMessageProvenance[]; - /** Default applied to the last new user message when no explicit array. */ - newMessageProvenance?: PiMessageProvenance; - }, -): Promise<{ sessionId: string; provenance: PiMessageProvenance[] }> { - const store = args.store ?? (await defaultStore()); - const entries = await store.read(args); - const existingProjection = project(entries); - const currentId = currentSessionId(entries); - const nextProvenance = resolveCommitProvenance({ - existing: existingProjection, - nextMessages: args.messages, - ...(args.provenance ? { explicitProvenance: args.provenance } : {}), - ...(args.trailingMessageProvenance - ? { trailingMessageProvenance: args.trailingMessageProvenance } - : {}), - ...(args.newMessageProvenance - ? { newMessageProvenance: args.newMessageProvenance } - : {}), - }); - const commit = commitEntries( - existingProjection, - args.messages, - nextProvenance, - currentId, - entries, - ); - await store.append({ - scope: args, - entries: commit.entries, - ttlMs: args.ttlMs, - }); - return { - sessionId: commit.sessionId, - provenance: nextProvenance, - }; -} - -/** Build an instruction-provenance record for the given actor. */ -export function instructionProvenanceFor( - actor: Actor | undefined, -): PiMessageProvenance { - return instructionProvenance(actor); -} - -/** Unattributed ambient-context provenance for non-instruction Pi messages. */ -export const contextProvenance: PiMessageProvenance = - unattributedContextProvenance; - -/** - * Parse durably-stored message provenance, failing closed on misalignment. - * - * A missing field is a legacy record: it materializes as unauthored context of - * the expected length. A present-but-malformed or wrong-length value returns - * undefined so callers reject the record rather than zip a mismatched array. - */ -export function parseStoredMessageProvenance( - value: unknown, - expectedLength: number, -): PiMessageProvenance[] | undefined { - if (value === undefined) { - return Array.from( - { length: expectedLength }, - () => unattributedContextProvenance, - ); - } - const parsed = z.array(piMessageProvenanceSchema).safeParse(value); - if (!parsed.success || parsed.data.length !== expectedLength) { - return undefined; - } - return parsed.data; -} diff --git a/packages/junior/src/chat/state/turn-session.ts b/packages/junior/src/chat/state/turn-session.ts index daacdae30..08ff78589 100644 --- a/packages/junior/src/chat/state/turn-session.ts +++ b/packages/junior/src/chat/state/turn-session.ts @@ -2,10 +2,10 @@ * Turn session records. * * These records track one user request across auth pauses, timeout slices, and - * completion. Full Pi messages live in the durable agent step store; this + * completion. Full Pi messages live in the durable conversation event store; this * record stores resumability metadata and a committed `seq` cursor into - * `junior_agent_steps` so resumes can materialize the exact continuable - * boundary without duplicating the step history. + * `junior_conversation_events` so resumes can materialize the exact continuable + * boundary without duplicating the event history. */ import { THREAD_STATE_TTL_MS, type StateAdapter } from "chat"; import { @@ -21,9 +21,8 @@ import { toStoredSlackActor, type Actor } from "@/chat/actor"; import { instructionActors, instructionProvenanceFor, - type PiMessageProvenance, - type SessionProjection, -} from "./session-log"; + type ConversationMessageProvenance, +} from "@/chat/conversations/provenance"; import { commitMessages, loadTurnProjection, @@ -55,6 +54,11 @@ export type AgentTurnSurface = "slack" | "api" | "scheduler" | "internal"; export type AgentTurnResumeReason = "timeout" | "auth" | "yield"; +interface ConversationMessageProjection { + messages: PiMessage[]; + provenance: ConversationMessageProvenance[]; +} + export interface AgentTurnSessionRecord { channelName?: string; version: number; @@ -70,7 +74,7 @@ export interface AgentTurnSessionRecord { reasoningLevel?: string; piMessages: PiMessage[]; /** Per-message provenance aligned one-to-one with `piMessages`. */ - piMessageProvenance: PiMessageProvenance[]; + piMessageProvenance: ConversationMessageProvenance[]; /** * All distinct actors annotated on this run's committed instruction-authority * messages, in first-seen order. Persisted as an attribution handle so a @@ -107,7 +111,7 @@ interface StoredAgentTurnSessionRecord extends Omit< > { actors?: Actor[]; /** - * `seq` of the last step in `junior_agent_steps` whose projection reproduces + * `seq` of the last event in `junior_conversation_events` whose projection reproduces * this record's committed Pi messages; -1 when nothing was committed. */ committedSeq: number; @@ -292,7 +296,7 @@ async function recordConversationActivityMetadata(args: { function materializeAgentTurnSessionRecord( stored: StoredAgentTurnSessionRecord, - piProjection: SessionProjection, + piProjection: ConversationMessageProjection, turnStartMessageIndex?: number, ): AgentTurnSessionRecord { return { @@ -448,7 +452,7 @@ async function setStoredRecord(args: { /** Source-confirmed destination visibility from the current event's signal. */ destinationVisibility?: ConversationPrivacy; piMessages: PiMessage[]; - piMessageProvenance: PiMessageProvenance[]; + piMessageProvenance: ConversationMessageProvenance[]; record: StoredAgentTurnSessionRecord; ttlMs: number; turnStartMessageIndex?: number; @@ -574,7 +578,7 @@ export async function upsertAgentTurnSessionRecord(args: { surface?: AgentTurnSurface; piMessages: PiMessage[]; /** Provenance for trailing newly committed messages, such as steering. */ - trailingMessageProvenance?: PiMessageProvenance[]; + trailingMessageProvenance?: ConversationMessageProvenance[]; actor?: Actor; actors?: Actor[]; resumeReason?: AgentTurnResumeReason; @@ -590,7 +594,7 @@ export async function upsertAgentTurnSessionRecord(args: { args.sessionId, ); const ttlMs = Math.max(1, args.ttlMs ?? AGENT_TURN_SESSION_TTL_MS); - // Attribute new user input to the turn's actor as an instruction; the step + // Attribute new user input to the turn's actor as an instruction; the event // store reuses committed provenance for the unchanged prefix and defaults the // rest to context. Platform-neutral so local identities are preserved too. const instructionActor = args.actor ?? existingRecord?.actor; diff --git a/packages/junior/src/chat/task-execution/state.ts b/packages/junior/src/chat/task-execution/state.ts index ba772d1dd..681c39fff 100644 --- a/packages/junior/src/chat/task-execution/state.ts +++ b/packages/junior/src/chat/task-execution/state.ts @@ -1053,7 +1053,7 @@ export async function getConversationWorkState(args: { return conversation ? conversationWorkState(conversation) : undefined; } -/** Count mailbox messages that have not yet reached the session log. */ +/** Count mailbox messages that have not yet reached the conversation event log. */ export function countPendingConversationMessages( conversation: Conversation, ): number { diff --git a/packages/junior/src/chat/task-execution/store.ts b/packages/junior/src/chat/task-execution/store.ts index 3ee221b8d..42e924427 100644 --- a/packages/junior/src/chat/task-execution/store.ts +++ b/packages/junior/src/chat/task-execution/store.ts @@ -126,7 +126,7 @@ export async function getConversationWorkState(args: { return await workState.getConversationWorkState(args); } -/** Count mailbox messages that have not yet reached the session log. */ +/** Count mailbox messages that have not yet reached the conversation event log. */ export function countPendingConversationMessages( conversation: Conversation, ): number { diff --git a/packages/junior/src/cli/upgrade.ts b/packages/junior/src/cli/upgrade.ts index f5688ee2d..744d4e8ef 100644 --- a/packages/junior/src/cli/upgrade.ts +++ b/packages/junior/src/cli/upgrade.ts @@ -13,6 +13,7 @@ import { resolveUpgradePlugins } from "./upgrade/migrations/upgrade-plugins"; import { redisConversationStateMigration } from "./upgrade/migrations/redis-conversation-state"; import { agentTurnSessionActorMigration } from "./upgrade/migrations/agent-turn-session-actor"; import { conversationUsageRepairMigration } from "./upgrade/migrations/conversation-usage"; +import { conversationEventDataMigration } from "./upgrade/migrations/conversation-event-data"; import type { MigrationContext, MigrationResult, @@ -31,6 +32,7 @@ const MIGRATIONS: UpgradeMigration[] = [ redisConversationStateMigration, coreSqlSchemaMigration, sqlConversationMigration, + conversationEventDataMigration, sqlConversationHistoryMigration, conversationUsageRepairMigration, sqlPluginMigration, diff --git a/packages/junior/src/cli/upgrade/migrations/conversation-event-data.ts b/packages/junior/src/cli/upgrade/migrations/conversation-event-data.ts new file mode 100644 index 000000000..6d454bf62 --- /dev/null +++ b/packages/junior/src/cli/upgrade/migrations/conversation-event-data.ts @@ -0,0 +1,114 @@ +import { getChatConfig } from "@/chat/config"; +import type { JuniorSqlExecutor } from "@/db/db"; +import { createJuniorSqlExecutor } from "@/db/executor"; +import type { MigrationContext, MigrationResult } from "../types"; + +const EVENT_DATA_BATCH_SIZE = 500; +const EVENT_DATA_REWRITE_LOCK = "junior:upgrade:conversation-event-data"; + +const REWRITE_EVENT_DATA_BATCH_SQL = ` +WITH candidates AS MATERIALIZED ( + SELECT conversation_id, seq + FROM junior_conversation_events + WHERE type = 'pi_message' + AND ( + $2::text IS NULL + OR (conversation_id, seq) > ($2::text, $3::integer) + ) + ORDER BY conversation_id, seq + LIMIT $1 + FOR UPDATE SKIP LOCKED +), updated AS ( + UPDATE junior_conversation_events AS event + SET + schema_version = 1, + type = 'message', + payload = CASE + WHEN jsonb_typeof(event.payload) = 'object' + THEN event.payload - 'schemaVersion' + ELSE event.payload + END + FROM candidates + WHERE event.conversation_id = candidates.conversation_id + AND event.seq = candidates.seq + RETURNING event.conversation_id, event.seq +) +SELECT conversation_id, seq +FROM updated +ORDER BY conversation_id, seq +`; + +const COUNT_LEGACY_EVENT_DATA_SQL = ` +SELECT count(*)::integer AS count +FROM junior_conversation_events +WHERE type = 'pi_message' +`; + +/** Rewrite legacy Pi message rows into canonical event data in bounded batches. */ +export async function migrateConversationEventData( + _context: MigrationContext, + options: { + batchSize?: number; + executor?: JuniorSqlExecutor; + } = {}, +): Promise { + let executor = options.executor; + let closeExecutor: (() => Promise) | undefined; + if (!executor) { + const { sql } = getChatConfig(); + executor = createJuniorSqlExecutor({ + connectionString: sql.databaseUrl, + driver: sql.driver, + }); + closeExecutor = () => executor!.close(); + } + + const batchSize = Math.max( + 1, + Math.floor(options.batchSize ?? EVENT_DATA_BATCH_SIZE), + ); + let migrated = 0; + let cursor: { conversationId: string; seq: number } | undefined; + try { + while (true) { + const rows = await executor.withLock(EVENT_DATA_REWRITE_LOCK, () => + executor.query<{ conversation_id: string; seq: number }>( + REWRITE_EVENT_DATA_BATCH_SQL, + [batchSize, cursor?.conversationId ?? null, cursor?.seq ?? null], + ), + ); + migrated += rows.length; + if (rows.length === 0) { + break; + } + const last = rows.at(-1)!; + cursor = { conversationId: last.conversation_id, seq: last.seq }; + } + const [remaining] = await executor.query<{ count: number }>( + COUNT_LEGACY_EVENT_DATA_SQL, + ); + if (!remaining) { + throw new Error( + "Conversation event migration could not verify that all legacy rows were rewritten", + ); + } + if (remaining.count > 0) { + throw new Error( + `Conversation event migration left ${remaining.count} locked legacy row(s); rerun junior upgrade`, + ); + } + return { + existing: 0, + migrated, + missing: 0, + scanned: migrated, + }; + } finally { + await closeExecutor?.(); + } +} + +export const conversationEventDataMigration = { + name: "rewrite-conversation-event-data", + run: migrateConversationEventData, +}; diff --git a/packages/junior/src/cli/upgrade/migrations/conversation-usage.ts b/packages/junior/src/cli/upgrade/migrations/conversation-usage.ts index f3fb5a7ce..a97fd31bb 100644 --- a/packages/junior/src/cli/upgrade/migrations/conversation-usage.ts +++ b/packages/junior/src/cli/upgrade/migrations/conversation-usage.ts @@ -1,5 +1,5 @@ /** - * Repair legacy usage from durable SQL steps in bounded, retry-safe batches. + * Repair legacy usage from durable SQL events in bounded, retry-safe batches. * Ephemeral run summaries are not an authority because TTL can erase evidence. */ import { getChatConfig } from "@/chat/config"; @@ -36,21 +36,21 @@ WITH candidates AS MATERIALIZED ( -- collapsing context copies of the same message across rebuilt epochs. message_occurrences AS ( SELECT - step.conversation_id, - step.payload -> 'message' AS message, + event.conversation_id, + event.payload -> 'message' AS message, row_number() OVER ( PARTITION BY - step.conversation_id, - step.context_epoch, - step.payload -> 'message' - ORDER BY step.seq + event.conversation_id, + event.context_epoch, + event.payload -> 'message' + ORDER BY event.seq ) AS occurrence - FROM junior_agent_steps AS step + FROM junior_conversation_events AS event INNER JOIN candidates AS candidate - ON candidate.conversation_id = step.conversation_id - WHERE step.type = 'pi_message' - AND step.role = 'assistant' - AND jsonb_typeof(step.payload -> 'message' -> 'usage') = 'object' + ON candidate.conversation_id = event.conversation_id + WHERE event.type = 'message' + AND event.role = 'assistant' + AND jsonb_typeof(event.payload -> 'message' -> 'usage') = 'object' ), canonical_messages AS ( SELECT DISTINCT conversation_id, message, occurrence diff --git a/packages/junior/src/cli/upgrade/migrations/conversations-history-sql.ts b/packages/junior/src/cli/upgrade/migrations/conversations-history-sql.ts index 66e3e39a0..a11c32535 100644 --- a/packages/junior/src/cli/upgrade/migrations/conversations-history-sql.ts +++ b/packages/junior/src/cli/upgrade/migrations/conversations-history-sql.ts @@ -15,7 +15,7 @@ const HISTORY_BACKFILL_LIMIT = 10_000; * Bulk-import legacy Redis conversation history (session logs, advisor blobs, * and visible messages) into SQL, bounded newest-first over the same activity * scan as the metadata backfill. Idempotent per conversation: it skips any - * conversation that already has step rows. + * conversation that already has event rows. */ export async function migrateConversationHistoryToSql( context: MigrationContext, @@ -82,6 +82,6 @@ export async function migrateConversationHistoryToSql( } export const sqlConversationHistoryMigration = { - name: "backfill-agent-steps-sql", + name: "backfill-conversation-events-sql", run: migrateConversationHistoryToSql, }; diff --git a/packages/junior/src/db/schema.ts b/packages/junior/src/db/schema.ts index 341c2d225..e284178fb 100644 --- a/packages/junior/src/db/schema.ts +++ b/packages/junior/src/db/schema.ts @@ -1,4 +1,4 @@ -import { juniorAgentSteps } from "./schema/agent-steps"; +import { juniorConversationEvents } from "./schema/conversation-events"; import { juniorConversationMessages } from "./schema/conversation-messages"; import { juniorConversations } from "./schema/conversations"; import { juniorDestinations } from "./schema/destinations"; @@ -6,7 +6,7 @@ import { juniorIdentities } from "./schema/identities"; import { juniorUsers } from "./schema/users"; export { - juniorAgentSteps, + juniorConversationEvents, juniorConversationMessages, juniorConversations, juniorDestinations, @@ -15,7 +15,7 @@ export { }; export const juniorSqlSchema = { - juniorAgentSteps, + juniorConversationEvents, juniorConversationMessages, juniorConversations, juniorDestinations, diff --git a/packages/junior/src/db/schema/agent-steps.ts b/packages/junior/src/db/schema/agent-steps.ts deleted file mode 100644 index a22061e08..000000000 --- a/packages/junior/src/db/schema/agent-steps.ts +++ /dev/null @@ -1,39 +0,0 @@ -import { - index, - integer, - jsonb, - pgTable, - primaryKey, - text, -} from "drizzle-orm/pg-core"; -import { juniorConversations } from "./conversations"; -import { timestamptz } from "./timestamps"; - -/** - * Append-only durable execution history: one row per agent step. `context_epoch` - * partitions the log into rebuild generations; the model context is the highest - * epoch's `pi_message` rows in `seq` order. The `(conversation_id, seq)` primary - * key doubles as the lease-fencing tripwire — a conflicting append fails loudly. - */ -export const juniorAgentSteps = pgTable( - "junior_agent_steps", - { - conversationId: text("conversation_id") - .notNull() - .references(() => juniorConversations.conversationId), - seq: integer("seq").notNull(), - contextEpoch: integer("context_epoch").notNull(), - type: text("type").notNull(), - role: text("role"), - payload: jsonb("payload").$type>().notNull(), - createdAt: timestamptz("created_at").notNull(), - }, - (table) => [ - primaryKey({ columns: [table.conversationId, table.seq] }), - index("junior_agent_steps_epoch_idx").on( - table.conversationId, - table.contextEpoch, - table.seq, - ), - ], -); diff --git a/packages/junior/src/db/schema/conversation-events.ts b/packages/junior/src/db/schema/conversation-events.ts new file mode 100644 index 000000000..9bb64004b --- /dev/null +++ b/packages/junior/src/db/schema/conversation-events.ts @@ -0,0 +1,46 @@ +import { + foreignKey, + index, + integer, + jsonb, + pgTable, + primaryKey, + text, +} from "drizzle-orm/pg-core"; +import { juniorConversations } from "./conversations"; +import { timestamptz } from "./timestamps"; + +/** + * Append-only canonical conversation history. `context_epoch` partitions the + * log into rebuild generations, while `(conversation_id, seq)` is the stable + * event identity and lease-fencing tripwire. + */ +export const juniorConversationEvents = pgTable( + "junior_conversation_events", + { + conversationId: text("conversation_id").notNull(), + seq: integer("seq").notNull(), + contextEpoch: integer("context_epoch").notNull(), + schemaVersion: integer("schema_version").default(1).notNull(), + type: text("type").notNull(), + role: text("role"), + payload: jsonb("payload").$type>().notNull(), + createdAt: timestamptz("created_at").notNull(), + }, + (table) => [ + primaryKey({ + name: "junior_conversation_events_conversation_id_seq_pk", + columns: [table.conversationId, table.seq], + }), + foreignKey({ + name: "junior_conversation_events_conversation_id_junior_conversations_conversation_id_fk", + columns: [table.conversationId], + foreignColumns: [juniorConversations.conversationId], + }), + index("junior_conversation_events_epoch_idx").on( + table.conversationId, + table.contextEpoch, + table.seq, + ), + ], +); diff --git a/packages/junior/src/handlers/mcp-oauth-callback.ts b/packages/junior/src/handlers/mcp-oauth-callback.ts index 05162d3a9..db1edb414 100644 --- a/packages/junior/src/handlers/mcp-oauth-callback.ts +++ b/packages/junior/src/handlers/mcp-oauth-callback.ts @@ -1,7 +1,7 @@ /** * MCP OAuth callback handler. * - * This handler finalizes provider OAuth, updates pending-auth/session-log state, + * This handler finalizes provider OAuth, updates pending-auth/event-log state, * and resumes the exact Slack turn that parked on MCP auth. Stale callbacks * must not resume newer thread work after another user message has superseded * the paused request. diff --git a/packages/junior/src/handlers/oauth-callback.ts b/packages/junior/src/handlers/oauth-callback.ts index 623b998ad..8b45a5217 100644 --- a/packages/junior/src/handlers/oauth-callback.ts +++ b/packages/junior/src/handlers/oauth-callback.ts @@ -537,7 +537,7 @@ async function resumePendingOAuthMessage( replyContext: { input: { conversationContext, - // Pi history is SQL-authoritative via the step-store projection. + // Pi history is SQL-authoritative via the event-store projection. piMessages: await loadProjection({ conversationId: threadId }), }, routing: { diff --git a/packages/junior/tests/component/cli/conversation-event-migration.test.ts b/packages/junior/tests/component/cli/conversation-event-migration.test.ts new file mode 100644 index 000000000..371068599 --- /dev/null +++ b/packages/junior/tests/component/cli/conversation-event-migration.test.ts @@ -0,0 +1,374 @@ +import path from "node:path"; +import { readFileSync } from "node:fs"; +import { describe, expect, it, vi } from "vitest"; +import { getStateAdapter } from "@/chat/state/adapter"; +import { migrateConversationEventData } from "@/cli/upgrade/migrations/conversation-event-data"; +import type { JuniorSqlExecutor } from "@/db/db"; +import { createLocalJuniorSqlFixture } from "../../fixtures/sql"; + +const migrationsFolder = path.resolve( + import.meta.dirname, + "../../../migrations", +); + +function migrationStatements(name: string): string[] { + return readFileSync(path.join(migrationsFolder, name), "utf8").split( + "--> statement-breakpoint", + ); +} + +async function executeStatements( + execute: (statement: string) => Promise, + statements: readonly string[], +): Promise { + for (const statement of statements) { + if (statement.trim()) { + await execute(statement); + } + } +} + +describe("conversation event migration", () => { + it("advances batches by the last rewritten event key", async () => { + const query = vi + .fn() + .mockResolvedValueOnce([ + { conversation_id: "conversation-one", seq: 3 }, + { conversation_id: "conversation-two", seq: 1 }, + ]) + .mockResolvedValueOnce([{ conversation_id: "conversation-two", seq: 4 }]) + .mockResolvedValueOnce([]) + .mockResolvedValueOnce([{ count: 0 }]); + const executor = { + query, + withLock: async (_name: string, callback: () => Promise) => + callback(), + } as unknown as JuniorSqlExecutor; + + await expect( + migrateConversationEventData( + { io: { info: () => {} }, stateAdapter: getStateAdapter() }, + { batchSize: 2, executor }, + ), + ).resolves.toEqual({ + existing: 0, + migrated: 3, + missing: 0, + scanned: 3, + }); + expect(query.mock.calls.map(([, parameters]) => parameters)).toEqual([ + [2, null, null], + [2, "conversation-two", 1], + [2, "conversation-two", 4], + undefined, + ]); + }); + + it("fails closed when skipped locked legacy rows remain", async () => { + const query = vi + .fn() + .mockResolvedValueOnce([]) + .mockResolvedValueOnce([{ count: 1 }]); + const executor = { + query, + withLock: async (_name: string, callback: () => Promise) => + callback(), + } as unknown as JuniorSqlExecutor; + + await expect( + migrateConversationEventData( + { io: { info: () => {} }, stateAdapter: getStateAdapter() }, + { batchSize: 10, executor }, + ), + ).rejects.toThrow( + "Conversation event migration left 1 locked legacy row(s)", + ); + }); + + it("fails closed when the remaining-row aggregate is missing", async () => { + const query = vi + .fn() + .mockResolvedValueOnce([]) + .mockResolvedValueOnce([]); + const executor = { + query, + withLock: async (_name: string, callback: () => Promise) => + callback(), + } as unknown as JuniorSqlExecutor; + + await expect( + migrateConversationEventData( + { io: { info: () => {} }, stateAdapter: getStateAdapter() }, + { executor }, + ), + ).rejects.toThrow( + "Conversation event migration could not verify that all legacy rows were rewritten", + ); + }); + + it("renames history, preserves rows, and provides rolling compatibility", async () => { + const fixture = await createLocalJuniorSqlFixture(); + const initial = migrationStatements("0000_initial.sql"); + const conversationEvents = migrationStatements( + "0004_conversation_events.sql", + ); + + try { + await executeStatements( + (statement) => fixture.sql.execute(statement), + initial, + ); + await fixture.sql.execute( + `INSERT INTO junior_conversations ( + conversation_id, + created_at, + last_activity_at, + updated_at, + execution_status + ) VALUES ($1, $2, $2, $2, 'idle')`, + ["conversation-one", new Date("2026-07-14T10:00:00.000Z")], + ); + await fixture.sql.execute( + `INSERT INTO junior_agent_steps ( + conversation_id, + seq, + context_epoch, + type, + role, + payload, + created_at + ) VALUES + ($1, 2, 3, 'authorization_completed', NULL, $2::jsonb, $4), + ($1, 1, 2, 'pi_message', 'assistant', $3::jsonb, $4)`, + [ + "conversation-one", + JSON.stringify({ requestId: "request-one" }), + JSON.stringify({ + schemaVersion: 7, + message: { + role: "assistant", + schemaVersion: "message-owned", + }, + }), + new Date("2026-07-14T10:01:00.000Z"), + ], + ); + + await executeStatements( + (statement) => fixture.sql.execute(statement), + conversationEvents, + ); + + await expect( + fixture.sql.query<{ + conversationId: string; + contextEpoch: number; + createdAtPreserved: boolean; + role: string | null; + schemaVersion: number; + seq: number; + type: string; + }>(` +SELECT + conversation_id AS "conversationId", + seq, + context_epoch AS "contextEpoch", + created_at = '2026-07-14T10:01:00.000Z'::timestamptz AS "createdAtPreserved", + schema_version AS "schemaVersion", + type, + role +FROM junior_conversation_events +ORDER BY seq +`), + ).resolves.toEqual([ + { + contextEpoch: 2, + conversationId: "conversation-one", + createdAtPreserved: true, + role: "assistant", + schemaVersion: 1, + seq: 1, + type: "pi_message", + }, + { + contextEpoch: 3, + conversationId: "conversation-one", + createdAtPreserved: true, + role: null, + schemaVersion: 1, + seq: 2, + type: "authorization_completed", + }, + ]); + + await expect( + migrateConversationEventData( + { io: { info: () => {} }, stateAdapter: getStateAdapter() }, + { batchSize: 1, executor: fixture.sql }, + ), + ).resolves.toEqual({ + existing: 0, + migrated: 1, + missing: 0, + scanned: 1, + }); + + const canonicalRows = await fixture.sql.query<{ + payload: Record; + schemaVersion: number; + seq: number; + type: string; + }>(` +SELECT + seq, + schema_version AS "schemaVersion", + type, + payload +FROM junior_conversation_events +ORDER BY seq +`); + expect(canonicalRows).toEqual([ + { + payload: { + message: { + role: "assistant", + schemaVersion: "message-owned", + }, + }, + schemaVersion: 1, + seq: 1, + type: "message", + }, + { + payload: { requestId: "request-one" }, + schemaVersion: 1, + seq: 2, + type: "authorization_completed", + }, + ]); + await expect( + fixture.sql.query<{ count: number }>( + `SELECT count(*)::integer AS count + FROM junior_conversation_events + WHERE type = 'pi_message'`, + ), + ).resolves.toEqual([{ count: 0 }]); + await expect( + fixture.sql.query<{ seq: number; type: string }>( + `SELECT seq, type FROM junior_agent_steps ORDER BY seq`, + ), + ).resolves.toEqual([ + { seq: 1, type: "pi_message" }, + { seq: 2, type: "authorization_completed" }, + ]); + + await fixture.sql.execute( + `INSERT INTO junior_agent_steps ( + conversation_id, + seq, + context_epoch, + type, + role, + payload, + created_at + ) VALUES ($1, 3, 3, 'pi_message', 'user', $2::jsonb, $3)`, + [ + "conversation-one", + JSON.stringify({ + schemaVersion: 99, + message: { role: "user", content: "hello" }, + }), + new Date("2026-07-14T10:02:00.000Z"), + ], + ); + await expect( + fixture.sql.query<{ + payload: Record; + schemaVersion: number; + type: string; + }>( + ` +SELECT + schema_version AS "schemaVersion", + type, + payload +FROM junior_conversation_events +WHERE conversation_id = $1 AND seq = 3 +`, + ["conversation-one"], + ), + ).resolves.toEqual([ + { + payload: { message: { role: "user", content: "hello" } }, + schemaVersion: 1, + type: "message", + }, + ]); + + await fixture.sql.execute( + `DELETE FROM junior_agent_steps + WHERE conversation_id = $1 AND seq = 3`, + ["conversation-one"], + ); + await expect( + fixture.sql.query<{ count: number }>( + `SELECT count(*)::integer AS count + FROM junior_conversation_events + WHERE conversation_id = $1 AND seq = 3`, + ["conversation-one"], + ), + ).resolves.toEqual([{ count: 0 }]); + + await expect( + migrateConversationEventData( + { io: { info: () => {} }, stateAdapter: getStateAdapter() }, + { batchSize: 1, executor: fixture.sql }, + ), + ).resolves.toEqual({ + existing: 0, + migrated: 0, + missing: 0, + scanned: 0, + }); + await expect( + fixture.sql.query<{ name: string; type: string }>(` +SELECT table_name AS name, table_type AS type +FROM information_schema.tables +WHERE table_schema = 'public' + AND table_name IN ('junior_agent_steps', 'junior_conversation_events') +ORDER BY table_name +`), + ).resolves.toEqual([ + { name: "junior_agent_steps", type: "VIEW" }, + { name: "junior_conversation_events", type: "BASE TABLE" }, + ]); + await expect( + fixture.sql.query<{ name: string }>(` +SELECT conname AS name +FROM pg_constraint +WHERE conrelid = 'junior_conversation_events'::regclass +ORDER BY conname +`), + ).resolves.toEqual([ + { + name: "junior_conversation_events_conversation_id_junior_conversations", + }, + { name: "junior_conversation_events_conversation_id_seq_pk" }, + ]); + await expect( + fixture.sql.query<{ name: string }>(` +SELECT indexname AS name +FROM pg_indexes +WHERE schemaname = 'public' + AND tablename = 'junior_conversation_events' +ORDER BY indexname +`), + ).resolves.toEqual([ + { name: "junior_conversation_events_conversation_id_seq_pk" }, + { name: "junior_conversation_events_epoch_idx" }, + ]); + } finally { + await fixture.close(); + } + }, 15_000); +}); diff --git a/packages/junior/tests/component/cli/upgrade-cli.test.ts b/packages/junior/tests/component/cli/upgrade-cli.test.ts index 0a3cbbd3b..7d6691e23 100644 --- a/packages/junior/tests/component/cli/upgrade-cli.test.ts +++ b/packages/junior/tests/component/cli/upgrade-cli.test.ts @@ -12,7 +12,7 @@ import { requestConversationWork, } from "@/chat/task-execution/store"; import { createSqlStore } from "@/chat/conversations/sql/store"; -import { createSqlAgentStepStore } from "@/chat/conversations/sql/history"; +import { createSqlConversationEventStore } from "@/chat/conversations/sql/history"; import { migrateSchema } from "@/chat/conversations/sql/migrations"; import type { ConversationStore } from "@/chat/conversations/store"; import type { PiMessage } from "@/chat/pi/messages"; @@ -551,7 +551,7 @@ WHERE conversation_id = $1 await stateAdapter.connect(); const fixture = await createLocalJuniorSqlFixture(); const sqlStore = createSqlStore(fixture.sql); - const stepStore = createSqlAgentStepStore(fixture.sql); + const eventStore = createSqlConversationEventStore(fixture.sql); const mixedConversationId = `${CONVERSATION_ID}:mixed`; const unsafeConversationId = `${CONVERSATION_ID}:unsafe`; const firstAssistant = { @@ -620,21 +620,21 @@ WHERE conversation_id = $1 }, updatedAtMs: 4_000, }); - await stepStore.startEpoch(CONVERSATION_ID, { + await eventStore.startEpoch(CONVERSATION_ID, { reason: "initial", modelProfile: "standard", modelId: "test-model", messages: [{ message: firstAssistant, createdAtMs: 2_000 }], }); - await stepStore.startEpoch(CONVERSATION_ID, { + await eventStore.startEpoch(CONVERSATION_ID, { reason: "compaction", modelProfile: "standard", modelId: "test-model", messages: [{ message: firstAssistant, createdAtMs: 2_000 }], }); - await stepStore.append(CONVERSATION_ID, [ + await eventStore.append(CONVERSATION_ID, [ { - entry: { type: "pi_message", message: secondAssistant }, + data: { type: "message", message: secondAssistant }, createdAtMs: 3_000, }, ]); @@ -646,10 +646,10 @@ WHERE conversation_id = $1 metrics: { durationMs: 1_000, usage: { totalTokens: 777 } }, updatedAtMs: 4_000, }); - await stepStore.append(mixedConversationId, [ + await eventStore.append(mixedConversationId, [ { - entry: { - type: "pi_message", + data: { + type: "message", message: { role: "assistant", usage: { input: 3, totalTokens: 999 }, @@ -658,8 +658,8 @@ WHERE conversation_id = $1 createdAtMs: 2_000, }, { - entry: { - type: "pi_message", + data: { + type: "message", message: { role: "assistant", usage: { totalTokens: 5 }, @@ -676,10 +676,10 @@ WHERE conversation_id = $1 metrics: { durationMs: 1_000, usage: { totalTokens: 777 } }, updatedAtMs: 4_000, }); - await stepStore.append(unsafeConversationId, [ + await eventStore.append(unsafeConversationId, [ { - entry: { - type: "pi_message", + data: { + type: "message", message: { role: "assistant", usage: { input: 9_007_199_254_740_992 }, @@ -765,7 +765,7 @@ WHERE conversation_id = $1 await stateAdapter.connect(); const fixture = await createLocalJuniorSqlFixture(); const sqlStore = createSqlStore(fixture.sql); - const stepStore = createSqlAgentStepStore(fixture.sql); + const eventStore = createSqlConversationEventStore(fixture.sql); const conversationIds = ["local:usage-batch-a", "local:usage-batch-b"]; try { @@ -779,10 +779,10 @@ WHERE conversation_id = $1 metrics: { durationMs: 1_000, usage: { totalTokens: 999 } }, updatedAtMs: 2_000, }); - await stepStore.append(conversationId, [ + await eventStore.append(conversationId, [ { - entry: { - type: "pi_message", + data: { + type: "message", message: { role: "assistant", usage: { input: index + 1, totalTokens: index + 1 }, @@ -901,7 +901,7 @@ WHERE conversation_id = $1 await stateAdapter.connect(); const fixture = await createLocalJuniorSqlFixture(); const sqlStore = createSqlStore(fixture.sql); - const stepStore = createSqlAgentStepStore(fixture.sql); + const eventStore = createSqlConversationEventStore(fixture.sql); try { await migrateSchema(fixture.sql); @@ -920,10 +920,10 @@ WHERE conversation_id = $1 }, updatedAtMs: 2_000, }); - await stepStore.append(CONVERSATION_ID, [ + await eventStore.append(CONVERSATION_ID, [ { - entry: { - type: "pi_message", + data: { + type: "message", message: { role: "assistant", usage: { input: 4, output: 2, totalTokens: 6 }, diff --git a/packages/junior/tests/component/conversation-transcripts-sql.test.ts b/packages/junior/tests/component/conversation-transcripts-sql.test.ts index 89d47b208..f46bac02d 100644 --- a/packages/junior/tests/component/conversation-transcripts-sql.test.ts +++ b/packages/junior/tests/component/conversation-transcripts-sql.test.ts @@ -1,8 +1,11 @@ import { eq } from "drizzle-orm"; import { describe, expect, it } from "vitest"; -import { createSqlAgentStepStore } from "@/chat/conversations/sql/history"; -import { agentStepEntrySchema } from "@/chat/conversations/history"; -import { getAgentStepStore } from "@/chat/db"; +import { createSqlConversationEventStore } from "@/chat/conversations/sql/history"; +import { + conversationEventDataSchema, + conversationEventSchema, +} from "@/chat/conversations/history"; +import { getConversationEventStore } from "@/chat/db"; import { purgeConversation } from "@/chat/conversations/retention"; import { createSqlConversationMessageStore } from "@/chat/conversations/sql/messages"; import { @@ -12,7 +15,7 @@ import { import { coerceThreadConversationState } from "@/chat/state/conversation"; import { migrateSchema } from "@/chat/conversations/sql/migrations"; import type { JuniorSqlDatabase } from "@/db/db"; -import { juniorAgentSteps, juniorConversations } from "@/db/schema"; +import { juniorConversationEvents, juniorConversations } from "@/db/schema"; import { buildJuniorSqlConversation, createLocalJuniorSqlFixture, @@ -29,14 +32,14 @@ const CHILD_CONVERSATION_ID = "advisor:child-1"; it("accepts legacy markers and validates current profile names", () => { expect( - agentStepEntrySchema.safeParse({ + conversationEventDataSchema.safeParse({ type: "context_epoch_started", reason: "initial", modelProfile: "standard", }).success, ).toBe(false); expect( - agentStepEntrySchema.safeParse({ + conversationEventDataSchema.safeParse({ type: "context_epoch_started", reason: "initial", modelProfile: "standard", @@ -44,20 +47,20 @@ it("accepts legacy markers and validates current profile names", () => { }).success, ).toBe(true); expect( - agentStepEntrySchema.safeParse({ + conversationEventDataSchema.safeParse({ type: "context_epoch_started", reason: "handoff", }).success, ).toBe(false); expect( - agentStepEntrySchema.safeParse({ + conversationEventDataSchema.safeParse({ type: "context_epoch_started", reason: "handoff", modelProfile: "handoff", }).success, ).toBe(false); expect( - agentStepEntrySchema.safeParse({ + conversationEventDataSchema.safeParse({ type: "context_epoch_started", reason: "handoff", modelProfile: "standard", @@ -65,7 +68,7 @@ it("accepts legacy markers and validates current profile names", () => { }).success, ).toBe(false); expect( - agentStepEntrySchema.safeParse({ + conversationEventDataSchema.safeParse({ type: "context_epoch_started", reason: "compaction", modelProfile: "Fast!", @@ -73,13 +76,13 @@ it("accepts legacy markers and validates current profile names", () => { }).success, ).toBe(false); expect( - agentStepEntrySchema.safeParse({ + conversationEventDataSchema.safeParse({ type: "context_epoch_started", reason: "compaction", }).success, ).toBe(true); expect( - agentStepEntrySchema.safeParse({ + conversationEventDataSchema.safeParse({ type: "context_epoch_started", reason: "compaction", modelProfile: "coding", @@ -87,14 +90,14 @@ it("accepts legacy markers and validates current profile names", () => { }).success, ).toBe(true); expect( - agentStepEntrySchema.safeParse({ + conversationEventDataSchema.safeParse({ type: "context_epoch_started", reason: "compaction", modelProfile: "coding", }).success, ).toBe(false); expect( - agentStepEntrySchema.safeParse({ + conversationEventDataSchema.safeParse({ type: "context_epoch_started", reason: "compaction", modelId: "openai/gpt-5.4", @@ -102,11 +105,53 @@ it("accepts legacy markers and validates current profile names", () => { ).toBe(false); }); +it("rejects unknown Junior event fields while retaining opaque message fields", () => { + expect( + conversationEventDataSchema.safeParse({ + type: "mcp_provider_connected", + provider: "github", + unknown: true, + }).success, + ).toBe(false); + expect( + conversationEventDataSchema.safeParse({ + type: "visible_context_compacted", + compactions: [ + { + coveredMessageIds: ["m1"], + createdAtMs: 1_000, + id: "compaction-1", + summary: "Earlier context", + unknown: true, + }, + ], + }).success, + ).toBe(false); + expect( + conversationEventDataSchema.safeParse({ + type: "message", + message: { role: "", providerOwnedField: true }, + }).success, + ).toBe(true); +}); + +it("rejects unsupported conversation event schema versions", () => { + expect( + conversationEventSchema.safeParse({ + schemaVersion: 2, + seq: 0, + contextEpoch: 0, + createdAtMs: 1_000, + data: { type: "mcp_provider_connected", provider: "github" }, + }).success, + ).toBe(false); +}); + it("rejects epoch markers through the ordinary append boundary", async () => { await expect( - getAgentStepStore().append("local:test:invalid-marker-append", [ + getConversationEventStore().append("local:test:invalid-marker-append", [ { - entry: { + data: { type: "context_epoch_started", reason: "compaction", }, @@ -119,14 +164,14 @@ it("rejects epoch markers through the ordinary append boundary", async () => { it("rejects incomplete markers through the epoch boundary", async () => { const conversationId = "local:test:invalid-marker-start"; await expect( - getAgentStepStore().startEpoch(conversationId, { + getConversationEventStore().startEpoch(conversationId, { reason: "handoff", modelProfile: "handoff", messages: [], } as never), ).rejects.toThrow("Invalid input"); await expect( - getAgentStepStore().loadHistory(conversationId), + getConversationEventStore().loadHistory(conversationId), ).resolves.toEqual([]); }); @@ -147,21 +192,23 @@ it("opens an explicit initial epoch without dropping earlier host facts", async await expect(loadConnectedMcpProviders({ conversationId })).resolves.toEqual([ "linear", ]); - expect(await getAgentStepStore().loadHistory(conversationId)).toEqual([ - expect.objectContaining({ - contextEpoch: 0, - entry: expect.objectContaining({ type: "mcp_provider_connected" }), - }), - expect.objectContaining({ - contextEpoch: 0, - entry: { - type: "context_epoch_started", - reason: "initial", - modelProfile: "standard", - modelId: "openai/gpt-5.4", - }, - }), - ]); + expect(await getConversationEventStore().loadHistory(conversationId)).toEqual( + [ + expect.objectContaining({ + contextEpoch: 0, + data: expect.objectContaining({ type: "mcp_provider_connected" }), + }), + expect.objectContaining({ + contextEpoch: 0, + data: { + type: "context_epoch_started", + reason: "initial", + modelProfile: "standard", + modelId: "openai/gpt-5.4", + }, + }), + ], + ); }); async function seedConversation( @@ -189,8 +236,8 @@ function userMessage(text: string) { } describe("conversation transcript SQL stores", () => { - it("persists visible-context compaction snapshots in agent history", async () => { - const steps = getAgentStepStore(); + it("persists visible-context compaction snapshots in conversation history", async () => { + const events = getConversationEventStore(); const conversation = coerceThreadConversationState({}); conversation.compactions = [ { @@ -216,7 +263,7 @@ describe("conversation transcript SQL stores", () => { conversationId: CONVERSATION_ID, }); expect(rehydrated.compactions).toEqual(conversation.compactions); - expect(await steps.loadHistory(CONVERSATION_ID)).toHaveLength(1); + expect(await events.loadHistory(CONVERSATION_ID)).toHaveLength(1); }); it("applies Drizzle migrations idempotently", async () => { @@ -241,30 +288,31 @@ describe("conversation transcript SQL stores", () => { try { await migrateSchema(fixture.sql); await seedConversation(fixture, CONVERSATION_ID); - const store = createSqlAgentStepStore(fixture.sql); + const store = createSqlConversationEventStore(fixture.sql); await store.append(CONVERSATION_ID, [ { - entry: { type: "pi_message", message: userMessage("one") }, + data: { type: "message", message: userMessage("one") }, createdAtMs: 1_000, }, { - entry: { type: "pi_message", message: userMessage("two") }, + data: { type: "message", message: userMessage("two") }, createdAtMs: 2_000, }, ]); await store.append(CONVERSATION_ID, [ { - entry: { type: "mcp_provider_connected", provider: "github" }, + data: { type: "mcp_provider_connected", provider: "github" }, createdAtMs: 3_000, }, ]); const history = await store.loadHistory(CONVERSATION_ID); - expect(history.map((step) => step.seq)).toEqual([0, 1, 2]); - expect(history.map((step) => step.entry.type)).toEqual([ - "pi_message", - "pi_message", + expect(history.map((event) => event.seq)).toEqual([0, 1, 2]); + expect(history.map((event) => event.schemaVersion)).toEqual([1, 1, 1]); + expect(history.map((event) => event.data.type)).toEqual([ + "message", + "message", "mcp_provider_connected", ]); @@ -272,12 +320,13 @@ describe("conversation transcript SQL stores", () => { await expect( fixture.sql .db() - .insert(juniorAgentSteps) + .insert(juniorConversationEvents) .values({ conversationId: CONVERSATION_ID, seq: 0, contextEpoch: 0, - type: "pi_message", + schemaVersion: 1, + type: "message", role: "user", payload: { message: userMessage("clobber") }, createdAt: new Date(4_000), @@ -288,32 +337,34 @@ describe("conversation transcript SQL stores", () => { } }); - it("replaces NUL characters before persisting agent steps", async () => { + it("replaces NUL characters before persisting conversation events", async () => { const fixture = await createLocalJuniorSqlFixture(); try { await migrateSchema(fixture.sql); await seedConversation(fixture, CONVERSATION_ID); - const store = createSqlAgentStepStore(fixture.sql); + const store = createSqlConversationEventStore(fixture.sql); await store.append(CONVERSATION_ID, [ { - entry: { - type: "pi_message", + data: { + type: "message", message: userMessage("before\u0000after and literal \\u0000"), }, createdAtMs: 1_000, }, ]); - expect( - (await store.loadHistory(CONVERSATION_ID))[0]?.entry, - ).toMatchObject({ - type: "pi_message", - message: { - content: [{ text: "before after and literal \\u0000", type: "text" }], + expect((await store.loadHistory(CONVERSATION_ID))[0]?.data).toMatchObject( + { + type: "message", + message: { + content: [ + { text: "before after and literal \\u0000", type: "text" }, + ], + }, }, - }); + ); } finally { await fixture.close(); } @@ -325,15 +376,15 @@ describe("conversation transcript SQL stores", () => { try { await migrateSchema(fixture.sql); await seedConversation(fixture, CONVERSATION_ID); - const store = createSqlAgentStepStore(fixture.sql); + const store = createSqlConversationEventStore(fixture.sql); await store.append(CONVERSATION_ID, [ { - entry: { type: "pi_message", message: userMessage("epoch0-a") }, + data: { type: "message", message: userMessage("epoch0-a") }, createdAtMs: 1_000, }, { - entry: { type: "pi_message", message: userMessage("epoch0-b") }, + data: { type: "message", message: userMessage("epoch0-b") }, createdAtMs: 2_000, }, ]); @@ -347,15 +398,15 @@ describe("conversation transcript SQL stores", () => { }); const current = await store.loadCurrentEpoch(CONVERSATION_ID); - expect(current.map((step) => step.contextEpoch)).toEqual([1, 1]); - expect(current.map((step) => step.entry.type)).toEqual([ + expect(current.map((event) => event.contextEpoch)).toEqual([1, 1]); + expect(current.map((event) => event.data.type)).toEqual([ "context_epoch_started", - "pi_message", + "message", ]); - expect(current.map((step) => step.seq)).toEqual([2, 3]); + expect(current.map((event) => event.seq)).toEqual([2, 3]); const history = await store.loadHistory(CONVERSATION_ID); - expect(history.map((step) => step.contextEpoch)).toEqual([0, 0, 1, 1]); + expect(history.map((event) => event.contextEpoch)).toEqual([0, 0, 1, 1]); } finally { await fixture.close(); } @@ -367,8 +418,8 @@ describe("conversation transcript SQL stores", () => { try { await migrateSchema(fixture.sql); await seedConversation(fixture, CONVERSATION_ID); - const store = createSqlAgentStepStore(fixture.sql); - const entry = { + const store = createSqlConversationEventStore(fixture.sql); + const data = { type: "subagent_started" as const, subagentInvocationId: "future-subagent-call", subagentKind: "task", @@ -376,11 +427,9 @@ describe("conversation transcript SQL stores", () => { historyMode: "isolated" as const, }; - await store.append(CONVERSATION_ID, [{ entry, createdAtMs: 1_000 }]); + await store.append(CONVERSATION_ID, [{ data, createdAtMs: 1_000 }]); - expect((await store.loadHistory(CONVERSATION_ID))[0]?.entry).toEqual( - entry, - ); + expect((await store.loadHistory(CONVERSATION_ID))[0]?.data).toEqual(data); } finally { await fixture.close(); } @@ -392,10 +441,10 @@ describe("conversation transcript SQL stores", () => { try { await migrateSchema(fixture.sql); await seedConversation(fixture, CONVERSATION_ID); - const store = createSqlAgentStepStore(fixture.sql); + const store = createSqlConversationEventStore(fixture.sql); await store.append(CONVERSATION_ID, [ { - entry: { type: "pi_message", message: userMessage("epoch0") }, + data: { type: "message", message: userMessage("epoch0") }, createdAtMs: 1_000, }, ]); @@ -410,7 +459,7 @@ describe("conversation transcript SQL stores", () => { throw new Error("epoch write failed"); }), }; - const failingStore = createSqlAgentStepStore(failing); + const failingStore = createSqlConversationEventStore(failing); await expect( failingStore.startEpoch(CONVERSATION_ID, { @@ -422,42 +471,90 @@ describe("conversation transcript SQL stores", () => { ).rejects.toThrow("epoch write failed"); const history = await store.loadHistory(CONVERSATION_ID); - expect(history.map((step) => step.contextEpoch)).toEqual([0]); + expect(history.map((event) => event.contextEpoch)).toEqual([0]); expect( - history.some((step) => step.entry.type === "context_epoch_started"), + history.some((event) => event.data.type === "context_epoch_started"), ).toBe(false); } finally { await fixture.close(); } }); - it("fails loudly when a stored step has an unknown type", async () => { + it.each([ + { schemaVersion: 1, type: "bogus_type" }, + { schemaVersion: 2, type: "mcp_provider_connected" }, + ])( + "fails loudly for unsupported stored event envelopes %#", + async ({ schemaVersion, type }) => { + const fixture = await createLocalJuniorSqlFixture(); + + try { + await migrateSchema(fixture.sql); + await seedConversation(fixture, CONVERSATION_ID); + const store = createSqlConversationEventStore(fixture.sql); + + await fixture.sql.execute( + ` +INSERT INTO junior_conversation_events ( + conversation_id, seq, context_epoch, schema_version, type, role, payload, created_at +) VALUES ($1, $2, $3, $4, $5, NULL, $6::jsonb, $7) +`, + [ + CONVERSATION_ID, + 0, + 0, + schemaVersion, + type, + JSON.stringify( + type === "mcp_provider_connected" ? { provider: "github" } : {}, + ), + new Date(1_000).toISOString(), + ], + ); + + await expect(store.loadHistory(CONVERSATION_ID)).rejects.toThrow( + /Invalid input/, + ); + } finally { + await fixture.close(); + } + }, + ); + + it("uses physical event columns as authoritative when decoding rows", async () => { const fixture = await createLocalJuniorSqlFixture(); try { await migrateSchema(fixture.sql); await seedConversation(fixture, CONVERSATION_ID); - const store = createSqlAgentStepStore(fixture.sql); + const store = createSqlConversationEventStore(fixture.sql); await fixture.sql.execute( ` -INSERT INTO junior_agent_steps ( - conversation_id, seq, context_epoch, type, role, payload, created_at -) VALUES ($1, $2, $3, $4, NULL, $5::jsonb, $6) +INSERT INTO junior_conversation_events ( + conversation_id, seq, context_epoch, schema_version, type, role, payload, created_at +) VALUES ($1, $2, $3, $4, $5, NULL, $6::jsonb, $7) `, [ CONVERSATION_ID, 0, 0, - "bogus_type", - "{}", + 1, + "mcp_provider_connected", + JSON.stringify({ type: "message", provider: "github" }), new Date(1_000).toISOString(), ], ); - await expect(store.loadHistory(CONVERSATION_ID)).rejects.toThrow( - /Invalid input/, - ); + await expect(store.loadHistory(CONVERSATION_ID)).resolves.toEqual([ + { + schemaVersion: 1, + seq: 0, + contextEpoch: 0, + createdAtMs: 1_000, + data: { type: "mcp_provider_connected", provider: "github" }, + }, + ]); } finally { await fixture.close(); } @@ -533,7 +630,7 @@ INSERT INTO junior_agent_steps ( .set({ lastActivityAt: new Date(1_000) }) .where(eq(juniorConversations.conversationId, CONVERSATION_ID)); const messages = createSqlConversationMessageStore(fixture.sql); - const steps = createSqlAgentStepStore(fixture.sql); + const events = createSqlConversationEventStore(fixture.sql); // A newer message advances the clock (append-refresh semantics). await messages.record(CONVERSATION_ID, [ @@ -562,17 +659,17 @@ INSERT INTO junior_agent_steps ( ]); expect(await lastActivityMs()).toBe(6_500); - // Step appends advance the clock too, and also never regress it. - await steps.append(CONVERSATION_ID, [ + // Event appends advance the clock too, and also never regress it. + await events.append(CONVERSATION_ID, [ { - entry: { type: "pi_message", message: userMessage("newest") }, + data: { type: "message", message: userMessage("newest") }, createdAtMs: 8_000, }, ]); expect(await lastActivityMs()).toBe(8_000); - await steps.append(CONVERSATION_ID, [ + await events.append(CONVERSATION_ID, [ { - entry: { type: "pi_message", message: userMessage("backdated") }, + data: { type: "message", message: userMessage("backdated") }, createdAtMs: 3_000, }, ]); @@ -582,20 +679,20 @@ INSERT INTO junior_agent_steps ( } }); - it("purges steps and messages for a conversation and its descendants", async () => { + it("purges events and messages for a conversation and its descendants", async () => { const fixture = await createLocalJuniorSqlFixture(); try { await migrateSchema(fixture.sql); await seedConversation(fixture, CONVERSATION_ID); await seedConversation(fixture, CHILD_CONVERSATION_ID, CONVERSATION_ID); - const steps = createSqlAgentStepStore(fixture.sql); + const events = createSqlConversationEventStore(fixture.sql); const messages = createSqlConversationMessageStore(fixture.sql); for (const conversationId of [CONVERSATION_ID, CHILD_CONVERSATION_ID]) { - await steps.append(conversationId, [ + await events.append(conversationId, [ { - entry: { type: "pi_message", message: userMessage("hi") }, + data: { type: "message", message: userMessage("hi") }, createdAtMs: 1_000, }, ]); @@ -609,7 +706,7 @@ INSERT INTO junior_agent_steps ( }); for (const conversationId of [CONVERSATION_ID, CHILD_CONVERSATION_ID]) { - expect(await steps.loadHistory(conversationId)).toEqual([]); + expect(await events.loadHistory(conversationId)).toEqual([]); expect(await messages.list(conversationId)).toEqual([]); } @@ -624,9 +721,9 @@ INSERT INTO junior_agent_steps ( expect(rows).toHaveLength(1); expect(rows[0]?.transcriptPurgedAt).toBeInstanceOf(Date); - await steps.append(CONVERSATION_ID, [ + await events.append(CONVERSATION_ID, [ { - entry: { type: "pi_message", message: userMessage("new history") }, + data: { type: "message", message: userMessage("new history") }, createdAtMs: 6_000, }, ]); @@ -639,7 +736,7 @@ INSERT INTO junior_agent_steps ( }, ]); - expect(await steps.loadHistory(CONVERSATION_ID)).toHaveLength(1); + expect(await events.loadHistory(CONVERSATION_ID)).toHaveLength(1); expect(await messages.list(CONVERSATION_ID)).toHaveLength(1); const reopened = await fixture.sql .db() diff --git a/packages/junior/tests/component/conversations/legacy-import.test.ts b/packages/junior/tests/component/conversations/legacy-import.test.ts index fb14774db..7a54f27e1 100644 --- a/packages/junior/tests/component/conversations/legacy-import.test.ts +++ b/packages/junior/tests/component/conversations/legacy-import.test.ts @@ -3,7 +3,7 @@ import { eq, inArray } from "drizzle-orm"; import { juniorConversations } from "@/db/schema"; import { closeDb, - getAgentStepStore, + getConversationEventStore, getConversationMessageStore, getSqlExecutor, } from "@/chat/db"; @@ -13,7 +13,7 @@ import { ensureLegacyConversationImport, importConversationFromLegacy, } from "@/chat/conversations/legacy-import"; -import { createSqlAgentStepStore } from "@/chat/conversations/sql/history"; +import { createSqlConversationEventStore } from "@/chat/conversations/sql/history"; import { createSqlConversationMessageStore } from "@/chat/conversations/sql/messages"; import { migrateSchema } from "@/chat/conversations/sql/migrations"; import { hydrateConversationMessages } from "@/chat/conversations/visible-messages"; @@ -43,7 +43,7 @@ async function processSqlStores() { await migrateSchema(executor); return { executor, - stepStore: getAgentStepStore(), + eventStore: getConversationEventStore(), messageStore: getConversationMessageStore(), }; } @@ -86,7 +86,6 @@ function conversationRecord(): Conversation { function staticSessionLogStore(entries: SessionLogEntry[]): SessionLogStore { return { read: async () => entries, - append: async () => {}, }; } @@ -110,10 +109,10 @@ describe("legacy conversation import", () => { vi.restoreAllMocks(); }); - it("imports steps, advisor child, and visible messages once, idempotently", async () => { + it("imports events, advisor child, and visible messages once, idempotently", async () => { const fixture = await createLocalJuniorSqlFixture(); await migrateSchema(fixture.sql); - const stepStore = createSqlAgentStepStore(fixture.sql); + const eventStore = createSqlConversationEventStore(fixture.sql); const messageStore = createSqlConversationMessageStore(fixture.sql); const childId = `advisor:${CONVERSATION_ID}`; @@ -175,31 +174,31 @@ describe("legacy conversation import", () => { importConversationFromLegacy(CONVERSATION_ID, deps), ).resolves.toEqual({ imported: true }); - const history = await stepStore.loadHistory(CONVERSATION_ID); + const history = await eventStore.loadHistory(CONVERSATION_ID); expect( - history.map((step) => ({ - seq: step.seq, - epoch: step.contextEpoch, - type: step.entry.type, + history.map((event) => ({ + seq: event.seq, + epoch: event.contextEpoch, + type: event.data.type, })), ).toEqual([ - { seq: 0, epoch: 0, type: "pi_message" }, + { seq: 0, epoch: 0, type: "message" }, { seq: 1, epoch: 1, type: "context_epoch_started" }, - { seq: 2, epoch: 1, type: "pi_message" }, + { seq: 2, epoch: 1, type: "message" }, { seq: 3, epoch: 1, type: "subagent_started" }, ]); // Current context is exactly the highest epoch's messages. - const current = await stepStore.loadCurrentEpoch(CONVERSATION_ID); + const current = await eventStore.loadCurrentEpoch(CONVERSATION_ID); expect( - current.filter((step) => step.entry.type === "pi_message"), + current.filter((event) => event.data.type === "message"), ).toHaveLength(1); - // Advisor child is its own conversation with epoch-0 pi_message rows. - const childHistory = await stepStore.loadHistory(childId); - expect(childHistory.map((step) => step.entry.type)).toEqual([ - "pi_message", - "pi_message", + // Advisor child is its own conversation with epoch-0 message events. + const childHistory = await eventStore.loadHistory(childId); + expect(childHistory.map((event) => event.data.type)).toEqual([ + "message", + "message", ]); expect(childHistory[0]!.createdAtMs).toBe(960); @@ -243,20 +242,20 @@ describe("legacy conversation import", () => { ]), ); - // Re-running is a no-op: step rows already exist. + // Re-running is a no-op: event rows already exist. await expect( importConversationFromLegacy(CONVERSATION_ID, deps), ).resolves.toEqual({ imported: false }); - expect(await stepStore.loadHistory(CONVERSATION_ID)).toHaveLength(4); + expect(await eventStore.loadHistory(CONVERSATION_ID)).toHaveLength(4); } finally { await fixture.close(); } }, 20_000); - it("rolls back steps when the transactional message import fails", async () => { + it("rolls back events when the transactional message import fails", async () => { const fixture = await createLocalJuniorSqlFixture(); await migrateSchema(fixture.sql); - const stepStore = createSqlAgentStepStore(fixture.sql); + const eventStore = createSqlConversationEventStore(fixture.sql); const messageStore = createSqlConversationMessageStore(fixture.sql); const entries = staticSessionLogStore([ @@ -285,8 +284,8 @@ describe("legacy conversation import", () => { 'Failed query: insert into "junior_conversation_messages"', ); - // Messages and steps share one transaction, so neither side commits. - expect(await stepStore.loadHistory(CONVERSATION_ID)).toHaveLength(0); + // Messages and events share one transaction, so neither side commits. + expect(await eventStore.loadHistory(CONVERSATION_ID)).toHaveLength(0); expect(await messageStore.list(CONVERSATION_ID)).toHaveLength(0); const visible: ThreadConversationMessage[] = [ @@ -303,7 +302,7 @@ describe("legacy conversation import", () => { }), ).resolves.toEqual({ imported: true }); - expect(await stepStore.loadHistory(CONVERSATION_ID)).toHaveLength(1); + expect(await eventStore.loadHistory(CONVERSATION_ID)).toHaveLength(1); const messages = await messageStore.list(CONVERSATION_ID); expect(messages.map((message) => message.messageId)).toEqual([ "m1", @@ -314,10 +313,10 @@ describe("legacy conversation import", () => { } }, 20_000); - it("seals a completed message-only import without step rows", async () => { + it("seals a completed message-only import without event rows", async () => { const fixture = await createLocalJuniorSqlFixture(); await migrateSchema(fixture.sql); - const stepStore = createSqlAgentStepStore(fixture.sql); + const eventStore = createSqlConversationEventStore(fixture.sql); const messageStore = createSqlConversationMessageStore(fixture.sql); const loadVisibleMessages = vi.fn(async () => [ { @@ -354,7 +353,7 @@ describe("legacy conversation import", () => { await expect( importConversationFromLegacy(CONVERSATION_ID, deps), ).resolves.toEqual({ imported: false }); - expect(await stepStore.loadHistory(CONVERSATION_ID)).toEqual([]); + expect(await eventStore.loadHistory(CONVERSATION_ID)).toEqual([]); expect(await messageStore.list(CONVERSATION_ID)).toMatchObject([ { messageId: "message-only", @@ -382,7 +381,7 @@ describe("legacy conversation import", () => { it("never fabricates import-time timestamps for timestamp-less rows", async () => { const fixture = await createLocalJuniorSqlFixture(); await migrateSchema(fixture.sql); - const stepStore = createSqlAgentStepStore(fixture.sql); + const eventStore = createSqlConversationEventStore(fixture.sql); const messageStore = createSqlConversationMessageStore(fixture.sql); const before = Date.now(); @@ -402,7 +401,7 @@ describe("legacy conversation import", () => { loadVisibleMessages: async () => [], }); - const history = await stepStore.loadHistory(CONVERSATION_ID); + const history = await eventStore.loadHistory(CONVERSATION_ID); expect(history).toHaveLength(1); // Falls back to the conversation record's createdAt, not Date.now(). expect(history[0]!.createdAtMs).toBe(500); @@ -413,7 +412,7 @@ describe("legacy conversation import", () => { }, 20_000); it("lazily imports a straggler with a Redis log but no SQL rows, once", async () => { - const { executor, stepStore } = await processSqlStores(); + const { executor, eventStore } = await processSqlStores(); const stateAdapter = getStateAdapter(); await stateAdapter.connect(); @@ -433,9 +432,9 @@ describe("legacy conversation import", () => { }); await ensureLegacyConversationImport({ conversationId: CONVERSATION_ID }); - const history = await stepStore.loadHistory(CONVERSATION_ID); + const history = await eventStore.loadHistory(CONVERSATION_ID); expect(history).toHaveLength(1); - expect(history[0]!.entry.type).toBe("pi_message"); + expect(history[0]!.data.type).toBe("message"); expect(history[0]!.createdAtMs).toBe(70); const [conversation] = await executor .db() @@ -446,7 +445,7 @@ describe("legacy conversation import", () => { // Second read is idempotent: no duplicate rows. await ensureLegacyConversationImport({ conversationId: CONVERSATION_ID }); - expect(await stepStore.loadHistory(CONVERSATION_ID)).toHaveLength(1); + expect(await eventStore.loadHistory(CONVERSATION_ID)).toHaveLength(1); }, 20_000); it("loadConnectedMcpProviders triggers the lazy import for a straggler", async () => { @@ -528,7 +527,7 @@ describe("legacy conversation import", () => { }, 20_000); it("does not resurrect purged SQL history from legacy Redis", async () => { - const { executor, stepStore } = await processSqlStores(); + const { executor, eventStore } = await processSqlStores(); await executor .db() @@ -558,13 +557,13 @@ describe("legacy conversation import", () => { conversationId: CONVERSATION_ID, }); expect(conversation.messages).toEqual([]); - expect(await stepStore.loadHistory(CONVERSATION_ID)).toEqual([]); + expect(await eventStore.loadHistory(CONVERSATION_ID)).toEqual([]); }, 20_000); it("rejects a legacy import when the SQL transcript was already purged", async () => { const fixture = await createLocalJuniorSqlFixture(); await migrateSchema(fixture.sql); - const stepStore = createSqlAgentStepStore(fixture.sql); + const eventStore = createSqlConversationEventStore(fixture.sql); const messageStore = createSqlConversationMessageStore(fixture.sql); await fixture.sql @@ -602,7 +601,7 @@ describe("legacy conversation import", () => { }); expect(result).toEqual({ imported: false }); - expect(await stepStore.loadHistory(CONVERSATION_ID)).toEqual([]); + expect(await eventStore.loadHistory(CONVERSATION_ID)).toEqual([]); expect(await messageStore.list(CONVERSATION_ID)).toEqual([]); } finally { await fixture.close(); @@ -659,11 +658,11 @@ describe("legacy conversation import", () => { scanned: 1, }); - const stepStore = createSqlAgentStepStore(fixture.sql); - const history = await stepStore.loadHistory(CONVERSATION_ID); - expect(history.map((step) => step.entry.type)).toEqual([ - "pi_message", - "pi_message", + const eventStore = createSqlConversationEventStore(fixture.sql); + const history = await eventStore.loadHistory(CONVERSATION_ID); + expect(history.map((event) => event.data.type)).toEqual([ + "message", + "message", ]); // Re-running the bounded scan imports nothing twice. @@ -680,10 +679,10 @@ describe("legacy conversation import", () => { } }, 20_000); - it("replaces NUL characters in imported agent steps", async () => { + it("replaces NUL characters in imported conversation events", async () => { const fixture = await createLocalJuniorSqlFixture(); await migrateSchema(fixture.sql); - const stepStore = createSqlAgentStepStore(fixture.sql); + const eventStore = createSqlConversationEventStore(fixture.sql); const messageStore = createSqlConversationMessageStore(fixture.sql); try { @@ -706,9 +705,9 @@ describe("legacy conversation import", () => { }); expect( - (await stepStore.loadHistory(CONVERSATION_ID))[0]?.entry, + (await eventStore.loadHistory(CONVERSATION_ID))[0]?.data, ).toMatchObject({ - type: "pi_message", + type: "message", message: { content: [{ text: "before after and literal \\u0000", type: "text" }], }, diff --git a/packages/junior/tests/component/conversations/retention.test.ts b/packages/junior/tests/component/conversations/retention.test.ts index 752b11b52..0b5a47335 100644 --- a/packages/junior/tests/component/conversations/retention.test.ts +++ b/packages/junior/tests/component/conversations/retention.test.ts @@ -11,7 +11,7 @@ import { selectExpiredRoots, } from "@/chat/conversations/sql/purge"; import { - juniorAgentSteps, + juniorConversationEvents, juniorConversationMessages, juniorConversations, juniorDestinations, @@ -79,12 +79,13 @@ async function seedConversation( if (args.withContent !== false) { await executor .db() - .insert(juniorAgentSteps) + .insert(juniorConversationEvents) .values({ conversationId: args.conversationId, seq: 0, contextEpoch: 0, - type: "pi_message", + schemaVersion: 1, + type: "message", role: "user", payload: { message: { role: "user", content: [] } }, createdAt: at, @@ -99,15 +100,15 @@ async function seedConversation( } } -async function stepCount( +async function eventCount( executor: JuniorSqlDatabase, conversationId: string, ): Promise { const rows = await executor .db() .select() - .from(juniorAgentSteps) - .where(eq(juniorAgentSteps.conversationId, conversationId)); + .from(juniorConversationEvents) + .where(eq(juniorConversationEvents.conversationId, conversationId)); return rows.length; } @@ -178,8 +179,8 @@ describe("retention purge job", () => { nowMs: BASE_MS + 15 * DAY_MS, }); expect(first.purged).toBe(1); - expect(await stepCount(fixture.sql, "priv")).toBe(0); - expect(await stepCount(fixture.sql, "pub")).toBe(1); + expect(await eventCount(fixture.sql, "priv")).toBe(0); + expect(await eventCount(fixture.sql, "pub")).toBe(1); expect( (await readConversation(fixture.sql, "pub")).transcriptPurgedAt, ).toBe(null); @@ -189,7 +190,7 @@ describe("retention purge job", () => { nowMs: BASE_MS + 91 * DAY_MS, }); expect(second.purged).toBe(1); - expect(await stepCount(fixture.sql, "pub")).toBe(0); + expect(await eventCount(fixture.sql, "pub")).toBe(0); }); it("shortens the window when visibility flips public to private", async () => { @@ -202,7 +203,7 @@ describe("retention purge job", () => { // Still public at 15 days: retained. await runRetentionPurge(fixture.sql, { nowMs: BASE_MS + 15 * DAY_MS }); - expect(await stepCount(fixture.sql, "flip")).toBe(1); + expect(await eventCount(fixture.sql, "flip")).toBe(1); // Flip to private, then the next pass at 15 days applies the 14-day window. await setVisibility(fixture.sql, dest, "private"); @@ -210,7 +211,7 @@ describe("retention purge job", () => { nowMs: BASE_MS + 15 * DAY_MS, }); expect(result.purged).toBe(1); - expect(await stepCount(fixture.sql, "flip")).toBe(0); + expect(await eventCount(fixture.sql, "flip")).toBe(0); }); it("rechecks activity and visibility inside the destructive transaction", async () => { @@ -247,7 +248,7 @@ describe("retention purge job", () => { }, }), ).resolves.toEqual({ purged: false, conversations: 0 }); - expect(await stepCount(fixture.sql, "raced")).toBe(1); + expect(await eventCount(fixture.sql, "raced")).toBe(1); }); it("rides children on the root window and purges them with the root", async () => { @@ -269,14 +270,14 @@ describe("retention purge job", () => { nowMs: BASE_MS + 30 * DAY_MS, }); expect(result.purged).toBe(0); - expect(await stepCount(fixture.sql, "child")).toBe(1); + expect(await eventCount(fixture.sql, "child")).toBe(1); // Erasing the root purges the child's content too. await purgeConversation(fixture.sql, "root", { nowMs: BASE_MS + 30 * DAY_MS, }); - expect(await stepCount(fixture.sql, "root")).toBe(0); - expect(await stepCount(fixture.sql, "child")).toBe(0); + expect(await eventCount(fixture.sql, "root")).toBe(0); + expect(await eventCount(fixture.sql, "child")).toBe(0); }); it("purges an expired root whose remaining content exists only on a child", async () => { @@ -305,7 +306,7 @@ describe("retention purge job", () => { }); expect(result.purged).toBe(1); - expect(await stepCount(fixture.sql, "remaining-child")).toBe(0); + expect(await eventCount(fixture.sql, "remaining-child")).toBe(0); expect(await messageCount(fixture.sql, "remaining-child")).toBe(0); }); @@ -332,12 +333,12 @@ describe("retention purge job", () => { expect(first.scanned).toBe(2); expect(first.purged).toBe(2); // Oldest-activity-first: "a" and "b" go, "c" remains. - expect(await stepCount(fixture.sql, "a")).toBe(0); - expect(await stepCount(fixture.sql, "c")).toBe(1); + expect(await eventCount(fixture.sql, "a")).toBe(0); + expect(await eventCount(fixture.sql, "c")).toBe(1); const second = await runRetentionPurge(fixture.sql, { nowMs, limit: 2 }); expect(second.purged).toBe(1); - expect(await stepCount(fixture.sql, "c")).toBe(0); + expect(await eventCount(fixture.sql, "c")).toBe(0); // Nothing left to do once everything is purged. const third = await runRetentionPurge(fixture.sql, { nowMs, limit: 2 }); @@ -389,10 +390,10 @@ describe("retention purge job", () => { // A daily pass at the same instant would keep it (well inside 14 days). const pass = await runRetentionPurge(fixture.sql, { nowMs: BASE_MS }); expect(pass.purged).toBe(0); - expect(await stepCount(fixture.sql, "fresh")).toBe(1); + expect(await eventCount(fixture.sql, "fresh")).toBe(1); await purgeConversation(fixture.sql, "fresh", { nowMs: BASE_MS }); - expect(await stepCount(fixture.sql, "fresh")).toBe(0); + expect(await eventCount(fixture.sql, "fresh")).toBe(0); expect(await messageCount(fixture.sql, "fresh")).toBe(0); const row = await readConversation(fixture.sql, "fresh"); expect(row.transcriptPurgedAt).not.toBe(null); diff --git a/packages/junior/tests/component/services/context-compaction.test.ts b/packages/junior/tests/component/services/context-compaction.test.ts index 7e1f862c2..bc449aef1 100644 --- a/packages/junior/tests/component/services/context-compaction.test.ts +++ b/packages/junior/tests/component/services/context-compaction.test.ts @@ -248,7 +248,7 @@ describe("context compaction projection reset", () => { await import("@/chat/conversations/projection"); const { coerceThreadConversationState } = await import("@/chat/state/conversation"); - const { getAgentStepStore } = await import("@/chat/db"); + const { getConversationEventStore } = await import("@/chat/db"); const { botConfig } = await import("@/chat/config"); const conversationId = "conversation-handoff"; const priorMessages = [ @@ -302,8 +302,10 @@ describe("context compaction projection reset", () => { expect( (await loadConversationProjection({ conversationId })).modelProfile, ).toBe("handoff"); - const marker = (await getAgentStepStore().loadHistory(conversationId)) - .map((step) => step.entry) + const marker = ( + await getConversationEventStore().loadHistory(conversationId) + ) + .map((event) => event.data) .find( (entry) => entry.type === "context_epoch_started" && entry.reason === "handoff", @@ -339,9 +341,9 @@ describe("context compaction projection reset", () => { (await loadConversationProjection({ conversationId })).modelProfile, ).toBe("handoff"); const projectionMarkers = ( - await getAgentStepStore().loadHistory(conversationId) + await getConversationEventStore().loadHistory(conversationId) ) - .map((step) => step.entry) + .map((event) => event.data) .filter((entry) => entry.type === "context_epoch_started"); expect( projectionMarkers.map(({ reason, modelProfile, modelId }) => ({ diff --git a/packages/junior/tests/component/services/turn-session-record.test.ts b/packages/junior/tests/component/services/turn-session-record.test.ts index df9037512..7d5b34c0e 100644 --- a/packages/junior/tests/component/services/turn-session-record.test.ts +++ b/packages/junior/tests/component/services/turn-session-record.test.ts @@ -409,7 +409,7 @@ describe("persistAuthPauseSessionRecord", () => { }); }); - it("persists turn transcript scope and actor in the session log", async () => { + it("persists turn transcript scope and actor in the event log", async () => { const { getAgentTurnSessionRecord, listAgentTurnSessionSummariesForConversation, diff --git a/packages/junior/tests/integration/agent-continue-slack.test.ts b/packages/junior/tests/integration/agent-continue-slack.test.ts index b0a0374e8..b8c92dcbc 100644 --- a/packages/junior/tests/integration/agent-continue-slack.test.ts +++ b/packages/junior/tests/integration/agent-continue-slack.test.ts @@ -1342,8 +1342,8 @@ describe("agent continuation Slack integration", () => { email: "testuser@example.com", }, }); - const { getAgentStepStore } = await import("@/chat/db"); - await getAgentStepStore().startEpoch(conversationId, { + const { getConversationEventStore } = await import("@/chat/db"); + await getConversationEventStore().startEpoch(conversationId, { modelId: "test/model", reason: "handoff", modelProfile: "handoff", @@ -1356,10 +1356,10 @@ describe("agent continuation Slack integration", () => { }); // A prior recovery can park runtime context in the handoff epoch before // another worker dies; the next recovery must replace rather than copy it. - await getAgentStepStore().append(conversationId, [ + await getConversationEventStore().append(conversationId, [ { - entry: { - type: "pi_message", + data: { + type: "message", message: { role: "user", content: [{ type: "text", text: previousRuntimeContext }], @@ -1464,15 +1464,16 @@ describe("agent continuation Slack integration", () => { expect(JSON.stringify(projection)).toContain("Handoff recovery completed."); expect(JSON.stringify(projection)).not.toContain(staleText); expect(JSON.stringify(projection)).not.toContain(""); - const history = await getAgentStepStore().loadHistory(conversationId); + const history = + await getConversationEventStore().loadHistory(conversationId); const rollbacks = history.filter( - (step) => - step.entry.type === "context_epoch_started" && - step.entry.reason === "rollback", + (event) => + event.data.type === "context_epoch_started" && + event.data.reason === "rollback", ); expect(rollbacks).toHaveLength(2); for (const rollback of rollbacks) { - expect(rollback.entry).toEqual( + expect(rollback.data).toEqual( expect.objectContaining({ modelProfile: "handoff" }), ); } diff --git a/packages/junior/tests/integration/dashboard-reporting.test.ts b/packages/junior/tests/integration/dashboard-reporting.test.ts index f202994b5..2c3364bcb 100644 --- a/packages/junior/tests/integration/dashboard-reporting.test.ts +++ b/packages/junior/tests/integration/dashboard-reporting.test.ts @@ -151,7 +151,85 @@ describe("dashboard reporting", () => { }); }); - it("uses SQL title and visible messages when agent steps are absent", async () => { + it("redacts private conversation summaries", async () => { + const { getConversationStore } = await import("@/chat/db"); + const { listRecentConversationSummaries } = + await import("@/reporting/plugin-conversations"); + const conversationStore = getConversationStore(); + + await conversationStore.recordActivity({ + conversationId: "slack:G1:222", + channelName: "private-incident-room", + nowMs: 1_000, + source: "slack", + title: "Sensitive escalation", + }); + + const summaries = await listRecentConversationSummaries(); + + expect(JSON.stringify(summaries)).not.toContain("private-incident-room"); + expect(JSON.stringify(summaries)).not.toContain("Sensitive escalation"); + expect(summaries[0]).toMatchObject({ + conversationId: "slack:G1:222", + status: "completed", + }); + }); + + it("redacts C-prefixed conversations Slack reports as private", async () => { + const { getConversationStore } = await import("@/chat/db"); + const { listRecentConversationSummaries } = + await import("@/reporting/plugin-conversations"); + const conversationStore = getConversationStore(); + + // Modern Slack private channels use C-prefixed ids; the event said + // channel_type: group, so the destination is confirmed private. + await conversationStore.recordActivity({ + conversationId: "slack:C9:333", + channelName: "stealth-project", + destination: { platform: "slack", teamId: "T1", channelId: "C9" }, + nowMs: 1_000, + source: "slack", + title: "Stealth planning", + visibility: "private", + }); + + const summaries = await listRecentConversationSummaries(); + + expect(JSON.stringify(summaries)).not.toContain("stealth-project"); + expect(JSON.stringify(summaries)).not.toContain("Stealth planning"); + expect(summaries[0]).toMatchObject({ + conversationId: "slack:C9:333", + }); + }); + + it("redacts C-prefixed conversations without public visibility", async () => { + const { getConversationStore } = await import("@/chat/db"); + const { listRecentConversationSummaries } = + await import("@/reporting/plugin-conversations"); + const conversationStore = getConversationStore(); + + // Legacy-style row: no live signal ever marked this channel public. + await conversationStore.recordActivity({ + conversationId: "slack:C9:444", + channelName: "maybe-private-room", + destination: { platform: "slack", teamId: "T1", channelId: "C9" }, + nowMs: 1_000, + source: "slack", + title: "Private by default", + }); + + const summaries = await listRecentConversationSummaries(); + + expect(JSON.stringify(summaries)).not.toContain("maybe-private-room"); + expect(JSON.stringify(summaries)).not.toContain("Private by default"); + expect(summaries[0]).toMatchObject({ + channelName: "Private Conversation", + channelNameRedacted: true, + displayTitle: "Private Conversation", + }); + }); + + it("uses SQL title and visible messages when conversation events are absent", async () => { const { getConversationMessageStore, getConversationStore } = await import("@/chat/db"); @@ -354,7 +432,7 @@ describe("dashboard reporting", () => { it("reports private execution activity as safe metadata", async () => { const { upsertAgentTurnSessionRecord } = await import("@/chat/state/turn-session"); - const { getAgentStepStore } = await import("@/chat/db"); + const { getConversationEventStore } = await import("@/chat/db"); await upsertAgentTurnSessionRecord({ modelId: "test/model", @@ -378,10 +456,10 @@ describe("dashboard reporting", () => { }, ] as PiMessage[], }); - // Activity now derives from durable agent steps, not the Redis session log. - await getAgentStepStore().append("slack:G1:activity", [ + // Activity now derives from durable conversation events, not the Redis session log. + await getConversationEventStore().append("slack:G1:activity", [ { - entry: { + data: { type: "tool_execution_started", toolCallId: "advisor-call-1", toolName: "advisor", @@ -390,7 +468,7 @@ describe("dashboard reporting", () => { createdAtMs: 2, }, { - entry: { + data: { type: "subagent_started", subagentInvocationId: "advisor-call-1", subagentKind: "advisor", @@ -403,7 +481,7 @@ describe("dashboard reporting", () => { createdAtMs: 3, }, { - entry: { + data: { type: "subagent_ended", subagentInvocationId: "advisor-call-1", outcome: "success", @@ -411,7 +489,7 @@ describe("dashboard reporting", () => { createdAtMs: 5, }, ]); - await getAgentStepStore().startEpoch("slack:G1:activity", { + await getConversationEventStore().startEpoch("slack:G1:activity", { reason: "compaction", modelProfile: "standard", modelId: "test/model", @@ -469,23 +547,23 @@ describe("dashboard reporting", () => { }); it("loads subagent transcript history from the child conversation", async () => { - const { getAgentStepStore, getConversationStore } = + const { getConversationEventStore, getConversationStore } = await import("@/chat/db"); const conversationId = "slack:C1:subagent-slices"; await confirmPublicSlackConversation(conversationId); const childConversationId = `task:${conversationId}`; const conversationStore = getConversationStore(); - const stepStore = getAgentStepStore(); + const eventStore = getConversationEventStore(); await conversationStore.ensureChildConversation({ conversationId: childConversationId, parentConversationId: conversationId, }); - await stepStore.append(childConversationId, [ + await eventStore.append(childConversationId, [ { - entry: { - type: "pi_message", + data: { + type: "message", message: { role: "user", content: [ @@ -501,8 +579,8 @@ describe("dashboard reporting", () => { createdAtMs: 10, }, { - entry: { - type: "pi_message", + data: { + type: "message", message: { role: "assistant", content: [{ type: "text", text: "first subagent answer" }], @@ -512,8 +590,8 @@ describe("dashboard reporting", () => { createdAtMs: 20, }, { - entry: { - type: "pi_message", + data: { + type: "message", message: { role: "user", content: [{ type: "text", text: "second subagent question" }], @@ -523,8 +601,8 @@ describe("dashboard reporting", () => { createdAtMs: 30, }, { - entry: { - type: "pi_message", + data: { + type: "message", message: { role: "assistant", content: [{ type: "text", text: "second subagent answer" }], @@ -534,8 +612,8 @@ describe("dashboard reporting", () => { createdAtMs: 40, }, { - entry: { - type: "pi_message", + data: { + type: "message", message: { role: "user", content: [ @@ -556,9 +634,9 @@ describe("dashboard reporting", () => { // Repeated subagent calls share one child conversation, so both // parent subagent markers name the same child history. for (const subagentId of ["task-plan", "task-review"]) { - await stepStore.append(conversationId, [ + await eventStore.append(conversationId, [ { - entry: { + data: { type: "subagent_started", subagentInvocationId: subagentId, subagentKind: "task", @@ -571,7 +649,7 @@ describe("dashboard reporting", () => { createdAtMs: subagentId === "task-plan" ? 3 : 31, }, { - entry: { + data: { type: "subagent_ended", subagentInvocationId: subagentId, outcome: "success", @@ -580,9 +658,9 @@ describe("dashboard reporting", () => { }, ]); } - await stepStore.append(conversationId, [ + await eventStore.append(conversationId, [ { - entry: { + data: { type: "subagent_started", subagentInvocationId: "legacy-advisor", subagentKind: "advisor", @@ -592,7 +670,7 @@ describe("dashboard reporting", () => { createdAtMs: 50, }, { - entry: { + data: { type: "subagent_ended", subagentInvocationId: "legacy-advisor", outcome: "success", @@ -644,7 +722,7 @@ describe("dashboard reporting", () => { }); it("redacts advisor subagent transcript history for private conversations", async () => { - const { getAgentStepStore, getConversationStore } = + const { getConversationEventStore, getConversationStore } = await import("@/chat/db"); const conversationId = "slack:D1:advisor-private"; @@ -652,16 +730,16 @@ describe("dashboard reporting", () => { const privateAdvisorText = "private advisor question"; const childConversationId = `advisor:${conversationId}`; const conversationStore = getConversationStore(); - const stepStore = getAgentStepStore(); + const eventStore = getConversationEventStore(); await conversationStore.ensureChildConversation({ conversationId: childConversationId, parentConversationId: conversationId, }); - await stepStore.append(childConversationId, [ + await eventStore.append(childConversationId, [ { - entry: { - type: "pi_message", + data: { + type: "message", message: { role: "user", content: [{ type: "text", text: privateAdvisorText }], @@ -671,9 +749,9 @@ describe("dashboard reporting", () => { createdAtMs: 10, }, ]); - await stepStore.append(conversationId, [ + await eventStore.append(conversationId, [ { - entry: { + data: { type: "subagent_started", subagentInvocationId: toolCallId, subagentKind: "advisor", @@ -684,7 +762,7 @@ describe("dashboard reporting", () => { createdAtMs: 3, }, { - entry: { + data: { type: "subagent_ended", subagentInvocationId: toolCallId, outcome: "success", @@ -708,7 +786,7 @@ describe("dashboard reporting", () => { it("derives unfinished subagent status from completed parent tool results", async () => { const { upsertAgentTurnSessionRecord } = await import("@/chat/state/turn-session"); - const { getAgentStepStore } = await import("@/chat/db"); + const { getConversationEventStore } = await import("@/chat/db"); await upsertAgentTurnSessionRecord({ modelId: "test/model", @@ -732,30 +810,33 @@ describe("dashboard reporting", () => { }, ] as PiMessage[], }); - // The subagent has no end step; its status derives from the parent tool's + // The subagent has no end event; its status derives from the parent tool's // completed result in the current epoch projection. - await getAgentStepStore().append("slack:C1:activity-parent-result", [ - { - entry: { - type: "tool_execution_started", - toolCallId: "advisor-call-parent", - toolName: "advisor", - args: { question: "public question" }, + await getConversationEventStore().append( + "slack:C1:activity-parent-result", + [ + { + data: { + type: "tool_execution_started", + toolCallId: "advisor-call-parent", + toolName: "advisor", + args: { question: "public question" }, + }, + createdAtMs: 2, }, - createdAtMs: 2, - }, - { - entry: { - type: "subagent_started", - subagentInvocationId: "advisor-call-parent", - subagentKind: "advisor", - parentToolCallId: "advisor-call-parent", - childConversationId: "advisor:slack:C1:activity-parent-result", - historyMode: "shared", + { + data: { + type: "subagent_started", + subagentInvocationId: "advisor-call-parent", + subagentKind: "advisor", + parentToolCallId: "advisor-call-parent", + childConversationId: "advisor:slack:C1:activity-parent-result", + historyMode: "shared", + }, + createdAtMs: 3, }, - createdAtMs: 3, - }, - ]); + ], + ); const report = await readConversationDetailReport( "slack:C1:activity-parent-result", @@ -864,7 +945,7 @@ describe("dashboard reporting", () => { }); it("reports a conversation directly from SQL without a secondary execution index", async () => { - const { getAgentStepStore, getConversationStore } = + const { getConversationEventStore, getConversationStore } = await import("@/chat/db"); await getConversationStore().recordActivity({ conversationId: "slack:C1:999", @@ -876,10 +957,10 @@ describe("dashboard reporting", () => { source: "slack", visibility: "public", }); - await getAgentStepStore().append("slack:C1:999", [ + await getConversationEventStore().append("slack:C1:999", [ { - entry: { - type: "pi_message", + data: { + type: "message", message: { role: "user", content: [{ type: "text", text: "target question" }], @@ -905,7 +986,7 @@ describe("dashboard reporting", () => { }); it("reports terminal assistant outcomes without exposing provider errors", async () => { - const { getAgentStepStore, getConversationStore } = + const { getConversationEventStore, getConversationStore } = await import("@/chat/db"); const errorConversationId = "slack:C1:terminal-error"; const abortedConversationId = "slack:D1:terminal-aborted"; @@ -913,10 +994,10 @@ describe("dashboard reporting", () => { "xAI 503 credential=secret-provider-token upstream payload"; await confirmPublicSlackConversation(errorConversationId); - await getAgentStepStore().append(errorConversationId, [ + await getConversationEventStore().append(errorConversationId, [ { - entry: { - type: "pi_message", + data: { + type: "message", message: { role: "assistant", api: "openai-responses", @@ -956,10 +1037,10 @@ describe("dashboard reporting", () => { source: "slack", visibility: "private", }); - await getAgentStepStore().append(abortedConversationId, [ + await getConversationEventStore().append(abortedConversationId, [ { - entry: { - type: "pi_message", + data: { + type: "message", message: { role: "assistant", api: "openai-responses", @@ -1025,7 +1106,7 @@ describe("dashboard reporting", () => { }); it("keeps SQL detail available when optional execution settings fail", async () => { - const { getAgentStepStore, getConversationStore } = + const { getConversationEventStore, getConversationStore } = await import("@/chat/db"); const { getStateAdapter } = await import("@/chat/state/adapter"); const conversationId = "slack:C1:settings-unavailable"; @@ -1039,10 +1120,10 @@ describe("dashboard reporting", () => { source: "slack", visibility: "public", }); - await getAgentStepStore().append(conversationId, [ + await getConversationEventStore().append(conversationId, [ { - entry: { - type: "pi_message", + data: { + type: "message", message: { role: "user", content: [{ type: "text", text: "available transcript" }], @@ -1339,17 +1420,17 @@ describe("dashboard reporting", () => { }); it("reports complete history around a compaction without copied messages", async () => { - const { getAgentStepStore } = await import("@/chat/db"); + const { getConversationEventStore } = await import("@/chat/db"); const conversationId = "slack:C1:compaction"; await confirmPublicSlackConversation(conversationId); - const stepStore = getAgentStepStore(); + const eventStore = getConversationEventStore(); // Epoch 0: execution that remains visible after a later context rebuild. - await stepStore.append(conversationId, [ + await eventStore.append(conversationId, [ { - entry: { - type: "pi_message", + data: { + type: "message", message: { role: "user", content: [ @@ -1364,8 +1445,8 @@ describe("dashboard reporting", () => { createdAtMs: 0, }, { - entry: { - type: "pi_message", + data: { + type: "message", message: { role: "user", content: [{ type: "text", text: "old question" }], @@ -1375,8 +1456,8 @@ describe("dashboard reporting", () => { createdAtMs: 1, }, { - entry: { - type: "pi_message", + data: { + type: "message", message: { role: "user", content: [{ type: "text", text: "current question" }], @@ -1387,8 +1468,8 @@ describe("dashboard reporting", () => { createdAtMs: 2, }, { - entry: { - type: "pi_message", + data: { + type: "message", message: { role: "user", content: [{ type: "text", text: "current question" }], @@ -1399,7 +1480,7 @@ describe("dashboard reporting", () => { createdAtMs: 2, }, { - entry: { + data: { type: "tool_execution_started", toolCallId: "old-tool", toolName: "search", @@ -1408,7 +1489,7 @@ describe("dashboard reporting", () => { }, ]); // Compaction copies the latest user intent and adds a generated summary. - await stepStore.startEpoch(conversationId, { + await eventStore.startEpoch(conversationId, { modelId: "test/model", reason: "compaction", modelProfile: "standard", @@ -1462,10 +1543,10 @@ describe("dashboard reporting", () => { ], }); // A current-epoch tool execution the report should surface. - await stepStore.append(conversationId, [ + await eventStore.append(conversationId, [ { - entry: { - type: "pi_message", + data: { + type: "message", message: { role: "user", content: [ @@ -1481,7 +1562,7 @@ describe("dashboard reporting", () => { createdAtMs: 4.5, }, { - entry: { + data: { type: "tool_execution_started", toolCallId: "new-tool", toolName: "search", @@ -1524,12 +1605,12 @@ describe("dashboard reporting", () => { }); it("reports the original execution and continuation around a model handoff", async () => { - const { getAgentStepStore } = await import("@/chat/db"); + const { getConversationEventStore } = await import("@/chat/db"); const conversationId = "slack:C1:handoff-reporting"; await confirmPublicSlackConversation(conversationId); - const stepStore = getAgentStepStore(); + const eventStore = getConversationEventStore(); - await stepStore.startEpoch(conversationId, { + await eventStore.startEpoch(conversationId, { reason: "initial", modelProfile: "standard", modelId: "openai/gpt-5.4", @@ -1580,7 +1661,7 @@ describe("dashboard reporting", () => { }, ], }); - await stepStore.startEpoch(conversationId, { + await eventStore.startEpoch(conversationId, { reason: "handoff", modelProfile: "handoff", modelId: "openai/gpt-5.6-sol", @@ -1613,10 +1694,10 @@ describe("dashboard reporting", () => { }, ], }); - await stepStore.append(conversationId, [ + await eventStore.append(conversationId, [ { - entry: { - type: "pi_message", + data: { + type: "message", message: { role: "assistant", content: [{ type: "text", text: "I prepared the ordering fix." }], @@ -1662,17 +1743,17 @@ describe("dashboard reporting", () => { }); it("reports divergent rollback history without repeating the shared prefix", async () => { - const { getAgentStepStore } = await import("@/chat/db"); + const { getConversationEventStore } = await import("@/chat/db"); const conversationId = "slack:C1:rollback-reporting"; await confirmPublicSlackConversation(conversationId); - const stepStore = getAgentStepStore(); + const eventStore = getConversationEventStore(); const shared = { role: "user", content: [{ type: "text", text: "Regenerate the answer" }], timestamp: 1, } as PiMessage; - await stepStore.startEpoch(conversationId, { + await eventStore.startEpoch(conversationId, { reason: "initial", modelProfile: "standard", modelId: "openai/gpt-5.4", @@ -1688,7 +1769,7 @@ describe("dashboard reporting", () => { }, ], }); - await stepStore.startEpoch(conversationId, { + await eventStore.startEpoch(conversationId, { reason: "rollback", modelProfile: "standard", modelId: "openai/gpt-5.4", @@ -1723,23 +1804,23 @@ describe("dashboard reporting", () => { }); it("reports ordered compaction and handoff events with a once-only transcript", async () => { - const { getAgentStepStore } = await import("@/chat/db"); + const { getConversationEventStore } = await import("@/chat/db"); const conversationId = "slack:C1:compaction-handoff-reporting"; await confirmPublicSlackConversation(conversationId); - const stepStore = getAgentStepStore(); + const eventStore = getConversationEventStore(); const original = { role: "user", content: [{ type: "text", text: "Finish the release work" }], timestamp: 1, } as PiMessage; - await stepStore.startEpoch(conversationId, { + await eventStore.startEpoch(conversationId, { reason: "initial", modelProfile: "standard", modelId: "openai/gpt-5.4", messages: [{ message: original, createdAtMs: 1 }], }); - await stepStore.startEpoch(conversationId, { + await eventStore.startEpoch(conversationId, { reason: "compaction", modelProfile: "standard", modelId: "openai/gpt-5.4", @@ -1760,10 +1841,10 @@ describe("dashboard reporting", () => { }, ], }); - await stepStore.append(conversationId, [ + await eventStore.append(conversationId, [ { - entry: { - type: "pi_message", + data: { + type: "message", message: { role: "assistant", content: [{ type: "text", text: "Compacted continuation" }], @@ -1773,7 +1854,7 @@ describe("dashboard reporting", () => { createdAtMs: 3, }, ]); - await stepStore.startEpoch(conversationId, { + await eventStore.startEpoch(conversationId, { reason: "handoff", modelProfile: "handoff", modelId: "openai/gpt-5.6-sol", @@ -1793,10 +1874,10 @@ describe("dashboard reporting", () => { }, ], }); - await stepStore.append(conversationId, [ + await eventStore.append(conversationId, [ { - entry: { - type: "pi_message", + data: { + type: "message", message: { role: "assistant", content: [{ type: "text", text: "Handoff continuation" }], diff --git a/packages/junior/tests/integration/runtime/agent-run-model-handoff.test.ts b/packages/junior/tests/integration/runtime/agent-run-model-handoff.test.ts index ef8d46430..be7a94291 100644 --- a/packages/junior/tests/integration/runtime/agent-run-model-handoff.test.ts +++ b/packages/junior/tests/integration/runtime/agent-run-model-handoff.test.ts @@ -149,7 +149,7 @@ import { } from "@/chat/conversations/projection"; import { disconnectStateAdapter } from "@/chat/state/adapter"; import { getAgentTurnSessionRecord } from "@/chat/state/turn-session"; -import { getAgentStepStore } from "@/chat/db"; +import { getConversationEventStore } from "@/chat/db"; const ORIGINAL_STATE_ADAPTER = process.env.JUNIOR_STATE_ADAPTER; @@ -229,8 +229,10 @@ describe("executeAgentRun model handoff", () => { expect( (await loadConversationProjection({ conversationId })).modelProfile, ).toBe("handoff"); - const epochMarkers = (await getAgentStepStore().loadHistory(conversationId)) - .map((step) => step.entry) + const epochMarkers = ( + await getConversationEventStore().loadHistory(conversationId) + ) + .map((event) => event.data) .filter((entry) => entry.type === "context_epoch_started"); expect(epochMarkers).toEqual([ { @@ -363,8 +365,8 @@ describe("executeAgentRun model handoff", () => { (await loadConversationProjection({ conversationId })).modelProfile, ).toBe("coding"); expect( - (await getAgentStepStore().loadHistory(conversationId)) - .map((step) => step.entry) + (await getConversationEventStore().loadHistory(conversationId)) + .map((event) => event.data) .filter((entry) => entry.type === "context_epoch_started"), ).toEqual([ { @@ -450,8 +452,8 @@ describe("executeAgentRun model handoff", () => { (await loadConversationProjection({ conversationId })).modelProfile, ).toBe("handoff"); expect( - (await getAgentStepStore().loadHistory(conversationId)) - .map((step) => step.entry) + (await getConversationEventStore().loadHistory(conversationId)) + .map((event) => event.data) .filter((entry) => entry.type === "context_epoch_started"), ).toEqual([ { diff --git a/packages/junior/tests/integration/slack/bot-handlers.test.ts b/packages/junior/tests/integration/slack/bot-handlers.test.ts index e2625931d..60b1492d6 100644 --- a/packages/junior/tests/integration/slack/bot-handlers.test.ts +++ b/packages/junior/tests/integration/slack/bot-handlers.test.ts @@ -6,7 +6,7 @@ import { getSlackInterruptionMarker } from "@/chat/slack/output"; import { completedAgentRun } from "@/chat/runtime/agent-run-outcome"; import { disconnectStateAdapter, getStateAdapter } from "@/chat/state/adapter"; import { acquireActiveLock } from "@/chat/state/locks"; -import { instructionActors } from "@/chat/state/session-log"; +import { instructionActors } from "@/chat/conversations/provenance"; import { loadProjection, loadConversationProjection, @@ -1278,7 +1278,7 @@ describe("bot handlers (integration)", () => { expect(conversation?.processing?.activeTurnId).toBeUndefined(); }); - it("appends a parked-conversation follow-up to the session log before consuming it", async () => { + it("appends a parked-conversation follow-up to the event log before consuming it", async () => { const conversationId = "slack:C9PARKEDLOG:1700000000.000"; const destination = slackDestination("C9PARKEDLOG"); const activeSessionId = "turn_msg-original"; @@ -1558,7 +1558,7 @@ describe("bot handlers (integration)", () => { }); expect(capturedActorUserId).toBe("U-alice"); - // The drain commits Bob's batched mention to the session log before the run + // The drain commits Bob's batched mention to the event log before the run // with his own instruction provenance, so he joins the run's actors instead // of being dropped as anonymous context under Alice's turn. const projection = await loadConversationProjection({ conversationId }); diff --git a/packages/junior/tests/unit/chat/pi/conversation-events.test.ts b/packages/junior/tests/unit/chat/pi/conversation-events.test.ts new file mode 100644 index 000000000..35a10fb5f --- /dev/null +++ b/packages/junior/tests/unit/chat/pi/conversation-events.test.ts @@ -0,0 +1,108 @@ +import { describe, expect, it } from "vitest"; +import { + conversationEventSchema, + type ConversationEvent, + type ConversationEventData, +} from "@/chat/conversations/history"; +import { projectConversationEvents } from "@/chat/pi/conversation-events"; + +function event(seq: number, data: ConversationEventData): ConversationEvent { + return conversationEventSchema.parse({ + schemaVersion: 1, + seq, + contextEpoch: 2, + createdAtMs: 1_000 + seq, + data, + }); +} + +describe("projectConversationEvents", () => { + const instructionProvenance = { + authority: "instruction" as const, + actor: { + platform: "slack" as const, + teamId: "T123", + userId: "U123", + }, + }; + const firstMessage = { + role: "user" as const, + content: [{ type: "text" as const, text: "Investigate this" }], + timestamp: 1_002, + }; + const lastMessage = { + role: "user" as const, + content: [{ type: "text" as const, text: "Continue" }], + timestamp: 1_006, + }; + const events = [ + event(0, { + type: "context_epoch_started", + reason: "handoff", + modelProfile: "coding", + modelId: "openai/gpt-5.4", + }), + event(1, { type: "mcp_provider_connected", provider: "github" }), + event(2, { + type: "message", + message: firstMessage, + provenance: instructionProvenance, + }), + event(3, { + type: "authorization_completed", + kind: "mcp", + provider: "linear", + actorId: "U123", + authorizationId: "auth-1", + }), + event(4, { + type: "tool_execution_started", + toolCallId: "tool-1", + toolName: "github_search", + }), + event(5, { + type: "authorization_requested", + kind: "plugin", + provider: "sentry", + actorId: "U123", + authorizationId: "auth-2", + delivery: "private_link_sent", + }), + event(6, { type: "message", message: lastMessage }), + ]; + + it("projects messages, authorization observations, provenance, and model binding", () => { + const projection = projectConversationEvents(events); + + expect(projection).toEqual({ + messages: [ + firstMessage, + expect.objectContaining({ + role: "user", + content: [expect.objectContaining({ type: "text" })], + timestamp: 1_003, + }), + lastMessage, + ], + provenance: [ + instructionProvenance, + { authority: "context" }, + { authority: "context" }, + ], + seqs: [2, 3, 6], + modelProfile: "coding", + modelId: "openai/gpt-5.4", + }); + }); + + it("stops at maxSeq while retaining the epoch model binding", () => { + const projection = projectConversationEvents(events, { maxSeq: 3 }); + + expect(projection).toMatchObject({ + seqs: [2, 3], + modelProfile: "coding", + modelId: "openai/gpt-5.4", + }); + expect(projection.messages).toHaveLength(2); + }); +}); diff --git a/packages/junior/tests/unit/conversations/legacy-history-import.test.ts b/packages/junior/tests/unit/conversations/legacy-history-import.test.ts index 2fed042b8..83a853765 100644 --- a/packages/junior/tests/unit/conversations/legacy-history-import.test.ts +++ b/packages/junior/tests/unit/conversations/legacy-history-import.test.ts @@ -41,7 +41,7 @@ function piEntry( describe("convertLegacySessionLog", () => { it("keeps a single session in epoch 0 with sequential seq and message timestamps", () => { - const { steps, advisorChildConversationId } = convertLegacySessionLog({ + const { events, advisorChildConversationId } = convertLegacySessionLog({ conversationId: CONVERSATION_ID, fallbackCreatedAtMs: FALLBACK_MS, entries: [ @@ -51,13 +51,13 @@ describe("convertLegacySessionLog", () => { }); expect(advisorChildConversationId).toBeUndefined(); - expect(steps).toEqual([ + expect(events).toEqual([ { seq: 0, contextEpoch: 0, createdAtMs: 10, - entry: { - type: "pi_message", + data: { + type: "message", message: userMessage("hello", 10), provenance: { authority: "context" }, }, @@ -66,8 +66,8 @@ describe("convertLegacySessionLog", () => { seq: 1, contextEpoch: 0, createdAtMs: 20, - entry: { - type: "pi_message", + data: { + type: "message", message: assistantMessage("hi", 20), provenance: { authority: "context" }, }, @@ -82,7 +82,7 @@ describe("convertLegacySessionLog", () => { teamId: "T1", slackUserName: "ada", }; - const { steps } = convertLegacySessionLog({ + const { events } = convertLegacySessionLog({ conversationId: CONVERSATION_ID, fallbackCreatedAtMs: FALLBACK_MS, entries: [ @@ -101,9 +101,9 @@ describe("convertLegacySessionLog", () => { // actor_recorded produces no row; the v1 pi_message decodes to an authored // instruction from the stored Slack actor. - expect(steps).toHaveLength(1); - expect(steps[0]!.entry).toEqual({ - type: "pi_message", + expect(events).toHaveLength(1); + expect(events[0]!.data).toEqual({ + type: "message", message: userMessage("do the thing", 30), provenance: { authority: "instruction", @@ -118,7 +118,7 @@ describe("convertLegacySessionLog", () => { }); it("explodes projection_reset into an epoch marker plus per-message rows and keeps stale sessions in their epoch", () => { - const { steps } = convertLegacySessionLog({ + const { events } = convertLegacySessionLog({ conversationId: CONVERSATION_ID, fallbackCreatedAtMs: FALLBACK_MS, entries: [ @@ -136,30 +136,30 @@ describe("convertLegacySessionLog", () => { }); expect( - steps.map((step) => ({ - seq: step.seq, - epoch: step.contextEpoch, - type: step.entry.type, + events.map((event) => ({ + seq: event.seq, + epoch: event.contextEpoch, + type: event.data.type, })), ).toEqual([ - { seq: 0, epoch: 0, type: "pi_message" }, + { seq: 0, epoch: 0, type: "message" }, { seq: 1, epoch: 1, type: "context_epoch_started" }, - { seq: 2, epoch: 1, type: "pi_message" }, - { seq: 3, epoch: 1, type: "pi_message" }, - { seq: 4, epoch: 0, type: "pi_message" }, - { seq: 5, epoch: 1, type: "pi_message" }, + { seq: 2, epoch: 1, type: "message" }, + { seq: 3, epoch: 1, type: "message" }, + { seq: 4, epoch: 0, type: "message" }, + { seq: 5, epoch: 1, type: "message" }, ]); - expect(steps[1]!.entry).toEqual({ + expect(events[1]!.data).toEqual({ type: "context_epoch_started", reason: "compaction", }); // Highest epoch (current context) is exactly the reset's session rows. - const currentEpoch = Math.max(...steps.map((step) => step.contextEpoch)); + const currentEpoch = Math.max(...events.map((event) => event.contextEpoch)); expect(currentEpoch).toBe(1); }); it("converts an advisor subagent transcriptRef to a child conversation link and drops transcript cursors", () => { - const { steps, advisorChildConversationId } = convertLegacySessionLog({ + const { events, advisorChildConversationId } = convertLegacySessionLog({ conversationId: CONVERSATION_ID, fallbackCreatedAtMs: FALLBACK_MS, entries: [ @@ -193,7 +193,7 @@ describe("convertLegacySessionLog", () => { }); expect(advisorChildConversationId).toBe(`advisor:${CONVERSATION_ID}`); - expect(steps[0]!.entry).toEqual({ + expect(events[0]!.data).toEqual({ type: "subagent_started", subagentInvocationId: "call-1", subagentKind: "advisor", @@ -201,8 +201,8 @@ describe("convertLegacySessionLog", () => { childConversationId: `advisor:${CONVERSATION_ID}`, historyMode: "shared", }); - expect(steps[0]!.createdAtMs).toBe(50); - expect(steps[1]!.entry).toEqual({ + expect(events[0]!.createdAtMs).toBe(50); + expect(events[1]!.data).toEqual({ type: "subagent_ended", subagentInvocationId: "call-1", outcome: "success", @@ -211,7 +211,7 @@ describe("convertLegacySessionLog", () => { it("falls back to the supplied conversation timestamp and never fabricates now", () => { const before = Date.now(); - const { steps } = convertLegacySessionLog({ + const { events } = convertLegacySessionLog({ conversationId: CONVERSATION_ID, fallbackCreatedAtMs: FALLBACK_MS, entries: [ @@ -225,13 +225,13 @@ describe("convertLegacySessionLog", () => { ], }); - expect(steps.map((step) => step.createdAtMs)).toEqual([ + expect(events.map((event) => event.createdAtMs)).toEqual([ FALLBACK_MS, FALLBACK_MS, ]); // Guard against any Date.now() creeping in as a timestamp source. - for (const step of steps) { - expect(step.createdAtMs).toBeLessThan(before); + for (const event of events) { + expect(event.createdAtMs).toBeLessThan(before); } }); }); @@ -256,8 +256,8 @@ describe("convertAdvisorMessages", () => { seq: 0, contextEpoch: 0, createdAtMs: 5, - entry: { - type: "pi_message", + data: { + type: "message", message: userMessage( `Review & "risk".\n\nExecutor context:\nUse owner='dashboard' & preserve .`, 5, @@ -269,8 +269,8 @@ describe("convertAdvisorMessages", () => { seq: 1, contextEpoch: 0, createdAtMs: 6, - entry: { - type: "pi_message", + data: { + type: "message", message: assistantMessage("a", 6), provenance: { authority: "context" }, }, @@ -279,8 +279,8 @@ describe("convertAdvisorMessages", () => { seq: 2, contextEpoch: 0, createdAtMs: FALLBACK_MS, - entry: { - type: "pi_message", + data: { + type: "message", message: assistantMessage("no ts"), provenance: { authority: "context" }, }, diff --git a/packages/junior/tests/unit/conversations/provenance.test.ts b/packages/junior/tests/unit/conversations/provenance.test.ts new file mode 100644 index 000000000..11b813035 --- /dev/null +++ b/packages/junior/tests/unit/conversations/provenance.test.ts @@ -0,0 +1,68 @@ +import { describe, expect, it } from "vitest"; +import { + instructionActors, + type ConversationMessageProvenance, +} from "@/chat/conversations/provenance"; + +describe("instructionActors", () => { + it("deduplicates actor identities while preserving the first representation", () => { + const provenance: ConversationMessageProvenance[] = [ + { + authority: "instruction", + actor: { + platform: "slack", + teamId: "T1", + userId: "U1", + userName: "first", + }, + }, + { + authority: "instruction", + actor: { + platform: "slack", + teamId: "T1", + userId: "U1", + userName: "renamed", + }, + }, + ]; + + expect(instructionActors(provenance)).toEqual([provenance[0]!.actor]); + }); + + it("keeps identities distinct across platforms and Slack workspaces", () => { + const provenance: ConversationMessageProvenance[] = [ + { + authority: "instruction", + actor: { platform: "slack", teamId: "T1", userId: "U1" }, + }, + { + authority: "instruction", + actor: { platform: "slack", teamId: "T2", userId: "U1" }, + }, + { + authority: "instruction", + actor: { platform: "local", userId: "U1" }, + }, + { + authority: "instruction", + actor: { platform: "system", name: "scheduler" }, + }, + { + authority: "instruction", + actor: { platform: "system", name: "scheduler" }, + }, + { + authority: "context", + actor: { platform: "local", userId: "ignored" }, + }, + ]; + + expect(instructionActors(provenance)).toEqual([ + provenance[0]!.actor, + provenance[1]!.actor, + provenance[2]!.actor, + provenance[3]!.actor, + ]); + }); +}); diff --git a/packages/junior/tests/unit/state/conversation-state.test.ts b/packages/junior/tests/unit/state/conversation-state.test.ts index 107202277..7678482f6 100644 --- a/packages/junior/tests/unit/state/conversation-state.test.ts +++ b/packages/junior/tests/unit/state/conversation-state.test.ts @@ -158,7 +158,7 @@ describe("conversation state", () => { const patch = buildConversationStatePatch(conversation); expect(patch.conversation).not.toHaveProperty("messages"); expect(patch.conversation).not.toHaveProperty("compactions"); - // Pi history lives in the SQL AgentStepStore; thread-state carries no mirror. + // Model history lives in the SQL ConversationEventStore; thread-state carries no mirror. expect(patch.conversation).not.toHaveProperty("piMessages"); // The count stat is still derived from the working set for reporting. expect(patch.conversation.stats.totalMessageCount).toBe(1); diff --git a/packages/junior/tests/unit/state/session-log.test.ts b/packages/junior/tests/unit/state/session-log.test.ts index b13322c84..a0b6c8273 100644 --- a/packages/junior/tests/unit/state/session-log.test.ts +++ b/packages/junior/tests/unit/state/session-log.test.ts @@ -1,995 +1,156 @@ -import { afterEach, describe, expect, it, vi } from "vitest"; +import { describe, expect, it } from "vitest"; import type { PiMessage } from "@/chat/pi/messages"; import { - commitMessages, - instructionActors, - loadActivityEntries, - loadConnectedMcpProviders, - loadMessages, - loadProjection, - loadProjectionWithActor, - loadProjectionWithProvenance, - recordAuthorizationCompleted, - recordAuthorizationRequested, - recordMcpProviderConnected, - recordSubagentEnded, - recordSubagentStarted, - recordToolExecutionStarted, - type PiMessageProvenance, + legacyActorProvenance, + readSessionLogEntries, type SessionLogEntry, type SessionLogStore, } from "@/chat/state/session-log"; -function memoryStore(): SessionLogStore & { - entries: SessionLogEntry[]; -} { - const entries: SessionLogEntry[] = []; +function userMessage(text: string): PiMessage { + return { + role: "user", + content: [{ type: "text", text }], + timestamp: 1, + } as PiMessage; +} +function memoryStore(values: unknown[]): SessionLogStore { return { - entries, - async append(args) { - entries.push(...args.entries); - }, - async read() { - return [...entries]; - }, + read: async () => values as SessionLogEntry[], }; } -describe("agent session log store", () => { - afterEach(() => { - vi.useRealTimers(); - }); - - it("appends Pi messages for a growing session projection", async () => { - const store = memoryStore(); - const first: PiMessage = { - role: "user", - content: [{ type: "text", text: "first" }], - timestamp: 1, - } as PiMessage; - const second: PiMessage = { - role: "assistant", - content: [{ type: "text", text: "second" }], - timestamp: 2, - } as PiMessage; - - await commitMessages({ - store, - conversationId: "conversation-1", - messages: [first], - ttlMs: 60_000, - }); - await commitMessages({ - store, - conversationId: "conversation-1", - messages: [first, second], - ttlMs: 60_000, - }); - - expect(store.entries).toEqual([ - { - schemaVersion: 2, - type: "pi_message", - sessionId: "session_0", - message: first, - }, - { - schemaVersion: 2, - type: "pi_message", - sessionId: "session_0", - message: second, - }, - ]); - await expect( - loadMessages({ - store, - conversationId: "conversation-1", - messageCount: 2, - }), - ).resolves.toEqual([first, second]); - }); - - it("keeps host activity entries out of Pi projections", async () => { - const store = memoryStore(); - const first: PiMessage = { - role: "user", - content: [{ type: "text", text: "first" }], - timestamp: 1, - } as PiMessage; - - await commitMessages({ - store, - conversationId: "conversation-1", - messages: [first], - ttlMs: 60_000, - }); - await recordToolExecutionStarted({ - store, - conversationId: "conversation-1", - createdAtMs: 2, - toolCallId: "call-1", - toolName: "advisor", - args: { question: "private" }, - ttlMs: 60_000, - }); - await recordSubagentStarted({ - store, - conversationId: "conversation-1", - createdAtMs: 3, - historyMode: "shared", - parentConversationId: "conversation-1", - parentToolCallId: "call-1", - subagentInvocationId: "call-1", - subagentKind: "advisor", - transcriptRef: { - type: "advisor_session", - parentConversationId: "conversation-1", - key: "junior:conversation-1:advisor_session", - }, - ttlMs: 60_000, - }); - await recordSubagentEnded({ - store, - conversationId: "conversation-1", - createdAtMs: 4, - outcome: "success", - subagentInvocationId: "call-1", - ttlMs: 60_000, - }); - - await expect( - loadProjection({ store, conversationId: "conversation-1" }), - ).resolves.toEqual([first]); - await expect( - loadActivityEntries({ store, conversationId: "conversation-1" }), - ).resolves.toEqual([ - expect.objectContaining({ - type: "tool_execution_started", - toolCallId: "call-1", - }), - expect.objectContaining({ - type: "subagent_started", - parentToolCallId: "call-1", - }), - expect.objectContaining({ - type: "subagent_ended", - subagentInvocationId: "call-1", - }), - ]); - }); - - it("records projection resets instead of rewriting unsafe history", async () => { - const store = memoryStore(); - const first: PiMessage = { - role: "user", - content: [{ type: "text", text: "first" }], - timestamp: 1, - } as PiMessage; - const unsafe: PiMessage = { - role: "assistant", - content: [{ type: "text", text: "unsafe" }], - timestamp: 2, - } as PiMessage; - const replacement: PiMessage = { - role: "toolResult", - toolCallId: "call-1", - content: [{ type: "text", text: "safe" }], - timestamp: 3, - } as PiMessage; - - await commitMessages({ - store, - conversationId: "conversation-1", - messages: [first, unsafe], - ttlMs: 60_000, - }); - await commitMessages({ - store, - conversationId: "conversation-1", - messages: [first, replacement], - ttlMs: 60_000, - }); - - expect(store.entries).toEqual([ +describe("legacy session log decode", () => { + it("preserves ordered legacy entry discriminants for the SQL importer", async () => { + const message = userMessage("legacy question"); + const values = [ { schemaVersion: 2, type: "pi_message", sessionId: "session_0", - message: first, - }, - { - schemaVersion: 2, - type: "pi_message", - sessionId: "session_0", - message: unsafe, + message, }, { schemaVersion: 2, type: "projection_reset", sessionId: "session_1", - messages: [first, replacement], - provenance: [{ authority: "context" }, { authority: "context" }], - }, - ]); - await expect( - loadMessages({ - store, - conversationId: "conversation-1", - messageCount: 2, - }), - ).resolves.toEqual([first, replacement]); - await expect( - loadMessages({ - store, - conversationId: "conversation-1", - messageCount: 3, - }), - ).resolves.toBeUndefined(); - }); - - it("filters prior session events after a reset", async () => { - const store = memoryStore(); - const first: PiMessage = { - role: "user", - content: [{ type: "text", text: "first" }], - timestamp: 1, - } as PiMessage; - const replacement: PiMessage = { - role: "user", - content: [{ type: "text", text: "replacement" }], - timestamp: 2, - } as PiMessage; - const lateOldMessage: PiMessage = { - role: "assistant", - content: [{ type: "text", text: "late old session" }], - timestamp: 3, - } as PiMessage; - const next: PiMessage = { - role: "assistant", - content: [{ type: "text", text: "next" }], - timestamp: 4, - } as PiMessage; - - await commitMessages({ - store, - conversationId: "conversation-1", - messages: [first], - ttlMs: 60_000, - }); - await recordMcpProviderConnected({ - store, - conversationId: "conversation-1", - provider: "old-provider", - ttlMs: 60_000, - }); - await commitMessages({ - store, - conversationId: "conversation-1", - messages: [replacement], - ttlMs: 60_000, - }); - - store.entries.push( - { - schemaVersion: 1, - type: "pi_message", - sessionId: "session_0", - message: lateOldMessage, + messages: [message], + provenance: [{ authority: "context" }], }, { - schemaVersion: 1, + schemaVersion: 2, type: "mcp_provider_connected", - sessionId: "session_0", - provider: "old-provider-late", + sessionId: "session_1", + provider: "github", }, - ); - await recordMcpProviderConnected({ - store, - conversationId: "conversation-1", - provider: "new-provider", - ttlMs: 60_000, - }); - await commitMessages({ - store, - conversationId: "conversation-1", - messages: [replacement, next], - ttlMs: 60_000, - }); + ]; await expect( - loadProjection({ - store, - conversationId: "conversation-1", + readSessionLogEntries({ + conversationId: "legacy-conversation", + store: memoryStore(values), }), - ).resolves.toEqual([replacement, next]); - await expect( - loadProjection({ - store, - conversationId: "conversation-1", - sessionId: "session_0", - }), - ).resolves.toEqual([first]); - await expect( - loadConnectedMcpProviders({ - store, - conversationId: "conversation-1", - }), - ).resolves.toEqual(["new-provider"]); + ).resolves.toEqual(values); }); - it("keeps legacy entries without session ids readable", async () => { - const store = memoryStore(); - const ignored: PiMessage = { - role: "assistant", - content: [{ type: "text", text: "ignored" }], - timestamp: 1, - } as PiMessage; - const replacement: PiMessage = { - role: "user", - content: [{ type: "text", text: "replacement" }], - timestamp: 2, - } as PiMessage; - const next: PiMessage = { - role: "assistant", - content: [{ type: "text", text: "next" }], - timestamp: 3, - } as PiMessage; - - // Simulate stored rows written before sessionId existed. - const legacyEntries = [ - { schemaVersion: 1, type: "pi_message", message: ignored }, - { - schemaVersion: 1, - type: "projection_reset", - messages: [replacement], - }, - { schemaVersion: 1, type: "pi_message", message: next }, - ] as unknown as SessionLogEntry[]; - store.entries.push(...legacyEntries); + it("wraps a pre-envelope Pi message in the legacy wire format", async () => { + const message = userMessage("raw legacy message"); await expect( - loadProjection({ - store, - conversationId: "conversation-1", + readSessionLogEntries({ + conversationId: "legacy-conversation", + store: memoryStore([message]), }), - ).resolves.toEqual([replacement, next]); - }); - - it("migrates legacy requester log metadata while reading", async () => { - const store = memoryStore(); - const first: PiMessage = { - role: "user", - content: [{ type: "text", text: "first" }], - timestamp: 1, - } as PiMessage; - - store.entries.push( - { - schemaVersion: 1, - type: "pi_message", - message: first, - requester: { slackUserId: "U123", email: "alice@sentry.io" }, - } as unknown as SessionLogEntry, - { - schemaVersion: 1, - type: "requester_recorded", - requester: { slackUserId: "U456", email: "bob@sentry.io" }, - } as unknown as SessionLogEntry, - { - schemaVersion: 1, - type: "authorization_completed", - createdAtMs: 2, - kind: "plugin", - provider: "github", - requesterId: "U456", - authorizationId: "auth-1", - } as unknown as SessionLogEntry, - ); - - const projection = await loadProjectionWithActor({ - store, - conversationId: "conversation-1", - }); - expect(projection.messages).toEqual([ - first, - { - role: "user", - content: [ - { - type: "text", - text: 'Authorization completed for provider "github". Continue the blocked request and retry the provider operation if needed.', - }, - ], - timestamp: 2, - }, - ]); - // Legacy latest-wins actor metadata cannot be aligned to a specific - // message, so attribution fails closed instead of adopting the recorded - // actor for the whole projection. - expect(projection.actor).toBeUndefined(); - }); - - it("records connected MCP providers outside the Pi projection", async () => { - const store = memoryStore(); - const message: PiMessage = { - role: "user", - content: [{ type: "text", text: "first" }], - timestamp: 1, - } as PiMessage; - - await commitMessages({ - store, - conversationId: "conversation-1", - messages: [message], - ttlMs: 60_000, - }); - await recordMcpProviderConnected({ - store, - conversationId: "conversation-1", - provider: "linear", - ttlMs: 60_000, - }); - await recordMcpProviderConnected({ - store, - conversationId: "conversation-1", - provider: "linear", - ttlMs: 60_000, - }); - - expect(store.entries).toEqual([ + ).resolves.toEqual([ { schemaVersion: 2, type: "pi_message", sessionId: "session_0", message, }, - { - schemaVersion: 2, - type: "mcp_provider_connected", - sessionId: "session_0", - provider: "linear", - }, ]); - await expect( - loadConnectedMcpProviders({ - store, - conversationId: "conversation-1", - }), - ).resolves.toEqual(["linear"]); - await expect( - loadMessages({ - store, - conversationId: "conversation-1", - messageCount: 1, - }), - ).resolves.toEqual([message]); }); - it("records authorization interrupts and projects completion to Pi", async () => { - vi.useFakeTimers(); - vi.setSystemTime(1_000); - const store = memoryStore(); - const message: PiMessage = { - role: "user", - content: [{ type: "text", text: "list my orgs" }], - timestamp: 1, - } as PiMessage; - - await commitMessages({ - store, - conversationId: "conversation-1", - messages: [message], - ttlMs: 60_000, - }); - await recordAuthorizationRequested({ - store, - conversationId: "conversation-1", - kind: "plugin", - provider: "sentry", - actorId: "U123", - authorizationId: "auth-1", - delivery: "private_link_sent", - ttlMs: 60_000, - }); - await recordAuthorizationRequested({ - store, - conversationId: "conversation-1", - kind: "plugin", - provider: "sentry", - actorId: "U123", - authorizationId: "auth-1", - delivery: "private_link_sent", - ttlMs: 60_000, - }); - await recordAuthorizationCompleted({ - store, - conversationId: "conversation-1", - kind: "plugin", - provider: "sentry", - actorId: "U123", - authorizationId: "auth-1", - ttlMs: 60_000, - }); - await recordAuthorizationCompleted({ - store, - conversationId: "conversation-1", - kind: "plugin", - provider: "sentry", - actorId: "U123", - authorizationId: "auth-1", - ttlMs: 60_000, - }); + it("decodes JSON strings and migrates requester fields", async () => { + const message = userMessage("authored legacy message"); + const requester = { + platform: "slack", + slackUserId: "U123", + slackUserName: "alice", + teamId: "T123", + }; - expect(store.entries).toEqual([ + await expect( + readSessionLogEntries({ + conversationId: "legacy-conversation", + store: memoryStore([ + JSON.stringify({ + schemaVersion: 1, + type: "pi_message", + message, + requester, + }), + { + schemaVersion: 1, + type: "requester_recorded", + requester, + }, + ]), + }), + ).resolves.toEqual([ { - schemaVersion: 2, + schemaVersion: 1, type: "pi_message", sessionId: "session_0", message, + actor: requester, }, { - schemaVersion: 2, - type: "authorization_requested", - sessionId: "session_0", - createdAtMs: 1_000, - kind: "plugin", - provider: "sentry", - actorId: "U123", - authorizationId: "auth-1", - delivery: "private_link_sent", - }, - { - schemaVersion: 2, - type: "authorization_completed", + schemaVersion: 1, + type: "actor_recorded", sessionId: "session_0", - createdAtMs: 1_000, - kind: "plugin", - provider: "sentry", - actorId: "U123", - authorizationId: "auth-1", - }, - ]); - - await expect( - loadProjection({ - store, - conversationId: "conversation-1", - }), - ).resolves.toEqual([ - message, - { - role: "user", - content: [ - { - type: "text", - text: 'Authorization completed for provider "sentry". Continue the blocked request and retry the provider operation if needed.', - }, - ], - timestamp: 1_000, + actor: requester, }, ]); + }); - const firstProjection = await loadProjection({ - store, - conversationId: "conversation-1", - }); - vi.setSystemTime(9_000); + it("rejects malformed legacy entries at the decode boundary", async () => { await expect( - loadProjection({ - store, - conversationId: "conversation-1", + readSessionLogEntries({ + conversationId: "legacy-conversation", + store: memoryStore([{ type: "unknown_legacy_entry" }]), }), - ).resolves.toEqual(firstProjection); + ).rejects.toThrow("Invalid input"); }); }); -describe("session log message provenance", () => { - const alice = { - platform: "slack" as const, - teamId: "T123", - userId: "U123", - userName: "alice", - fullName: "Alice Example", - email: "alice@sentry.io", - }; - const bob = { - platform: "slack" as const, - teamId: "T123", - userId: "U456", - userName: "bob", - }; - - it("attaches instruction provenance to the last new user message on commit", async () => { - const store = memoryStore(); - const contextMsg: PiMessage = { - role: "user", - content: [{ type: "text", text: "prior context" }], - timestamp: 1, - } as PiMessage; - const turnMsg: PiMessage = { - role: "user", - content: [{ type: "text", text: "current question" }], - timestamp: 2, - } as PiMessage; - - await commitMessages({ - store, - conversationId: "conv-prov-1", - messages: [contextMsg, turnMsg], - ttlMs: 60_000, - newMessageProvenance: { authority: "instruction", actor: alice }, - }); - - // Instruction provenance lands on the LAST new user message (turnMsg); - // the earlier context message stays unauthored context (field omitted). - const piEntries = store.entries.filter((e) => e.type === "pi_message"); - expect(piEntries).toHaveLength(2); +describe("legacyActorProvenance", () => { + it("recovers an instruction actor when identity fields are intact", () => { expect( - (piEntries[0] as { provenance?: unknown }).provenance, - ).toBeUndefined(); - expect((piEntries[1] as { provenance?: unknown }).provenance).toEqual({ - authority: "instruction", - actor: alice, - }); - - // Provenance is not on the model-visible Pi message object. - const msgPayload = piEntries[1] as { - message?: { provenance?: unknown }; - }; - expect(msgPayload.message?.provenance).toBeUndefined(); - }); - - it("returns aligned per-message provenance from loadMessagesWithProvenance", async () => { - const store = memoryStore(); - const contextMsg: PiMessage = { - role: "user", - content: [{ type: "text", text: "prior context" }], - timestamp: 1, - } as PiMessage; - const turnMsg: PiMessage = { - role: "user", - content: [{ type: "text", text: "current question" }], - timestamp: 2, - } as PiMessage; - - await commitMessages({ - store, - conversationId: "conv-prov-2", - messages: [contextMsg, turnMsg], - ttlMs: 60_000, - newMessageProvenance: { authority: "instruction", actor: alice }, - }); - - const { loadMessagesWithProvenance } = - await import("@/chat/state/session-log"); - await expect( - loadMessagesWithProvenance({ - store, - conversationId: "conv-prov-2", - messageCount: 2, + legacyActorProvenance({ + platform: "slack", + slackUserId: "U123", + slackUserName: "alice", + teamId: "T123", }), - ).resolves.toEqual({ - messages: [contextMsg, turnMsg], - provenance: [ - { authority: "context" }, - { authority: "instruction", actor: alice }, - ], - }); - }); - - it("lets trailing provenance override the run actor default for steered messages", async () => { - const store = memoryStore(); - const initial: PiMessage = { - role: "user", - content: [{ type: "text", text: "start the deploy" }], - timestamp: 1, - } as PiMessage; - const steered: PiMessage = { - role: "user", - content: [{ type: "text", text: "actually run tests first" }], - timestamp: 2, - } as PiMessage; - - await commitMessages({ - store, - conversationId: "conv-prov-steering", - messages: [initial], - ttlMs: 60_000, - newMessageProvenance: { authority: "instruction", actor: alice }, - }); - await commitMessages({ - store, - conversationId: "conv-prov-steering", - messages: [initial, steered], - ttlMs: 60_000, - newMessageProvenance: { authority: "instruction", actor: alice }, - trailingMessageProvenance: [{ authority: "instruction", actor: bob }], - }); - - await expect( - loadProjectionWithProvenance({ - store, - conversationId: "conv-prov-steering", - }), - ).resolves.toEqual({ - messages: [initial, steered], - provenance: [ - { authority: "instruction", actor: alice }, - { authority: "instruction", actor: bob }, - ], - }); - }); - - it("derives the latest instruction actor via loadProjectionWithActor", async () => { - const store = memoryStore(); - const turnMsg: PiMessage = { - role: "user", - content: [{ type: "text", text: "question" }], - timestamp: 1, - } as PiMessage; - - await commitMessages({ - store, - conversationId: "conv-prov-3", - messages: [turnMsg], - ttlMs: 60_000, - newMessageProvenance: { authority: "instruction", actor: alice }, - }); - - const { loadProjectionWithActor } = - await import("@/chat/state/session-log"); - const projection = await loadProjectionWithActor({ - store, - conversationId: "conv-prov-3", - }); - - expect(projection.actor).toMatchObject({ - slackUserId: "U123", - slackUserName: "alice", - fullName: "Alice Example", - email: "alice@sentry.io", - }); - expect(projection.messages).toHaveLength(1); - }); - - it("writes aligned provenance through a projection reset", async () => { - const store = memoryStore(); - const first: PiMessage = { - role: "user", - content: [{ type: "text", text: "first" }], - timestamp: 1, - } as PiMessage; - const replacementUser: PiMessage = { - role: "user", - content: [{ type: "text", text: "replacement user" }], - timestamp: 2, - } as PiMessage; - const replacementSummary: PiMessage = { - role: "user", - content: [{ type: "text", text: "context handoff" }], - timestamp: 3, - } as PiMessage; - - await commitMessages({ - store, - conversationId: "conv-prov-4", - messages: [first], - ttlMs: 60_000, - }); - // A reset with explicit per-message provenance keeps the retained actor - // as an instruction while the synthetic summary is unauthored context. - await commitMessages({ - store, - conversationId: "conv-prov-4", - messages: [replacementUser, replacementSummary], - ttlMs: 60_000, - provenance: [ - { authority: "instruction", actor: alice }, - { authority: "context" }, - ], - }); - - const resetEntry = store.entries.find( - (entry) => entry.type === "projection_reset", - ); - expect(resetEntry).toMatchObject({ - messages: [replacementUser, replacementSummary], - provenance: [ - { authority: "instruction", actor: alice }, - { authority: "context" }, - ], + ).toEqual({ + authority: "instruction", + actor: { + platform: "slack", + teamId: "T123", + userId: "U123", + userName: "alice", + }, }); - await expect( - loadProjectionWithActor({ store, conversationId: "conv-prov-4" }), - ).resolves.toMatchObject({ actor: { slackUserId: "U123" } }); }); - it("fails closed on misaligned explicit provenance", async () => { - const store = memoryStore(); - const turnMsg: PiMessage = { - role: "user", - content: [{ type: "text", text: "question" }], - timestamp: 1, - } as PiMessage; - - await expect( - commitMessages({ - store, - conversationId: "conv-prov-5", - messages: [turnMsg], - ttlMs: 60_000, - provenance: [ - { authority: "instruction", actor: alice }, - { authority: "context" }, - ], - }), - ).rejects.toThrow(/align/); - }); - - it("returns a single actor for a single-actor run", () => { + it("fails closed to ambient context when identity is incomplete", () => { expect( - instructionActors([ - { authority: "context" }, - { authority: "instruction", actor: alice }, - { authority: "context" }, - ]), - ).toEqual([alice]); - }); - - it("collects batched multi-actor input in first-seen order, distinct by ids", () => { - const bob = { - platform: "slack" as const, - teamId: "T123", - userId: "U456", - userName: "bob", - }; - // Same human as alice by identity ids, but a different display profile; - // distinctness is by ids only, so it must collapse onto the first entry. - const aliceRenamed = { - platform: "slack" as const, - teamId: "T123", - userId: "U123", - userName: "alice-mobile", - fullName: "Alice On Mobile", - }; - const provenance: PiMessageProvenance[] = [ - { authority: "instruction", actor: alice }, - { authority: "instruction", actor: bob }, - { authority: "instruction", actor: aliceRenamed }, - ]; - - // First-seen order preserved; the second alice profile does not re-add. - expect(instructionActors(provenance)).toEqual([alice, bob]); - }); - - it("treats matching user ids on different teams as distinct authors", () => { - const aliceTeamB = { - platform: "slack" as const, - teamId: "T999", - userId: "U123", - }; - expect( - instructionActors([ - { authority: "instruction", actor: alice }, - { authority: "instruction", actor: aliceTeamB }, - ]), - ).toEqual([alice, aliceTeamB]); - }); - - it("does not collapse Slack actors whose concatenated team and user ids collide", () => { - const first = { - platform: "slack" as const, - teamId: "T1", - userId: "1U2", - }; - const second = { - platform: "slack" as const, - teamId: "T11", - userId: "U2", - }; - - expect( - instructionActors([ - { authority: "instruction", actor: first }, - { authority: "instruction", actor: second }, - ]), - ).toEqual([first, second]); - }); - - it("excludes context and unattributable instruction messages", () => { - // A system-actor / no-human run: nothing carries an instruction actor. - expect( - instructionActors([ - { authority: "context" }, - { authority: "context", actor: alice }, - { authority: "instruction" }, - ]), - ).toEqual([]); - }); - - it("is monotonic across a growing prefix, so continuation reuses the prefix set", () => { - const bob = { - platform: "slack" as const, - teamId: "T123", - userId: "U456", - userName: "bob", - }; - const full: PiMessageProvenance[] = [ - { authority: "instruction", actor: alice }, - { authority: "context" }, - { authority: "instruction", actor: bob }, - ]; - const committedPrefix = full.slice(0, 2); - - // The prefix set is a prefix of the full set: mid-run readers see a lower - // bound and a continuation derived from the committed prefix is stable. - expect(instructionActors(committedPrefix)).toEqual([alice]); - expect(instructionActors(full)).toEqual([alice, bob]); - }); - - it("decodes legacy v1 entries as unauthored context", async () => { - const store = memoryStore(); - const legacyContext: PiMessage = { - role: "user", - content: [{ type: "text", text: "legacy context" }], - timestamp: 1, - } as PiMessage; - const legacyInstruction: PiMessage = { - role: "user", - content: [{ type: "text", text: "legacy instruction" }], - timestamp: 2, - } as PiMessage; - - store.entries.push( - { - schemaVersion: 1, - type: "pi_message", - sessionId: "session_0", - message: legacyContext, - } as unknown as SessionLogEntry, - { - schemaVersion: 1, - type: "pi_message", - sessionId: "session_0", - message: legacyInstruction, - requester: { - platform: "slack", - slackUserId: "U123", - slackUserName: "alice", - teamId: "T123", - }, - } as unknown as SessionLogEntry, - ); - - const { loadMessagesWithProvenance, loadProjectionWithActor } = - await import("@/chat/state/session-log"); - await expect( - loadMessagesWithProvenance({ - store, - conversationId: "conv-prov-legacy", - messageCount: 2, - }), - ).resolves.toEqual({ - messages: [legacyContext, legacyInstruction], - provenance: [ - { authority: "context" }, - { - authority: "instruction", - actor: { - platform: "slack", - teamId: "T123", - userId: "U123", - userName: "alice", - }, - }, - ], - }); - await expect( - loadProjectionWithActor({ - store, - conversationId: "conv-prov-legacy", + legacyActorProvenance({ + platform: "slack", + slackUserId: "U123", }), - ).resolves.toMatchObject({ actor: { slackUserId: "U123" } }); + ).toEqual({ authority: "context" }); }); }); diff --git a/policies/data-redaction.md b/policies/data-redaction.md index 177f3eb7e..590d35892 100644 --- a/policies/data-redaction.md +++ b/policies/data-redaction.md @@ -10,7 +10,7 @@ conversation is authorized to retain. - Private channels, direct messages, local conversations, and unknown destinations are private by default. - Missing visibility metadata never widens access. -- Child conversations, agent steps, generated artifacts, and plugin records +- Child conversations, conversation events, generated artifacts, and plugin records inherit the visibility and actor boundaries of their owning conversation. ## Storage diff --git a/policies/interface-design.md b/policies/interface-design.md index 2b9b195fa..70587549e 100644 --- a/policies/interface-design.md +++ b/policies/interface-design.md @@ -31,7 +31,9 @@ Interfaces should expose the smallest useful capability while keeping ownership, they serve multiple consumers. `conversation:active` and `conversation:by-activity` describe the data contract better than process-specific names such as `recovery`. -- Keep exported interfaces role-shaped and small. `SessionLogStore` with `read` and `append` is clearer than a broad adapter that exposes unrelated state, Redis, or queue details. +- Keep exported interfaces role-shaped and small. A bounded legacy-import port + with only `read` is clearer than a broad adapter that exposes unrelated state, + Redis, or queue details. - Prefer import-site readability over globally unique names. If a name is only clear because it includes five qualifiers, the module boundary is probably doing too little work. - When a term is overloaded in the product or platform, define it once in the owning module documentation and avoid using it for nearby concepts. From 1fd83971970ffa28fb3a0a529d0315db86ef698b Mon Sep 17 00:00:00 2001 From: David Cramer Date: Tue, 14 Jul 2026 21:13:42 -0700 Subject: [PATCH 02/58] ref(conversations): Decouple reporting from Pi projection Move provider-neutral event reduction and runtime-context shaping into the conversations domain. Keep the Pi adapter responsible only for validating projected model messages. Build reporting transcripts, activity, and child history from canonical events while preserving lazy legacy import, privacy gates, redaction, and the existing API contract. Co-Authored-By: GPT-5 Codex --- .../junior/src/api/conversations/activity.ts | 15 ++- .../api/conversations/detail-projection.ts | 35 +++---- .../src/api/conversations/transcript.ts | 12 +-- .../chat/conversations/event-projection.ts | 96 +++++++++++++++++++ .../src/chat/conversations/history-loader.ts | 23 +++++ .../junior/src/chat/conversations/history.ts | 5 + .../src/chat/conversations/model-messages.ts | 63 ++++++++++++ .../src/chat/conversations/projection.ts | 37 ++----- .../junior/src/chat/pi/conversation-events.ts | 83 +++------------- packages/junior/src/chat/pi/transcript.ts | 60 ++---------- .../unit/chat/pi/conversation-events.test.ts | 4 + .../unit/conversations/model-messages.test.ts | 40 ++++++++ 12 files changed, 293 insertions(+), 180 deletions(-) create mode 100644 packages/junior/src/chat/conversations/event-projection.ts create mode 100644 packages/junior/src/chat/conversations/history-loader.ts create mode 100644 packages/junior/src/chat/conversations/model-messages.ts create mode 100644 packages/junior/tests/unit/conversations/model-messages.test.ts diff --git a/packages/junior/src/api/conversations/activity.ts b/packages/junior/src/api/conversations/activity.ts index 8c2c65624..c07b54b41 100644 --- a/packages/junior/src/api/conversations/activity.ts +++ b/packages/junior/src/api/conversations/activity.ts @@ -2,7 +2,6 @@ import type { ConversationEvent, ConversationEventData, } from "@/chat/conversations/history"; -import type { PiMessage } from "@/chat/pi/messages"; import { redactedPayloadFields } from "./transcript"; import type { ConversationActivityReport, @@ -17,11 +16,12 @@ interface ActivityPayloadMetadata { inputType?: string; } function toolResultStatuses( - messages: PiMessage[], + events: ConversationEvent[], ): Map { const statuses = new Map(); - for (const message of messages) { - const record = message as unknown as Record; + for (const event of events) { + if (event.data.type !== "message") continue; + const record = event.data.message as Record; if (record.role !== "toolResult" || typeof record.toolCallId !== "string") { continue; } @@ -47,15 +47,14 @@ function activityPayloadFields( * * Tool executions, subagent starts/ends, and their nesting are derived from the * conversation's durable events instead of the legacy Redis session log; tool - * statuses come from aligned model-message tool results. Redaction - * stays byte-compatible with the prior session-log path. + * statuses come from durable model-message events. Redaction stays + * byte-compatible with the prior session-log path. */ export function buildConversationActivityFromEvents(args: { canExposePayload: boolean; events: ConversationEvent[]; - messages: PiMessage[]; }): ConversationActivityReport[] { - const toolStatuses = toolResultStatuses(args.messages); + const toolStatuses = toolResultStatuses(args.events); const subagentEnds = new Map(); const subagentsByToolCallId = new Map< string, diff --git a/packages/junior/src/api/conversations/detail-projection.ts b/packages/junior/src/api/conversations/detail-projection.ts index 036c5d3ef..3e39d538a 100644 --- a/packages/junior/src/api/conversations/detail-projection.ts +++ b/packages/junior/src/api/conversations/detail-projection.ts @@ -7,16 +7,16 @@ import type { import type { ConversationEventStore, ConversationEvent, + ConversationModelMessage, } from "@/chat/conversations/history"; import type { Conversation } from "@/chat/conversations/store"; -import { loadProjection } from "@/chat/conversations/projection"; -import { projectConversationEvents } from "@/chat/pi/conversation-events"; +import { projectConversationEventHistory } from "@/chat/conversations/event-projection"; +import { loadCurrentConversationEvents } from "@/chat/conversations/history-loader"; +import { stripRuntimeTurnContextMessages } from "@/chat/conversations/model-messages"; import { getConversationEventStore, getConversationMessageStore, } from "@/chat/db"; -import type { PiMessage } from "@/chat/pi/messages"; -import { stripRuntimeTurnContext } from "@/chat/pi/transcript"; import { buildSentryConversationUrl, buildSentryTraceUrl, @@ -55,7 +55,7 @@ type EpochStartedEvent = ConversationEvent & { data: Extract; }; -function messageText(message: PiMessage): string { +function messageText(message: ConversationModelMessage): string { return normalizeTranscriptMessage(message) .parts.filter((part) => part.type === "text") .map((part) => part.text ?? "") @@ -64,7 +64,7 @@ function messageText(message: PiMessage): string { } function summaryAfterPrefix( - message: PiMessage, + message: ConversationModelMessage, prefixes: readonly string[], ): string | undefined { const text = messageText(message); @@ -74,7 +74,7 @@ function summaryAfterPrefix( } function summaryIndex( - messages: PiMessage[], + messages: ConversationModelMessage[], provenance: Array<{ authority: "context" | "instruction" }>, prefixes: readonly string[], ): number { @@ -89,7 +89,10 @@ function summaryIndex( return -1; } -function matchingPrefix(left: PiMessage[], right: PiMessage[]): number { +function matchingPrefix( + left: ConversationModelMessage[], + right: ConversationModelMessage[], +): number { const limit = Math.min(left.length, right.length); for (let index = 0; index < limit; index += 1) { if (!isDeepStrictEqual(left[index], right[index])) return index; @@ -107,10 +110,10 @@ function historyContent(args: { events: ConversationEvent[]; }): { contextEvents: ConversationContextEvent[]; - messages: PiMessage[]; + messages: ConversationModelMessage[]; } { const contextEvents: ConversationContextEvent[] = []; - const messages: PiMessage[] = []; + const messages: ConversationModelMessage[] = []; const epochs = new Map(); for (const event of args.events) { const epoch = epochs.get(event.contextEpoch); @@ -119,17 +122,17 @@ function historyContent(args: { } let previousModelId: string | undefined; - let previousProjection: PiMessage[] = []; + let previousProjection: ConversationModelMessage[] = []; for (const events of epochs.values()) { const marker = events.find( (event): event is EpochStartedEvent => event.data.type === "context_epoch_started", ); - const projection = projectConversationEvents(events); - const projected: PiMessage[] = []; + const projection = projectConversationEventHistory(events); + const projected: ConversationModelMessage[] = []; const projectedProvenance: typeof projection.provenance = []; projection.messages.forEach((message, index) => { - for (const retained of stripRuntimeTurnContext([message])) { + for (const retained of stripRuntimeTurnContextMessages([message])) { projected.push(retained); projectedProvenance.push(projection.provenance[index]!); } @@ -237,7 +240,6 @@ async function conversationContent(args: { activity: buildConversationActivityFromEvents({ canExposePayload: args.canExposePayload, events, - messages, }), contextEvents: history.contextEvents, transcript, @@ -410,9 +412,10 @@ export async function buildConversationSubagent( }); } - const childMessages: PiMessage[] = await loadProjection({ + const childEvents = await loadCurrentConversationEvents({ conversationId: childConversationId, }); + const childMessages = projectConversationEventHistory(childEvents).messages; if (childMessages.length === 0) { return subagentTranscriptReport(activity, { ...conversationFields, diff --git a/packages/junior/src/api/conversations/transcript.ts b/packages/junior/src/api/conversations/transcript.ts index e2ffce705..def1a0d59 100644 --- a/packages/junior/src/api/conversations/transcript.ts +++ b/packages/junior/src/api/conversations/transcript.ts @@ -1,6 +1,6 @@ import { isRecord } from "@/chat/coerce"; +import type { ConversationModelMessage } from "@/chat/conversations/history"; import { unwrapCurrentInstruction } from "@/chat/current-instruction"; -import type { PiMessage } from "@/chat/pi/messages"; import { unescapeXml } from "@/chat/xml"; import type { ConversationSubagentActivityReport, @@ -70,7 +70,7 @@ function isDisplayableTranscriptPart(part: TranscriptPart): boolean { return typeof part.output === "string" && part.output.trim().length > 0; } -/** Normalize Pi content parts for user-facing transcript output. */ +/** Normalize opaque model content parts for user-facing transcript output. */ function normalizeTranscriptPart( part: unknown, options: { @@ -161,16 +161,16 @@ function normalizeToolResultMessage( }; } -/** Normalize one provider transcript message into the reporting contract. */ +/** Normalize one durable model message into the reporting contract. */ export function normalizeTranscriptMessage( - message: PiMessage, + message: ConversationModelMessage, ): TranscriptMessage { return normalizeMessage(message); } /** Normalize a stored subagent message, including bounded legacy formats. */ export function normalizeSubagentTranscriptMessage( - message: PiMessage, + message: ConversationModelMessage, subagentKind: string, ): TranscriptMessage { return normalizeMessage(message, { @@ -179,7 +179,7 @@ export function normalizeSubagentTranscriptMessage( } function normalizeMessage( - message: PiMessage, + message: ConversationModelMessage, options: { unwrapLegacyAdvisorTask?: boolean } = {}, ): TranscriptMessage { const record = message as unknown as Record; diff --git a/packages/junior/src/chat/conversations/event-projection.ts b/packages/junior/src/chat/conversations/event-projection.ts new file mode 100644 index 000000000..77fc9d9bd --- /dev/null +++ b/packages/junior/src/chat/conversations/event-projection.ts @@ -0,0 +1,96 @@ +/** + * Provider-neutral projection of Junior conversation events. + * + * This reducer preserves opaque model messages while aligning their durable + * provenance and event sequence. Provider adapters validate the opaque payload + * only when they turn this projection into provider-specific state. + */ +import type { ModelProfile } from "@/chat/model-profile"; +import { + contextProvenance, + type ConversationMessageProvenance, +} from "./provenance"; +import type { + ConversationEvent, + ConversationEventData, + ConversationModelMessage, +} from "./history"; + +type MessageEventData = Extract; +type AuthorizationCompletedEventData = Extract< + ConversationEventData, + { type: "authorization_completed" } +>; + +/** Opaque model context projected from ordered Junior conversation events. */ +export interface ConversationEventProjection { + messages: ConversationModelMessage[]; + provenance: ConversationMessageProvenance[]; + seqs: number[]; + modelProfile: ModelProfile; + modelId: string | undefined; +} + +function authorizationObservationMessage( + data: AuthorizationCompletedEventData, + createdAtMs: number, +): ConversationModelMessage { + const label = data.kind === "mcp" ? "MCP authorization" : "Authorization"; + return { + role: "user", + content: [ + { + type: "text", + text: `${label} completed for provider "${data.provider}". Continue the blocked request and retry the provider operation if needed.`, + }, + ], + timestamp: createdAtMs, + } as ConversationModelMessage; +} + +function messageEventProvenance( + data: MessageEventData, +): ConversationMessageProvenance { + return data.provenance ?? contextProvenance; +} + +/** + * Project ordered Junior events into provider-neutral model context. + * + * Host-only events are filtered, completed authorization becomes a synthetic + * observation, and `maxSeq` reproduces an exact committed boundary. + */ +export function projectConversationEventHistory( + events: ConversationEvent[], + options?: { maxSeq?: number }, +): ConversationEventProjection { + const messages: ConversationModelMessage[] = []; + const provenance: ConversationMessageProvenance[] = []; + const seqs: number[] = []; + let modelProfile: ModelProfile = "standard"; + let modelId: string | undefined; + + for (const event of events) { + if (options?.maxSeq !== undefined && event.seq > options.maxSeq) break; + if (event.data.type === "context_epoch_started") { + modelProfile = event.data.modelProfile ?? "standard"; + modelId = event.data.modelId; + continue; + } + if (event.data.type === "message") { + messages.push(event.data.message); + provenance.push(messageEventProvenance(event.data)); + seqs.push(event.seq); + continue; + } + if (event.data.type === "authorization_completed") { + messages.push( + authorizationObservationMessage(event.data, event.createdAtMs), + ); + provenance.push(contextProvenance); + seqs.push(event.seq); + } + } + + return { messages, provenance, seqs, modelProfile, modelId }; +} diff --git a/packages/junior/src/chat/conversations/history-loader.ts b/packages/junior/src/chat/conversations/history-loader.ts new file mode 100644 index 000000000..7ed6808c5 --- /dev/null +++ b/packages/junior/src/chat/conversations/history-loader.ts @@ -0,0 +1,23 @@ +/** Bounded legacy import followed by canonical conversation-event reads. */ +import { getConversationEventStore } from "@/chat/db"; +import { ensureLegacyConversationImport } from "./legacy-import"; +import type { ConversationEvent } from "./history"; + +interface ScopedConversation { + conversationId: string; +} + +/** Ensure any pre-cutover transcript is represented in the canonical event log. */ +export async function ensureConversationEventHistory( + args: ScopedConversation, +): Promise { + await ensureLegacyConversationImport({ conversationId: args.conversationId }); +} + +/** Load the current event epoch after the bounded lazy legacy import. */ +export async function loadCurrentConversationEvents( + args: ScopedConversation, +): Promise { + await ensureConversationEventHistory(args); + return getConversationEventStore().loadCurrentEpoch(args.conversationId); +} diff --git a/packages/junior/src/chat/conversations/history.ts b/packages/junior/src/chat/conversations/history.ts index d4a923954..0d766e6b1 100644 --- a/packages/junior/src/chat/conversations/history.ts +++ b/packages/junior/src/chat/conversations/history.ts @@ -23,6 +23,11 @@ export const conversationModelMessageSchema = z .passthrough() .transform((value) => value as { role: string }); +/** Opaque model-continuity message stored by a Junior conversation event. */ +export type ConversationModelMessage = z.output< + typeof conversationModelMessageSchema +>; + const messageEventDataSchema = z .object({ type: z.literal("message"), diff --git a/packages/junior/src/chat/conversations/model-messages.ts b/packages/junior/src/chat/conversations/model-messages.ts new file mode 100644 index 000000000..3c05ea9e4 --- /dev/null +++ b/packages/junior/src/chat/conversations/model-messages.ts @@ -0,0 +1,63 @@ +/** Shape-only utilities for opaque model messages stored in conversation events. */ +import { TURN_CONTEXT_TAG } from "@/chat/turn-context-tag"; +import type { ConversationModelMessage } from "./history"; + +const RUNTIME_TURN_CONTEXT_START = `<${TURN_CONTEXT_TAG}>`; + +function userMessageContent( + message: ConversationModelMessage, +): unknown[] | undefined { + const record = message as { role?: unknown; content?: unknown }; + return record.role === "user" && Array.isArray(record.content) + ? record.content + : undefined; +} + +function isRuntimeTurnContextPart(part: unknown): boolean { + return ( + part !== null && + typeof part === "object" && + (part as { type?: unknown }).type === "text" && + typeof (part as { text?: unknown }).text === "string" && + (part as { text: string }).text.startsWith(RUNTIME_TURN_CONTEXT_START) + ); +} + +/** Return whether opaque model history carries volatile runtime bootstrap context. */ +export function hasRuntimeTurnContextMessages( + messages: ConversationModelMessage[], +): boolean { + return messages.some((message) => + userMessageContent(message)?.some(isRuntimeTurnContextPart), + ); +} + +/** Keep only volatile runtime bootstrap messages needed during context replacement. */ +export function retainRuntimeTurnContextMessages< + T extends ConversationModelMessage, +>(messages: T[]): T[] { + return messages.flatMap((message) => { + const runtimeContent = + userMessageContent(message)?.filter(isRuntimeTurnContextPart) ?? []; + return runtimeContent.length > 0 + ? ([{ ...message, content: runtimeContent }] as T[]) + : []; + }); +} + +/** Remove volatile runtime bootstrap parts without interpreting provider fields. */ +export function stripRuntimeTurnContextMessages< + T extends ConversationModelMessage, +>(messages: T[]): T[] { + return messages.flatMap((message) => { + const content = userMessageContent(message); + if (!content) return [message]; + + const nextContent = content.filter( + (part) => !isRuntimeTurnContextPart(part), + ); + if (nextContent.length === content.length) return [message]; + if (nextContent.length === 0) return []; + return [{ ...message, content: nextContent } as T]; + }); +} diff --git a/packages/junior/src/chat/conversations/projection.ts b/packages/junior/src/chat/conversations/projection.ts index a28de60f7..9858025f1 100644 --- a/packages/junior/src/chat/conversations/projection.ts +++ b/packages/junior/src/chat/conversations/projection.ts @@ -3,7 +3,7 @@ * * Materializes model-visible Pi context and derived run facts such as connected * providers. Storage and commit lifecycle stay in the conversations domain; - * Pi-specific event reduction lives in `pi/conversation-events`. + * provider-neutral event reduction stays here while Pi validates at its adapter. */ import { isDeepStrictEqual } from "node:util"; import type { PiMessage } from "@/chat/pi/messages"; @@ -16,7 +16,10 @@ import type { ConversationEvent, } from "@/chat/conversations/history"; import { getConversationEventStore } from "@/chat/db"; -import { ensureLegacyConversationImport } from "@/chat/conversations/legacy-import"; +import { + ensureConversationEventHistory, + loadCurrentConversationEvents, +} from "@/chat/conversations/history-loader"; import { projectConversationEvents, type PiConversationEventProjection, @@ -114,25 +117,11 @@ interface ScopedConversation { conversationId: string; } -/** - * Bridge a straggler's legacy Redis history into SQL before an execution read. - * - * Runtime reads run under the conversation lease the worker already holds, so - * this is the once-only lazy-import seam. Removed with the rest of the one-time - * import after the Redis TTL horizon. - */ -async function importLegacyIfNeeded(args: ScopedConversation): Promise { - await ensureLegacyConversationImport({ conversationId: args.conversationId }); -} - /** Load the current-epoch Pi projection for a conversation. */ export async function loadProjection( args: ScopedConversation, ): Promise { - await importLegacyIfNeeded(args); - const events = await getConversationEventStore().loadCurrentEpoch( - args.conversationId, - ); + const events = await loadCurrentConversationEvents(args); return projectConversationEvents(events).messages; } @@ -140,10 +129,7 @@ export async function loadProjection( export async function loadConversationProjection( args: ScopedConversation, ): Promise { - await importLegacyIfNeeded(args); - const events = await getConversationEventStore().loadCurrentEpoch( - args.conversationId, - ); + const events = await loadCurrentConversationEvents(args); const { messages, provenance, modelProfile, modelId } = projectConversationEvents(events); return { messages, provenance, modelProfile, modelId }; @@ -153,7 +139,7 @@ export async function loadConversationProjection( export async function openConversationProjection( args: ScopedConversation & { modelId: string }, ): Promise { - await importLegacyIfNeeded(args); + await ensureConversationEventHistory(args); const eventStore = getConversationEventStore(); const events = await eventStore.loadCurrentEpoch(args.conversationId); const projection = projectConversationEvents(events); @@ -199,7 +185,7 @@ export async function loadTurnProjection(args: { committedSeq: number; includeTail: boolean; }): Promise { - await importLegacyIfNeeded(args); + await ensureConversationEventHistory(args); const eventStore = getConversationEventStore(); // A record that committed no messages materializes the live projection, the // same way count-based records with a zero cursor did. @@ -227,10 +213,7 @@ export async function loadTurnProjection(args: { export async function loadConnectedMcpProviders( args: ScopedConversation, ): Promise { - await importLegacyIfNeeded(args); - const events = await getConversationEventStore().loadCurrentEpoch( - args.conversationId, - ); + const events = await loadCurrentConversationEvents(args); return connectedMcpProvidersFromEvents(events); } diff --git a/packages/junior/src/chat/pi/conversation-events.ts b/packages/junior/src/chat/pi/conversation-events.ts index 8f889b6fa..2c12c8ea8 100644 --- a/packages/junior/src/chat/pi/conversation-events.ts +++ b/packages/junior/src/chat/pi/conversation-events.ts @@ -1,26 +1,14 @@ /** * Pi adapter for Junior conversation events. * - * Conversation storage owns the canonical ordered event log. This module is - * the only reducer that translates those Junior events into Pi messages, - * aligned provenance, source sequence numbers, and epoch model binding. + * Conversation storage owns the canonical ordered event log and its generic + * projection. This module validates the opaque projected messages as Pi state. */ import type { ModelProfile } from "@/chat/model-profile"; -import type { - ConversationEvent, - ConversationEventData, -} from "@/chat/conversations/history"; +import type { ConversationEvent } from "@/chat/conversations/history"; +import { projectConversationEventHistory } from "@/chat/conversations/event-projection"; import { piMessageSchema, type PiMessage } from "@/chat/pi/messages"; -import { - contextProvenance, - type ConversationMessageProvenance, -} from "@/chat/conversations/provenance"; - -type MessageEventData = Extract; -type AuthorizationCompletedEventData = Extract< - ConversationEventData, - { type: "authorization_completed" } ->; +import type { ConversationMessageProvenance } from "@/chat/conversations/provenance"; /** Pi context projected from one Junior conversation epoch. */ export interface PiConversationProjection { @@ -35,29 +23,6 @@ export interface PiConversationEventProjection extends PiConversationProjection seqs: number[]; } -function authorizationObservationMessage( - data: AuthorizationCompletedEventData, - createdAtMs: number, -): PiMessage { - const label = data.kind === "mcp" ? "MCP authorization" : "Authorization"; - return { - role: "user", - content: [ - { - type: "text", - text: `${label} completed for provider "${data.provider}". Continue the blocked request and retry the provider operation if needed.`, - }, - ], - timestamp: createdAtMs, - } as PiMessage; -} - -function messageEventProvenance( - data: MessageEventData, -): ConversationMessageProvenance { - return data.provenance ?? contextProvenance; -} - /** * Project ordered Junior events into Pi context. * @@ -68,35 +33,11 @@ export function projectConversationEvents( events: ConversationEvent[], options?: { maxSeq?: number }, ): PiConversationEventProjection { - const messages: PiMessage[] = []; - const provenance: ConversationMessageProvenance[] = []; - const seqs: number[] = []; - let modelProfile: ModelProfile = "standard"; - let modelId: string | undefined; - - for (const event of events) { - if (options?.maxSeq !== undefined && event.seq > options.maxSeq) { - break; - } - if (event.data.type === "context_epoch_started") { - modelProfile = event.data.modelProfile ?? "standard"; - modelId = event.data.modelId; - continue; - } - if (event.data.type === "message") { - messages.push(piMessageSchema.parse(event.data.message)); - provenance.push(messageEventProvenance(event.data)); - seqs.push(event.seq); - continue; - } - if (event.data.type === "authorization_completed") { - messages.push( - authorizationObservationMessage(event.data, event.createdAtMs), - ); - provenance.push(contextProvenance); - seqs.push(event.seq); - } - } - - return { messages, provenance, seqs, modelProfile, modelId }; + const projection = projectConversationEventHistory(events, options); + return { + ...projection, + messages: projection.messages.map((message) => + piMessageSchema.parse(message), + ), + }; } diff --git a/packages/junior/src/chat/pi/transcript.ts b/packages/junior/src/chat/pi/transcript.ts index 9d7357752..39704c863 100644 --- a/packages/junior/src/chat/pi/transcript.ts +++ b/packages/junior/src/chat/pi/transcript.ts @@ -12,10 +12,12 @@ import type { ToolResultMessage, } from "@earendil-works/pi-ai"; import { unwrapCurrentInstruction } from "@/chat/current-instruction"; +import { + hasRuntimeTurnContextMessages, + retainRuntimeTurnContextMessages, + stripRuntimeTurnContextMessages, +} from "@/chat/conversations/model-messages"; import type { PiMessage } from "@/chat/pi/messages"; -import { TURN_CONTEXT_TAG } from "@/chat/turn-context-tag"; - -const RUNTIME_TURN_CONTEXT_START = `<${TURN_CONTEXT_TAG}>`; // Prior-thread context blocks the runtime embeds inside the same user-turn text // that carries the block (see buildUserTurnText and @@ -123,44 +125,14 @@ export function trimTrailingAssistantMessages( return end === messages.length ? [...messages] : messages.slice(0, end); } -function getUserMessageContent(message: PiMessage): unknown[] | undefined { - const record = message as { role?: unknown; content?: unknown }; - return record.role === "user" && Array.isArray(record.content) - ? record.content - : undefined; -} - -function isRuntimeTurnContextPart(part: unknown): boolean { - return ( - part !== null && - typeof part === "object" && - (part as { type?: unknown }).type === "text" && - typeof (part as { text?: unknown }).text === "string" && - (part as { text: string }).text.startsWith(RUNTIME_TURN_CONTEXT_START) - ); -} - /** Return whether Pi history already carries session bootstrap context. */ export function hasRuntimeTurnContext(messages: PiMessage[]): boolean { - return messages.some((message) => - getUserMessageContent(message)?.some((part) => - isRuntimeTurnContextPart(part), - ), - ); + return hasRuntimeTurnContextMessages(messages); } /** Keep only volatile bootstrap messages needed by an in-turn context replacement. */ export function retainRuntimeTurnContext(messages: PiMessage[]): PiMessage[] { - return messages.flatMap((message) => { - const content = getUserMessageContent(message); - if (!content) { - return []; - } - const runtimeContent = content.filter(isRuntimeTurnContextPart); - return runtimeContent.length > 0 - ? [{ ...message, content: runtimeContent } as PiMessage] - : []; - }); + return retainRuntimeTurnContextMessages(messages); } /** @@ -186,21 +158,5 @@ export function instructionTextForProjection(text: string): string { /** Remove volatile runtime context before reusing messages as history. */ export function stripRuntimeTurnContext(messages: PiMessage[]): PiMessage[] { - return messages.flatMap((message) => { - const content = getUserMessageContent(message); - if (!content) { - return [message]; - } - - const nextContent = content.filter( - (part) => !isRuntimeTurnContextPart(part), - ); - if (nextContent.length === content.length) { - return [message]; - } - if (nextContent.length === 0) { - return []; - } - return [{ ...message, content: nextContent } as PiMessage]; - }); + return stripRuntimeTurnContextMessages(messages); } diff --git a/packages/junior/tests/unit/chat/pi/conversation-events.test.ts b/packages/junior/tests/unit/chat/pi/conversation-events.test.ts index 35a10fb5f..15bbb3652 100644 --- a/packages/junior/tests/unit/chat/pi/conversation-events.test.ts +++ b/packages/junior/tests/unit/chat/pi/conversation-events.test.ts @@ -4,6 +4,7 @@ import { type ConversationEvent, type ConversationEventData, } from "@/chat/conversations/history"; +import { projectConversationEventHistory } from "@/chat/conversations/event-projection"; import { projectConversationEvents } from "@/chat/pi/conversation-events"; function event(seq: number, data: ConversationEventData): ConversationEvent { @@ -72,8 +73,11 @@ describe("projectConversationEvents", () => { ]; it("projects messages, authorization observations, provenance, and model binding", () => { + const genericProjection = projectConversationEventHistory(events); const projection = projectConversationEvents(events); + expect(projection).toEqual(genericProjection); + expect(projection).toEqual({ messages: [ firstMessage, diff --git a/packages/junior/tests/unit/conversations/model-messages.test.ts b/packages/junior/tests/unit/conversations/model-messages.test.ts new file mode 100644 index 000000000..4b66c72b8 --- /dev/null +++ b/packages/junior/tests/unit/conversations/model-messages.test.ts @@ -0,0 +1,40 @@ +import { describe, expect, it } from "vitest"; +import { + hasRuntimeTurnContextMessages, + retainRuntimeTurnContextMessages, + stripRuntimeTurnContextMessages, +} from "@/chat/conversations/model-messages"; + +const runtimePart = { + type: "text", + text: "volatile", +}; +const instructionPart = { type: "text", text: "keep me" }; + +describe("opaque conversation model messages", () => { + it("detects and retains only user runtime context", () => { + const messages = [ + { role: "assistant", content: [runtimePart] }, + { role: "user", content: [runtimePart, instructionPart] }, + ]; + + expect(hasRuntimeTurnContextMessages(messages)).toBe(true); + expect(retainRuntimeTurnContextMessages(messages)).toEqual([ + { role: "user", content: [runtimePart] }, + ]); + }); + + it("strips runtime context and drops empty user messages", () => { + const assistant = { role: "assistant", content: [runtimePart] }; + const messages = [ + assistant, + { role: "user", content: [runtimePart] }, + { role: "user", content: [runtimePart, instructionPart] }, + ]; + + expect(stripRuntimeTurnContextMessages(messages)).toEqual([ + assistant, + { role: "user", content: [instructionPart] }, + ]); + }); +}); From ec2bc43f7b27944aa0e1240195c8f5a6f415e156 Mon Sep 17 00:00:00 2001 From: David Cramer Date: Tue, 14 Jul 2026 21:46:24 -0700 Subject: [PATCH 03/58] ref(conversations): Canonicalize visible message history Persist visible message, metadata, and reply facts in the canonical conversation event log while maintaining the SQL message table as an atomic hydration and search projection. Add idempotent event keys, shared sequence locking, a bounded zero-gap backfill, and the stopped-worker schema cut that removes the legacy step compatibility view. Co-Authored-By: GPT-5 Codex --- .../conversation-event-log/proposal.md | 17 +- .../changes/conversation-event-log/tasks.md | 8 +- .../migrations/0005_conversation_events.sql | 4 +- .../0005_visible_message_events.sql | 5 + packages/junior/migrations/README.md | 13 +- .../junior/migrations/meta/0005_snapshot.json | 119 ++++---- packages/junior/migrations/meta/_journal.json | 7 + .../junior/src/chat/conversations/README.md | 15 +- .../junior/src/chat/conversations/history.ts | 37 +++ .../junior/src/chat/conversations/messages.ts | 12 +- .../src/chat/conversations/sql/event-lock.ts | 13 + .../src/chat/conversations/sql/history.ts | 63 ++++- .../sql/legacy-history-import.ts | 234 +++++++++------- .../src/chat/conversations/sql/messages.ts | 129 ++++++++- .../chat/conversations/visible-messages.ts | 9 +- packages/junior/src/cli/upgrade.ts | 2 + .../conversation-visible-message-events.ts | 205 ++++++++++++++ .../src/db/schema/conversation-events.ts | 6 + .../cli/conversation-event-migration.test.ts | 253 ++++++++++++++++++ .../conversation-transcripts-sql.test.ts | 153 ++++++++++- .../conversations/legacy-import.test.ts | 21 +- 21 files changed, 1088 insertions(+), 237 deletions(-) create mode 100644 packages/junior/migrations/0005_visible_message_events.sql create mode 100644 packages/junior/src/chat/conversations/sql/event-lock.ts create mode 100644 packages/junior/src/cli/upgrade/migrations/conversation-visible-message-events.ts diff --git a/openspec/changes/conversation-event-log/proposal.md b/openspec/changes/conversation-event-log/proposal.md index 0e70066d4..b63e1519f 100644 --- a/openspec/changes/conversation-event-log/proposal.md +++ b/openspec/changes/conversation-event-log/proposal.md @@ -56,9 +56,10 @@ child event stream. 1. Formalize the event contract, isolate Pi projection, rename the physical table, persist schema versions, and cut every application reader and writer over to canonical Junior events. -2. Rewrite legacy SQL message rows in bounded batches. A database-only view - translates old worker reads and writes during the rolling-deploy window; it - stores no duplicate rows and is removed in the next release. +2. Rewrite legacy model-message rows, drain all 0.103.x workers, then apply the + hard schema cut that removes their compatibility view. Backfill + visible-message rows while workers remain stopped; start new workers only + after fail-closed zero-gap verification passes. 3. Expose privacy-safe events from the detail API and move timeline shaping to the dashboard. 4. Add child-conversation lineage and context-fork projection. @@ -75,8 +76,8 @@ child event stream. ## Compatibility -The in-place table rename preserves sequence identity and payload bytes. Old -workers remain compatible through the temporary SQL view while the application -uses only `junior_conversation_events`; the view and triggers are dropped after -the rolling-deploy window. Internal runtime and dashboard contracts may use -hard cutovers because their producers and consumers ship together. +The in-place table rename preserves sequence identity and payload bytes. A +temporary SQL view supports the first migration phase, but the visible-message +cutover is intentionally not rolling: 0.103.x workers must be drained before +the view and its functions are dropped. New workers start only after the final +backfill verifies zero gaps. diff --git a/openspec/changes/conversation-event-log/tasks.md b/openspec/changes/conversation-event-log/tasks.md index 6cecab296..6730166e0 100644 --- a/openspec/changes/conversation-event-log/tasks.md +++ b/openspec/changes/conversation-event-log/tasks.md @@ -24,6 +24,11 @@ database-only compatibility view supports rolling old workers. - [x] Keep Pi messages derived rather than separately persisted. - [ ] Add turn outcome, delivery, and explicit turn-correlation event variants. +- [x] Persist visible-message and reply facts as canonical events while updating + the hydration/search table as an atomic read model. +- [x] Backfill existing visible-message rows with a bounded, verified migration + while workers are stopped, after removing the drained 0.103.x workers' + compatibility view. ## 3. Event API And Dashboard @@ -43,7 +48,8 @@ ## 5. Cleanup And Verification -- [ ] Make remaining visible-message and aggregate stores explicit read models. +- [x] Make the visible-message table an explicit event-backed read model. +- [ ] Make remaining aggregate stores explicit read models. - [ ] Remove obsolete transcript reconstruction and legacy compatibility after migration completion. - [ ] Verify retention, purge, redaction, idempotency, replay parity, and event diff --git a/packages/junior/migrations/0005_conversation_events.sql b/packages/junior/migrations/0005_conversation_events.sql index 2c817dd2d..bbb9923ac 100644 --- a/packages/junior/migrations/0005_conversation_events.sql +++ b/packages/junior/migrations/0005_conversation_events.sql @@ -3,8 +3,8 @@ ALTER TABLE "junior_conversation_events" RENAME CONSTRAINT "junior_agent_steps_c ALTER TABLE "junior_conversation_events" RENAME CONSTRAINT "junior_agent_steps_conversation_id_junior_conversations_conversation_id_fk" TO "junior_conversation_events_conversation_id_junior_conversations_conversation_id_fk";--> statement-breakpoint ALTER INDEX "junior_agent_steps_epoch_idx" RENAME TO "junior_conversation_events_epoch_idx";--> statement-breakpoint ALTER TABLE "junior_conversation_events" ADD COLUMN "schema_version" integer DEFAULT 1 NOT NULL;--> statement-breakpoint --- TODO(v0.104.0): Remove the junior_agent_steps compatibility view and trigger --- functions after the 0.103.x rolling-deploy window. +-- Temporary 0.103.x compatibility; 0005 removes the view and functions after +-- the required worker drain. CREATE VIEW "junior_agent_steps" AS SELECT "conversation_id", diff --git a/packages/junior/migrations/0005_visible_message_events.sql b/packages/junior/migrations/0005_visible_message_events.sql new file mode 100644 index 000000000..02046117f --- /dev/null +++ b/packages/junior/migrations/0005_visible_message_events.sql @@ -0,0 +1,5 @@ +ALTER TABLE "junior_conversation_events" ADD COLUMN "idempotency_key" text;--> statement-breakpoint +CREATE UNIQUE INDEX "junior_conversation_events_idempotency_idx" ON "junior_conversation_events" USING btree ("conversation_id","idempotency_key");--> statement-breakpoint +DROP VIEW "junior_agent_steps";--> statement-breakpoint +DROP FUNCTION "junior_agent_steps_insert_compat"();--> statement-breakpoint +DROP FUNCTION "junior_agent_steps_delete_compat"(); diff --git a/packages/junior/migrations/README.md b/packages/junior/migrations/README.md index 4974f66da..ab0483b1e 100644 --- a/packages/junior/migrations/README.md +++ b/packages/junior/migrations/README.md @@ -13,8 +13,11 @@ pre-Drizzle Junior migration runner. During upgrade, existing installations adopt that baseline once; new installations execute it normally. All later migrations are applied by Drizzle in journal order. -`0004_conversation_events.sql` keeps `junior_agent_steps` as an updatable -compatibility view for the 0.103.x rolling-deploy window. It maps legacy -`pi_message` reads and writes to canonical `message` events while the CLI -upgrade backfills existing rows in bounded batches. Remove the view and its -trigger functions in 0.104.0 after 0.103.x workers are no longer supported. +`0004_conversation_events.sql` temporarily creates `junior_agent_steps` as an +updatable 0.103.x compatibility view. It maps legacy `pi_message` reads and +writes to canonical `message` events while the first event rewrite runs. + +`0005_visible_message_events.sql` is the hard cut: drain every 0.103.x worker +before applying it because it drops the legacy view and its functions. Run the +final visible-message backfill next and require zero-gap verification before +starting new workers. diff --git a/packages/junior/migrations/meta/0005_snapshot.json b/packages/junior/migrations/meta/0005_snapshot.json index f50845eb1..d7dfee2b9 100644 --- a/packages/junior/migrations/meta/0005_snapshot.json +++ b/packages/junior/migrations/meta/0005_snapshot.json @@ -1,6 +1,6 @@ { - "id": "5bcf5afe-2cc5-497a-a873-08c54ab1dfb0", - "prevId": "f4f247d8-7b7d-420b-a78f-7a36ba8c1472", + "id": "cf8b6b75-b49a-40db-b801-dfaac587ae2d", + "prevId": "5bcf5afe-2cc5-497a-a873-08c54ab1dfb0", "version": "7", "dialect": "postgresql", "tables": { @@ -33,6 +33,12 @@ "notNull": true, "default": 1 }, + "idempotency_key": { + "name": "idempotency_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, "type": { "name": "type", "type": "text", @@ -85,6 +91,27 @@ "concurrently": false, "method": "btree", "with": {} + }, + "junior_conversation_events_idempotency_idx": { + "name": "junior_conversation_events_idempotency_idx", + "columns": [ + { + "expression": "conversation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "idempotency_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} } }, "foreignKeys": { @@ -92,12 +119,8 @@ "name": "junior_conversation_events_conversation_id_junior_conversations_conversation_id_fk", "tableFrom": "junior_conversation_events", "tableTo": "junior_conversations", - "columnsFrom": [ - "conversation_id" - ], - "columnsTo": [ - "conversation_id" - ], + "columnsFrom": ["conversation_id"], + "columnsTo": ["conversation_id"], "onDelete": "no action", "onUpdate": "no action" } @@ -105,10 +128,7 @@ "compositePrimaryKeys": { "junior_conversation_events_conversation_id_seq_pk": { "name": "junior_conversation_events_conversation_id_seq_pk", - "columns": [ - "conversation_id", - "seq" - ] + "columns": ["conversation_id", "seq"] } }, "uniqueConstraints": {}, @@ -212,12 +232,8 @@ "name": "junior_conversation_messages_conversation_id_junior_conversations_conversation_id_fk", "tableFrom": "junior_conversation_messages", "tableTo": "junior_conversations", - "columnsFrom": [ - "conversation_id" - ], - "columnsTo": [ - "conversation_id" - ], + "columnsFrom": ["conversation_id"], + "columnsTo": ["conversation_id"], "onDelete": "no action", "onUpdate": "no action" }, @@ -225,12 +241,8 @@ "name": "junior_conversation_messages_author_identity_id_junior_identities_id_fk", "tableFrom": "junior_conversation_messages", "tableTo": "junior_identities", - "columnsFrom": [ - "author_identity_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["author_identity_id"], + "columnsTo": ["id"], "onDelete": "no action", "onUpdate": "no action" } @@ -238,10 +250,7 @@ "compositePrimaryKeys": { "junior_conversation_messages_conversation_id_message_id_pk": { "name": "junior_conversation_messages_conversation_id_message_id_pk", - "columns": [ - "conversation_id", - "message_id" - ] + "columns": ["conversation_id", "message_id"] } }, "uniqueConstraints": {}, @@ -429,12 +438,6 @@ "type": "text", "primaryKey": false, "notNull": false - }, - "archived_at": { - "name": "archived_at", - "type": "timestamp with time zone", - "primaryKey": false, - "notNull": false } }, "indexes": { @@ -571,12 +574,8 @@ "name": "junior_conversations_destination_id_junior_destinations_id_fk", "tableFrom": "junior_conversations", "tableTo": "junior_destinations", - "columnsFrom": [ - "destination_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["destination_id"], + "columnsTo": ["id"], "onDelete": "no action", "onUpdate": "no action" }, @@ -584,12 +583,8 @@ "name": "junior_conversations_actor_identity_id_junior_identities_id_fk", "tableFrom": "junior_conversations", "tableTo": "junior_identities", - "columnsFrom": [ - "actor_identity_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["actor_identity_id"], + "columnsTo": ["id"], "onDelete": "no action", "onUpdate": "no action" }, @@ -597,12 +592,8 @@ "name": "junior_conversations_creator_identity_id_junior_identities_id_fk", "tableFrom": "junior_conversations", "tableTo": "junior_identities", - "columnsFrom": [ - "creator_identity_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["creator_identity_id"], + "columnsTo": ["id"], "onDelete": "no action", "onUpdate": "no action" }, @@ -610,12 +601,8 @@ "name": "junior_conversations_credential_subject_identity_id_junior_identities_id_fk", "tableFrom": "junior_conversations", "tableTo": "junior_identities", - "columnsFrom": [ - "credential_subject_identity_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["credential_subject_identity_id"], + "columnsTo": ["id"], "onDelete": "no action", "onUpdate": "no action" }, @@ -623,12 +610,8 @@ "name": "junior_conversations_parent_conversation_id_junior_conversations_conversation_id_fk", "tableFrom": "junior_conversations", "tableTo": "junior_conversations", - "columnsFrom": [ - "parent_conversation_id" - ], - "columnsTo": [ - "conversation_id" - ], + "columnsFrom": ["parent_conversation_id"], + "columnsTo": ["conversation_id"], "onDelete": "no action", "onUpdate": "no action" } @@ -952,12 +935,8 @@ "name": "junior_identities_user_id_junior_users_id_fk", "tableFrom": "junior_identities", "tableTo": "junior_users", - "columnsFrom": [ - "user_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["user_id"], + "columnsTo": ["id"], "onDelete": "no action", "onUpdate": "no action" } diff --git a/packages/junior/migrations/meta/_journal.json b/packages/junior/migrations/meta/_journal.json index 7f4c61ea1..d5e259957 100644 --- a/packages/junior/migrations/meta/_journal.json +++ b/packages/junior/migrations/meta/_journal.json @@ -43,6 +43,13 @@ "when": 1784137322421, "tag": "0005_conversation_events", "breakpoints": true + }, + { + "idx": 5, + "version": "7", + "when": 1784089383071, + "tag": "0005_visible_message_events", + "breakpoints": true } ] } diff --git a/packages/junior/src/chat/conversations/README.md b/packages/junior/src/chat/conversations/README.md index b32c2b47b..19502e004 100644 --- a/packages/junior/src/chat/conversations/README.md +++ b/packages/junior/src/chat/conversations/README.md @@ -7,7 +7,9 @@ conversation events, compaction boundaries, search, retention, and legacy import - Conversation rows identify the source, destination, participants, visibility, and lifecycle metadata. -- Visible messages are the destination-facing user and assistant history. +- Visible-message events are the destination-facing user and assistant history. + `junior_conversation_messages` is their rebuildable hydration and search read + model, never a second history authority. - Conversation events are the versioned, append-only execution history used to restore Pi state. Pi messages and context are projections of this log, not a second authority. @@ -36,6 +38,8 @@ must not become a dashboard or external API payload. - Persist user input before agent execution. - Persist assistant text only after successful destination delivery. +- Append each visible-message fact and update its SQL read model in one + transaction under the conversation event lock. - Append conversation events in monotonic sequence order. - Restore state from durable events rather than a duplicate transcript cache. - Compaction replaces prior model context without rewriting visible history. @@ -58,9 +62,12 @@ Follow `../../../../../policies/data-redaction.md` and - Data rewrites use explicit migrations or resumable import code. - Legacy fields remain readable only for the migration window and are removed after the new authority is verified. -- The database-only `junior_agent_steps` compatibility view supports old workers - during the 0.103.x rollout and is removed in 0.104.0; it stores no duplicate - rows and is never an application read or write path. +- Drain every 0.103.x worker before applying the visible-message schema cut, + which drops the temporary `junior_agent_steps` compatibility view and its + functions. +- Run the final rerunnable visible-message backfill while workers remain + stopped. Its fail-closed zero-gap verification is the cutover gate; start new + workers only after it passes. - Purge and migration jobs operate in bounded batches and are safe to retry. Representative coverage lives in diff --git a/packages/junior/src/chat/conversations/history.ts b/packages/junior/src/chat/conversations/history.ts index 0d766e6b1..46c3b052c 100644 --- a/packages/junior/src/chat/conversations/history.ts +++ b/packages/junior/src/chat/conversations/history.ts @@ -162,6 +162,38 @@ const toolExecutionStartedEventDataSchema = z }) .strict(); +const visibleMessageRoleSchema = z.union([ + z.literal("user"), + z.literal("assistant"), + z.literal("system"), +]); + +const visibleMessageRecordedEventDataSchema = z + .object({ + type: z.literal("visible_message_recorded"), + messageId: z.string().min(1), + role: visibleMessageRoleSchema, + text: z.string(), + authorIdentityId: z.string().min(1).optional(), + meta: z.record(z.string(), z.unknown()).optional(), + }) + .strict(); + +const visibleMessageMetadataUpdatedEventDataSchema = z + .object({ + type: z.literal("visible_message_metadata_updated"), + messageId: z.string().min(1), + meta: z.record(z.string(), z.unknown()), + }) + .strict(); + +const visibleMessageRepliedEventDataSchema = z + .object({ + type: z.literal("visible_message_replied"), + messageId: z.string().min(1), + }) + .strict(); + const visibleContextCompactedEventDataSchema = z .object({ type: z.literal("visible_context_compacted"), @@ -213,6 +245,9 @@ const appendableConversationEventDataSchema = z.union([ authorizationRequestedEventDataSchema, authorizationCompletedEventDataSchema, toolExecutionStartedEventDataSchema, + visibleMessageRecordedEventDataSchema, + visibleMessageMetadataUpdatedEventDataSchema, + visibleMessageRepliedEventDataSchema, visibleContextCompactedEventDataSchema, subagentStartedEventDataSchema, subagentEndedEventDataSchema, @@ -238,6 +273,7 @@ export const conversationEventSchema = z schemaVersion: z.literal(1), seq: z.number().int().nonnegative(), contextEpoch: z.number().int().nonnegative(), + idempotencyKey: z.string().min(1).optional(), createdAtMs: z.number().finite(), data: conversationEventDataSchema, }) @@ -250,6 +286,7 @@ export type ConversationEvent = z.output; export const newConversationEventSchema = z .object({ data: appendableConversationEventDataSchema, + idempotencyKey: z.string().min(1).optional(), createdAtMs: z.number().finite(), }) .strict(); diff --git a/packages/junior/src/chat/conversations/messages.ts b/packages/junior/src/chat/conversations/messages.ts index f32ea8f01..eb8b70790 100644 --- a/packages/junior/src/chat/conversations/messages.ts +++ b/packages/junior/src/chat/conversations/messages.ts @@ -1,8 +1,8 @@ /** * Visible conversation message port. * - * Messages are immutable source facts recorded idempotently by source identity; - * the only mutable bookkeeping is the `replied_at` delivery mark. + * Conversation events are the source facts. This port atomically appends those + * facts and maintains the SQL table used for hydration and indexed search. */ /** Author role of a visible conversation message. */ @@ -30,20 +30,20 @@ export interface ConversationMessage { createdAtMs: number; } -/** Persist and read the visible per-conversation message transcript. */ +/** Persist event-backed visible messages and read their SQL projection. */ export interface ConversationMessageStore { - /** Record source messages idempotently by `(conversation_id, message_id)`. */ + /** Append source facts and project them idempotently by message identity. */ record( conversationId: string, messages: NewConversationMessage[], ): Promise; - /** Set the mutable `replied_at` mark; content columns are never touched. */ + /** Append a reply fact and project its immutable first timestamp. */ markReplied( conversationId: string, messageId: string, repliedAtMs: number, ): Promise; - /** List messages in `created_at` order. */ + /** List the materialized read model in `created_at` order. */ list( conversationId: string, opts?: { limit?: number }, diff --git a/packages/junior/src/chat/conversations/sql/event-lock.ts b/packages/junior/src/chat/conversations/sql/event-lock.ts new file mode 100644 index 000000000..1462fe721 --- /dev/null +++ b/packages/junior/src/chat/conversations/sql/event-lock.ts @@ -0,0 +1,13 @@ +import type { JuniorSqlDatabase } from "@/db/db"; + +/** Serialize all event sequence allocation and visible projection writes. */ +export async function withConversationEventLock( + executor: JuniorSqlDatabase, + conversationId: string, + callback: () => Promise, +): Promise { + return executor.withLock( + `junior_conversation:event:${conversationId}`, + callback, + ); +} diff --git a/packages/junior/src/chat/conversations/sql/history.ts b/packages/junior/src/chat/conversations/sql/history.ts index 0d8b525e9..5a52ad80d 100644 --- a/packages/junior/src/chat/conversations/sql/history.ts +++ b/packages/junior/src/chat/conversations/sql/history.ts @@ -1,4 +1,4 @@ -import { and, asc, eq, isNotNull, sql } from "drizzle-orm"; +import { and, asc, eq, inArray, isNotNull, sql } from "drizzle-orm"; import type { JuniorSqlDatabase } from "@/db/db"; import { conversationEventDataSchema, @@ -15,19 +15,21 @@ import { import { ensureConversationRow } from "./conversation-row"; import { juniorConversationEvents, juniorConversations } from "@/db/schema"; import { sanitizePostgresJson } from "@/db/postgres-json"; +import { withConversationEventLock } from "./event-lock"; type ConversationEventRow = typeof juniorConversationEvents.$inferSelect; type ConversationEventInsert = typeof juniorConversationEvents.$inferInsert; type PersistedConversationEvent = { data: ConversationEventData; + idempotencyKey?: string; createdAtMs: number; }; function messageRole(data: ConversationEventData): string | null { - if (data.type !== "message") { - return null; + if (data.type === "message") { + return data.message.role; } - return data.message.role; + return data.type === "visible_message_recorded" ? data.role : null; } /** Split validated event data into column-lifted and JSON payload fields. */ @@ -43,6 +45,7 @@ function insertFromEvent( seq, contextEpoch, schemaVersion: 1, + idempotencyKey: event.idempotencyKey ?? null, type, role: messageRole(event.data), payload: sanitizePostgresJson(payload), @@ -56,6 +59,7 @@ function eventFromRow(row: ConversationEventRow): ConversationEvent { schemaVersion: row.schemaVersion, seq: row.seq, contextEpoch: row.contextEpoch, + ...(row.idempotencyKey ? { idempotencyKey: row.idempotencyKey } : {}), createdAtMs: row.createdAt.getTime(), data: { ...row.payload, type: row.type }, }); @@ -88,7 +92,7 @@ class SqlConversationEventStore implements ConversationEventStore { const newestCreatedAtMs = Math.max( ...parsed.map((event) => event.createdAtMs), ); - await this.executor.transaction(async () => { + await withConversationEventLock(this.executor, conversationId, async () => { await ensureConversationRow( this.executor, conversationId, @@ -104,13 +108,56 @@ class SqlConversationEventStore implements ConversationEventStore { isNotNull(juniorConversations.archivedAt), ), ); + const existingKeys = parsed + .map((event) => event.idempotencyKey) + .filter((key): key is string => key !== undefined); + const persistedKeys = + existingKeys.length === 0 + ? new Set() + : new Set( + ( + await this.executor + .db() + .select({ key: juniorConversationEvents.idempotencyKey }) + .from(juniorConversationEvents) + .where( + and( + eq( + juniorConversationEvents.conversationId, + conversationId, + ), + inArray( + juniorConversationEvents.idempotencyKey, + existingKeys, + ), + ), + ) + ).flatMap((row) => (row.key ? [row.key] : [])), + ); + const pending = parsed.filter( + (event) => + event.idempotencyKey === undefined || + !persistedKeys.has(event.idempotencyKey), + ); + if (pending.length === 0) { + return; + } const cursor = await this.readCursor(conversationId); const contextEpoch = cursor.maxEpoch ?? 0; let seq = cursor.nextSeq; - const rows = parsed.map((event) => + const rows = pending.map((event) => insertFromEvent(conversationId, seq++, contextEpoch, event), ); - await this.executor.db().insert(juniorConversationEvents).values(rows); + await this.executor + .db() + .insert(juniorConversationEvents) + .values(rows) + .onConflictDoNothing({ + target: [ + juniorConversationEvents.conversationId, + juniorConversationEvents.idempotencyKey, + ], + }); }); } @@ -119,7 +166,7 @@ class SqlConversationEventStore implements ConversationEventStore { opts: ContextEpochStart, ): Promise { const parsed = contextEpochStartSchema.parse(opts); - await this.executor.transaction(async () => { + await withConversationEventLock(this.executor, conversationId, async () => { await ensureConversationRow(this.executor, conversationId, Date.now()); await this.executor .db() diff --git a/packages/junior/src/chat/conversations/sql/legacy-history-import.ts b/packages/junior/src/chat/conversations/sql/legacy-history-import.ts index c2f95f74a..533efdfc1 100644 --- a/packages/junior/src/chat/conversations/sql/legacy-history-import.ts +++ b/packages/junior/src/chat/conversations/sql/legacy-history-import.ts @@ -33,6 +33,7 @@ import { juniorConversationMessages, juniorConversations, } from "@/db/schema"; +import { withConversationEventLock } from "./event-lock"; const INITIAL_SESSION_ID = "session_0"; const ADVISOR_TASK_OPEN = "\n"; @@ -44,6 +45,7 @@ const ADVISOR_CONTEXT_CLOSE = "\n"; export interface ImportedEvent { seq: number; contextEpoch: number; + idempotencyKey?: string; data: ConversationEventData; createdAtMs: number; } @@ -321,10 +323,10 @@ export function convertAdvisorMessages( type ConversationEventInsert = typeof juniorConversationEvents.$inferInsert; function messageRole(entry: ConversationEventData): string | null { - if (entry.type !== "message") { - return null; + if (entry.type === "message") { + return entry.message.role; } - return entry.message.role; + return entry.type === "visible_message_recorded" ? entry.role : null; } /** Encode one validated imported event as a physical SQL row. */ @@ -338,6 +340,7 @@ function insertRow( seq: event.seq, contextEpoch: event.contextEpoch, schemaVersion: 1, + idempotencyKey: event.idempotencyKey ?? null, type, role: messageRole(event.data), payload: sanitizePostgresJson(payload), @@ -367,103 +370,138 @@ export async function writeLegacyImport( executor: JuniorSqlDatabase, args: LegacyImportWrite, ): Promise { - return executor.withLock( - `junior_conversation:legacy-import:${args.conversationId}`, - () => - executor.transaction(async () => { - const db = executor.db(); - const conversations = await db - .select({ - transcriptPurgedAt: juniorConversations.transcriptPurgedAt, - }) - .from(juniorConversations) - .where(eq(juniorConversations.conversationId, args.conversationId)) - .for("update"); - if (conversations[0]?.transcriptPurgedAt) { - return false; - } - const existing = await db - .select({ seq: juniorConversationEvents.seq }) - .from(juniorConversationEvents) - .where( - eq(juniorConversationEvents.conversationId, args.conversationId), - ) - .limit(1); - if (existing.length > 0) { - return false; - } - const createdAt = new Date(args.fallbackCreatedAtMs); - await ensureConversationRow( - executor, - args.conversationId, - createdAt, - new Date(args.lastActivityAtMs), + return withConversationEventLock(executor, args.conversationId, async () => { + const db = executor.db(); + const conversations = await db + .select({ + transcriptPurgedAt: juniorConversations.transcriptPurgedAt, + }) + .from(juniorConversations) + .where(eq(juniorConversations.conversationId, args.conversationId)) + .for("update"); + if (conversations[0]?.transcriptPurgedAt) { + return false; + } + const existing = await db + .select({ seq: juniorConversationEvents.seq }) + .from(juniorConversationEvents) + .where(eq(juniorConversationEvents.conversationId, args.conversationId)) + .limit(1); + if (existing.length > 0) { + return false; + } + const createdAt = new Date(args.fallbackCreatedAtMs); + await ensureConversationRow( + executor, + args.conversationId, + createdAt, + new Date(args.lastActivityAtMs), + ); + const lastEvent = args.events.at(-1); + let nextSeq = (lastEvent?.seq ?? -1) + 1; + const contextEpoch = lastEvent?.contextEpoch ?? 0; + const visibleEvents: ImportedEvent[] = []; + for (const message of args.messages ?? []) { + visibleEvents.push({ + seq: nextSeq++, + contextEpoch, + idempotencyKey: `visible-message:${message.messageId}:recorded`, + data: { + type: "visible_message_recorded", + messageId: message.messageId, + role: message.role, + text: message.text, + ...(message.authorIdentityId + ? { authorIdentityId: message.authorIdentityId } + : {}), + ...(message.meta ? { meta: message.meta } : {}), + }, + createdAtMs: message.createdAtMs, + }); + if (message.repliedAtMs !== undefined) { + visibleEvents.push({ + seq: nextSeq++, + contextEpoch, + idempotencyKey: `visible-message:${message.messageId}:replied`, + data: { + type: "visible_message_replied", + messageId: message.messageId, + }, + createdAtMs: message.repliedAtMs, + }); + } + } + if (visibleEvents.length > 0) { + await db + .insert(juniorConversationEvents) + .values( + visibleEvents.map((event) => insertRow(args.conversationId, event)), ); - if (args.messages && args.messages.length > 0) { - await db - .insert(juniorConversationMessages) - .values( - args.messages.map((message) => ({ - conversationId: args.conversationId, - messageId: message.messageId, - role: message.role, - authorIdentityId: message.authorIdentityId ?? null, - text: message.text, - meta: message.meta ?? null, - repliedAt: - message.repliedAtMs === undefined - ? null - : new Date(message.repliedAtMs), - createdAt: new Date(message.createdAtMs), - })), - ) - .onConflictDoUpdate({ - target: [ - juniorConversationMessages.conversationId, - juniorConversationMessages.messageId, - ], - set: { - meta: sql`nullif(coalesce(${juniorConversationMessages.meta}, '{}'::jsonb) || coalesce(excluded.meta, '{}'::jsonb), '{}'::jsonb)`, - repliedAt: sql`coalesce(${juniorConversationMessages.repliedAt}, excluded.replied_at)`, - }, - }); - } - if (args.events.length > 0) { - await db - .insert(juniorConversationEvents) - .values( - args.events.map((event) => insertRow(args.conversationId, event)), - ); - } - if (args.child) { - const childCreatedAtMs = - args.child.events.length > 0 - ? Math.min(...args.child.events.map((event) => event.createdAtMs)) - : args.fallbackCreatedAtMs; - const childLastActivityAtMs = - args.child.events.length > 0 - ? Math.max(...args.child.events.map((event) => event.createdAtMs)) - : childCreatedAtMs; - await ensureChildConversationRow( - executor, - args.child.conversationId, - args.conversationId, - new Date(childCreatedAtMs), - new Date(childLastActivityAtMs), + } + if (args.messages && args.messages.length > 0) { + await db + .insert(juniorConversationMessages) + .values( + args.messages.map((message) => ({ + conversationId: args.conversationId, + messageId: message.messageId, + role: message.role, + authorIdentityId: message.authorIdentityId ?? null, + text: message.text, + meta: message.meta ?? null, + repliedAt: + message.repliedAtMs === undefined + ? null + : new Date(message.repliedAtMs), + createdAt: new Date(message.createdAtMs), + })), + ) + .onConflictDoUpdate({ + target: [ + juniorConversationMessages.conversationId, + juniorConversationMessages.messageId, + ], + set: { + meta: sql`nullif(coalesce(${juniorConversationMessages.meta}, '{}'::jsonb) || coalesce(excluded.meta, '{}'::jsonb), '{}'::jsonb)`, + repliedAt: sql`coalesce(${juniorConversationMessages.repliedAt}, excluded.replied_at)`, + }, + }); + } + if (args.events.length > 0) { + await db + .insert(juniorConversationEvents) + .values( + args.events.map((event) => insertRow(args.conversationId, event)), + ); + } + if (args.child) { + const childCreatedAtMs = + args.child.events.length > 0 + ? Math.min(...args.child.events.map((event) => event.createdAtMs)) + : args.fallbackCreatedAtMs; + const childLastActivityAtMs = + args.child.events.length > 0 + ? Math.max(...args.child.events.map((event) => event.createdAtMs)) + : childCreatedAtMs; + await ensureChildConversationRow( + executor, + args.child.conversationId, + args.conversationId, + new Date(childCreatedAtMs), + new Date(childLastActivityAtMs), + ); + if (args.child.events.length > 0) { + await db + .insert(juniorConversationEvents) + .values( + args.child.events.map((event) => + insertRow(args.child!.conversationId, event), + ), ); - if (args.child.events.length > 0) { - await db - .insert(juniorConversationEvents) - .values( - args.child.events.map((event) => - insertRow(args.child!.conversationId, event), - ), - ); - } - } - return true; - }), - ); + } + } + return true; + }); } async function ensureConversationRow( diff --git a/packages/junior/src/chat/conversations/sql/messages.ts b/packages/junior/src/chat/conversations/sql/messages.ts index 8310e56eb..b672c99aa 100644 --- a/packages/junior/src/chat/conversations/sql/messages.ts +++ b/packages/junior/src/chat/conversations/sql/messages.ts @@ -1,4 +1,5 @@ -import { and, asc, eq, isNotNull, isNull, sql } from "drizzle-orm"; +import { isDeepStrictEqual } from "node:util"; +import { and, asc, eq, inArray, isNotNull, isNull, sql } from "drizzle-orm"; import type { JuniorSqlDatabase } from "@/db/db"; import type { ConversationMessage, @@ -6,7 +7,10 @@ import type { NewConversationMessage, } from "../messages"; import { ensureConversationRow } from "./conversation-row"; +import { withConversationEventLock } from "./event-lock"; +import { createSqlConversationEventStore } from "./history"; import { juniorConversationMessages, juniorConversations } from "@/db/schema"; +import type { NewConversationEvent } from "../history"; type ConversationMessageRow = typeof juniorConversationMessages.$inferSelect; @@ -39,12 +43,68 @@ class SqlConversationMessageStore implements ConversationMessageStore { const newestCreatedAtMs = Math.max( ...messages.map((message) => message.createdAtMs), ); - await this.executor.transaction(async () => { + await withConversationEventLock(this.executor, conversationId, async () => { await ensureConversationRow( this.executor, conversationId, newestCreatedAtMs, ); + const messageIds = messages.map((message) => message.messageId); + const existing = new Map( + ( + await this.executor + .db() + .select() + .from(juniorConversationMessages) + .where( + and( + eq(juniorConversationMessages.conversationId, conversationId), + inArray(juniorConversationMessages.messageId, messageIds), + ), + ) + ).map((row) => [row.messageId, row]), + ); + const events: NewConversationEvent[] = []; + for (const message of messages) { + const current = existing.get(message.messageId); + const recorded = current + ? messageFromRow(current) + : { conversationId, ...message }; + events.push({ + idempotencyKey: `visible-message:${message.messageId}:recorded`, + data: { + type: "visible_message_recorded", + messageId: recorded.messageId, + role: recorded.role, + text: recorded.text, + ...(recorded.authorIdentityId + ? { authorIdentityId: recorded.authorIdentityId } + : {}), + ...(recorded.meta ? { meta: recorded.meta } : {}), + }, + createdAtMs: recorded.createdAtMs, + }); + if (current && message.meta) { + const mergedMeta = { + ...(current.meta ?? {}), + ...message.meta, + }; + if (!isDeepStrictEqual(current.meta ?? {}, mergedMeta)) { + events.push({ + data: { + type: "visible_message_metadata_updated", + messageId: message.messageId, + meta: mergedMeta, + }, + createdAtMs: Date.now(), + }); + } + } + } + await createSqlConversationEventStore(this.executor).append( + conversationId, + events, + ); await this.executor .db() .update(juniorConversations) @@ -94,19 +154,60 @@ class SqlConversationMessageStore implements ConversationMessageStore { messageId: string, repliedAtMs: number, ): Promise { - await this.executor - .db() - .update(juniorConversationMessages) - .set({ - repliedAt: sql`coalesce(${juniorConversationMessages.repliedAt}, ${new Date(repliedAtMs)})`, - }) - .where( - and( - eq(juniorConversationMessages.conversationId, conversationId), - eq(juniorConversationMessages.messageId, messageId), - isNull(juniorConversationMessages.repliedAt), - ), + await withConversationEventLock(this.executor, conversationId, async () => { + const rows = await this.executor + .db() + .select() + .from(juniorConversationMessages) + .where( + and( + eq(juniorConversationMessages.conversationId, conversationId), + eq(juniorConversationMessages.messageId, messageId), + ), + ) + .limit(1); + if (!rows[0] || rows[0].repliedAt) { + return; + } + const recorded = messageFromRow(rows[0]); + await createSqlConversationEventStore(this.executor).append( + conversationId, + [ + { + idempotencyKey: `visible-message:${messageId}:recorded`, + data: { + type: "visible_message_recorded", + messageId: recorded.messageId, + role: recorded.role, + text: recorded.text, + ...(recorded.authorIdentityId + ? { authorIdentityId: recorded.authorIdentityId } + : {}), + ...(recorded.meta ? { meta: recorded.meta } : {}), + }, + createdAtMs: recorded.createdAtMs, + }, + { + idempotencyKey: `visible-message:${messageId}:replied`, + data: { type: "visible_message_replied", messageId }, + createdAtMs: repliedAtMs, + }, + ], ); + await this.executor + .db() + .update(juniorConversationMessages) + .set({ + repliedAt: sql`coalesce(${juniorConversationMessages.repliedAt}, ${new Date(repliedAtMs)})`, + }) + .where( + and( + eq(juniorConversationMessages.conversationId, conversationId), + eq(juniorConversationMessages.messageId, messageId), + isNull(juniorConversationMessages.repliedAt), + ), + ); + }); } async list( diff --git a/packages/junior/src/chat/conversations/visible-messages.ts b/packages/junior/src/chat/conversations/visible-messages.ts index 395cbc03b..e6f659624 100644 --- a/packages/junior/src/chat/conversations/visible-messages.ts +++ b/packages/junior/src/chat/conversations/visible-messages.ts @@ -1,11 +1,10 @@ /** * Visible-transcript sync between the in-memory turn working set and SQL. * - * The durable authority for the visible conversation transcript is the - * `ConversationMessageStore`; `ThreadConversationState.messages` is only the - * in-memory working set for the current turn. These helpers hydrate that - * working set from SQL at load boundaries and write new/updated messages back - * through the store, so no transcript data is persisted to `thread-state`. + * Visible-message events are the durable transcript authority; + * `ConversationMessageStore` maintains their SQL hydration/search read model, + * and `ThreadConversationState.messages` is only the current-turn working set. + * No transcript data is persisted to `thread-state`. */ import type { ConversationMessage as StoredConversationMessage, diff --git a/packages/junior/src/cli/upgrade.ts b/packages/junior/src/cli/upgrade.ts index 744d4e8ef..18decfe05 100644 --- a/packages/junior/src/cli/upgrade.ts +++ b/packages/junior/src/cli/upgrade.ts @@ -14,6 +14,7 @@ import { redisConversationStateMigration } from "./upgrade/migrations/redis-conv import { agentTurnSessionActorMigration } from "./upgrade/migrations/agent-turn-session-actor"; import { conversationUsageRepairMigration } from "./upgrade/migrations/conversation-usage"; import { conversationEventDataMigration } from "./upgrade/migrations/conversation-event-data"; +import { conversationVisibleMessageEventsMigration } from "./upgrade/migrations/conversation-visible-message-events"; import type { MigrationContext, MigrationResult, @@ -34,6 +35,7 @@ const MIGRATIONS: UpgradeMigration[] = [ sqlConversationMigration, conversationEventDataMigration, sqlConversationHistoryMigration, + conversationVisibleMessageEventsMigration, conversationUsageRepairMigration, sqlPluginMigration, pluginStorageMigration, diff --git a/packages/junior/src/cli/upgrade/migrations/conversation-visible-message-events.ts b/packages/junior/src/cli/upgrade/migrations/conversation-visible-message-events.ts new file mode 100644 index 000000000..8afc7cf0a --- /dev/null +++ b/packages/junior/src/cli/upgrade/migrations/conversation-visible-message-events.ts @@ -0,0 +1,205 @@ +import { getChatConfig } from "@/chat/config"; +import { createSqlConversationEventStore } from "@/chat/conversations/sql/history"; +import type { ConversationMessageRole } from "@/chat/conversations/messages"; +import type { + ConversationEventStore, + NewConversationEvent, +} from "@/chat/conversations/history"; +import type { JuniorSqlExecutor } from "@/db/db"; +import { createJuniorSqlExecutor } from "@/db/executor"; +import type { MigrationContext, MigrationResult } from "../types"; + +const VISIBLE_EVENT_BATCH_SIZE = 500; +const VISIBLE_EVENT_BACKFILL_LOCK = + "junior:upgrade:conversation-visible-message-events"; + +const LOAD_MISSING_VISIBLE_EVENTS_SQL = ` +SELECT + message.conversation_id, + message.message_id, + message.role, + message.text, + message.author_identity_id, + message.meta, + message.replied_at, + message.created_at +FROM junior_conversation_messages message +WHERE ( + NOT EXISTS ( + SELECT 1 + FROM junior_conversation_events event + WHERE event.conversation_id = message.conversation_id + AND event.idempotency_key = + 'visible-message:' || message.message_id || ':recorded' + ) + OR ( + message.replied_at IS NOT NULL + AND NOT EXISTS ( + SELECT 1 + FROM junior_conversation_events event + WHERE event.conversation_id = message.conversation_id + AND event.idempotency_key = + 'visible-message:' || message.message_id || ':replied' + ) + ) +) +AND ( + $2::text IS NULL + OR (message.conversation_id, message.created_at, message.message_id) > + ($2::text, $3::timestamptz, $4::text) +) +ORDER BY message.conversation_id, message.created_at, message.message_id +LIMIT $1 +`; + +const COUNT_MISSING_VISIBLE_EVENTS_SQL = ` +SELECT count(*)::integer AS count +FROM junior_conversation_messages message +WHERE NOT EXISTS ( + SELECT 1 + FROM junior_conversation_events event + WHERE event.conversation_id = message.conversation_id + AND event.idempotency_key = + 'visible-message:' || message.message_id || ':recorded' +) +OR ( + message.replied_at IS NOT NULL + AND NOT EXISTS ( + SELECT 1 + FROM junior_conversation_events event + WHERE event.conversation_id = message.conversation_id + AND event.idempotency_key = + 'visible-message:' || message.message_id || ':replied' + ) +) +`; + +interface VisibleMessageRow { + conversation_id: string; + message_id: string; + role: ConversationMessageRole; + text: string; + author_identity_id: string | null; + meta: Record | null; + replied_at: Date | string | null; + created_at: Date | string; +} + +function timestampMs(value: Date | string): number { + return value instanceof Date ? value.getTime() : new Date(value).getTime(); +} + +function eventsForRow(row: VisibleMessageRow): NewConversationEvent[] { + const events: NewConversationEvent[] = [ + { + idempotencyKey: `visible-message:${row.message_id}:recorded`, + data: { + type: "visible_message_recorded", + messageId: row.message_id, + role: row.role, + text: row.text, + ...(row.author_identity_id + ? { authorIdentityId: row.author_identity_id } + : {}), + ...(row.meta ? { meta: row.meta } : {}), + }, + createdAtMs: timestampMs(row.created_at), + }, + ]; + if (row.replied_at) { + events.push({ + idempotencyKey: `visible-message:${row.message_id}:replied`, + data: { type: "visible_message_replied", messageId: row.message_id }, + createdAtMs: timestampMs(row.replied_at), + }); + } + return events; +} + +/** Backfill canonical visible-message events from the SQL read model. */ +export async function migrateConversationVisibleMessageEvents( + _context: MigrationContext, + options: { + batchSize?: number; + eventStore?: Pick; + executor?: JuniorSqlExecutor; + } = {}, +): Promise { + let executor = options.executor; + let closeExecutor: (() => Promise) | undefined; + if (!executor) { + const { sql } = getChatConfig(); + executor = createJuniorSqlExecutor({ + connectionString: sql.databaseUrl, + driver: sql.driver, + }); + closeExecutor = () => executor!.close(); + } + const batchSize = Math.max( + 1, + Math.floor(options.batchSize ?? VISIBLE_EVENT_BATCH_SIZE), + ); + let migrated = 0; + let cursor: + | { conversationId: string; createdAt: Date | string; messageId: string } + | undefined; + try { + while (true) { + const rows = await executor.withLock(VISIBLE_EVENT_BACKFILL_LOCK, () => + executor.query(LOAD_MISSING_VISIBLE_EVENTS_SQL, [ + batchSize, + cursor?.conversationId ?? null, + cursor?.createdAt ?? null, + cursor?.messageId ?? null, + ]), + ); + if (rows.length === 0) { + break; + } + const grouped = new Map(); + for (const row of rows) { + const events = grouped.get(row.conversation_id) ?? []; + events.push(...eventsForRow(row)); + grouped.set(row.conversation_id, events); + } + const store = + options.eventStore ?? createSqlConversationEventStore(executor); + for (const [conversationId, events] of grouped) { + await store.append(conversationId, events); + } + migrated += rows.length; + const last = rows.at(-1)!; + cursor = { + conversationId: last.conversation_id, + createdAt: last.created_at, + messageId: last.message_id, + }; + } + const [remaining] = await executor.query<{ count: number }>( + COUNT_MISSING_VISIBLE_EVENTS_SQL, + ); + if (!remaining) { + throw new Error( + "Visible-message event migration could not verify its projection", + ); + } + if (remaining.count > 0) { + throw new Error( + `Visible-message event migration left ${remaining.count} message row(s) without canonical events`, + ); + } + return { + existing: 0, + migrated, + missing: 0, + scanned: migrated, + }; + } finally { + await closeExecutor?.(); + } +} + +export const conversationVisibleMessageEventsMigration = { + name: "backfill-conversation-visible-message-events", + run: migrateConversationVisibleMessageEvents, +}; diff --git a/packages/junior/src/db/schema/conversation-events.ts b/packages/junior/src/db/schema/conversation-events.ts index 9bb64004b..52ebf840c 100644 --- a/packages/junior/src/db/schema/conversation-events.ts +++ b/packages/junior/src/db/schema/conversation-events.ts @@ -6,6 +6,7 @@ import { pgTable, primaryKey, text, + uniqueIndex, } from "drizzle-orm/pg-core"; import { juniorConversations } from "./conversations"; import { timestamptz } from "./timestamps"; @@ -22,6 +23,7 @@ export const juniorConversationEvents = pgTable( seq: integer("seq").notNull(), contextEpoch: integer("context_epoch").notNull(), schemaVersion: integer("schema_version").default(1).notNull(), + idempotencyKey: text("idempotency_key"), type: text("type").notNull(), role: text("role"), payload: jsonb("payload").$type>().notNull(), @@ -42,5 +44,9 @@ export const juniorConversationEvents = pgTable( table.contextEpoch, table.seq, ), + uniqueIndex("junior_conversation_events_idempotency_idx").on( + table.conversationId, + table.idempotencyKey, + ), ], ); diff --git a/packages/junior/tests/component/cli/conversation-event-migration.test.ts b/packages/junior/tests/component/cli/conversation-event-migration.test.ts index 371068599..f382ad5e5 100644 --- a/packages/junior/tests/component/cli/conversation-event-migration.test.ts +++ b/packages/junior/tests/component/cli/conversation-event-migration.test.ts @@ -3,7 +3,10 @@ import { readFileSync } from "node:fs"; import { describe, expect, it, vi } from "vitest"; import { getStateAdapter } from "@/chat/state/adapter"; import { migrateConversationEventData } from "@/cli/upgrade/migrations/conversation-event-data"; +import { migrateConversationVisibleMessageEvents } from "@/cli/upgrade/migrations/conversation-visible-message-events"; import type { JuniorSqlExecutor } from "@/db/db"; +import { createSqlConversationEventStore } from "@/chat/conversations/sql/history"; +import { createSqlConversationMessageStore } from "@/chat/conversations/sql/messages"; import { createLocalJuniorSqlFixture } from "../../fixtures/sql"; const migrationsFolder = path.resolve( @@ -106,6 +109,256 @@ describe("conversation event migration", () => { ); }); + it("fails closed when visible-message rows remain unbackfilled", async () => { + const query = vi + .fn() + .mockResolvedValueOnce([]) + .mockResolvedValueOnce([{ count: 1 }]); + const executor = { + query, + withLock: async (_name: string, callback: () => Promise) => + callback(), + } as unknown as JuniorSqlExecutor; + + await expect( + migrateConversationVisibleMessageEvents( + { io: { info: () => {} }, stateAdapter: getStateAdapter() }, + { executor }, + ), + ).rejects.toThrow("Visible-message event migration left 1 message row(s)"); + }); + + it("advances visible-message batches by the last stable row key", async () => { + const first = { + conversation_id: "conversation-one", + message_id: "m1", + role: "user", + text: "one", + author_identity_id: null, + meta: null, + replied_at: null, + created_at: "2026-07-14T10:00:01.000Z", + } as const; + const second = { + ...first, + conversation_id: "conversation-two", + message_id: "m2", + text: "two", + created_at: "2026-07-14T10:00:02.000Z", + } as const; + const query = vi + .fn() + .mockResolvedValueOnce([first]) + .mockResolvedValueOnce([second]) + .mockResolvedValueOnce([]) + .mockResolvedValueOnce([{ count: 0 }]); + const executor = { + query, + withLock: async (_name: string, callback: () => Promise) => + callback(), + } as unknown as JuniorSqlExecutor; + const append = vi.fn(async () => {}); + + await expect( + migrateConversationVisibleMessageEvents( + { io: { info: () => {} }, stateAdapter: getStateAdapter() }, + { batchSize: 1, eventStore: { append }, executor }, + ), + ).resolves.toMatchObject({ migrated: 2, missing: 0 }); + expect(query.mock.calls.map(([, parameters]) => parameters)).toEqual([ + [1, null, null, null], + [1, first.conversation_id, first.created_at, first.message_id], + [1, second.conversation_id, second.created_at, second.message_id], + undefined, + ]); + expect(append).toHaveBeenCalledTimes(2); + }); + + it("backfills drained old-worker rows after removing their compatibility view", async () => { + const fixture = await createLocalJuniorSqlFixture(); + const migrationsBeforeVisibleEvents = [ + "0000_initial.sql", + "0001_conversation_metrics.sql", + "0002_conversation_message_search.sql", + "0003_peaceful_scalphunter.sql", + "0004_conversation_events.sql", + ].flatMap(migrationStatements); + const visibleMessageEvents = migrationStatements( + "0005_visible_message_events.sql", + ); + const conversationId = "conversation-visible-events"; + + try { + await executeStatements( + (statement) => fixture.sql.execute(statement), + migrationsBeforeVisibleEvents, + ); + await fixture.sql.execute( + `INSERT INTO junior_conversations ( + conversation_id, created_at, last_activity_at, updated_at, + execution_status + ) VALUES ($1, $2, $2, $2, 'idle')`, + [conversationId, new Date("2026-07-14T10:00:00.000Z")], + ); + await fixture.sql.execute( + `INSERT INTO junior_conversation_messages ( + conversation_id, message_id, role, text, meta, replied_at, + created_at + ) VALUES ($1, 'before', 'user', 'before backfill', $2::jsonb, $3, $4)`, + [ + conversationId, + JSON.stringify({ imagesHydrated: true }), + new Date("2026-07-14T10:00:02.000Z"), + new Date("2026-07-14T10:00:01.000Z"), + ], + ); + await executeStatements( + (statement) => fixture.sql.execute(statement), + visibleMessageEvents, + ); + + const context = { + io: { info: () => {} }, + stateAdapter: getStateAdapter(), + }; + await expect( + migrateConversationVisibleMessageEvents(context, { + batchSize: 1, + executor: fixture.sql, + }), + ).resolves.toMatchObject({ migrated: 1, missing: 0 }); + await expect( + migrateConversationVisibleMessageEvents(context, { + batchSize: 1, + executor: fixture.sql, + }), + ).resolves.toMatchObject({ migrated: 0, missing: 0 }); + + await fixture.sql.execute( + `INSERT INTO junior_conversation_messages ( + conversation_id, message_id, role, text, created_at + ) VALUES ($1, 'old-worker', 'assistant', 'compatibility', $2)`, + [conversationId, new Date("2026-07-14T10:00:03.000Z")], + ); + await fixture.sql.execute( + `UPDATE junior_conversation_messages + SET meta = $2::jsonb, replied_at = $3 + WHERE conversation_id = $1 AND message_id = 'old-worker'`, + [ + conversationId, + JSON.stringify({ slackTs: "123.456" }), + new Date("2026-07-14T10:00:04.000Z"), + ], + ); + await fixture.sql.execute( + `INSERT INTO junior_conversation_messages ( + conversation_id, message_id, role, text, replied_at, created_at + ) VALUES ($1, 'old-worker-replied', 'user', 'already replied', $2, $3)`, + [ + conversationId, + new Date("2026-07-14T10:00:04.500Z"), + new Date("2026-07-14T10:00:04.000Z"), + ], + ); + + await expect( + migrateConversationVisibleMessageEvents(context, { + batchSize: 1, + executor: fixture.sql, + }), + ).resolves.toMatchObject({ migrated: 2, missing: 0 }); + + const store = createSqlConversationMessageStore(fixture.sql); + await store.record(conversationId, [ + { + messageId: "application", + role: "user", + text: "new writer", + meta: { imagesHydrated: false }, + createdAtMs: Date.parse("2026-07-14T10:00:05.000Z"), + }, + ]); + await store.record(conversationId, [ + { + messageId: "application", + role: "user", + text: "new writer", + meta: { imagesHydrated: true }, + createdAtMs: Date.parse("2026-07-14T10:00:05.000Z"), + }, + ]); + await store.markReplied( + conversationId, + "application", + Date.parse("2026-07-14T10:00:06.000Z"), + ); + + const events = await createSqlConversationEventStore( + fixture.sql, + ).loadHistory(conversationId); + const [compatibility] = await fixture.sql.query<{ + relation: string | null; + functions: number; + }>( + `SELECT + to_regclass('public.junior_agent_steps')::text AS relation, + ( + SELECT count(*)::integer + FROM pg_proc + WHERE proname IN ( + 'junior_agent_steps_insert_compat', + 'junior_agent_steps_delete_compat' + ) + ) AS functions`, + ); + expect(compatibility).toEqual({ relation: null, functions: 0 }); + expect( + events.filter( + (event) => + event.data.type === "visible_message_recorded" && + event.data.messageId === "old-worker", + ), + ).toHaveLength(1); + expect( + events.filter( + (event) => + event.data.type === "visible_message_replied" && + event.data.messageId === "old-worker-replied", + ), + ).toHaveLength(1); + expect( + events.filter( + (event) => + event.data.type === "visible_message_metadata_updated" && + event.data.messageId === "old-worker", + ), + ).toHaveLength(0); + expect( + events.filter( + (event) => + event.data.type === "visible_message_replied" && + event.data.messageId === "old-worker", + ), + ).toHaveLength(1); + expect( + events.filter( + (event) => + event.data.type === "visible_message_metadata_updated" && + event.data.messageId === "application", + ), + ).toHaveLength(1); + expect( + events.filter( + (event) => + event.data.type === "visible_message_replied" && + event.data.messageId === "application", + ), + ).toHaveLength(1); + } finally { + await fixture.close(); + } + }, 20_000); + it("renames history, preserves rows, and provides rolling compatibility", async () => { const fixture = await createLocalJuniorSqlFixture(); const initial = migrationStatements("0000_initial.sql"); diff --git a/packages/junior/tests/component/conversation-transcripts-sql.test.ts b/packages/junior/tests/component/conversation-transcripts-sql.test.ts index f46bac02d..26c1d5c08 100644 --- a/packages/junior/tests/component/conversation-transcripts-sql.test.ts +++ b/packages/junior/tests/component/conversation-transcripts-sql.test.ts @@ -106,6 +106,22 @@ it("accepts legacy markers and validates current profile names", () => { }); it("rejects unknown Junior event fields while retaining opaque message fields", () => { + expect( + conversationEventDataSchema.safeParse({ + type: "visible_message_recorded", + messageId: "m1", + role: "user", + text: "hello", + unknown: true, + }).success, + ).toBe(false); + expect( + conversationEventDataSchema.safeParse({ + type: "visible_message_metadata_updated", + messageId: "m1", + meta: { imagesHydrated: true }, + }).success, + ).toBe(true); expect( conversationEventDataSchema.safeParse({ type: "mcp_provider_connected", @@ -276,7 +292,7 @@ describe("conversation transcript SQL stores", () => { const [applied] = await fixture.sql.query<{ count: number }>( "SELECT count(*)::integer AS count FROM drizzle.__drizzle_junior_core", ); - expect(applied?.count).toBe(5); + expect(applied?.count).toBe(6); } finally { await fixture.close(); } @@ -452,12 +468,12 @@ describe("conversation transcript SQL stores", () => { // Force a failure inside the startEpoch transaction after its writes. const failing: JuniorSqlDatabase = { db: () => fixture.sql.db(), - withLock: (name, callback) => fixture.sql.withLock(name, callback), - transaction: (callback) => - fixture.sql.transaction(async () => { + withLock: (name, callback) => + fixture.sql.withLock(name, async () => { await callback(); throw new Error("epoch write failed"); }), + transaction: (callback) => fixture.sql.transaction(callback), }; const failingStore = createSqlConversationEventStore(failing); @@ -603,6 +619,133 @@ INSERT INTO junior_conversation_events ( createdAtMs: 2_000, }, ]); + const visibleEvents = ( + await createSqlConversationEventStore(fixture.sql).loadHistory( + CONVERSATION_ID, + ) + ).filter((event) => event.data.type.startsWith("visible_message_")); + expect(visibleEvents).toEqual([ + expect.objectContaining({ + idempotencyKey: "visible-message:m1:recorded", + data: expect.objectContaining({ + type: "visible_message_recorded", + messageId: "m1", + text: "first", + }), + }), + expect.objectContaining({ + idempotencyKey: "visible-message:m2:recorded", + data: expect.objectContaining({ + type: "visible_message_recorded", + messageId: "m2", + }), + }), + expect.objectContaining({ + idempotencyKey: "visible-message:m1:replied", + data: { type: "visible_message_replied", messageId: "m1" }, + createdAtMs: 5_000, + }), + ]); + } finally { + await fixture.close(); + } + }); + + it("rolls back a recorded event when its message projection fails", async () => { + const fixture = await createLocalJuniorSqlFixture(); + + try { + await migrateSchema(fixture.sql); + await seedConversation(fixture, CONVERSATION_ID); + await fixture.sql.execute(` + CREATE FUNCTION fail_visible_message_insert() + RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN + RAISE EXCEPTION 'projection insert failed'; + END; + $$ + `); + await fixture.sql.execute(` + CREATE TRIGGER fail_visible_message_insert_trigger + BEFORE INSERT ON junior_conversation_messages + FOR EACH ROW EXECUTE FUNCTION fail_visible_message_insert() + `); + const messages = createSqlConversationMessageStore(fixture.sql); + + await expect( + messages.record(CONVERSATION_ID, [ + { + messageId: "projection-failure", + role: "user", + text: "must roll back", + createdAtMs: 1_000, + }, + ]), + ).rejects.toThrow( + 'Failed query: insert into "junior_conversation_messages"', + ); + + expect(await messages.list(CONVERSATION_ID)).toEqual([]); + expect( + await createSqlConversationEventStore(fixture.sql).loadHistory( + CONVERSATION_ID, + ), + ).toEqual([]); + } finally { + await fixture.close(); + } + }); + + it("rolls back a replied event when its message projection fails", async () => { + const fixture = await createLocalJuniorSqlFixture(); + + try { + await migrateSchema(fixture.sql); + await seedConversation(fixture, CONVERSATION_ID); + const messages = createSqlConversationMessageStore(fixture.sql); + await messages.record(CONVERSATION_ID, [ + { + messageId: "reply-projection-failure", + role: "user", + text: "must stay unreplied", + createdAtMs: 1_000, + }, + ]); + await fixture.sql.execute(` + CREATE FUNCTION fail_visible_message_reply_update() + RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN + RAISE EXCEPTION 'projection reply update failed'; + END; + $$ + `); + await fixture.sql.execute(` + CREATE TRIGGER fail_visible_message_reply_update_trigger + BEFORE UPDATE OF replied_at ON junior_conversation_messages + FOR EACH ROW EXECUTE FUNCTION fail_visible_message_reply_update() + `); + + await expect( + messages.markReplied( + CONVERSATION_ID, + "reply-projection-failure", + 2_000, + ), + ).rejects.toThrow('Failed query: update "junior_conversation_messages"'); + + const projected = await messages.list(CONVERSATION_ID); + expect(projected).toEqual([ + expect.objectContaining({ messageId: "reply-projection-failure" }), + ]); + expect(projected[0]?.repliedAtMs).toBeUndefined(); + const visibleEvents = ( + await createSqlConversationEventStore(fixture.sql).loadHistory( + CONVERSATION_ID, + ) + ).filter((event) => event.data.type.startsWith("visible_message_")); + expect(visibleEvents.map((event) => event.data.type)).toEqual([ + "visible_message_recorded", + ]); } finally { await fixture.close(); } @@ -736,7 +879,7 @@ INSERT INTO junior_conversation_events ( }, ]); - expect(await events.loadHistory(CONVERSATION_ID)).toHaveLength(1); + expect(await events.loadHistory(CONVERSATION_ID)).toHaveLength(2); expect(await messages.list(CONVERSATION_ID)).toHaveLength(1); const reopened = await fixture.sql .db() diff --git a/packages/junior/tests/component/conversations/legacy-import.test.ts b/packages/junior/tests/component/conversations/legacy-import.test.ts index 7a54f27e1..e1ca04f29 100644 --- a/packages/junior/tests/component/conversations/legacy-import.test.ts +++ b/packages/junior/tests/component/conversations/legacy-import.test.ts @@ -186,6 +186,9 @@ describe("legacy conversation import", () => { { seq: 1, epoch: 1, type: "context_epoch_started" }, { seq: 2, epoch: 1, type: "message" }, { seq: 3, epoch: 1, type: "subagent_started" }, + { seq: 4, epoch: 1, type: "visible_message_recorded" }, + { seq: 5, epoch: 1, type: "visible_message_replied" }, + { seq: 6, epoch: 1, type: "visible_message_recorded" }, ]); // Current context is exactly the highest epoch's messages. @@ -246,7 +249,7 @@ describe("legacy conversation import", () => { await expect( importConversationFromLegacy(CONVERSATION_ID, deps), ).resolves.toEqual({ imported: false }); - expect(await eventStore.loadHistory(CONVERSATION_ID)).toHaveLength(4); + expect(await eventStore.loadHistory(CONVERSATION_ID)).toHaveLength(7); } finally { await fixture.close(); } @@ -281,7 +284,7 @@ describe("legacy conversation import", () => { loadVisibleMessages: async () => invalidVisible, }), ).rejects.toThrow( - 'Failed query: insert into "junior_conversation_messages"', + 'Failed query: insert into "junior_conversation_events"', ); // Messages and events share one transaction, so neither side commits. @@ -302,7 +305,7 @@ describe("legacy conversation import", () => { }), ).resolves.toEqual({ imported: true }); - expect(await eventStore.loadHistory(CONVERSATION_ID)).toHaveLength(1); + expect(await eventStore.loadHistory(CONVERSATION_ID)).toHaveLength(3); const messages = await messageStore.list(CONVERSATION_ID); expect(messages.map((message) => message.messageId)).toEqual([ "m1", @@ -313,7 +316,7 @@ describe("legacy conversation import", () => { } }, 20_000); - it("seals a completed message-only import without event rows", async () => { + it("treats event-backed visible-message writes as an import seal", async () => { const fixture = await createLocalJuniorSqlFixture(); await migrateSchema(fixture.sql); const eventStore = createSqlConversationEventStore(fixture.sql); @@ -347,17 +350,13 @@ describe("legacy conversation import", () => { ]); await messageStore.markReplied(CONVERSATION_ID, "message-only", 100); - await expect( - importConversationFromLegacy(CONVERSATION_ID, deps), - ).resolves.toEqual({ imported: true }); await expect( importConversationFromLegacy(CONVERSATION_ID, deps), ).resolves.toEqual({ imported: false }); - expect(await eventStore.loadHistory(CONVERSATION_ID)).toEqual([]); + expect(await eventStore.loadHistory(CONVERSATION_ID)).toHaveLength(2); expect(await messageStore.list(CONVERSATION_ID)).toMatchObject([ { messageId: "message-only", - meta: { author: { fullName: "Legacy User" } }, repliedAtMs: 100, }, ]); @@ -370,8 +369,8 @@ describe("legacy conversation import", () => { .from(juniorConversations) .where(eq(juniorConversations.conversationId, CONVERSATION_ID)); expect(conversation).toMatchObject({ - lastActivityAt: new Date(900), - updatedAt: new Date(900), + lastActivityAt: new Date(100), + updatedAt: new Date(100), }); } finally { await fixture.close(); From 235099ceb91fa50b02d845efac559f79b03ece60 Mon Sep 17 00:00:00 2001 From: David Cramer Date: Tue, 14 Jul 2026 22:15:26 -0700 Subject: [PATCH 04/58] ref(conversations): Read visible history from events Project runtime hydration, reporting, and eval transcripts from canonical visible-message events while retaining safe terminal outcome and trace metadata. Remove message-table transcript readers and make assistant message identities stable across persistence retries. Co-Authored-By: GPT-5 Codex --- .../changes/conversation-event-log/tasks.md | 2 + packages/junior-evals/src/helpers.ts | 16 +- .../api/conversations/detail-projection.ts | 94 ++++-- .../junior/src/chat/agent-dispatch/runner.ts | 4 +- .../junior/src/chat/conversations/README.md | 4 +- .../src/chat/conversations/history-loader.ts | 8 + .../src/chat/conversations/legacy-import.ts | 50 +--- .../junior/src/chat/conversations/messages.ts | 19 +- .../src/chat/conversations/sql/messages.ts | 34 +-- .../chat/conversations/visible-compactions.ts | 18 +- .../visible-message-projection.ts | 116 ++++++++ .../visible-message-serializer.ts | 24 ++ .../chat/conversations/visible-messages.ts | 109 ++----- .../src/chat/runtime/delivered-turn-state.ts | 4 +- .../junior/src/chat/runtime/reply-executor.ts | 12 +- .../junior/src/chat/state/conversation.ts | 13 +- packages/junior/src/chat/state/turn-id.ts | 5 + .../migrations/conversations-history-sql.ts | 3 - .../conversation-transcripts-sql.test.ts | 58 ++-- .../conversations/legacy-import.test.ts | 106 ++++--- .../integration/dashboard-reporting.test.ts | 277 ++++++++++++++---- .../visible-message-projection.test.ts | 126 ++++++++ .../unit/runtime/delivered-turn-state.test.ts | 60 ++++ .../unit/state/conversation-state.test.ts | 8 +- 24 files changed, 821 insertions(+), 349 deletions(-) create mode 100644 packages/junior/src/chat/conversations/visible-message-projection.ts create mode 100644 packages/junior/src/chat/conversations/visible-message-serializer.ts create mode 100644 packages/junior/tests/unit/conversations/visible-message-projection.test.ts create mode 100644 packages/junior/tests/unit/runtime/delivered-turn-state.test.ts diff --git a/openspec/changes/conversation-event-log/tasks.md b/openspec/changes/conversation-event-log/tasks.md index 6730166e0..ea9c093aa 100644 --- a/openspec/changes/conversation-event-log/tasks.md +++ b/openspec/changes/conversation-event-log/tasks.md @@ -49,6 +49,8 @@ ## 5. Cleanup And Verification - [x] Make the visible-message table an explicit event-backed read model. +- [x] Hydrate runtime and primary conversation-detail transcripts from visible + events instead of the message-table projection or model messages. - [ ] Make remaining aggregate stores explicit read models. - [ ] Remove obsolete transcript reconstruction and legacy compatibility after migration completion. diff --git a/packages/junior-evals/src/helpers.ts b/packages/junior-evals/src/helpers.ts index 8fb653ca1..60b912e82 100644 --- a/packages/junior-evals/src/helpers.ts +++ b/packages/junior-evals/src/helpers.ts @@ -20,12 +20,10 @@ import { type ToolCallRecord, } from "vitest-evals/harness"; import { registerLogRecordSink, type EmittedLogRecord } from "@/chat/logging"; -import { - getConversationEventStore, - getConversationMessageStore, -} from "@/chat/db"; +import { getConversationEventStore } from "@/chat/db"; import type { ConversationEvent } from "@/chat/conversations/history"; -import type { ConversationMessage } from "@/chat/conversations/messages"; +import type { ConversationMessage } from "@/chat/state/conversation"; +import { projectVisibleConversationMessages } from "@/chat/conversations/visible-message-projection"; import { renderResourceEventNotificationText } from "@/chat/resource-events/notification"; import { slackEventThread, @@ -1078,13 +1076,15 @@ export async function conversationEvents( } /** - * The run's visible conversation message rows in `created_at` order, read via - * `ConversationMessageStore.list`. Defaults to the run's sole conversation. + * The run's visible conversation messages projected from canonical events. + * Defaults to the run's sole conversation. */ export async function conversationMessages( session: NormalizedSession, conversationIdOverride?: string, ): Promise { const id = conversationIdOverride ?? conversationId(session); - return await getConversationMessageStore().list(id); + return projectVisibleConversationMessages( + await getConversationEventStore().loadHistory(id), + ); } diff --git a/packages/junior/src/api/conversations/detail-projection.ts b/packages/junior/src/api/conversations/detail-projection.ts index 3e39d538a..a08e5e852 100644 --- a/packages/junior/src/api/conversations/detail-projection.ts +++ b/packages/junior/src/api/conversations/detail-projection.ts @@ -1,9 +1,5 @@ import { isDeepStrictEqual } from "node:util"; import { canExposeConversationPayload } from "@/chat/conversation-privacy"; -import type { - ConversationMessage, - ConversationMessageStore, -} from "@/chat/conversations/messages"; import type { ConversationEventStore, ConversationEvent, @@ -13,10 +9,9 @@ import type { Conversation } from "@/chat/conversations/store"; import { projectConversationEventHistory } from "@/chat/conversations/event-projection"; import { loadCurrentConversationEvents } from "@/chat/conversations/history-loader"; import { stripRuntimeTurnContextMessages } from "@/chat/conversations/model-messages"; -import { - getConversationEventStore, - getConversationMessageStore, -} from "@/chat/db"; +import { getConversationEventStore } from "@/chat/db"; +import { projectVisibleConversationMessages } from "@/chat/conversations/visible-message-projection"; +import type { ConversationMessage } from "@/chat/state/conversation"; import { buildSentryConversationUrl, buildSentryTraceUrl, @@ -214,14 +209,52 @@ function historyContent(args: { return { contextEvents, messages }; } +function terminalOutcomeTranscript( + messages: ConversationModelMessage[], +): TranscriptMessage[] { + return messages.flatMap((message) => { + const normalized = normalizeTranscriptMessage(message); + if ( + normalized.role !== "assistant" || + (normalized.outcome !== "error" && normalized.outcome !== "aborted") + ) { + return []; + } + return [ + { + role: "assistant" as const, + outcome: normalized.outcome, + parts: [], + ...(normalized.timestamp === undefined + ? {} + : { timestamp: normalized.timestamp }), + }, + ]; + }); +} + +function mergeTranscriptChronologically( + messages: TranscriptMessage[], +): TranscriptMessage[] { + return messages + .map((message, index) => ({ message, index })) + .sort( + (left, right) => + (left.message.timestamp ?? Number.POSITIVE_INFINITY) - + (right.message.timestamp ?? Number.POSITIVE_INFINITY) || + left.index - right.index, + ) + .map(({ message }) => message); +} + async function conversationContent(args: { conversationId: string; - messageStore: ConversationMessageStore; eventStore: ConversationEventStore; canExposePayload: boolean; }): Promise<{ activity: ConversationActivityReport[]; contextEvents: ConversationContextEvent[]; + traceId?: string; transcript: TranscriptMessage[]; }> { const events = await args.eventStore.loadHistory(args.conversationId); @@ -229,19 +262,38 @@ async function conversationContent(args: { canExposePayload: args.canExposePayload, events, }); - const messages = history.messages; - const transcript = - messages.length > 0 - ? messages.map((message) => normalizeTranscriptMessage(message)) - : (await args.messageStore.list(args.conversationId)).map( - visibleMessageTranscript, - ); + const visibleMessages = projectVisibleConversationMessages(events); + const visibleTranscript = visibleMessages.map(visibleMessageTranscript); + const modelTranscript = history.messages.map(normalizeTranscriptMessage); + const transcript = mergeTranscriptChronologically([ + ...visibleTranscript, + ...terminalOutcomeTranscript(history.messages), + ]); + const contextEvents = history.contextEvents.map((event) => ({ + ...event, + transcriptIndex: transcript.findIndex( + (message) => + (message.timestamp ?? Number.POSITIVE_INFINITY) > + new Date(event.createdAt).getTime(), + ), + })); + for (const event of contextEvents) { + if (event.transcriptIndex < 0) event.transcriptIndex = transcript.length; + } return { activity: buildConversationActivityFromEvents({ canExposePayload: args.canExposePayload, events, }), - contextEvents: history.contextEvents, + contextEvents, + ...(args.canExposePayload + ? { + traceId: traceIdFromTranscript([ + ...modelTranscript, + ...visibleTranscript, + ]), + } + : {}), transcript, }; } @@ -267,7 +319,6 @@ export async function buildConversationDetail(args: { const conversationId = conversation.conversationId; const nowMs = Date.now(); const eventStore = getConversationEventStore(); - const messageStore = getConversationMessageStore(); const transcriptPurgedAtMs = conversation.transcriptPurgedAtMs; const transcriptExpiredAt = transcriptPurgedAtMs !== undefined @@ -281,20 +332,17 @@ export async function buildConversationDetail(args: { conversationId, visibility: conversation.visibility, }); - const currentContent = + const currentContent: Awaited> = transcriptPurgedAtMs === undefined ? await conversationContent({ conversationId, - messageStore, eventStore, canExposePayload: canExposeSqlContent, }) : { activity: [], contextEvents: [], transcript: [] }; const currentTranscript = currentContent.transcript; - const traceId = canExposeSqlContent - ? traceIdFromTranscript(currentTranscript) - : undefined; + const traceId = canExposeSqlContent ? currentContent.traceId : undefined; const sentryTraceUrl = traceId ? buildSentryTraceUrl(traceId) : undefined; const sentryConversationUrl = buildSentryConversationUrl(conversationId); diff --git a/packages/junior/src/chat/agent-dispatch/runner.ts b/packages/junior/src/chat/agent-dispatch/runner.ts index 93931643d..0bca65946 100644 --- a/packages/junior/src/chat/agent-dispatch/runner.ts +++ b/packages/junior/src/chat/agent-dispatch/runner.ts @@ -430,8 +430,8 @@ export async function runAgentDispatchSlice( // (`meta.slackTs`, checked by the redelivery guard above) immediately and // durably before the dispatch is marked terminal so the crash window // between post and marker stays as small as possible. The retry-and-swallow - // `persistRuntimePatch` below write-throughs the SQL transcript, so no - // separate transcript persist runs outside that guarded block. + // `persistRuntimePatch` below appends canonical visible-message facts, so + // no separate transcript persist runs outside that guarded block. markConversationMessage(conversation, userMessageId, { replied: true, skippedReason: undefined, diff --git a/packages/junior/src/chat/conversations/README.md b/packages/junior/src/chat/conversations/README.md index 19502e004..c29af97d6 100644 --- a/packages/junior/src/chat/conversations/README.md +++ b/packages/junior/src/chat/conversations/README.md @@ -8,8 +8,8 @@ conversation events, compaction boundaries, search, retention, and legacy import - Conversation rows identify the source, destination, participants, visibility, and lifecycle metadata. - Visible-message events are the destination-facing user and assistant history. - `junior_conversation_messages` is their rebuildable hydration and search read - model, never a second history authority. + `junior_conversation_messages` is their rebuildable search read model, never + a hydration source or second history authority. - Conversation events are the versioned, append-only execution history used to restore Pi state. Pi messages and context are projections of this log, not a second authority. diff --git a/packages/junior/src/chat/conversations/history-loader.ts b/packages/junior/src/chat/conversations/history-loader.ts index 7ed6808c5..8eb854a28 100644 --- a/packages/junior/src/chat/conversations/history-loader.ts +++ b/packages/junior/src/chat/conversations/history-loader.ts @@ -21,3 +21,11 @@ export async function loadCurrentConversationEvents( await ensureConversationEventHistory(args); return getConversationEventStore().loadCurrentEpoch(args.conversationId); } + +/** Load complete event history after the bounded lazy legacy import. */ +export async function loadConversationEventHistory( + args: ScopedConversation, +): Promise { + await ensureConversationEventHistory(args); + return getConversationEventStore().loadHistory(args.conversationId); +} diff --git a/packages/junior/src/chat/conversations/legacy-import.ts b/packages/junior/src/chat/conversations/legacy-import.ts index d5c63ea9c..19a34ad35 100644 --- a/packages/junior/src/chat/conversations/legacy-import.ts +++ b/packages/junior/src/chat/conversations/legacy-import.ts @@ -4,24 +4,23 @@ * A single per-conversation import unit shared by `junior upgrade` (bulk, * bounded newest-first) and the lazy first-read straggler path. It converts the * legacy session log into `junior_conversation_events`, imports the advisor - * session blob as a child conversation, and copies the `thread-state` visible messages into - * `junior_conversation_messages`. Import is idempotent per conversation: event - * rows seal normal imports, while message-only imports verify their complete - * SQL projection before skipping. It never fabricates import-time timestamps. + * session blob as a child conversation, and converts `thread-state` visible + * messages into canonical events plus their rebuildable SQL search projection. + * Import is idempotent per conversation: canonical event rows seal completed + * imports. It never fabricates import-time timestamps. * * This module and its lazy hook are removed wholesale after the legacy Redis TTL * horizon passes; keeping it separate keeps that deletion mechanical. */ // TODO(v0.104.0): Remove this module and its advisor-session reader after the // legacy Redis-to-SQL import horizon. -import { isDeepStrictEqual } from "node:util"; import { eq } from "drizzle-orm"; import { z } from "zod"; import { type ConversationCompaction, type ConversationMessage as ThreadConversationMessage, } from "@/chat/state/conversation"; -import { toStoredConversationMessage } from "@/chat/conversations/visible-messages"; +import { toStoredConversationMessage } from "@/chat/conversations/visible-message-serializer"; import { getStateAdapter } from "@/chat/state/adapter"; import type { PiMessage } from "@/chat/pi/messages"; import { @@ -35,13 +34,8 @@ import { } from "@/chat/conversations/legacy-advisor-session"; import type { JuniorSqlDatabase } from "@/db/db"; import { juniorConversations } from "@/db/schema"; -import { - getConversationEventStore, - getConversationMessageStore, - getSqlExecutor, -} from "@/chat/db"; +import { getConversationEventStore, getSqlExecutor } from "@/chat/db"; import { createSqlConversationEventStore } from "./sql/history"; -import type { ConversationMessageStore } from "./messages"; import type { Conversation } from "./store"; import { convertAdvisorMessages, @@ -82,7 +76,6 @@ const legacyThreadStateSnapshotSchema = z.object({ /** Legacy source seams used by the one-time migration. */ export interface LegacyImportDeps { executor: JuniorSqlDatabase; - messageStore: ConversationMessageStore; sessionLogStore?: SessionLogStore; advisorSessionStore?: LegacyAdvisorSessionReader; loadVisibleMessages?: ( @@ -220,36 +213,6 @@ export async function importConversationFromLegacy( }); } - if (converted.events.length === 0 && visible.length > 0) { - const existingMessages = new Map( - (await deps.messageStore.list(conversationId)).map((message) => [ - message.messageId, - message, - ]), - ); - const fullyImported = visible.every((message) => { - const existingMessage = existingMessages.get(message.id); - const projected = toStoredConversationMessage(message); - return ( - existingMessage !== undefined && - Object.entries(projected.meta ?? {}).every(([key, value]) => - isDeepStrictEqual(existingMessage.meta?.[key], value), - ) && - (message.meta?.replied !== true || - existingMessage.repliedAtMs !== undefined) - ); - }); - if (fullyImported) { - await writeLegacyImport(deps.executor, { - conversationId, - fallbackCreatedAtMs, - lastActivityAtMs, - events: [], - }); - return { imported: false }; - } - } - let child: | { conversationId: string; @@ -319,7 +282,6 @@ export async function ensureLegacyConversationImport(args: { } await importConversationFromLegacy(args.conversationId, { executor, - messageStore: getConversationMessageStore(), sessionLogStore: { read: async () => entries }, loadVisibleMessages: async () => visible, legacyCompactions: snapshot.compactions, diff --git a/packages/junior/src/chat/conversations/messages.ts b/packages/junior/src/chat/conversations/messages.ts index eb8b70790..147d24128 100644 --- a/packages/junior/src/chat/conversations/messages.ts +++ b/packages/junior/src/chat/conversations/messages.ts @@ -18,19 +18,7 @@ export interface NewConversationMessage { createdAtMs: number; } -/** A visible message read back from storage. */ -export interface ConversationMessage { - conversationId: string; - messageId: string; - role: ConversationMessageRole; - text: string; - authorIdentityId?: string; - meta?: Record; - repliedAtMs?: number; - createdAtMs: number; -} - -/** Persist event-backed visible messages and read their SQL projection. */ +/** Persist event-backed visible messages and maintain their SQL read model. */ export interface ConversationMessageStore { /** Append source facts and project them idempotently by message identity. */ record( @@ -43,9 +31,4 @@ export interface ConversationMessageStore { messageId: string, repliedAtMs: number, ): Promise; - /** List the materialized read model in `created_at` order. */ - list( - conversationId: string, - opts?: { limit?: number }, - ): Promise; } diff --git a/packages/junior/src/chat/conversations/sql/messages.ts b/packages/junior/src/chat/conversations/sql/messages.ts index b672c99aa..c3104e0ee 100644 --- a/packages/junior/src/chat/conversations/sql/messages.ts +++ b/packages/junior/src/chat/conversations/sql/messages.ts @@ -1,8 +1,7 @@ import { isDeepStrictEqual } from "node:util"; -import { and, asc, eq, inArray, isNotNull, isNull, sql } from "drizzle-orm"; +import { and, eq, inArray, isNotNull, isNull, sql } from "drizzle-orm"; import type { JuniorSqlDatabase } from "@/db/db"; import type { - ConversationMessage, ConversationMessageStore, NewConversationMessage, } from "../messages"; @@ -14,6 +13,17 @@ import type { NewConversationEvent } from "../history"; type ConversationMessageRow = typeof juniorConversationMessages.$inferSelect; +interface ConversationMessage { + conversationId: string; + messageId: string; + role: NewConversationMessage["role"]; + text: string; + authorIdentityId?: string; + meta?: Record; + repliedAtMs?: number; + createdAtMs: number; +} + function messageFromRow(row: ConversationMessageRow): ConversationMessage { return { conversationId: row.conversationId, @@ -209,26 +219,6 @@ class SqlConversationMessageStore implements ConversationMessageStore { ); }); } - - async list( - conversationId: string, - opts: { limit?: number } = {}, - ): Promise { - const query = this.executor - .db() - .select() - .from(juniorConversationMessages) - .where(eq(juniorConversationMessages.conversationId, conversationId)) - .orderBy( - asc(juniorConversationMessages.createdAt), - asc(juniorConversationMessages.messageId), - ); - const rows = - opts.limit === undefined - ? await query - : await query.limit(Math.max(0, opts.limit)); - return rows.map(messageFromRow); - } } /** Create a SQL-backed conversation message store. */ diff --git a/packages/junior/src/chat/conversations/visible-compactions.ts b/packages/junior/src/chat/conversations/visible-compactions.ts index 501b6c443..87983c3d5 100644 --- a/packages/junior/src/chat/conversations/visible-compactions.ts +++ b/packages/junior/src/chat/conversations/visible-compactions.ts @@ -6,7 +6,10 @@ import type { ThreadConversationState, } from "@/chat/state/conversation"; -function latestSnapshot(events: ConversationEvent[]): ConversationCompaction[] { +/** Project the latest visible-context compaction snapshot from event history. */ +export function projectVisibleConversationCompactions( + events: ConversationEvent[], +): ConversationCompaction[] { for (let index = events.length - 1; index >= 0; index -= 1) { const data = events[index]?.data; if (data?.type === "visible_context_compacted") { @@ -16,24 +19,13 @@ function latestSnapshot(events: ConversationEvent[]): ConversationCompaction[] { return []; } -/** Hydrate the durable visible-context compaction snapshot from SQL. */ -export async function hydrateConversationCompactions(args: { - conversation: ThreadConversationState; - conversationId: string; -}): Promise { - const events = await getConversationEventStore().loadHistory( - args.conversationId, - ); - args.conversation.compactions = latestSnapshot(events); -} - /** Persist a changed visible-context compaction snapshot in event history. */ export async function persistConversationCompactions(args: { conversation: ThreadConversationState; conversationId: string; }): Promise { const eventStore = getConversationEventStore(); - const existing = latestSnapshot( + const existing = projectVisibleConversationCompactions( await eventStore.loadHistory(args.conversationId), ); if (isDeepStrictEqual(existing, args.conversation.compactions)) { diff --git a/packages/junior/src/chat/conversations/visible-message-projection.ts b/packages/junior/src/chat/conversations/visible-message-projection.ts new file mode 100644 index 000000000..fb7c48537 --- /dev/null +++ b/packages/junior/src/chat/conversations/visible-message-projection.ts @@ -0,0 +1,116 @@ +import type { ConversationEvent } from "./history"; +import type { + ConversationAuthor, + ConversationMessage, + ConversationMessageMeta, +} from "@/chat/state/conversation"; + +type VisibleMessageEvent = ConversationEvent & { + data: Extract< + ConversationEvent["data"], + { + type: + | "visible_message_recorded" + | "visible_message_metadata_updated" + | "visible_message_replied"; + } + >; +}; + +interface ProjectedMessage { + message: ConversationMessage; + repliedAtMs?: number; +} + +function isVisibleMessageEvent( + event: ConversationEvent, +): event is VisibleMessageEvent { + return ( + event.data.type === "visible_message_recorded" || + event.data.type === "visible_message_metadata_updated" || + event.data.type === "visible_message_replied" + ); +} + +function splitMeta(meta: Record | undefined): { + author?: ConversationAuthor; + meta?: ConversationMessageMeta; +} { + const rest = { ...(meta ?? {}) }; + const author = rest.author as ConversationAuthor | undefined; + delete rest.author; + return { + ...(author ? { author } : {}), + ...(Object.keys(rest).length > 0 + ? { meta: rest as ConversationMessageMeta } + : {}), + }; +} + +function storedMeta(message: ConversationMessage): Record { + return { + ...(message.author ? { author: message.author } : {}), + ...(message.meta ?? {}), + }; +} + +function missingBaselineError(event: VisibleMessageEvent): Error { + return new Error( + `Visible message event ${event.data.type} at seq ${event.seq} references ${event.data.messageId} before visible_message_recorded`, + ); +} + +/** Reduce canonical visible-message facts into the destination-facing transcript. */ +export function projectVisibleConversationMessages( + events: ConversationEvent[], +): ConversationMessage[] { + const byId = new Map(); + + for (const event of events) { + if (!isVisibleMessageEvent(event)) continue; + const data = event.data; + const current = byId.get(data.messageId); + + if (data.type === "visible_message_recorded") { + const message: ConversationMessage = { + id: data.messageId, + role: data.role, + text: data.text, + createdAtMs: event.createdAtMs, + ...splitMeta(data.meta), + }; + if (current) + throw new Error( + `Duplicate visible_message_recorded event for ${data.messageId} at seq ${event.seq}`, + ); + byId.set(data.messageId, { message }); + continue; + } + + if (!current) throw missingBaselineError(event); + if (data.type === "visible_message_metadata_updated") { + const merged = { ...storedMeta(current.message), ...data.meta }; + const { author: _author, meta: _meta, ...baseline } = current.message; + current.message = { + ...baseline, + ...splitMeta(merged), + }; + continue; + } + + current.repliedAtMs ??= event.createdAtMs; + } + + return [...byId.values()] + .map(({ message, repliedAtMs }) => { + if (repliedAtMs === undefined) return message; + return { + ...message, + meta: { ...(message.meta ?? {}), replied: true }, + }; + }) + .sort( + (left, right) => + left.createdAtMs - right.createdAtMs || left.id.localeCompare(right.id), + ); +} diff --git a/packages/junior/src/chat/conversations/visible-message-serializer.ts b/packages/junior/src/chat/conversations/visible-message-serializer.ts new file mode 100644 index 000000000..28affe219 --- /dev/null +++ b/packages/junior/src/chat/conversations/visible-message-serializer.ts @@ -0,0 +1,24 @@ +import type { NewConversationMessage } from "./messages"; +import type { ConversationMessage } from "@/chat/state/conversation"; + +/** Serialize one in-memory visible message into its event-backed write shape. */ +export function toStoredConversationMessage( + message: ConversationMessage, +): NewConversationMessage { + const meta: Record = {}; + if (message.author) { + meta.author = message.author; + } + const { replied, ...restMeta } = message.meta ?? {}; + Object.assign(meta, restMeta); + if (replied === false) { + meta.replied = false; + } + return { + messageId: message.id, + role: message.role, + text: message.text, + ...(Object.keys(meta).length > 0 ? { meta } : {}), + createdAtMs: message.createdAtMs, + }; +} diff --git a/packages/junior/src/chat/conversations/visible-messages.ts b/packages/junior/src/chat/conversations/visible-messages.ts index e6f659624..2c8e6825b 100644 --- a/packages/junior/src/chat/conversations/visible-messages.ts +++ b/packages/junior/src/chat/conversations/visible-messages.ts @@ -1,81 +1,25 @@ /** * Visible-transcript sync between the in-memory turn working set and SQL. * - * Visible-message events are the durable transcript authority; - * `ConversationMessageStore` maintains their SQL hydration/search read model, - * and `ThreadConversationState.messages` is only the current-turn working set. - * No transcript data is persisted to `thread-state`. + * Visible-message events are the durable transcript and hydration authority; + * `ConversationMessageStore` maintains only their SQL search read model, and + * `ThreadConversationState.messages` is the current-turn working set. No + * transcript data is persisted to `thread-state`. */ -import type { - ConversationMessage as StoredConversationMessage, - NewConversationMessage, -} from "@/chat/conversations/messages"; import { getConversationMessageStore } from "@/chat/db"; -import { hydrateConversationCompactions } from "./visible-compactions"; +import { loadConversationEventHistory } from "./history-loader"; +import { projectVisibleConversationCompactions } from "./visible-compactions"; +import { projectVisibleConversationMessages } from "./visible-message-projection"; +import { toStoredConversationMessage } from "./visible-message-serializer"; import { updateConversationStats } from "@/chat/services/conversation-memory"; -import type { - ConversationAuthor, - ConversationMessage, - ConversationMessageMeta, - ThreadConversationState, -} from "@/chat/state/conversation"; +import type { ThreadConversationState } from "@/chat/state/conversation"; /** - * Project the in-memory message onto the store insert shape. This is the single - * serialization point for visible messages, so both live turn persistence and - * the one-time legacy import produce identical rows: author display facts and - * bounded source meta ride in the `meta` JSON so the working set rehydrates with - * identical rendering, and `replied === true` is not stored in meta because - * `replied_at` is its authority. - */ -export function toStoredConversationMessage( - message: ConversationMessage, -): NewConversationMessage { - const meta: Record = {}; - if (message.author) { - meta.author = message.author; - } - const { replied, ...restMeta } = message.meta ?? {}; - Object.assign(meta, restMeta); - if (replied === false) { - meta.replied = false; - } - return { - messageId: message.id, - role: message.role, - text: message.text, - ...(Object.keys(meta).length > 0 ? { meta } : {}), - createdAtMs: message.createdAtMs, - }; -} - -/** Rebuild the in-memory message from a stored row, deriving `replied` from `replied_at`. */ -function fromStoredMessage( - row: StoredConversationMessage, -): ConversationMessage { - const rawMeta: Record = { ...(row.meta ?? {}) }; - const author = rawMeta.author as ConversationAuthor | undefined; - delete rawMeta.author; - const meta = { ...rawMeta } as ConversationMessageMeta; - if (row.repliedAtMs !== undefined) { - meta.replied = true; - } - return { - id: row.messageId, - role: row.role, - text: row.text, - createdAtMs: row.createdAtMs, - ...(author ? { author } : {}), - ...(Object.keys(meta).length > 0 ? { meta } : {}), - }; -} - -/** - * Replace the in-memory working set with the durable transcript from SQL, - * excluding messages already folded into a thread-state compaction summary. + * Replace the in-memory working set with the durable event-log transcript, + * excluding messages already folded into a visible compaction summary. * * Hydrate is a first-read boundary, so it must trigger the once-only Redis→SQL - * lazy import before reading SQL: consumers that hydrate before any step + * lazy import before reading events: consumers that hydrate before any model * projection read (turn-dedupe, delivered-message redelivery guards, * channel-context assembly) would otherwise make correctness decisions on an * empty transcript for promotion-window stragglers whose history is still only @@ -90,38 +34,25 @@ export async function hydrateConversationMessages(args: { args.conversation.messages = []; return; } - // Lazy Redis→SQL import for promotion-window stragglers, run before the SQL - // read so hydrate never observes an empty transcript for a conversation whose - // history is still only in legacy Redis. The dynamic import is deliberate: - // legacy-import.ts statically imports `toStoredConversationMessage` from this - // module, so a static import back would create a cycle; a function-level - // dynamic import keeps this seam trivially deletable when the legacy-import - // module is removed wholesale after the legacy Redis TTL horizon. - const { ensureLegacyConversationImport } = - await import("@/chat/conversations/legacy-import"); - await ensureLegacyConversationImport({ conversationId: args.conversationId }); - await hydrateConversationCompactions({ - conversation: args.conversation, + const events = await loadConversationEventHistory({ conversationId: args.conversationId, }); - const store = getConversationMessageStore(); - const rows = await store.list(args.conversationId); + args.conversation.compactions = projectVisibleConversationCompactions(events); const coveredIds = new Set( args.conversation.compactions.flatMap( (compaction) => compaction.coveredMessageIds, ), ); - args.conversation.messages = rows - .filter((row) => !coveredIds.has(row.messageId)) - .map(fromStoredMessage); + args.conversation.messages = projectVisibleConversationMessages( + events, + ).filter((message) => !coveredIds.has(message.id)); updateConversationStats(args.conversation); } /** - * Write the working set back to SQL: record every message idempotently and set - * the `replied_at` mark for messages the turn has answered. Content columns are - * insert-only and `meta` merges key-wise on conflict, so repeated calls across - * a turn's persist points are safe. + * Append the working set's source facts and refresh the SQL search read model. + * Content columns are insert-only and `meta` merges key-wise on conflict, so + * repeated calls across a turn's persist points are safe. */ export async function persistConversationMessages(args: { conversation: ThreadConversationState; diff --git a/packages/junior/src/chat/runtime/delivered-turn-state.ts b/packages/junior/src/chat/runtime/delivered-turn-state.ts index ca2b2baaf..fa3d66388 100644 --- a/packages/junior/src/chat/runtime/delivered-turn-state.ts +++ b/packages/junior/src/chat/runtime/delivered-turn-state.ts @@ -8,13 +8,13 @@ import { } from "@/chat/runtime/thread-state"; import { markTurnCompleted } from "@/chat/runtime/turn"; import { - generateConversationId, markConversationMessage, normalizeConversationText, upsertConversationMessage, updateConversationStats, } from "@/chat/services/conversation-memory"; import { clearPendingAuth } from "@/chat/services/pending-auth"; +import { buildDeterministicAssistantMessageId } from "@/chat/state/turn-id"; const NO_VISIBLE_REPLY_CONVERSATION_TEXT = "[no visible reply]"; @@ -48,7 +48,7 @@ export function buildDeliveredTurnStatePatch(args: { skippedReason: undefined, }); upsertConversationMessage(conversation, { - id: generateConversationId("assistant"), + id: buildDeterministicAssistantMessageId(args.sessionId), role: "assistant", text: assistantText, createdAtMs: Date.now(), diff --git a/packages/junior/src/chat/runtime/reply-executor.ts b/packages/junior/src/chat/runtime/reply-executor.ts index d5d723dec..40c48839f 100644 --- a/packages/junior/src/chat/runtime/reply-executor.ts +++ b/packages/junior/src/chat/runtime/reply-executor.ts @@ -62,7 +62,6 @@ import { markConversationMessage, normalizeConversationText, upsertConversationMessage, - generateConversationId, updateConversationStats, } from "@/chat/services/conversation-memory"; import type { ContextCompactor } from "@/chat/services/context-compaction"; @@ -96,6 +95,7 @@ import { TurnInputDeferredError, } from "@/chat/runtime/turn"; import { buildDeterministicTurnId } from "@/chat/runtime/turn"; +import { buildDeterministicAssistantMessageId } from "@/chat/state/turn-id"; import { markTurnClosed, markTurnFailed } from "@/chat/runtime/turn"; import { startActiveTurn } from "@/chat/runtime/turn"; import { @@ -858,7 +858,7 @@ export function createReplyToThread(deps: ReplyExecutorDeps) { }, ); upsertConversationMessage(preparedState.conversation, { - id: generateConversationId("assistant"), + id: buildDeterministicAssistantMessageId(turnId), role: "assistant", text: normalizeConversationText(configReply.text), createdAtMs: Date.now(), @@ -1295,7 +1295,7 @@ export function createReplyToThread(deps: ReplyExecutorDeps) { }, ); upsertConversationMessage(preparedState.conversation, { - id: generateConversationId("assistant"), + id: buildDeterministicAssistantMessageId(turnId), role: "assistant", text: normalizeConversationText(text), createdAtMs: Date.now(), @@ -1484,8 +1484,8 @@ export function createReplyToThread(deps: ReplyExecutorDeps) { } try { // Commit the durable delivery record first so recovery cannot - // regenerate an accepted reply. Persist the SQL-visible transcript - // next, then update Redis runtime scratch independently. + // regenerate an accepted reply. Persist canonical visible-message + // facts next, then update Redis runtime scratch independently. if (conversationId && reply.piMessages?.length) { await completeDeliveredTurn({ channelName, @@ -1661,7 +1661,7 @@ export function createReplyToThread(deps: ReplyExecutorDeps) { }, ); upsertConversationMessage(preparedState.conversation, { - id: generateConversationId("assistant"), + id: buildDeterministicAssistantMessageId(turnId), role: "assistant", text: normalizeConversationText(recoveryText), createdAtMs: Date.now(), diff --git a/packages/junior/src/chat/state/conversation.ts b/packages/junior/src/chat/state/conversation.ts index 49430bf37..8c4c3e34f 100644 --- a/packages/junior/src/chat/state/conversation.ts +++ b/packages/junior/src/chat/state/conversation.ts @@ -162,10 +162,9 @@ export function coerceThreadConversationState( const rawConversation = isRecord(root.conversation) ? root.conversation : {}; const base = defaultConversationState(); - // The visible transcript lives in the ConversationMessageStore and Pi - // history lives in the ConversationEventStore (both SQL). Legacy `messages` / - // `piMessages` mirrors left in old thread-state payloads are ignored on - // read; consumers hydrate from SQL instead. + // Visible and model history live in the SQL ConversationEventStore. Legacy + // `messages` / `piMessages` mirrors left in old thread-state payloads are + // ignored on read; consumers hydrate from canonical events instead. const messages: ConversationMessage[] = []; const rawCompactions = Array.isArray(rawConversation.compactions) @@ -260,9 +259,9 @@ export function coerceThreadConversationState( /** * Wrap a conversation state into the storage envelope for persistence. The - * visible transcript (`messages`) is not written to `thread-state`; it lives - * in SQL (`ConversationMessageStore`), and Pi history lives in the SQL - * `ConversationEventStore`. Only runtime scratch is persisted here. + * visible transcript (`messages`) is not written to `thread-state`; visible and + * model history live in the SQL `ConversationEventStore`. Only runtime scratch + * is persisted here. */ export function buildConversationStatePatch( conversation: ThreadConversationState, diff --git a/packages/junior/src/chat/state/turn-id.ts b/packages/junior/src/chat/state/turn-id.ts index 58348d46c..61e574139 100644 --- a/packages/junior/src/chat/state/turn-id.ts +++ b/packages/junior/src/chat/state/turn-id.ts @@ -3,3 +3,8 @@ export function buildDeterministicTurnId(messageId: string): string { const sanitized = messageId.replace(/[^a-zA-Z0-9_-]/g, "_"); return `turn_${sanitized}`; } + +/** Build the stable visible assistant-message identity for one logical turn. */ +export function buildDeterministicAssistantMessageId(turnId: string): string { + return `assistant:${turnId}`; +} diff --git a/packages/junior/src/cli/upgrade/migrations/conversations-history-sql.ts b/packages/junior/src/cli/upgrade/migrations/conversations-history-sql.ts index a11c32535..ea544ac11 100644 --- a/packages/junior/src/cli/upgrade/migrations/conversations-history-sql.ts +++ b/packages/junior/src/cli/upgrade/migrations/conversations-history-sql.ts @@ -1,6 +1,5 @@ import { getChatConfig } from "@/chat/config"; import { importConversationFromLegacy } from "@/chat/conversations/legacy-import"; -import { createSqlConversationMessageStore } from "@/chat/conversations/sql/messages"; import { createStateConversationStore } from "@/chat/conversations/state"; import type { LegacyAdvisorSessionReader } from "@/chat/conversations/legacy-advisor-session"; import type { ConversationMessage as ThreadConversationMessage } from "@/chat/state/conversation"; @@ -42,7 +41,6 @@ export async function migrateConversationHistoryToSql( } const limit = Math.max(1, options.batchSize ?? HISTORY_BACKFILL_LIMIT); try { - const messageStore = createSqlConversationMessageStore(executor); const conversations = await source.listByActivity({ limit }); let migrated = 0; let existing = 0; @@ -51,7 +49,6 @@ export async function migrateConversationHistoryToSql( conversation.conversationId, { executor, - messageStore, conversationRecord: conversation, ...(options.sessionLogStore ? { sessionLogStore: options.sessionLogStore } diff --git a/packages/junior/tests/component/conversation-transcripts-sql.test.ts b/packages/junior/tests/component/conversation-transcripts-sql.test.ts index 26c1d5c08..45aaf6572 100644 --- a/packages/junior/tests/component/conversation-transcripts-sql.test.ts +++ b/packages/junior/tests/component/conversation-transcripts-sql.test.ts @@ -1,4 +1,4 @@ -import { eq } from "drizzle-orm"; +import { asc, eq } from "drizzle-orm"; import { describe, expect, it } from "vitest"; import { createSqlConversationEventStore } from "@/chat/conversations/sql/history"; import { @@ -9,13 +9,17 @@ import { getConversationEventStore } from "@/chat/db"; import { purgeConversation } from "@/chat/conversations/retention"; import { createSqlConversationMessageStore } from "@/chat/conversations/sql/messages"; import { - hydrateConversationCompactions, persistConversationCompactions, + projectVisibleConversationCompactions, } from "@/chat/conversations/visible-compactions"; import { coerceThreadConversationState } from "@/chat/state/conversation"; import { migrateSchema } from "@/chat/conversations/sql/migrations"; import type { JuniorSqlDatabase } from "@/db/db"; -import { juniorConversationEvents, juniorConversations } from "@/db/schema"; +import { + juniorConversationEvents, + juniorConversationMessages, + juniorConversations, +} from "@/db/schema"; import { buildJuniorSqlConversation, createLocalJuniorSqlFixture, @@ -30,6 +34,21 @@ import { const CONVERSATION_ID = "slack:C123:1718123456.000000"; const CHILD_CONVERSATION_ID = "advisor:child-1"; +async function listMessageRows( + fixture: LocalJuniorSqlFixture, + conversationId: string, +) { + return fixture.sql + .db() + .select() + .from(juniorConversationMessages) + .where(eq(juniorConversationMessages.conversationId, conversationId)) + .orderBy( + asc(juniorConversationMessages.createdAt), + asc(juniorConversationMessages.messageId), + ); +} + it("accepts legacy markers and validates current profile names", () => { expect( conversationEventDataSchema.safeParse({ @@ -273,13 +292,11 @@ describe("conversation transcript SQL stores", () => { conversationId: CONVERSATION_ID, }); - const rehydrated = coerceThreadConversationState({}); - await hydrateConversationCompactions({ - conversation: rehydrated, - conversationId: CONVERSATION_ID, - }); - expect(rehydrated.compactions).toEqual(conversation.compactions); - expect(await events.loadHistory(CONVERSATION_ID)).toHaveLength(1); + const history = await events.loadHistory(CONVERSATION_ID); + expect(projectVisibleConversationCompactions(history)).toEqual( + conversation.compactions, + ); + expect(history).toHaveLength(1); }); it("applies Drizzle migrations idempotently", async () => { @@ -601,22 +618,23 @@ INSERT INTO junior_conversation_events ( await store.markReplied(CONVERSATION_ID, "m1", 5_000); await store.markReplied(CONVERSATION_ID, "m1", 9_000); - const listed = await store.list(CONVERSATION_ID); - expect(listed).toEqual([ + const listed = await listMessageRows(fixture, CONVERSATION_ID); + expect(listed).toMatchObject([ { conversationId: CONVERSATION_ID, messageId: "m1", role: "user", text: "first", - createdAtMs: 1_000, - repliedAtMs: 5_000, + createdAt: new Date(1_000), + repliedAt: new Date(5_000), }, { conversationId: CONVERSATION_ID, messageId: "m2", role: "assistant", text: "reply", - createdAtMs: 2_000, + createdAt: new Date(2_000), + repliedAt: null, }, ]); const visibleEvents = ( @@ -685,7 +703,7 @@ INSERT INTO junior_conversation_events ( 'Failed query: insert into "junior_conversation_messages"', ); - expect(await messages.list(CONVERSATION_ID)).toEqual([]); + expect(await listMessageRows(fixture, CONVERSATION_ID)).toEqual([]); expect( await createSqlConversationEventStore(fixture.sql).loadHistory( CONVERSATION_ID, @@ -733,11 +751,11 @@ INSERT INTO junior_conversation_events ( ), ).rejects.toThrow('Failed query: update "junior_conversation_messages"'); - const projected = await messages.list(CONVERSATION_ID); + const projected = await listMessageRows(fixture, CONVERSATION_ID); expect(projected).toEqual([ expect.objectContaining({ messageId: "reply-projection-failure" }), ]); - expect(projected[0]?.repliedAtMs).toBeUndefined(); + expect(projected[0]?.repliedAt).toBe(null); const visibleEvents = ( await createSqlConversationEventStore(fixture.sql).loadHistory( CONVERSATION_ID, @@ -850,7 +868,7 @@ INSERT INTO junior_conversation_events ( for (const conversationId of [CONVERSATION_ID, CHILD_CONVERSATION_ID]) { expect(await events.loadHistory(conversationId)).toEqual([]); - expect(await messages.list(conversationId)).toEqual([]); + expect(await listMessageRows(fixture, conversationId)).toEqual([]); } const rows = await fixture.sql @@ -880,7 +898,7 @@ INSERT INTO junior_conversation_events ( ]); expect(await events.loadHistory(CONVERSATION_ID)).toHaveLength(2); - expect(await messages.list(CONVERSATION_ID)).toHaveLength(1); + expect(await listMessageRows(fixture, CONVERSATION_ID)).toHaveLength(1); const reopened = await fixture.sql .db() .select({ diff --git a/packages/junior/tests/component/conversations/legacy-import.test.ts b/packages/junior/tests/component/conversations/legacy-import.test.ts index e1ca04f29..49268647d 100644 --- a/packages/junior/tests/component/conversations/legacy-import.test.ts +++ b/packages/junior/tests/component/conversations/legacy-import.test.ts @@ -1,6 +1,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { eq, inArray } from "drizzle-orm"; -import { juniorConversations } from "@/db/schema"; +import { asc, eq, inArray } from "drizzle-orm"; +import { juniorConversationMessages, juniorConversations } from "@/db/schema"; +import type { JuniorSqlDatabase } from "@/db/db"; import { closeDb, getConversationEventStore, @@ -48,6 +49,26 @@ async function processSqlStores() { }; } +async function listMessageRows( + executor: JuniorSqlDatabase, + conversationId: string, +) { + const rows = await executor + .db() + .select() + .from(juniorConversationMessages) + .where(eq(juniorConversationMessages.conversationId, conversationId)) + .orderBy( + asc(juniorConversationMessages.createdAt), + asc(juniorConversationMessages.messageId), + ); + return rows.map((row) => ({ + messageId: row.messageId, + text: row.text, + repliedAtMs: row.repliedAt?.getTime(), + })); +} + function restoreEnv(name: string, value: string | undefined): void { if (value === undefined) { delete process.env[name]; @@ -113,7 +134,6 @@ describe("legacy conversation import", () => { const fixture = await createLocalJuniorSqlFixture(); await migrateSchema(fixture.sql); const eventStore = createSqlConversationEventStore(fixture.sql); - const messageStore = createSqlConversationMessageStore(fixture.sql); const childId = `advisor:${CONVERSATION_ID}`; const entries: SessionLogEntry[] = [ @@ -159,7 +179,6 @@ describe("legacy conversation import", () => { const deps = { executor: fixture.sql, - messageStore, conversationRecord: conversationRecord(), sessionLogStore: staticSessionLogStore(entries), advisorSessionStore: staticAdvisorStore([ @@ -206,7 +225,7 @@ describe("legacy conversation import", () => { expect(childHistory[0]!.createdAtMs).toBe(960); // Visible messages recorded; meta.replied becomes replied_at. - const messages = await messageStore.list(CONVERSATION_ID); + const messages = await listMessageRows(fixture.sql, CONVERSATION_ID); expect(messages.map((message) => message.messageId)).toEqual([ "m1", "m2", @@ -259,7 +278,6 @@ describe("legacy conversation import", () => { const fixture = await createLocalJuniorSqlFixture(); await migrateSchema(fixture.sql); const eventStore = createSqlConversationEventStore(fixture.sql); - const messageStore = createSqlConversationMessageStore(fixture.sql); const entries = staticSessionLogStore([ { @@ -278,7 +296,6 @@ describe("legacy conversation import", () => { await expect( importConversationFromLegacy(CONVERSATION_ID, { executor: fixture.sql, - messageStore, conversationRecord: conversationRecord(), sessionLogStore: entries, loadVisibleMessages: async () => invalidVisible, @@ -289,7 +306,9 @@ describe("legacy conversation import", () => { // Messages and events share one transaction, so neither side commits. expect(await eventStore.loadHistory(CONVERSATION_ID)).toHaveLength(0); - expect(await messageStore.list(CONVERSATION_ID)).toHaveLength(0); + expect(await listMessageRows(fixture.sql, CONVERSATION_ID)).toHaveLength( + 0, + ); const visible: ThreadConversationMessage[] = [ { id: "m1", role: "user", text: "hi there", createdAtMs: 100 }, @@ -298,7 +317,6 @@ describe("legacy conversation import", () => { await expect( importConversationFromLegacy(CONVERSATION_ID, { executor: fixture.sql, - messageStore, conversationRecord: conversationRecord(), sessionLogStore: entries, loadVisibleMessages: async () => visible, @@ -306,7 +324,7 @@ describe("legacy conversation import", () => { ).resolves.toEqual({ imported: true }); expect(await eventStore.loadHistory(CONVERSATION_ID)).toHaveLength(3); - const messages = await messageStore.list(CONVERSATION_ID); + const messages = await listMessageRows(fixture.sql, CONVERSATION_ID); expect(messages.map((message) => message.messageId)).toEqual([ "m1", "m2", @@ -333,7 +351,6 @@ describe("legacy conversation import", () => { ]); const deps = { executor: fixture.sql, - messageStore, conversationRecord: conversationRecord(), sessionLogStore: staticSessionLogStore([]), loadVisibleMessages, @@ -354,12 +371,14 @@ describe("legacy conversation import", () => { importConversationFromLegacy(CONVERSATION_ID, deps), ).resolves.toEqual({ imported: false }); expect(await eventStore.loadHistory(CONVERSATION_ID)).toHaveLength(2); - expect(await messageStore.list(CONVERSATION_ID)).toMatchObject([ - { - messageId: "message-only", - repliedAtMs: 100, - }, - ]); + expect(await listMessageRows(fixture.sql, CONVERSATION_ID)).toMatchObject( + [ + { + messageId: "message-only", + repliedAtMs: 100, + }, + ], + ); const [conversation] = await fixture.sql .db() .select({ @@ -381,13 +400,11 @@ describe("legacy conversation import", () => { const fixture = await createLocalJuniorSqlFixture(); await migrateSchema(fixture.sql); const eventStore = createSqlConversationEventStore(fixture.sql); - const messageStore = createSqlConversationMessageStore(fixture.sql); const before = Date.now(); try { await importConversationFromLegacy(CONVERSATION_ID, { executor: fixture.sql, - messageStore, conversationRecord: conversationRecord(), sessionLogStore: staticSessionLogStore([ { @@ -525,6 +542,36 @@ describe("legacy conversation import", () => { ]); }, 20_000); + it("hydrates visible messages from events without message-table rows", async () => { + const { eventStore } = await processSqlStores(); + await eventStore.append(CONVERSATION_ID, [ + { + data: { + type: "visible_message_recorded", + messageId: "event-only", + role: "user", + text: "canonical event text", + }, + createdAtMs: 100, + }, + ]); + + const conversation = coerceThreadConversationState({}); + await hydrateConversationMessages({ + conversation, + conversationId: CONVERSATION_ID, + }); + + expect(conversation.messages).toEqual([ + { + id: "event-only", + role: "user", + text: "canonical event text", + createdAtMs: 100, + }, + ]); + }, 20_000); + it("does not resurrect purged SQL history from legacy Redis", async () => { const { executor, eventStore } = await processSqlStores(); @@ -559,11 +606,10 @@ describe("legacy conversation import", () => { expect(await eventStore.loadHistory(CONVERSATION_ID)).toEqual([]); }, 20_000); - it("rejects a legacy import when the SQL transcript was already purged", async () => { + it("rejects a legacy import when event history was already purged", async () => { const fixture = await createLocalJuniorSqlFixture(); await migrateSchema(fixture.sql); const eventStore = createSqlConversationEventStore(fixture.sql); - const messageStore = createSqlConversationMessageStore(fixture.sql); await fixture.sql .db() @@ -580,7 +626,6 @@ describe("legacy conversation import", () => { try { const result = await importConversationFromLegacy(CONVERSATION_ID, { executor: fixture.sql, - messageStore, sessionLogStore: staticSessionLogStore([ { schemaVersion: 2, @@ -601,7 +646,7 @@ describe("legacy conversation import", () => { expect(result).toEqual({ imported: false }); expect(await eventStore.loadHistory(CONVERSATION_ID)).toEqual([]); - expect(await messageStore.list(CONVERSATION_ID)).toEqual([]); + expect(await listMessageRows(fixture.sql, CONVERSATION_ID)).toEqual([]); } finally { await fixture.close(); } @@ -682,12 +727,10 @@ describe("legacy conversation import", () => { const fixture = await createLocalJuniorSqlFixture(); await migrateSchema(fixture.sql); const eventStore = createSqlConversationEventStore(fixture.sql); - const messageStore = createSqlConversationMessageStore(fixture.sql); try { await importConversationFromLegacy(CONVERSATION_ID, { executor: fixture.sql, - messageStore, conversationRecord: conversationRecord(), sessionLogStore: staticSessionLogStore([ { @@ -719,7 +762,6 @@ describe("legacy conversation import", () => { it("reads legacy visible messages from a real thread-state payload", async () => { const fixture = await createLocalJuniorSqlFixture(); await migrateSchema(fixture.sql); - const messageStore = createSqlConversationMessageStore(fixture.sql); // Persisted pre-cutover shape: the visible transcript nested under // `conversation.messages`, which the live thread-state contract no longer @@ -750,7 +792,6 @@ describe("legacy conversation import", () => { try { await importConversationFromLegacy(CONVERSATION_ID, { executor: fixture.sql, - messageStore, conversationRecord: conversationRecord(), sessionLogStore: staticSessionLogStore([ { @@ -763,7 +804,7 @@ describe("legacy conversation import", () => { advisorSessionStore: staticAdvisorStore([]), }); - const imported = await messageStore.list(CONVERSATION_ID); + const imported = await listMessageRows(fixture.sql, CONVERSATION_ID); expect( imported.map((message) => ({ messageId: message.messageId, @@ -782,7 +823,6 @@ describe("legacy conversation import", () => { it("rejects malformed legacy visible messages", async () => { const fixture = await createLocalJuniorSqlFixture(); await migrateSchema(fixture.sql); - const messageStore = createSqlConversationMessageStore(fixture.sql); const stateAdapter = getStateAdapter(); await stateAdapter.connect(); await stateAdapter.set(`thread-state:${CONVERSATION_ID}`, { @@ -795,20 +835,21 @@ describe("legacy conversation import", () => { await expect( importConversationFromLegacy(CONVERSATION_ID, { executor: fixture.sql, - messageStore, conversationRecord: conversationRecord(), sessionLogStore: staticSessionLogStore([]), advisorSessionStore: staticAdvisorStore([]), }), ).rejects.toThrow("Invalid input"); - await expect(messageStore.list(CONVERSATION_ID)).resolves.toEqual([]); + await expect( + listMessageRows(fixture.sql, CONVERSATION_ID), + ).resolves.toEqual([]); } finally { await fixture.close(); } }, 20_000); it("preserves message author identity through import and hydration", async () => { - const { executor, messageStore } = await processSqlStores(); + const { executor } = await processSqlStores(); // The resume/continuation paths key off the persisted user message's // author userId, so the import must fold `author` into `meta.author` just @@ -827,7 +868,6 @@ describe("legacy conversation import", () => { await importConversationFromLegacy(CONVERSATION_ID, { executor, - messageStore, conversationRecord: conversationRecord(), sessionLogStore: staticSessionLogStore([ { diff --git a/packages/junior/tests/integration/dashboard-reporting.test.ts b/packages/junior/tests/integration/dashboard-reporting.test.ts index 2c3364bcb..35ac28994 100644 --- a/packages/junior/tests/integration/dashboard-reporting.test.ts +++ b/packages/junior/tests/integration/dashboard-reporting.test.ts @@ -41,6 +41,30 @@ async function confirmPublicSlackConversation( }); } +async function recordVisibleTranscript( + conversationId: string, + messages: Array<{ + role: "assistant" | "system" | "user"; + text: string; + timestamp: number; + }>, +) { + const { getConversationEventStore } = await import("@/chat/db"); + await getConversationEventStore().append( + conversationId, + messages.map((message, index) => ({ + idempotencyKey: `test-visible:${message.timestamp}:${index}:${message.role}`, + data: { + type: "visible_message_recorded" as const, + messageId: `visible:${message.timestamp}:${index}:${message.role}`, + role: message.role, + text: message.text, + }, + createdAtMs: message.timestamp, + })), + ); +} + describe("dashboard reporting", () => { beforeEach(async () => { process.env = { @@ -229,8 +253,8 @@ describe("dashboard reporting", () => { }); }); - it("uses SQL title and visible messages when conversation events are absent", async () => { - const { getConversationMessageStore, getConversationStore } = + it("uses SQL title and visible events when model history is absent", async () => { + const { getConversationEventStore, getConversationStore } = await import("@/chat/db"); await confirmPublicSlackConversation("slack:C1:details-only"); @@ -240,11 +264,14 @@ describe("dashboard reporting", () => { source: "slack", title: "SQL Title", }); - await getConversationMessageStore().record("slack:C1:details-only", [ + await getConversationEventStore().append("slack:C1:details-only", [ { - messageId: "visible-only", - role: "user", - text: "Visible SQL message", + data: { + type: "visible_message_recorded", + messageId: "visible-only", + role: "user", + text: "Visible SQL message", + }, createdAtMs: 1_000, }, ]); @@ -372,6 +399,12 @@ describe("dashboard reporting", () => { sliceId: 1, state: "running", }); + await recordVisibleTranscript("slack:C1:222", [ + { role: "user", text: "previous question", timestamp: 1 }, + { role: "assistant", text: "previous answer", timestamp: 2 }, + { role: "user", text: "current question", timestamp: 3 }, + { role: "assistant", text: "current answer", timestamp: 6 }, + ]); const report = await readConversationDetailFromSql("slack:C1:222"); expect(report).toMatchObject({ @@ -397,30 +430,6 @@ describe("dashboard reporting", () => { timestamp: 3, parts: [{ type: "text", text: "current question" }], }, - { - role: "assistant", - timestamp: 4, - parts: [ - { type: "thinking", output: "I should use a tool" }, - { - type: "tool_call", - name: "search", - input: { query: "current question" }, - }, - ], - }, - { - role: "toolResult", - timestamp: 5, - parts: [ - { - type: "tool_result", - id: "search-1", - name: "search", - output: "tool result", - }, - ], - }, { role: "assistant", timestamp: 6, @@ -858,7 +867,7 @@ describe("dashboard reporting", () => { ]); }); - it("keeps the complete SQL transcript when steering adds a message", async () => { + it("keeps the complete visible transcript when steering adds a message", async () => { const { upsertAgentTurnSessionRecord } = await import("@/chat/state/turn-session"); @@ -903,6 +912,14 @@ describe("dashboard reporting", () => { }, ] as PiMessage[], }); + await recordVisibleTranscript("slack:C1:steering-transcript", [ + { role: "user", text: "previous question", timestamp: 1 }, + { role: "assistant", text: "previous answer", timestamp: 2 }, + { role: "user", text: "hello", timestamp: 3 }, + { role: "assistant", text: "working", timestamp: 4 }, + { role: "user", text: "steering message", timestamp: 5 }, + { role: "assistant", text: "done", timestamp: 6 }, + ]); const report = await readConversationDetailReport( "slack:C1:steering-transcript", @@ -960,12 +977,10 @@ describe("dashboard reporting", () => { await getConversationEventStore().append("slack:C1:999", [ { data: { - type: "message", - message: { - role: "user", - content: [{ type: "text", text: "target question" }], - timestamp: 1, - } as PiMessage, + type: "visible_message_recorded", + messageId: "target-question", + role: "user", + text: "target question", }, createdAtMs: 1, }, @@ -985,6 +1000,50 @@ describe("dashboard reporting", () => { ]); }); + it("extracts trace ids from authorized model history outside the visible transcript", async () => { + const { getConversationEventStore } = await import("@/chat/db"); + const conversationId = "slack:C1:model-trace"; + const traceId = "0123456789abcdef0123456789abcdef"; + await confirmPublicSlackConversation(conversationId); + await getConversationEventStore().append(conversationId, [ + { + data: { + type: "message", + message: { + role: "toolResult", + toolCallId: "trace-tool", + toolName: "lookup", + isError: false, + content: [{ type: "text", text: `trace_id=${traceId}` }], + timestamp: 2, + } as PiMessage, + }, + createdAtMs: 2, + }, + ]); + await recordVisibleTranscript(conversationId, [ + { role: "user", text: "look it up", timestamp: 1 }, + { role: "assistant", text: "done", timestamp: 3 }, + ]); + + const report = await readConversationDetailReport(conversationId); + + expect(report.traceId).toBe(traceId); + expect(JSON.stringify(report.transcript)).not.toContain(traceId); + expect(report.transcript).toEqual([ + { + role: "user", + timestamp: 1, + parts: [{ type: "text", text: "look it up" }], + }, + { + role: "assistant", + timestamp: 3, + parts: [{ type: "text", text: "done" }], + }, + ]); + }); + it("reports terminal assistant outcomes without exposing provider errors", async () => { const { getConversationEventStore, getConversationStore } = await import("@/chat/db"); @@ -1105,6 +1164,68 @@ describe("dashboard reporting", () => { ); }); + it("merges safe terminal outcomes chronologically with visible messages", async () => { + const { getConversationEventStore } = await import("@/chat/db"); + const conversationId = "slack:C1:terminal-ordered"; + const sensitiveError = "provider=xai credential=secret"; + await confirmPublicSlackConversation(conversationId); + await getConversationEventStore().append(conversationId, [ + { + data: { + type: "message", + message: { + role: "assistant", + api: "openai-responses", + content: [], + model: "grok-4.5", + provider: "xai", + stopReason: "error", + errorMessage: sensitiveError, + timestamp: 2, + usage: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + total: 0, + }, + }, + } as PiMessage, + }, + createdAtMs: 2, + }, + ]); + await recordVisibleTranscript(conversationId, [ + { role: "user", text: "please retry", timestamp: 1 }, + { role: "assistant", text: "safe fallback", timestamp: 3 }, + ]); + + const report = await readConversationDetailReport(conversationId); + + expect(report.transcript).toEqual([ + { + role: "user", + timestamp: 1, + parts: [{ type: "text", text: "please retry" }], + }, + { role: "assistant", outcome: "error", parts: [], timestamp: 2 }, + { + role: "assistant", + timestamp: 3, + parts: [{ type: "text", text: "safe fallback" }], + }, + ]); + expect(report.transcriptMessageCount).toBe(2); + expect(JSON.stringify(report)).not.toContain(sensitiveError); + expect(JSON.stringify(report)).not.toContain('"provider":"xai"'); + }); + it("keeps SQL detail available when optional execution settings fail", async () => { const { getConversationEventStore, getConversationStore } = await import("@/chat/db"); @@ -1123,12 +1244,10 @@ describe("dashboard reporting", () => { await getConversationEventStore().append(conversationId, [ { data: { - type: "message", - message: { - role: "user", - content: [{ type: "text", text: "available transcript" }], - timestamp: 1, - } as PiMessage, + type: "visible_message_recorded", + messageId: "available-transcript", + role: "user", + text: "available transcript", }, createdAtMs: 1, }, @@ -1154,7 +1273,7 @@ describe("dashboard reporting", () => { expect(report).not.toHaveProperty("reasoningLevel"); }); - it("reports multiple message exchanges as one complete SQL transcript", async () => { + it("reports multiple exchanges as one complete visible transcript", async () => { const { upsertAgentTurnSessionRecord } = await import("@/chat/state/turn-session"); @@ -1231,6 +1350,12 @@ describe("dashboard reporting", () => { }, ] as PiMessage[], }); + await recordVisibleTranscript("slack:C1:333", [ + { role: "user", text: "first question", timestamp: 1 }, + { role: "assistant", text: "first answer", timestamp: 2 }, + { role: "user", text: "second question", timestamp: 3 }, + { role: "assistant", text: "second answer", timestamp: 4 }, + ]); const report = await readConversationDetailReport("slack:C1:333"); expect(report).toMatchObject({ conversationId: "slack:C1:333" }); @@ -1309,6 +1434,10 @@ describe("dashboard reporting", () => { ] as PiMessage[], traceId: "0123456789abcdef0123456789abcdef", }); + await recordVisibleTranscript("slack:D1:222", [ + { role: "user", text: "private question", timestamp: 1 }, + { role: "assistant", text: "private answer", timestamp: 2 }, + ]); const report = await readConversationDetailReport("slack:D1:222"); @@ -1335,12 +1464,18 @@ describe("dashboard reporting", () => { "sensitive generated thread title", ); expect(JSON.stringify(report)).not.toContain("secret-dm-name"); - const toolCall = report.transcriptMetadata?.[1]?.parts.find( - (part) => part.type === "tool_call", - ); - expect(toolCall?.inputKeys).toHaveLength(20); - expect(toolCall?.inputKeys).toContain("privateKey0"); - expect(toolCall?.inputKeys).not.toContain("privateKey20"); + expect(report.transcriptMetadata).toEqual([ + { + role: "user", + parts: [{ type: "text", bytes: 16, chars: 16, redacted: true }], + timestamp: 1, + }, + { + role: "assistant", + parts: [{ type: "text", bytes: 14, chars: 14, redacted: true }], + timestamp: 2, + }, + ]); }); it("marks expired private transcripts as privacy redacted", async () => { @@ -1571,6 +1706,20 @@ describe("dashboard reporting", () => { createdAtMs: 5, }, ]); + await recordVisibleTranscript(conversationId, [ + { + role: "user", + text: "Context compaction summary for future Junior turns:\nThis is quoted documentation, not a generated summary.", + timestamp: 0, + }, + { role: "user", text: "old question", timestamp: 1 }, + { role: "user", text: "current question", timestamp: 2 }, + { + role: "user", + text: "Context compaction summary for future Junior turns:\nPlease explain this marker to the user.", + timestamp: 4.5, + }, + ]); const report = await readConversationDetailReport(conversationId); const currentRun = report; @@ -1601,7 +1750,7 @@ describe("dashboard reporting", () => { currentRun.transcript.filter((message) => JSON.stringify(message).includes("current question"), ), - ).toHaveLength(2); + ).toHaveLength(1); }); it("reports the original execution and continuation around a model handoff", async () => { @@ -1707,6 +1856,19 @@ describe("dashboard reporting", () => { createdAtMs: 4, }, ]); + await recordVisibleTranscript(conversationId, [ + { + role: "user", + text: "Model handoff checkpoint. Continue the outstanding request now using this summary as the complete prior context:\nThis is quoted documentation, not a generated checkpoint.", + timestamp: 0, + }, + { role: "user", text: "Investigate the release", timestamp: 1 }, + { + role: "assistant", + text: "I prepared the ordering fix.", + timestamp: 4, + }, + ]); const report = await readConversationDetailReport(conversationId); @@ -1729,8 +1891,7 @@ describe("dashboard reporting", () => { report.transcript.filter((message) => JSON.stringify(message).includes("Investigate the release"), ), - ).toHaveLength(2); - expect(JSON.stringify(report.transcript)).toContain("handoff-call"); + ).toHaveLength(1); expect(JSON.stringify(report.transcript)).toContain( "I prepared the ordering fix.", ); @@ -1769,6 +1930,11 @@ describe("dashboard reporting", () => { }, ], }); + await recordVisibleTranscript(conversationId, [ + { role: "user", text: "Regenerate the answer", timestamp: 1 }, + { role: "assistant", text: "Original answer", timestamp: 2 }, + { role: "assistant", text: "Regenerated answer", timestamp: 3 }, + ]); await eventStore.startEpoch(conversationId, { reason: "rollback", modelProfile: "standard", @@ -1887,6 +2053,11 @@ describe("dashboard reporting", () => { createdAtMs: 5, }, ]); + await recordVisibleTranscript(conversationId, [ + { role: "user", text: "Finish the release work", timestamp: 1 }, + { role: "assistant", text: "Compacted continuation", timestamp: 3 }, + { role: "assistant", text: "Handoff continuation", timestamp: 5 }, + ]); const report = await readConversationDetailReport(conversationId); const transcript = JSON.stringify(report.transcript); diff --git a/packages/junior/tests/unit/conversations/visible-message-projection.test.ts b/packages/junior/tests/unit/conversations/visible-message-projection.test.ts new file mode 100644 index 000000000..394b39d8f --- /dev/null +++ b/packages/junior/tests/unit/conversations/visible-message-projection.test.ts @@ -0,0 +1,126 @@ +import { describe, expect, it } from "vitest"; +import { + conversationEventSchema, + type ConversationEvent, + type ConversationEventData, +} from "@/chat/conversations/history"; +import { projectVisibleConversationMessages } from "@/chat/conversations/visible-message-projection"; + +function event( + seq: number, + createdAtMs: number, + data: ConversationEventData, +): ConversationEvent { + return conversationEventSchema.parse({ + schemaVersion: 1, + seq, + contextEpoch: 0, + createdAtMs, + data, + }); +} + +describe("visible message projection", () => { + it("merges metadata and preserves the first replied fact", () => { + expect( + projectVisibleConversationMessages([ + event(0, 1_000, { + type: "visible_message_recorded", + messageId: "m1", + role: "user", + text: "hello", + meta: { + author: { userId: "U1", userName: "alice" }, + explicitMention: true, + }, + }), + event(1, 2_000, { + type: "visible_message_metadata_updated", + messageId: "m1", + meta: { + author: { userId: "U1", userName: "alice", fullName: "Alice" }, + imageFileIds: ["F1"], + }, + }), + event(2, 3_000, { + type: "visible_message_replied", + messageId: "m1", + }), + event(3, 4_000, { + type: "visible_message_replied", + messageId: "m1", + }), + ]), + ).toEqual([ + { + id: "m1", + role: "user", + text: "hello", + createdAtMs: 1_000, + author: { userId: "U1", userName: "alice", fullName: "Alice" }, + meta: { + explicitMention: true, + imageFileIds: ["F1"], + replied: true, + }, + }, + ]); + }); + + it.each([ + "visible_message_metadata_updated", + "visible_message_replied", + ] as const)("rejects %s before its recorded baseline", (type) => { + const data = + type === "visible_message_metadata_updated" + ? { type, messageId: "missing", meta: { late: true } } + : { type, messageId: "missing" }; + expect(() => + projectVisibleConversationMessages([event(0, 1_000, data)]), + ).toThrow("before visible_message_recorded"); + }); + + it("rejects duplicate recorded baselines", () => { + expect(() => + projectVisibleConversationMessages([ + event(0, 1_000, { + type: "visible_message_recorded", + messageId: "m1", + role: "user", + text: "first", + }), + event(1, 1_000, { + type: "visible_message_recorded", + messageId: "m1", + role: "user", + text: "conflict", + }), + ]), + ).toThrow("Duplicate visible_message_recorded"); + }); + + it("orders equal timestamps by message id", () => { + expect( + projectVisibleConversationMessages([ + event(0, 2_000, { + type: "visible_message_recorded", + messageId: "later", + role: "assistant", + text: "later", + }), + event(1, 1_000, { + type: "visible_message_recorded", + messageId: "b", + role: "user", + text: "b", + }), + event(2, 1_000, { + type: "visible_message_recorded", + messageId: "a", + role: "user", + text: "a", + }), + ]).map((message) => message.id), + ).toEqual(["a", "b", "later"]); + }); +}); diff --git a/packages/junior/tests/unit/runtime/delivered-turn-state.test.ts b/packages/junior/tests/unit/runtime/delivered-turn-state.test.ts new file mode 100644 index 000000000..34e167860 --- /dev/null +++ b/packages/junior/tests/unit/runtime/delivered-turn-state.test.ts @@ -0,0 +1,60 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { buildDeliveredTurnStatePatch } from "@/chat/runtime/delivered-turn-state"; +import { coerceThreadArtifactsState } from "@/chat/state/artifacts"; +import { coerceThreadConversationState } from "@/chat/state/conversation"; + +describe("delivered turn state", () => { + afterEach(() => vi.useRealTimers()); + + it("reuses the logical turn assistant id across persistence retries", () => { + vi.useFakeTimers(); + vi.setSystemTime(1_000); + const first = buildDeliveredTurnStatePatch({ + artifacts: coerceThreadArtifactsState({}), + conversation: coerceThreadConversationState({}), + reply: { + text: "delivered reply", + diagnostics: { + assistantMessageCount: 1, + modelId: "test/model", + outcome: "success", + toolCalls: [], + toolErrorCount: 0, + toolResultCount: 0, + usedPrimaryText: true, + }, + }, + sessionId: "turn_msg-1", + }); + + vi.setSystemTime(2_000); + const retried = buildDeliveredTurnStatePatch({ + artifacts: coerceThreadArtifactsState({}), + conversation: first.conversation, + reply: { + text: "delivered reply", + diagnostics: { + assistantMessageCount: 1, + modelId: "test/model", + outcome: "success", + toolCalls: [], + toolErrorCount: 0, + toolResultCount: 0, + usedPrimaryText: true, + }, + }, + sessionId: "turn_msg-1", + }); + + expect( + retried.conversation.messages.filter( + (message) => message.role === "assistant", + ), + ).toEqual([ + expect.objectContaining({ + id: "assistant:turn_msg-1", + text: "delivered reply", + }), + ]); + }); +}); diff --git a/packages/junior/tests/unit/state/conversation-state.test.ts b/packages/junior/tests/unit/state/conversation-state.test.ts index 7678482f6..714d7cb9d 100644 --- a/packages/junior/tests/unit/state/conversation-state.test.ts +++ b/packages/junior/tests/unit/state/conversation-state.test.ts @@ -44,8 +44,8 @@ describe("conversation state", () => { }, }); - // The visible transcript lives in SQL now; a legacy transcript mirror in a - // persisted payload is dropped on read. + // The visible transcript lives in SQL events now; a legacy transcript + // mirror in a persisted payload is dropped on read. expect(conversation.messages).toEqual([]); expect(conversation.vision.byFileId).toEqual({ F123: { @@ -100,8 +100,8 @@ describe("conversation state", () => { }, }); - // The visible transcript lives in SQL now; a legacy transcript mirror in a - // persisted payload is dropped on read. + // The visible transcript lives in SQL events now; a legacy transcript + // mirror in a persisted payload is dropped on read. expect(conversation.messages).toEqual([]); expect(conversation.vision.byFileId).toEqual({ F123: { From 5f72744c6afb0d48794a803216eddd9952155380 Mon Sep 17 00:00:00 2001 From: David Cramer Date: Tue, 14 Jul 2026 23:00:57 -0700 Subject: [PATCH 05/58] feat(conversations): Record correlated turn lifecycle Add strict start and first-terminal-wins outcome facts, then wire the local runtime through durable input, delivery, and persistence boundaries. Surface privacy-safe lifecycle failures in conversation detail, preserve incident correlation, and remove synthetic no-reply transcript messages. Co-Authored-By: GPT-5 Codex --- .../conversation-event-log/proposal.md | 16 +- .../changes/conversation-event-log/tasks.md | 6 +- .../api/conversations/detail-projection.ts | 62 ++- .../junior/src/chat/agent-dispatch/runner.ts | 31 +- .../junior/src/chat/conversations/README.md | 21 + .../junior/src/chat/conversations/history.ts | 58 +++ .../src/chat/conversations/turn-lifecycle.ts | 89 ++++ packages/junior/src/chat/local/README.md | 8 + packages/junior/src/chat/local/runner.ts | 236 ++++++++--- packages/junior/src/chat/runtime/README.md | 10 + .../src/chat/runtime/delivered-turn-state.ts | 36 +- .../chat/services/turn-failure-response.ts | 40 +- .../conversation-transcripts-sql.test.ts | 95 +++++ .../integration/agent-dispatch-runner.test.ts | 14 +- .../integration/dashboard-reporting.test.ts | 143 +++++++ .../integration/local-agent-runner.test.ts | 384 +++++++++++++++++- .../unit/chat/pi/conversation-events.test.ts | 17 + .../unit/conversations/turn-lifecycle.test.ts | 187 +++++++++ .../unit/runtime/delivered-turn-state.test.ts | 23 ++ 19 files changed, 1355 insertions(+), 121 deletions(-) create mode 100644 packages/junior/src/chat/conversations/turn-lifecycle.ts create mode 100644 packages/junior/tests/unit/conversations/turn-lifecycle.test.ts diff --git a/openspec/changes/conversation-event-log/proposal.md b/openspec/changes/conversation-event-log/proposal.md index b63e1519f..a89304766 100644 --- a/openspec/changes/conversation-event-log/proposal.md +++ b/openspec/changes/conversation-event-log/proposal.md @@ -22,9 +22,10 @@ rather than creating another transcript or lifecycle sidecar. `ConversationEvent` is the canonical persisted history contract. Every event has stable conversation ordering, a timestamp, and a versioned payload. The -first slice formalizes only the durable event kinds Junior already stores; turn -lifecycle, delivery, correlation, and lineage variants are added with their -own write boundaries in later slices. +initial cutover formalized the durable event kinds Junior already stored. A +later vertical slice adds correlated start and first-terminal-wins outcome +events to the local runtime; Slack, dispatch, delivery attempts, and lineage +follow at their own owning boundaries. The storage cutover rewrites existing `pi_message` rows into Junior-owned `message` events. Their opaque continuity payload is interpreted as a Pi @@ -60,10 +61,13 @@ child event stream. hard schema cut that removes their compatibility view. Backfill visible-message rows while workers remain stopped; start new workers only after fail-closed zero-gap verification passes. -3. Expose privacy-safe events from the detail API and move timeline shaping to +3. Add correlated local lifecycle events and project failures as safe detail + markers. Add Slack/dispatch writers only with durable delivery + intent/receipt reconciliation so process death cannot strand accepted work. +4. Expose privacy-safe events from the detail API and move timeline shaping to the dashboard. -4. Add child-conversation lineage and context-fork projection. -5. Remove obsolete transcript reconstruction and demote remaining message +5. Add child-conversation lineage and context-fork projection. +6. Remove obsolete transcript reconstruction and demote remaining message stores to explicit read models. ## Non-Goals diff --git a/openspec/changes/conversation-event-log/tasks.md b/openspec/changes/conversation-event-log/tasks.md index ea9c093aa..be5201112 100644 --- a/openspec/changes/conversation-event-log/tasks.md +++ b/openspec/changes/conversation-event-log/tasks.md @@ -23,7 +23,11 @@ - [x] Rewrite legacy SQL message rows in bounded, retry-safe batches while a database-only compatibility view supports rolling old workers. - [x] Keep Pi messages derived rather than separately persisted. -- [ ] Add turn outcome, delivery, and explicit turn-correlation event variants. +- [x] Add strict turn-start and mutually exclusive terminal outcome variants, + then cut the local runtime over at its input/delivery persistence bounds. +- [ ] Add durable delivery intent/receipt reconciliation and delivery-attempt + events, then cut Slack/dispatch lifecycle writers over without a + destination-acceptance/process-crash gap. - [x] Persist visible-message and reply facts as canonical events while updating the hydration/search table as an atomic read model. - [x] Backfill existing visible-message rows with a bounded, verified migration diff --git a/packages/junior/src/api/conversations/detail-projection.ts b/packages/junior/src/api/conversations/detail-projection.ts index a08e5e852..0d71b584b 100644 --- a/packages/junior/src/api/conversations/detail-projection.ts +++ b/packages/junior/src/api/conversations/detail-projection.ts @@ -209,10 +209,49 @@ function historyContent(args: { return { contextEvents, messages }; } -function terminalOutcomeTranscript( - messages: ConversationModelMessage[], -): TranscriptMessage[] { - return messages.flatMap((message) => { +function completedTurnIntervals(events: ConversationEvent[]): Array<{ + startSeq: number; + terminalSeq: number; +}> { + const starts = new Map(); + const intervals: Array<{ startSeq: number; terminalSeq: number }> = []; + for (const event of events) { + if (event.data.type === "turn_started") { + starts.set(event.data.turnId, event.seq); + continue; + } + if ( + event.data.type !== "turn_completed" && + event.data.type !== "turn_failed" + ) { + continue; + } + const startSeq = starts.get(event.data.turnId); + if (startSeq !== undefined && startSeq <= event.seq) { + intervals.push({ startSeq, terminalSeq: event.seq }); + } + } + return intervals; +} + +function terminalOutcomeTranscript(args: { + events: ConversationEvent[]; + messages: ConversationModelMessage[]; +}): TranscriptMessage[] { + const intervals = completedTurnIntervals(args.events); + const lifecycleCoveredMessages = new Set( + args.events.flatMap((event) => + event.data.type === "message" && + intervals.some( + (interval) => + event.seq >= interval.startSeq && event.seq <= interval.terminalSeq, + ) + ? [event.data.message] + : [], + ), + ); + const legacyOutcomes = args.messages.flatMap((message) => { + if (lifecycleCoveredMessages.has(message)) return []; const normalized = normalizeTranscriptMessage(message); if ( normalized.role !== "assistant" || @@ -231,6 +270,19 @@ function terminalOutcomeTranscript( }, ]; }); + const lifecycleFailures = args.events.flatMap((event) => + event.data.type === "turn_failed" + ? [ + { + role: "assistant" as const, + outcome: "error" as const, + parts: [], + timestamp: event.createdAtMs, + }, + ] + : [], + ); + return [...legacyOutcomes, ...lifecycleFailures]; } function mergeTranscriptChronologically( @@ -267,7 +319,7 @@ async function conversationContent(args: { const modelTranscript = history.messages.map(normalizeTranscriptMessage); const transcript = mergeTranscriptChronologically([ ...visibleTranscript, - ...terminalOutcomeTranscript(history.messages), + ...terminalOutcomeTranscript({ events, messages: history.messages }), ]); const contextEvents = history.contextEvents.map((event) => ({ ...event, diff --git a/packages/junior/src/chat/agent-dispatch/runner.ts b/packages/junior/src/chat/agent-dispatch/runner.ts index 0bca65946..518fdbb07 100644 --- a/packages/junior/src/chat/agent-dispatch/runner.ts +++ b/packages/junior/src/chat/agent-dispatch/runner.ts @@ -436,20 +436,23 @@ export async function runAgentDispatchSlice( replied: true, skippedReason: undefined, }); - upsertConversationMessage(conversation, { - id: getAssistantMessageId(dispatch), - role: "assistant", - text: normalizeConversationText(deliveryReply.text) || "[empty response]", - createdAtMs: nowMs, - author: { - userName: botConfig.userName, - isBot: true, - }, - meta: { - replied: true, - slackTs: resultMessageTs, - }, - }); + if (reply.deliveryPlan?.postThreadText !== false) { + upsertConversationMessage(conversation, { + id: getAssistantMessageId(dispatch), + role: "assistant", + text: + normalizeConversationText(deliveryReply.text) || "[empty response]", + createdAtMs: nowMs, + author: { + userName: botConfig.userName, + isBot: true, + }, + meta: { + replied: true, + slackTs: resultMessageTs, + }, + }); + } updateConversationStats(conversation); const nextArtifacts = reply.artifactStatePatch ? mergeArtifactsState(artifacts, reply.artifactStatePatch) diff --git a/packages/junior/src/chat/conversations/README.md b/packages/junior/src/chat/conversations/README.md index c29af97d6..0dd4d79b8 100644 --- a/packages/junior/src/chat/conversations/README.md +++ b/packages/junior/src/chat/conversations/README.md @@ -41,6 +41,10 @@ must not become a dashboard or external API payload. - Append each visible-message fact and update its SQL read model in one transaction under the conversation event lock. - Append conversation events in monotonic sequence order. +- Correlate each turn with one `turn_started` event after its input messages and + one first-writer-wins `turn_completed` or `turn_failed` terminal event. +- Persist only allowlisted failure classifications and opaque Sentry event IDs; + raw exceptions, provider payloads, and URLs are not lifecycle data. - Restore state from durable events rather than a duplicate transcript cache. - Compaction replaces prior model context without rewriting visible history. - Imports and migrations are idempotent and preserve stable conversation IDs. @@ -73,3 +77,20 @@ Follow `../../../../../policies/data-redaction.md` and Representative coverage lives in `packages/junior/tests/integration/conversation-sql.test.ts` and the conversation storage component tests. + +The local runtime is the first lifecycle-event writer, and current detail +reporting reduces `turn_failed` to one privacy-safe error marker. Slack, +dispatch, delivery-attempt events, and the ordered safe event API remain +follow-up cutovers. + +The structural failure marker never exposes failure code or event ID. An +independently delivered fallback remains ordinary visible content, so a public +conversation preserves its approved `event_id` reference while private detail +redaction removes the fallback text and retains only structural metadata. + +Lifecycle appends have stable idempotency keys so explicitly retried calls are +safe, but they are not an outbox transaction with an external destination. A +process death after destination acceptance and before the +visible/session/terminal writes can still leave a started turn without a +terminal event. The next delivery slice must add durable intent/receipt +reconciliation for Slack before claiming crash-safe terminality. diff --git a/packages/junior/src/chat/conversations/history.ts b/packages/junior/src/chat/conversations/history.ts index 46c3b052c..ecabe052c 100644 --- a/packages/junior/src/chat/conversations/history.ts +++ b/packages/junior/src/chat/conversations/history.ts @@ -210,6 +210,61 @@ const visibleContextCompactedEventDataSchema = z }) .strict(); +/** Product surface that owns a durable conversation turn. */ +export const conversationTurnSurfaceSchema = z.enum([ + "slack", + "api", + "scheduler", + "internal", +]); + +/** Stable, privacy-safe classification for a failed turn. */ +export const conversationTurnFailureCodeSchema = z.enum([ + "agent_run_failed", + "delivery_failed", + "model_execution_failed", + "persistence_failed", +]); + +/** Failure classification persisted without raw provider or exception data. */ +export type ConversationTurnFailureCode = z.output< + typeof conversationTurnFailureCodeSchema +>; + +const turnStartedEventDataSchema = z + .object({ + type: z.literal("turn_started"), + turnId: z.string().min(1), + inputMessageIds: z.array(z.string().min(1)).min(1), + surface: conversationTurnSurfaceSchema, + }) + .strict() + .refine( + (data) => + new Set(data.inputMessageIds).size === data.inputMessageIds.length, + "turn input message ids must be unique", + ); + +const turnCompletedEventDataSchema = z + .object({ + type: z.literal("turn_completed"), + turnId: z.string().min(1), + outcome: z.enum(["success", "no_reply"]), + }) + .strict(); + +const turnFailedEventDataSchema = z + .object({ + type: z.literal("turn_failed"), + turnId: z.string().min(1), + failureCode: conversationTurnFailureCodeSchema, + eventId: z + .string() + .regex(/^[a-f0-9]{32}$/i) + .optional(), + }) + .strict(); + // Subagent histories are child conversations; the marker references the child by // its own conversation id rather than a polymorphic transcript locator. const subagentStartedEventDataSchema = z @@ -249,6 +304,9 @@ const appendableConversationEventDataSchema = z.union([ visibleMessageMetadataUpdatedEventDataSchema, visibleMessageRepliedEventDataSchema, visibleContextCompactedEventDataSchema, + turnStartedEventDataSchema, + turnCompletedEventDataSchema, + turnFailedEventDataSchema, subagentStartedEventDataSchema, subagentEndedEventDataSchema, ]); diff --git a/packages/junior/src/chat/conversations/turn-lifecycle.ts b/packages/junior/src/chat/conversations/turn-lifecycle.ts new file mode 100644 index 000000000..771913b37 --- /dev/null +++ b/packages/junior/src/chat/conversations/turn-lifecycle.ts @@ -0,0 +1,89 @@ +import type { + ConversationEventStore, + ConversationTurnFailureCode, +} from "./history"; + +/** Product-owned inputs for opening one correlated conversation turn. */ +export interface StartConversationTurnInput { + conversationId: string; + createdAtMs: number; + inputMessageIds: string[]; + surface: "slack" | "api" | "scheduler" | "internal"; + turnId: string; +} + +/** Product-owned inputs for closing one correlated conversation turn. */ +export interface CompleteConversationTurnInput { + conversationId: string; + createdAtMs: number; + outcome: "success" | "no_reply"; + turnId: string; +} + +/** Privacy-safe inputs for failing one correlated conversation turn. */ +export interface FailConversationTurnInput { + conversationId: string; + createdAtMs: number; + eventId?: string; + failureCode: ConversationTurnFailureCode; + turnId: string; +} + +/** Runtime port for correlated canonical turn lifecycle persistence. */ +export interface ConversationTurnLifecycle { + start(input: StartConversationTurnInput): Promise; + complete(input: CompleteConversationTurnInput): Promise; + fail(input: FailConversationTurnInput): Promise; +} + +/** Persist correlated turn lifecycle facts with retry-safe terminal exclusion. */ +export class ConversationTurnLifecycleService implements ConversationTurnLifecycle { + constructor(private readonly events: ConversationEventStore) {} + + /** Record the turn start once after every input message is durable. */ + async start(input: StartConversationTurnInput): Promise { + await this.events.append(input.conversationId, [ + { + idempotencyKey: `turn:${input.turnId}:started`, + createdAtMs: input.createdAtMs, + data: { + type: "turn_started", + turnId: input.turnId, + inputMessageIds: input.inputMessageIds, + surface: input.surface, + }, + }, + ]); + } + + /** Record successful or intentional-silence completion once. */ + async complete(input: CompleteConversationTurnInput): Promise { + await this.events.append(input.conversationId, [ + { + idempotencyKey: `turn:${input.turnId}:terminal`, + createdAtMs: input.createdAtMs, + data: { + type: "turn_completed", + turnId: input.turnId, + outcome: input.outcome, + }, + }, + ]); + } + + /** Record a classified failure once without accepting raw error details. */ + async fail(input: FailConversationTurnInput): Promise { + await this.events.append(input.conversationId, [ + { + idempotencyKey: `turn:${input.turnId}:terminal`, + createdAtMs: input.createdAtMs, + data: { + type: "turn_failed", + turnId: input.turnId, + failureCode: input.failureCode, + ...(input.eventId ? { eventId: input.eventId } : {}), + }, + }, + ]); + } +} diff --git a/packages/junior/src/chat/local/README.md b/packages/junior/src/chat/local/README.md index b562f9cf7..7728e0599 100644 --- a/packages/junior/src/chat/local/README.md +++ b/packages/junior/src/chat/local/README.md @@ -12,6 +12,14 @@ provider mailbox worker. and Slack-only authorization or delivery surfaces are disabled. - User input is persisted before execution; finalized assistant output is persisted after stdout delivery succeeds. +- Each invocation uses a collision-resistant turn ID independent of transcript + length. It records `turn_started` after durable input, then a terminal + success, no-reply, or privacy-safe failure after the owning boundary. +- Intentional no-reply turns do not call the stdout sink and do not synthesize + an assistant transcript message. +- Event appends are idempotent when explicitly retried, but stdout acceptance + and SQL persistence are not one transaction. A process death in that interval + can strand a started turn; lifecycle history does not claim otherwise. - New CLI invocations do not promise restoration of prior interactive history. - Status and diagnostics go to stderr; the final answer goes to stdout. - Local file requests use paths named by the user. The adapter does not diff --git a/packages/junior/src/chat/local/runner.ts b/packages/junior/src/chat/local/runner.ts index b7c880bb9..c6a525152 100644 --- a/packages/junior/src/chat/local/runner.ts +++ b/packages/junior/src/chat/local/runner.ts @@ -7,6 +7,7 @@ * accepts the final output. */ import type { AgentRunResult } from "@/chat/services/turn-result"; +import { randomUUID } from "node:crypto"; import type { AgentRunner } from "@/chat/runtime/agent-runner"; import type { PiMessage } from "@/chat/pi/messages"; import { @@ -31,7 +32,7 @@ import { persistThreadStateById, } from "@/chat/runtime/thread-state"; import { startActiveTurn, markTurnFailed } from "@/chat/runtime/turn"; -import { finalizeFailedTurnReply } from "@/chat/services/turn-failure-response"; +import { finalizeFailedTurnReplyWithEvent } from "@/chat/services/turn-failure-response"; import { completeDeliveredTurn } from "@/chat/services/turn-session-record"; import { buildConversationContext, @@ -48,8 +49,23 @@ import { loadProjection, } from "@/chat/conversations/projection"; import { sleep } from "@/chat/sleep"; +import { getConversationEventStore } from "@/chat/db"; +import { + ConversationTurnLifecycleService, + type ConversationTurnLifecycle, +} from "@/chat/conversations/turn-lifecycle"; +import type { ConversationTurnFailureCode } from "@/chat/conversations/history"; +import { persistConversationMessages } from "@/chat/conversations/visible-messages"; const DELIVERED_STATE_PERSIST_ATTEMPTS = 3; +const SENTRY_EVENT_ID_PATTERN = /^[a-f0-9]{32}$/i; + +const LOCAL_FAILURE_EVENT_NAMES: Record = { + agent_run_failed: "local_agent_run_failed", + delivery_failed: "local_reply_delivery_failed", + model_execution_failed: "local_model_execution_failed", + persistence_failed: "local_turn_persistence_failed", +}; export interface LocalAgentTurnInput { conversationId: string; @@ -69,7 +85,13 @@ export type LocalToolResult = ToolExecutionReport; export interface LocalAgentTurnDeps { agentRunner: AgentRunner; + /** Post-delivery Pi/session persistence boundary. */ + completeDeliveredTurn?: typeof completeDeliveredTurn; deliverReply: (reply: LocalAgentReply) => Promise; + /** Injectable failure capture boundary for deterministic runtime integration tests. */ + logException?: typeof logException; + /** Canonical lifecycle writer; defaults to the production SQL service. */ + turnLifecycle?: ConversationTurnLifecycle; now?: () => number; onStatus?: (status: string) => void | Promise; onTextDelta?: (deltaText: string) => void | Promise; @@ -93,8 +115,8 @@ function localDestination(conversationId: string): LocalDestination { return parsed.data; } -function localTurnId(sequence: number): string { - return `local-turn-${sequence}`; +function localTurnId(): string { + return `local-turn-${randomUUID()}`; } function localReply(reply: AgentRunResult): LocalAgentReply { @@ -103,13 +125,21 @@ function localReply(reply: AgentRunResult): LocalAgentReply { }; } -function nextUserMessageSequence( - conversation: ReturnType, -): number { - return ( - conversation.messages.filter((message) => message.role === "user").length + - 1 +function captureLocalBoundaryFailure(args: { + capture: typeof logException; + conversationId: string; + error: unknown; + failureCode: ConversationTurnFailureCode; + turnId: string; +}): string | undefined { + const eventId = args.capture( + args.error, + LOCAL_FAILURE_EVENT_NAMES[args.failureCode], + { conversationId: args.conversationId, runId: args.turnId }, + { "app.ai.failure_code": args.failureCode }, + "Local agent turn failed at its owning runtime boundary", ); + return eventId && SENTRY_EVENT_ID_PATTERN.test(eventId) ? eventId : undefined; } /** Load durable local Pi history from the conversation-event projection. */ @@ -163,6 +193,9 @@ export async function runLocalAgentTurn( } const destination = localDestination(input.conversationId); const source = createLocalSource(destination.conversationId); + const lifecycle = + deps.turnLifecycle ?? + new ConversationTurnLifecycleService(getConversationEventStore()); const now = deps.now ?? (() => Date.now()); const persisted = await getPersistedThreadState(input.conversationId); @@ -178,8 +211,7 @@ export async function runLocalAgentTurn( const initialSandboxId = sandboxId; const initialSandboxDependencyProfileHash = sandboxDependencyProfileHash; - const sequence = nextUserMessageSequence(conversation); - const turnId = localTurnId(sequence); + const turnId = localTurnId(); const userMessageId = `${turnId}:user`; const startedAtMs = now(); upsertConversationMessage(conversation, { @@ -197,15 +229,30 @@ export async function runLocalAgentTurn( replied: false, }, }); + // The source message is durable before execution begins or an active turn is + // advertised. A caller may safely retry lifecycle start by its stable key. + await persistConversationMessages({ + conversation, + conversationId: input.conversationId, + }); + await lifecycle.start({ + conversationId: input.conversationId, + createdAtMs: now(), + inputMessageIds: [userMessageId], + surface: "internal", + turnId, + }); startActiveTurn({ conversation, nextTurnId: turnId, updateConversationStats, }); - await persistThreadStateById(input.conversationId, { conversation }); let reply: AgentRunResult | undefined; let completedState: ReturnType; + let failureCode: ConversationTurnFailureCode = "agent_run_failed"; + let modelFailureEventId: string | undefined; + let modelFailureCaptureAttempted = false; let piMessagesBeforeRun: | Awaited> | undefined; @@ -216,6 +263,7 @@ export async function runLocalAgentTurn( userName: "local", }; try { + await persistThreadStateById(input.conversationId, { conversation }); const piMessages = await loadLocalPiMessages({ conversationId: input.conversationId, }); @@ -293,11 +341,14 @@ export async function runLocalAgentTurn( // Failed turns deliver the sanitized fallback (or genuine partial model // text), never raw exception strings and never silence. - reply = finalizeFailedTurnReply({ + modelFailureCaptureAttempted = reply.diagnostics.outcome !== "success"; + const finalized = finalizeFailedTurnReplyWithEvent({ reply, - logException, + logException: deps.logException ?? logException, context: { conversationId: input.conversationId }, }); + reply = finalized.reply; + modelFailureEventId = finalized.eventId; completedState = buildDeliveredTurnStatePatch({ artifacts, @@ -306,57 +357,134 @@ export async function runLocalAgentTurn( sessionId: turnId, userMessageId, }); - await deps.deliverReply(localReply(reply)); + if (reply.deliveryPlan?.postThreadText !== false) { + failureCode = "delivery_failed"; + await deps.deliverReply(localReply(reply)); + } } catch (error) { - if (reply) { - await commitMessages({ + const failureEventId = + modelFailureCaptureAttempted && failureCode === "agent_run_failed" + ? modelFailureEventId + : captureLocalBoundaryFailure({ + capture: deps.logException ?? logException, + conversationId: input.conversationId, + error, + failureCode, + turnId, + }); + try { + if (reply) { + await commitMessages({ + conversationId: input.conversationId, + modelId: reply.diagnostics.modelId, + messages: piMessagesBeforeRun ?? [], + }); + } + markTurnFailed({ + conversation, + nowMs: now(), + sessionId: turnId, + userMessageId, + markConversationMessage, + updateConversationStats, + }); + await persistThreadStateById(input.conversationId, { + artifacts: initialArtifacts, + conversation, + sandboxId: initialSandboxId ?? "", + sandboxDependencyProfileHash: initialSandboxDependencyProfileHash ?? "", + }); + } catch (persistenceError) { + const persistenceEventId = captureLocalBoundaryFailure({ + capture: deps.logException ?? logException, + conversationId: input.conversationId, + error: persistenceError, + failureCode: "persistence_failed", + turnId, + }); + await lifecycle.fail({ + conversationId: input.conversationId, + createdAtMs: now(), + ...(persistenceEventId ? { eventId: persistenceEventId } : {}), + failureCode: "persistence_failed", + turnId, + }); + throw new AggregateError( + [error, persistenceError], + "Local turn failure state could not be persisted", + ); + } + await lifecycle.fail({ + conversationId: input.conversationId, + createdAtMs: now(), + ...(failureEventId ? { eventId: failureEventId } : {}), + failureCode, + turnId, + }); + throw error; + } + + try { + await persistDeliveredLocalTurnState(input.conversationId, { + artifacts: completedState.artifacts ?? artifacts, + conversation: completedState.conversation, + sandboxId: reply.sandboxId ?? sandboxId, + sandboxDependencyProfileHash: + reply.sandboxDependencyProfileHash ?? sandboxDependencyProfileHash, + }); + if (reply.piMessages?.length) { + // Destination acceptance is the completion boundary: this first commits + // the final assistant messages to the event log and marks the session + // record completed only after the CLI sink accepted the reply. + await (deps.completeDeliveredTurn ?? completeDeliveredTurn)({ conversationId: input.conversationId, + sessionId: turnId, + sliceId: 1, + messages: reply.piMessages, modelId: reply.diagnostics.modelId, - messages: piMessagesBeforeRun ?? [], + durationMs: reply.diagnostics.durationMs, + usage: reply.diagnostics.usage, + reasoningLevel: reply.diagnostics.reasoningLevel, + destination, + source, + actor: localActor, + surface: "internal", + logContext: {}, }); } - markTurnFailed({ - conversation, - nowMs: now(), - sessionId: turnId, - userMessageId, - markConversationMessage, - updateConversationStats, + } catch (error) { + const persistenceEventId = captureLocalBoundaryFailure({ + capture: deps.logException ?? logException, + conversationId: input.conversationId, + error, + failureCode: "persistence_failed", + turnId, }); - await persistThreadStateById(input.conversationId, { - artifacts: initialArtifacts, - conversation, - sandboxId: initialSandboxId ?? "", - sandboxDependencyProfileHash: initialSandboxDependencyProfileHash ?? "", + await lifecycle.fail({ + conversationId: input.conversationId, + createdAtMs: now(), + ...(persistenceEventId ? { eventId: persistenceEventId } : {}), + failureCode: "persistence_failed", + turnId, }); throw error; } - await persistDeliveredLocalTurnState(input.conversationId, { - artifacts: completedState.artifacts ?? artifacts, - conversation: completedState.conversation, - sandboxId: reply.sandboxId ?? sandboxId, - sandboxDependencyProfileHash: - reply.sandboxDependencyProfileHash ?? sandboxDependencyProfileHash, - }); - if (reply.piMessages?.length) { - // Destination acceptance is the completion boundary: this first commits - // the final assistant messages to the event log and marks the session - // record completed only after the CLI sink accepted the reply. - await completeDeliveredTurn({ + if (reply.diagnostics.outcome === "success") { + await lifecycle.complete({ conversationId: input.conversationId, - sessionId: turnId, - sliceId: 1, - messages: reply.piMessages, - modelId: reply.diagnostics.modelId, - durationMs: reply.diagnostics.durationMs, - usage: reply.diagnostics.usage, - reasoningLevel: reply.diagnostics.reasoningLevel, - destination, - source, - actor: localActor, - surface: "internal", - logContext: {}, + createdAtMs: now(), + outcome: + reply.deliveryPlan?.postThreadText === false ? "no_reply" : "success", + turnId, + }); + } else { + await lifecycle.fail({ + conversationId: input.conversationId, + createdAtMs: now(), + ...(modelFailureEventId ? { eventId: modelFailureEventId } : {}), + failureCode: "model_execution_failed", + turnId, }); } if (reply.diagnostics.outcome === "success") { diff --git a/packages/junior/src/chat/runtime/README.md b/packages/junior/src/chat/runtime/README.md index 93773a2eb..4e60aca3b 100644 --- a/packages/junior/src/chat/runtime/README.md +++ b/packages/junior/src/chat/runtime/README.md @@ -26,6 +26,16 @@ this directory owns product orchestration around it. - Auth pauses persist the pending authorization state and end the live run; callbacks append new work and start a later run. - Completion and delivery markers make retries idempotent. +- Canonical turn lifecycle uses stable correlation IDs: input is durable before + `turn_started`, and completion/failure is appended only at the owning + delivery-and-persistence boundary. Competing terminal writes share one + idempotency key, so the first committed outcome wins. +- Intentional silence is a `turn_completed` `no_reply` outcome and does not + create a synthetic visible assistant message. +- Stable lifecycle keys make explicit retries idempotent; they do not cover + process death between external delivery and persistence. Slack needs a + durable delivery outbox/receipt reconciler before terminal events are + crash-safe across that boundary. ## Prompt Ownership diff --git a/packages/junior/src/chat/runtime/delivered-turn-state.ts b/packages/junior/src/chat/runtime/delivered-turn-state.ts index fa3d66388..216e3f8d0 100644 --- a/packages/junior/src/chat/runtime/delivered-turn-state.ts +++ b/packages/junior/src/chat/runtime/delivered-turn-state.ts @@ -16,8 +16,6 @@ import { import { clearPendingAuth } from "@/chat/services/pending-auth"; import { buildDeterministicAssistantMessageId } from "@/chat/state/turn-id"; -const NO_VISIBLE_REPLY_CONVERSATION_TEXT = "[no visible reply]"; - /** Build the canonical thread-state patch after final Slack delivery succeeds. */ export function buildDeliveredTurnStatePatch(args: { artifactStatePatch?: Partial; @@ -39,27 +37,27 @@ export function buildDeliveredTurnStatePatch(args: { clearPendingAuth(conversation, args.sessionId); const assistantText = - normalizeConversationText(args.reply.text) || - (args.reply.deliveryPlan?.postThreadText === false - ? NO_VISIBLE_REPLY_CONVERSATION_TEXT - : "[empty response]"); + normalizeConversationText(args.reply.text) || "[empty response]"; markConversationMessage(conversation, args.userMessageId, { replied: true, skippedReason: undefined, }); - upsertConversationMessage(conversation, { - id: buildDeterministicAssistantMessageId(args.sessionId), - role: "assistant", - text: assistantText, - createdAtMs: Date.now(), - author: { - userName: botConfig.userName, - isBot: true, - }, - meta: { - replied: true, - }, - }); + const intentionalSilence = args.reply.deliveryPlan?.postThreadText === false; + if (!intentionalSilence) { + upsertConversationMessage(conversation, { + id: buildDeterministicAssistantMessageId(args.sessionId), + role: "assistant", + text: assistantText, + createdAtMs: Date.now(), + author: { + userName: botConfig.userName, + isBot: true, + }, + meta: { + replied: true, + }, + }); + } markTurnCompleted({ conversation, nowMs: Date.now(), diff --git a/packages/junior/src/chat/services/turn-failure-response.ts b/packages/junior/src/chat/services/turn-failure-response.ts index b9c02a9b4..c1290423b 100644 --- a/packages/junior/src/chat/services/turn-failure-response.ts +++ b/packages/junior/src/chat/services/turn-failure-response.ts @@ -100,15 +100,21 @@ export function getAgentTurnDiagnosticsAttributes( }; } -/** Enforce one captured, event-ID-bearing failure response before delivery. */ -export function finalizeFailedTurnReply(args: { +/** Sanitized final reply plus its optional captured failure correlation. */ +export interface FinalizedTurnFailure { + eventId?: string; + reply: AgentRunResult; +} + +/** Enforce one captured failure response and return its structured correlation. */ +export function finalizeFailedTurnReplyWithEvent(args: { reply: AgentRunResult; logException: LogException; context: LogContext; attributes?: Record; -}): AgentRunResult { +}): FinalizedTurnFailure { if (args.reply.diagnostics.outcome === "success") { - return args.reply; + return { reply: args.reply }; } const capture = getFailureCapture(args.reply); @@ -138,14 +144,24 @@ export function finalizeFailedTurnReply(args: { : ""; return { - ...args.reply, - text: providerPartialText - ? `${providerPartialText}${getInterruptionMarker()}` - : buildTurnFailureResponse(eventId), - deliveryMode: "thread", - deliveryPlan: { - mode: "thread", - postThreadText: true, + eventId, + reply: { + ...args.reply, + text: providerPartialText + ? `${providerPartialText}${getInterruptionMarker()}` + : buildTurnFailureResponse(eventId), + deliveryMode: "thread", + deliveryPlan: { + mode: "thread", + postThreadText: true, + }, }, }; } + +/** Enforce one captured, event-ID-bearing failure response before delivery. */ +export function finalizeFailedTurnReply( + args: Parameters[0], +): AgentRunResult { + return finalizeFailedTurnReplyWithEvent(args).reply; +} diff --git a/packages/junior/tests/component/conversation-transcripts-sql.test.ts b/packages/junior/tests/component/conversation-transcripts-sql.test.ts index 45aaf6572..a3592cb8e 100644 --- a/packages/junior/tests/component/conversation-transcripts-sql.test.ts +++ b/packages/junior/tests/component/conversation-transcripts-sql.test.ts @@ -30,6 +30,7 @@ import { openConversationProjection, recordMcpProviderConnected, } from "@/chat/conversations/projection"; +import { ConversationTurnLifecycleService } from "@/chat/conversations/turn-lifecycle"; const CONVERSATION_ID = "slack:C123:1718123456.000000"; const CHILD_CONVERSATION_ID = "advisor:child-1"; @@ -170,6 +171,56 @@ it("rejects unknown Junior event fields while retaining opaque message fields", ).toBe(true); }); +it("validates strict privacy-safe turn lifecycle event shapes", () => { + expect( + conversationEventDataSchema.safeParse({ + type: "turn_started", + turnId: "turn-1", + inputMessageIds: ["message-1"], + surface: "internal", + }).success, + ).toBe(true); + expect( + conversationEventDataSchema.safeParse({ + type: "turn_completed", + turnId: "turn-1", + outcome: "no_reply", + }).success, + ).toBe(true); + expect( + conversationEventDataSchema.safeParse({ + type: "turn_failed", + turnId: "turn-1", + failureCode: "model_execution_failed", + eventId: "0123456789abcdef0123456789abcdef", + }).success, + ).toBe(true); + expect( + conversationEventDataSchema.safeParse({ + type: "turn_started", + turnId: "turn-1", + inputMessageIds: ["message-1", "message-1"], + surface: "internal", + }).success, + ).toBe(false); + expect( + conversationEventDataSchema.safeParse({ + type: "turn_failed", + turnId: "turn-1", + failureCode: "model_execution_failed", + eventId: "https://errors.invalid/raw-error-sentinel", + }).success, + ).toBe(false); + expect( + conversationEventDataSchema.safeParse({ + type: "turn_failed", + turnId: "turn-1", + failureCode: "provider_error: raw-error-sentinel", + providerError: { message: "raw-error-sentinel" }, + }).success, + ).toBe(false); +}); + it("rejects unsupported conversation event schema versions", () => { expect( conversationEventSchema.safeParse({ @@ -370,6 +421,50 @@ describe("conversation transcript SQL stores", () => { } }); + it("persists only the first conflicting terminal turn event", async () => { + const fixture = await createLocalJuniorSqlFixture(); + + try { + await migrateSchema(fixture.sql); + await seedConversation(fixture, CONVERSATION_ID); + const store = createSqlConversationEventStore(fixture.sql); + const lifecycle = new ConversationTurnLifecycleService(store); + await lifecycle.start({ + conversationId: CONVERSATION_ID, + createdAtMs: 1_000, + inputMessageIds: ["message-1"], + surface: "internal", + turnId: "turn-conflict", + }); + + await Promise.all([ + lifecycle.complete({ + conversationId: CONVERSATION_ID, + createdAtMs: 2_000, + outcome: "success", + turnId: "turn-conflict", + }), + lifecycle.fail({ + conversationId: CONVERSATION_ID, + createdAtMs: 2_000, + failureCode: "delivery_failed", + turnId: "turn-conflict", + }), + ]); + + const history = await store.loadHistory(CONVERSATION_ID); + expect(history.map((event) => event.data.type)).toEqual([ + "turn_started", + expect.stringMatching(/^turn_(?:completed|failed)$/), + ]); + expect( + history.filter((event) => event.idempotencyKey?.endsWith(":terminal")), + ).toHaveLength(1); + } finally { + await fixture.close(); + } + }); + it("replaces NUL characters before persisting conversation events", async () => { const fixture = await createLocalJuniorSqlFixture(); diff --git a/packages/junior/tests/integration/agent-dispatch-runner.test.ts b/packages/junior/tests/integration/agent-dispatch-runner.test.ts index 5c8a355fb..42990ed71 100644 --- a/packages/junior/tests/integration/agent-dispatch-runner.test.ts +++ b/packages/junior/tests/integration/agent-dispatch-runner.test.ts @@ -433,14 +433,14 @@ describe("agent dispatch runner", () => { conversation: sideEffectConversation, conversationId: dispatchConversationId, }); - expect(sideEffectConversation.messages).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - id: `dispatch:${created.record.id}:assistant`, - text: "[empty response]", - }), - ]), + expect(sideEffectConversation.messages).toContainEqual( + expect.objectContaining({ id: `dispatch:${created.record.id}:user` }), ); + expect( + sideEffectConversation.messages.find( + (message) => message.role === "assistant", + ), + ).toBeUndefined(); }); it("preserves task-scoped creator credentials across dispatch slices", async () => { diff --git a/packages/junior/tests/integration/dashboard-reporting.test.ts b/packages/junior/tests/integration/dashboard-reporting.test.ts index 35ac28994..c19d038a0 100644 --- a/packages/junior/tests/integration/dashboard-reporting.test.ts +++ b/packages/junior/tests/integration/dashboard-reporting.test.ts @@ -2,6 +2,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import type { PiMessage } from "@/chat/pi/messages"; import { readConversationDetail } from "@/api/conversations/detail"; import { readConversationSubagent as readConversationSubagentTranscriptReport } from "@/api/conversations/subagent"; +import { buildTurnFailureResponse } from "@/chat/logging"; vi.mock("@/chat/prompt", () => ({ buildSystemPrompt: vi.fn(() => "[system prompt]"), @@ -1164,6 +1165,148 @@ describe("dashboard reporting", () => { ); }); + it("reports one safe lifecycle failure marker for public and private details", async () => { + const { getConversationEventStore, getConversationStore } = + await import("@/chat/db"); + const publicConversationId = "slack:C1:lifecycle-failure-public"; + const privateConversationId = "slack:D1:lifecycle-failure-private"; + const sensitiveError = + "raw-error-sentinel https://provider.invalid/private?token=secret"; + const eventId = "0123456789abcdef0123456789abcdef"; + + await confirmPublicSlackConversation(publicConversationId); + await getConversationEventStore().append(publicConversationId, [ + { + data: { + type: "visible_message_recorded", + messageId: "public-user", + role: "user", + text: "please retry", + }, + createdAtMs: 1, + }, + { + data: { + type: "turn_started", + turnId: "turn-public", + inputMessageIds: ["public-user"], + surface: "slack", + }, + createdAtMs: 2, + }, + { + data: { + type: "message", + message: { + role: "assistant", + content: [], + stopReason: "error", + errorMessage: sensitiveError, + timestamp: 3, + } as unknown as PiMessage, + }, + createdAtMs: 3, + }, + { + data: { + type: "visible_message_recorded", + messageId: "public-fallback", + role: "assistant", + text: buildTurnFailureResponse(eventId), + }, + createdAtMs: 4, + }, + { + data: { + type: "turn_failed", + turnId: "turn-public", + failureCode: "model_execution_failed", + eventId, + }, + createdAtMs: 5, + }, + ]); + + await getConversationStore().recordActivity({ + conversationId: privateConversationId, + destination: { + platform: "slack", + teamId: "T1", + channelId: "D1", + }, + source: "slack", + visibility: "private", + }); + await getConversationEventStore().append(privateConversationId, [ + { + data: { + type: "turn_started", + turnId: "turn-private", + inputMessageIds: ["private-user"], + surface: "slack", + }, + createdAtMs: 10, + }, + { + data: { + type: "visible_message_recorded", + messageId: "private-fallback", + role: "assistant", + text: buildTurnFailureResponse(eventId), + }, + createdAtMs: 10_500, + }, + { + data: { + type: "turn_failed", + turnId: "turn-private", + failureCode: "delivery_failed", + eventId, + }, + createdAtMs: 11_000, + }, + ]); + + const publicReport = + await readConversationDetailReport(publicConversationId); + const publicOutcomeMarkers = publicReport.transcript.filter( + (message) => message.outcome === "error", + ); + expect(publicOutcomeMarkers).toEqual([ + { role: "assistant", outcome: "error", parts: [], timestamp: 5 }, + ]); + expect(publicReport.transcript).toContainEqual({ + role: "assistant", + parts: [{ type: "text", text: buildTurnFailureResponse(eventId) }], + timestamp: 4, + }); + + const privateReport = await readConversationDetailReport( + privateConversationId, + ); + expect(privateReport.transcript).toEqual([]); + expect(privateReport.transcriptMetadata).toContainEqual({ + role: "assistant", + parts: [expect.objectContaining({ type: "text", redacted: true })], + timestamp: 10_500, + }); + expect(privateReport.transcriptMetadata).toContainEqual({ + role: "assistant", + outcome: "error", + parts: [], + timestamp: 11_000, + }); + const publicSerialized = JSON.stringify(publicReport); + expect(publicSerialized).toContain(eventId); + expect(publicSerialized).not.toContain(sensitiveError); + expect(publicSerialized).not.toContain("model_execution_failed"); + + const privateSerialized = JSON.stringify(privateReport); + expect(privateSerialized).not.toContain(eventId); + expect(privateSerialized).not.toContain("delivery_failed"); + expect(privateSerialized).not.toContain(sensitiveError); + }); + it("merges safe terminal outcomes chronologically with visible messages", async () => { const { getConversationEventStore } = await import("@/chat/db"); const conversationId = "slack:C1:terminal-ordered"; diff --git a/packages/junior/tests/integration/local-agent-runner.test.ts b/packages/junior/tests/integration/local-agent-runner.test.ts index f0d9e9ddb..b68630cb1 100644 --- a/packages/junior/tests/integration/local-agent-runner.test.ts +++ b/packages/junior/tests/integration/local-agent-runner.test.ts @@ -8,6 +8,7 @@ import { normalizeLocalConversationId } from "@/chat/local/conversation"; import { runLocalAgentTurn, type LocalAgentReply, + type LocalAgentTurnDeps, type LocalToolInvocation, type LocalToolResult, } from "@/chat/local/runner"; @@ -28,6 +29,7 @@ import { coerceThreadArtifactsState } from "@/chat/state/artifacts"; import { setPlugins } from "@/chat/plugins/agent-hooks"; import { completedAgentRun } from "@/chat/runtime/agent-run-outcome"; import { flattenAgentRunRequestForTest } from "../fixtures/agent-runner"; +import { getConversationEventStore } from "@/chat/db"; function successReply( text: string, @@ -52,6 +54,29 @@ function successReply( }; } +function providerFailureReply(rawError: string): AgentRunResult { + return { + text: rawError, + diagnostics: { + assistantMessageCount: 0, + errorMessage: rawError, + modelId: "fake-local-agent", + outcome: "provider_error", + providerError: new Error(rawError), + toolCalls: [], + toolErrorCount: 0, + toolResultCount: 0, + usedPrimaryText: false, + }, + }; +} + +async function loadLifecycleEvents(conversationId: string) { + return (await getConversationEventStore().loadHistory(conversationId)).filter( + (event) => event.data.type.startsWith("turn_"), + ); +} + type FlatAgentRunRequest = ReturnType; async function persistCompletedSessionForFakeReply( @@ -172,6 +197,300 @@ describe("local agent runner", () => { replied: true, }, }); + + const history = await getConversationEventStore().loadHistory( + conversationId!, + ); + const userRecorded = history.findIndex( + (event) => + event.data.type === "visible_message_recorded" && + event.data.role === "user", + ); + const started = history.findIndex( + (event) => event.data.type === "turn_started", + ); + const assistantRecorded = history.findIndex( + (event) => + event.data.type === "visible_message_recorded" && + event.data.role === "assistant", + ); + const completed = history.findIndex( + (event) => event.data.type === "turn_completed", + ); + expect(userRecorded).toBeLessThan(started); + expect(started).toBeLessThan(assistantRecorded); + expect(assistantRecorded).toBeLessThan(completed); + expect(history[completed]?.data).toMatchObject({ + type: "turn_completed", + outcome: "success", + }); + }); + + it("records intentional silence without delivering or inventing a message", async () => { + const conversationId = normalizeLocalConversationId({ + alias: "no-reply", + cwd: "/tmp/local-agent-runner-no-reply", + }); + const deliverReply = vi.fn(); + + await runLocalAgentTurn( + { conversationId: conversationId!, message: "react only" }, + { + agentRunner: { + run: async () => + completedAgentRun({ + ...successReply(""), + deliveryPlan: { mode: "thread", postThreadText: false }, + }), + }, + deliverReply, + }, + ); + + expect(deliverReply).not.toHaveBeenCalled(); + const state = await getPersistedThreadState(conversationId!); + const conversation = coerceThreadConversationState(state); + await hydrateConversationMessages({ + conversation, + conversationId: conversationId!, + }); + expect(conversation.messages.map((message) => message.text)).toEqual([ + "react only", + ]); + expect(JSON.stringify(conversation.messages)).not.toContain( + "[no visible reply]", + ); + expect(await loadLifecycleEvents(conversationId!)).toEqual([ + expect.objectContaining({ + data: expect.objectContaining({ type: "turn_started" }), + }), + expect.objectContaining({ + data: expect.objectContaining({ + type: "turn_completed", + outcome: "no_reply", + }), + }), + ]); + }); + + it("records a delivered sanitized model failure without raw error data", async () => { + const conversationId = normalizeLocalConversationId({ + alias: "model-failure", + cwd: "/tmp/local-agent-runner-model-failure", + }); + const rawError = + "raw-error-sentinel https://provider.invalid/private?token=secret"; + const delivered: LocalAgentReply[] = []; + + await runLocalAgentTurn( + { conversationId: conversationId!, message: "please try" }, + { + agentRunner: { + run: async () => completedAgentRun(providerFailureReply(rawError)), + }, + deliverReply: async (reply) => { + delivered.push(reply); + }, + logException: vi + .fn() + .mockReturnValue("0123456789abcdef0123456789abcdef"), + }, + ); + + expect(delivered).toHaveLength(1); + expect(delivered[0]?.text).toContain( + "event_id=0123456789abcdef0123456789abcdef", + ); + expect(delivered[0]?.text).not.toContain("raw-error-sentinel"); + const lifecycle = await loadLifecycleEvents(conversationId!); + expect(lifecycle.at(-1)?.data).toEqual({ + type: "turn_failed", + turnId: expect.stringMatching(/^local-turn-/), + failureCode: "model_execution_failed", + eventId: "0123456789abcdef0123456789abcdef", + }); + expect(JSON.stringify(lifecycle)).not.toContain("raw-error-sentinel"); + expect(JSON.stringify(lifecycle)).not.toContain("provider.invalid"); + }); + + it("classifies a thrown agent run without persisting exception details", async () => { + const conversationId = normalizeLocalConversationId({ + alias: "runner-throw", + cwd: "/tmp/local-agent-runner-throw", + }); + const rawError = "raw-run-error-sentinel token=secret"; + const eventId = "11111111111111111111111111111111"; + const capture = vi.fn().mockReturnValue(eventId); + + await expect( + runLocalAgentTurn( + { conversationId: conversationId!, message: "please try" }, + { + agentRunner: { + run: async () => { + throw new Error(rawError); + }, + }, + deliverReply: async () => undefined, + logException: capture, + }, + ), + ).rejects.toThrow(rawError); + + const lifecycle = await loadLifecycleEvents(conversationId!); + expect(lifecycle.at(-1)?.data).toMatchObject({ + type: "turn_failed", + failureCode: "agent_run_failed", + eventId, + }); + expect(capture).toHaveBeenCalledOnce(); + expect(JSON.stringify(lifecycle)).not.toContain(rawError); + }); + + it("retains the model incident when post-delivery persistence fails", async () => { + const conversationId = normalizeLocalConversationId({ + alias: "model-persistence-failure", + cwd: "/tmp/local-agent-model-persistence-failure", + }); + const modelEventId = "22222222222222222222222222222222"; + const persistenceEventId = "55555555555555555555555555555555"; + const capture = vi + .fn() + .mockReturnValueOnce(modelEventId) + .mockReturnValueOnce(persistenceEventId); + const reply = providerFailureReply("raw-model-error-sentinel"); + reply.piMessages = [ + { + role: "user", + content: [{ type: "text", text: "please try" }], + } as PiMessage, + ]; + + await expect( + runLocalAgentTurn( + { conversationId: conversationId!, message: "please try" }, + { + agentRunner: { + run: async () => completedAgentRun(reply), + }, + completeDeliveredTurn: async () => { + throw new Error("session-persistence-error-sentinel"); + }, + deliverReply: async () => undefined, + logException: capture, + }, + ), + ).rejects.toThrow("session-persistence-error-sentinel"); + + expect(capture).toHaveBeenCalledTimes(2); + const lifecycle = await loadLifecycleEvents(conversationId!); + expect(lifecycle.at(-1)?.data).toMatchObject({ + type: "turn_failed", + eventId: persistenceEventId, + failureCode: "persistence_failed", + }); + const visible = coerceThreadConversationState( + await getPersistedThreadState(conversationId!), + ); + await hydrateConversationMessages({ + conversation: visible, + conversationId: conversationId!, + }); + expect(JSON.stringify(visible.messages)).toContain(modelEventId); + expect(JSON.stringify(visible.messages)).not.toContain(persistenceEventId); + expect(JSON.stringify(lifecycle)).not.toContain("raw-model-error-sentinel"); + expect(JSON.stringify(lifecycle)).not.toContain( + "session-persistence-error-sentinel", + ); + }); + + it("captures post-delivery persistence failure when no model incident exists", async () => { + const conversationId = normalizeLocalConversationId({ + alias: "success-persistence-failure", + cwd: "/tmp/local-agent-success-persistence-failure", + }); + const eventId = "44444444444444444444444444444444"; + const capture = vi.fn().mockReturnValue(eventId); + const rawError = "raw-session-persistence-error-sentinel"; + const reply = successReply("delivered", { + piMessages: [ + { + role: "assistant", + content: [{ type: "text", text: "delivered" }], + } as PiMessage, + ], + }); + + await expect( + runLocalAgentTurn( + { conversationId: conversationId!, message: "please try" }, + { + agentRunner: { run: async () => completedAgentRun(reply) }, + completeDeliveredTurn: async () => { + throw new Error(rawError); + }, + deliverReply: async () => undefined, + logException: capture, + }, + ), + ).rejects.toThrow(rawError); + + expect(capture).toHaveBeenCalledOnce(); + const lifecycle = await loadLifecycleEvents(conversationId!); + expect(lifecycle.at(-1)?.data).toMatchObject({ + type: "turn_failed", + eventId, + failureCode: "persistence_failed", + }); + expect(JSON.stringify(lifecycle)).not.toContain(rawError); + }); + + it("assigns distinct correlated ids to concurrent turns", async () => { + const conversationId = normalizeLocalConversationId({ + alias: "concurrent-turns", + cwd: "/tmp/local-agent-runner-concurrent-turns", + }); + const runIds: string[] = []; + const agentRunner: AgentRunner = { + run: async (request) => { + runIds.push(request.routing.correlation?.turnId ?? ""); + return completedAgentRun(successReply(`reply ${runIds.length}`)); + }, + }; + + await Promise.all([ + runLocalAgentTurn( + { conversationId: conversationId!, message: "first" }, + { agentRunner, deliverReply: async () => undefined }, + ), + runLocalAgentTurn( + { conversationId: conversationId!, message: "second" }, + { agentRunner, deliverReply: async () => undefined }, + ), + ]); + + expect(new Set(runIds).size).toBe(2); + expect(runIds).toEqual([ + expect.stringMatching(/^local-turn-[0-9a-f-]{36}$/), + expect.stringMatching(/^local-turn-[0-9a-f-]{36}$/), + ]); + const starts = (await loadLifecycleEvents(conversationId!)).filter( + (event) => event.data.type === "turn_started", + ); + expect( + new Set( + starts.flatMap((event) => + event.data.type === "turn_started" ? [event.data.turnId] : [], + ), + ).size, + ).toBe(2); + expect( + new Set( + starts.flatMap((event) => + event.data.type === "turn_started" ? event.data.inputMessageIds : [], + ), + ).size, + ).toBe(2); }); it("forwards tool events from the shared reply boundary", async () => { @@ -303,7 +622,7 @@ describe("local agent runner", () => { platform: "local", conversationId, }, - runId: "local-turn-1", + runId: expect.stringMatching(/^local-turn-[0-9a-f-]{36}$/), transcript: [ { type: "message", @@ -428,6 +747,45 @@ describe("local agent runner", () => { expect(conversation.messages).toEqual([]); }); + it("does not advertise an active turn when lifecycle start persistence fails", async () => { + const conversationId = normalizeLocalConversationId({ + alias: "start-failure", + cwd: "/tmp/local-agent-runner-start-failure", + }); + const agentRunner = { run: vi.fn() }; + const startError = new Error("lifecycle store unavailable"); + const turnLifecycle: NonNullable = { + start: vi.fn().mockRejectedValue(startError), + complete: vi.fn(), + fail: vi.fn(), + }; + + await expect( + runLocalAgentTurn( + { conversationId: conversationId!, message: "durable input" }, + { + agentRunner, + deliverReply: async () => undefined, + turnLifecycle, + }, + ), + ).rejects.toBe(startError); + + expect(agentRunner.run).not.toHaveBeenCalled(); + expect(turnLifecycle.complete).not.toHaveBeenCalled(); + expect(turnLifecycle.fail).not.toHaveBeenCalled(); + const state = await getPersistedThreadState(conversationId!); + const conversation = coerceThreadConversationState(state); + expect(conversation.processing.activeTurnId).toBeUndefined(); + await hydrateConversationMessages({ + conversation, + conversationId: conversationId!, + }); + expect(conversation.messages).toEqual([ + expect.objectContaining({ role: "user", text: "durable input" }), + ]); + }); + it("rejects malformed local conversation ids before generation", async () => { const generateReply = vi.fn(async () => { throw new Error("generation should not run"); @@ -679,6 +1037,9 @@ describe("local agent runner", () => { cwd: "/tmp/local-agent-runner-five", }); expect(conversationId).toBeDefined(); + const rawDeliveryError = "raw-delivery-error-sentinel stdout closed"; + const eventId = "33333333333333333333333333333333"; + const capture = vi.fn().mockReturnValue(eventId); const assistantMessage = { role: "assistant", @@ -711,12 +1072,13 @@ describe("local agent runner", () => { }, { deliverReply: async () => { - throw new Error("stdout closed"); + throw new Error(rawDeliveryError); }, agentRunner: { run: generateReply }, + logException: capture, }, ), - ).rejects.toThrow("stdout closed"); + ).rejects.toThrow(rawDeliveryError); expect(await loadProjection({ conversationId: conversationId! })).toEqual( [], @@ -724,5 +1086,21 @@ describe("local agent runner", () => { const state = await getPersistedThreadState(conversationId!); expect(coerceThreadArtifactsState(state).lastCanvasId).toBeUndefined(); expect(getPersistedSandboxState(state)).toEqual({}); + const visible = coerceThreadConversationState(state); + await hydrateConversationMessages({ + conversation: visible, + conversationId: conversationId!, + }); + expect(visible.messages).toEqual([ + expect.objectContaining({ role: "user", text: "hello" }), + ]); + const lifecycle = await loadLifecycleEvents(conversationId!); + expect(lifecycle.at(-1)?.data).toMatchObject({ + type: "turn_failed", + failureCode: "delivery_failed", + eventId, + }); + expect(capture).toHaveBeenCalledOnce(); + expect(JSON.stringify(lifecycle)).not.toContain(rawDeliveryError); }); }); diff --git a/packages/junior/tests/unit/chat/pi/conversation-events.test.ts b/packages/junior/tests/unit/chat/pi/conversation-events.test.ts index 15bbb3652..608a0187f 100644 --- a/packages/junior/tests/unit/chat/pi/conversation-events.test.ts +++ b/packages/junior/tests/unit/chat/pi/conversation-events.test.ts @@ -70,6 +70,23 @@ describe("projectConversationEvents", () => { delivery: "private_link_sent", }), event(6, { type: "message", message: lastMessage }), + event(7, { + type: "turn_started", + turnId: "turn-1", + inputMessageIds: ["message-1"], + surface: "slack", + }), + event(8, { + type: "turn_failed", + turnId: "turn-1", + failureCode: "model_execution_failed", + eventId: "0123456789abcdef0123456789abcdef", + }), + event(9, { + type: "turn_completed", + turnId: "turn-2", + outcome: "no_reply", + }), ]; it("projects messages, authorization observations, provenance, and model binding", () => { diff --git a/packages/junior/tests/unit/conversations/turn-lifecycle.test.ts b/packages/junior/tests/unit/conversations/turn-lifecycle.test.ts new file mode 100644 index 000000000..435755dfd --- /dev/null +++ b/packages/junior/tests/unit/conversations/turn-lifecycle.test.ts @@ -0,0 +1,187 @@ +import { describe, expect, it } from "vitest"; +import type { + ContextEpochStart, + ConversationEvent, + ConversationEventStore, + NewConversationEvent, +} from "@/chat/conversations/history"; +import { ConversationTurnLifecycleService } from "@/chat/conversations/turn-lifecycle"; + +class MemoryConversationEventStore implements ConversationEventStore { + readonly history: ConversationEvent[] = []; + private readonly idempotencyKeys = new Set(); + + async append( + _conversationId: string, + events: NewConversationEvent[], + ): Promise { + for (const event of events) { + if ( + event.idempotencyKey && + this.idempotencyKeys.has(event.idempotencyKey) + ) { + continue; + } + if (event.idempotencyKey) { + this.idempotencyKeys.add(event.idempotencyKey); + } + this.history.push({ + schemaVersion: 1, + seq: this.history.length, + contextEpoch: 0, + ...(event.idempotencyKey + ? { idempotencyKey: event.idempotencyKey } + : {}), + createdAtMs: event.createdAtMs, + data: event.data, + }); + } + } + + async startEpoch( + _conversationId: string, + _opts: ContextEpochStart, + ): Promise { + throw new Error("not implemented"); + } + + async loadCurrentEpoch(): Promise { + return this.history; + } + + async loadHistory(): Promise { + return this.history; + } +} + +describe("conversation turn lifecycle", () => { + it("records start retries once with stable correlation", async () => { + const store = new MemoryConversationEventStore(); + const lifecycle = new ConversationTurnLifecycleService(store); + const input = { + conversationId: "local:test:turn", + createdAtMs: 100, + inputMessageIds: ["message-1"], + surface: "internal" as const, + turnId: "turn-1", + }; + + await lifecycle.start(input); + await lifecycle.start(input); + + expect(store.history).toEqual([ + expect.objectContaining({ + idempotencyKey: "turn:turn-1:started", + data: { + type: "turn_started", + turnId: "turn-1", + inputMessageIds: ["message-1"], + surface: "internal", + }, + }), + ]); + }); + + it("keeps the first terminal fact across retries and conflicts", async () => { + const store = new MemoryConversationEventStore(); + const lifecycle = new ConversationTurnLifecycleService(store); + + await lifecycle.complete({ + conversationId: "local:test:turn", + createdAtMs: 200, + outcome: "success", + turnId: "turn-1", + }); + await lifecycle.complete({ + conversationId: "local:test:turn", + createdAtMs: 200, + outcome: "success", + turnId: "turn-1", + }); + await lifecycle.fail({ + conversationId: "local:test:turn", + createdAtMs: 300, + failureCode: "delivery_failed", + turnId: "turn-1", + }); + + expect(store.history).toHaveLength(1); + expect(store.history[0]).toMatchObject({ + idempotencyKey: "turn:turn-1:terminal", + data: { + type: "turn_completed", + turnId: "turn-1", + outcome: "success", + }, + }); + }); + + it("does not replace a first failure with later completion", async () => { + const store = new MemoryConversationEventStore(); + const lifecycle = new ConversationTurnLifecycleService(store); + + await lifecycle.fail({ + conversationId: "local:test:turn", + createdAtMs: 200, + failureCode: "agent_run_failed", + turnId: "turn-1", + }); + await lifecycle.complete({ + conversationId: "local:test:turn", + createdAtMs: 300, + outcome: "success", + turnId: "turn-1", + }); + + expect(store.history).toHaveLength(1); + expect(store.history[0]).toMatchObject({ + idempotencyKey: "turn:turn-1:terminal", + data: { + type: "turn_failed", + turnId: "turn-1", + failureCode: "agent_run_failed", + }, + }); + }); + + it("persists only the allowlisted failure correlation", async () => { + const store = new MemoryConversationEventStore(); + const lifecycle = new ConversationTurnLifecycleService(store); + + await lifecycle.fail({ + conversationId: "local:test:turn", + createdAtMs: 300, + eventId: "0123456789abcdef0123456789abcdef", + failureCode: "model_execution_failed", + turnId: "turn-1", + }); + + const serialized = JSON.stringify(store.history); + expect(serialized).toContain("0123456789abcdef0123456789abcdef"); + expect(serialized).not.toContain("providerError"); + expect(serialized).not.toContain("raw error sentinel"); + expect(serialized).not.toContain("https://"); + }); + + it("surfaces append failures to the owning runtime boundary", async () => { + const appendError = new Error("event store unavailable"); + const store = new MemoryConversationEventStore(); + let attempts = 0; + store.append = async () => { + attempts += 1; + throw appendError; + }; + const lifecycle = new ConversationTurnLifecycleService(store); + + await expect( + lifecycle.start({ + conversationId: "local:test:turn", + createdAtMs: 100, + inputMessageIds: ["message-1"], + surface: "internal", + turnId: "turn-1", + }), + ).rejects.toBe(appendError); + expect(attempts).toBe(1); + }); +}); diff --git a/packages/junior/tests/unit/runtime/delivered-turn-state.test.ts b/packages/junior/tests/unit/runtime/delivered-turn-state.test.ts index 34e167860..1b0c85323 100644 --- a/packages/junior/tests/unit/runtime/delivered-turn-state.test.ts +++ b/packages/junior/tests/unit/runtime/delivered-turn-state.test.ts @@ -57,4 +57,27 @@ describe("delivered turn state", () => { }), ]); }); + + it("does not invent an assistant message for intentional silence", () => { + const state = buildDeliveredTurnStatePatch({ + artifacts: coerceThreadArtifactsState({}), + conversation: coerceThreadConversationState({}), + reply: { + text: "", + deliveryPlan: { mode: "thread", postThreadText: false }, + diagnostics: { + assistantMessageCount: 1, + modelId: "test/model", + outcome: "success", + toolCalls: [], + toolErrorCount: 0, + toolResultCount: 0, + usedPrimaryText: true, + }, + }, + sessionId: "turn_silent", + }); + + expect(state.conversation.messages).toEqual([]); + }); }); From ac68fce35d069c7f83a4c8d8429259d2c1a9262d Mon Sep 17 00:00:00 2001 From: David Cramer Date: Wed, 15 Jul 2026 08:19:40 -0700 Subject: [PATCH 06/58] feat(conversations): Add immutable child lineage Co-Authored-By: GPT-5 Codex --- .../changes/conversation-event-log/tasks.md | 6 +- .../migrations/0007_conversation_lineage.sql | 13 + packages/junior/migrations/README.md | 4 + .../junior/migrations/meta/0007_snapshot.json | 1305 +++++++++++++++++ packages/junior/migrations/meta/_journal.json | 7 + .../junior/src/chat/conversations/README.md | 14 + .../junior/src/chat/conversations/history.ts | 2 + .../sql/legacy-history-import.ts | 86 +- .../src/chat/conversations/sql/store.ts | 63 +- .../junior/src/chat/conversations/state.ts | 3 - .../junior/src/chat/conversations/store.ts | 23 +- packages/junior/src/chat/db.ts | 9 + .../src/chat/services/subagent-lineage.ts | 418 ++++++ packages/junior/src/cli/upgrade.ts | 2 + .../migrations/conversation-lineage.ts | 130 ++ .../junior/src/db/schema/conversations.ts | 36 + .../tests/component/cli/upgrade-cli.test.ts | 1 - .../conversations/legacy-import.test.ts | 33 + .../services/subagent-lineage.test.ts | 495 +++++++ .../services/turn-session-record.test.ts | 1 - .../task-execution/conversation-work.test.ts | 2 - .../api/conversations/stats.test.ts | 24 +- .../integration/dashboard-reporting.test.ts | 137 +- .../legacy-history-import.test.ts | 2 +- 24 files changed, 2691 insertions(+), 125 deletions(-) create mode 100644 packages/junior/migrations/0007_conversation_lineage.sql create mode 100644 packages/junior/migrations/meta/0007_snapshot.json create mode 100644 packages/junior/src/chat/services/subagent-lineage.ts create mode 100644 packages/junior/src/cli/upgrade/migrations/conversation-lineage.ts create mode 100644 packages/junior/tests/component/services/subagent-lineage.test.ts diff --git a/openspec/changes/conversation-event-log/tasks.md b/openspec/changes/conversation-event-log/tasks.md index be5201112..e293c9b54 100644 --- a/openspec/changes/conversation-event-log/tasks.md +++ b/openspec/changes/conversation-event-log/tasks.md @@ -45,9 +45,9 @@ ## 4. Child Conversations -- [ ] Add parent/root conversation lineage and parent turn/event correlation. -- [ ] Record subagent start and finish references without copying child events. -- [ ] Represent shared context with an immutable parent sequence fork point. +- [x] Add parent/root conversation lineage and parent turn/event correlation. +- [x] Record subagent start and finish references without copying child events. +- [x] Represent shared context with an immutable parent sequence fork point. - [ ] Verify isolated and shared child Pi projections and inherited privacy. ## 5. Cleanup And Verification diff --git a/packages/junior/migrations/0007_conversation_lineage.sql b/packages/junior/migrations/0007_conversation_lineage.sql new file mode 100644 index 000000000..396f49826 --- /dev/null +++ b/packages/junior/migrations/0007_conversation_lineage.sql @@ -0,0 +1,13 @@ +ALTER TABLE "junior_conversations" ADD COLUMN "root_conversation_id" text;--> statement-breakpoint +ALTER TABLE "junior_conversations" ADD COLUMN "parent_turn_id" text;--> statement-breakpoint +ALTER TABLE "junior_conversations" ADD COLUMN "parent_event_seq" integer;--> statement-breakpoint +ALTER TABLE "junior_conversations" ADD COLUMN "context_fork_seq" integer;--> statement-breakpoint +ALTER TABLE "junior_conversations" ADD CONSTRAINT "junior_conversations_root_conversation_id_junior_conversations_conversation_id_fk" FOREIGN KEY ("root_conversation_id") REFERENCES "public"."junior_conversations"("conversation_id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint +CREATE INDEX "junior_conversations_root_idx" ON "junior_conversations" USING btree ("root_conversation_id");--> statement-breakpoint +CREATE INDEX "junior_conversations_parent_event_idx" ON "junior_conversations" USING btree ("parent_conversation_id","parent_event_seq");--> statement-breakpoint +ALTER TABLE "junior_conversations" ADD CONSTRAINT "junior_conversations_not_own_parent_check" CHECK ("junior_conversations"."parent_conversation_id" is null or "junior_conversations"."parent_conversation_id" <> "junior_conversations"."conversation_id");--> statement-breakpoint +ALTER TABLE "junior_conversations" ADD CONSTRAINT "junior_conversations_not_own_root_check" CHECK ("junior_conversations"."root_conversation_id" is null or "junior_conversations"."root_conversation_id" <> "junior_conversations"."conversation_id");--> statement-breakpoint +ALTER TABLE "junior_conversations" ADD CONSTRAINT "junior_conversations_context_fork_check" CHECK ("junior_conversations"."context_fork_seq" is null or ("junior_conversations"."parent_event_seq" is not null and "junior_conversations"."context_fork_seq" <= "junior_conversations"."parent_event_seq"));--> statement-breakpoint +ALTER TABLE "junior_conversations" ADD CONSTRAINT "junior_conversations_top_level_lineage_check" CHECK ("junior_conversations"."parent_conversation_id" is not null or ("junior_conversations"."root_conversation_id" is null and "junior_conversations"."parent_turn_id" is null and "junior_conversations"."parent_event_seq" is null and "junior_conversations"."context_fork_seq" is null));--> statement-breakpoint +ALTER TABLE "junior_conversations" ADD CONSTRAINT "junior_conversations_parent_correlation_check" CHECK (("junior_conversations"."parent_turn_id" is null) = ("junior_conversations"."parent_event_seq" is null));--> statement-breakpoint +ALTER TABLE "junior_conversations" ADD CONSTRAINT "junior_conversations_correlated_root_check" CHECK ("junior_conversations"."parent_turn_id" is null or "junior_conversations"."root_conversation_id" is not null); diff --git a/packages/junior/migrations/README.md b/packages/junior/migrations/README.md index ab0483b1e..dfea30191 100644 --- a/packages/junior/migrations/README.md +++ b/packages/junior/migrations/README.md @@ -21,3 +21,7 @@ writes to canonical `message` events while the first event rewrite runs. before applying it because it drops the legacy view and its functions. Run the final visible-message backfill next and require zero-gap verification before starting new workers. + +`0007_conversation_lineage.sql` expands conversation metadata with immutable +child lineage and fork correlation. The subsequent bounded upgrade backfill +fills historical root IDs only; unknown historical fork points stay null. diff --git a/packages/junior/migrations/meta/0007_snapshot.json b/packages/junior/migrations/meta/0007_snapshot.json new file mode 100644 index 000000000..533b36fd0 --- /dev/null +++ b/packages/junior/migrations/meta/0007_snapshot.json @@ -0,0 +1,1305 @@ +{ + "id": "f0793000-d703-4d2c-bc57-362bfa1a50a1", + "prevId": "46db5bc7-e30a-41bc-8194-abe7b42f6699", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.junior_conversation_events": { + "name": "junior_conversation_events", + "schema": "", + "columns": { + "conversation_id": { + "name": "conversation_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "seq": { + "name": "seq", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "context_epoch": { + "name": "context_epoch", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "schema_version": { + "name": "schema_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "idempotency_key": { + "name": "idempotency_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "junior_conversation_events_epoch_idx": { + "name": "junior_conversation_events_epoch_idx", + "columns": [ + { + "expression": "conversation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "context_epoch", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "seq", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "junior_conversation_events_idempotency_idx": { + "name": "junior_conversation_events_idempotency_idx", + "columns": [ + { + "expression": "conversation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "idempotency_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "junior_conversation_events_conversation_id_junior_conversations_conversation_id_fk": { + "name": "junior_conversation_events_conversation_id_junior_conversations_conversation_id_fk", + "tableFrom": "junior_conversation_events", + "tableTo": "junior_conversations", + "columnsFrom": ["conversation_id"], + "columnsTo": ["conversation_id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "junior_conversation_events_conversation_id_seq_pk": { + "name": "junior_conversation_events_conversation_id_seq_pk", + "columns": ["conversation_id", "seq"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.junior_conversation_messages": { + "name": "junior_conversation_messages", + "schema": "", + "columns": { + "conversation_id": { + "name": "conversation_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "message_id": { + "name": "message_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "author_identity_id": { + "name": "author_identity_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "text": { + "name": "text", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "meta": { + "name": "meta", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "replied_at": { + "name": "replied_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "junior_conversation_messages_activity_idx": { + "name": "junior_conversation_messages_activity_idx", + "columns": [ + { + "expression": "conversation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "junior_conversation_messages_search_idx": { + "name": "junior_conversation_messages_search_idx", + "columns": [ + { + "expression": "to_tsvector('english', \"text\")", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gin", + "with": {} + } + }, + "foreignKeys": { + "junior_conversation_messages_conversation_id_junior_conversations_conversation_id_fk": { + "name": "junior_conversation_messages_conversation_id_junior_conversations_conversation_id_fk", + "tableFrom": "junior_conversation_messages", + "tableTo": "junior_conversations", + "columnsFrom": ["conversation_id"], + "columnsTo": ["conversation_id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "junior_conversation_messages_author_identity_id_junior_identities_id_fk": { + "name": "junior_conversation_messages_author_identity_id_junior_identities_id_fk", + "tableFrom": "junior_conversation_messages", + "tableTo": "junior_identities", + "columnsFrom": ["author_identity_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "junior_conversation_messages_conversation_id_message_id_pk": { + "name": "junior_conversation_messages_conversation_id_message_id_pk", + "columns": ["conversation_id", "message_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.junior_conversations": { + "name": "junior_conversations", + "schema": "", + "columns": { + "conversation_id": { + "name": "conversation_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "schema_version": { + "name": "schema_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "origin_type": { + "name": "origin_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "origin_id": { + "name": "origin_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "origin_run_id": { + "name": "origin_run_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "destination_id": { + "name": "destination_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "destination_json": { + "name": "destination_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "actor_identity_id": { + "name": "actor_identity_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "creator_identity_id": { + "name": "creator_identity_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "credential_subject_identity_id": { + "name": "credential_subject_identity_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "actor_json": { + "name": "actor_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "channel_name": { + "name": "channel_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "last_activity_at": { + "name": "last_activity_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "execution_updated_at": { + "name": "execution_updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "execution_status": { + "name": "execution_status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_checkpoint_at": { + "name": "last_checkpoint_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_enqueued_at": { + "name": "last_enqueued_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "parent_conversation_id": { + "name": "parent_conversation_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "transcript_purged_at": { + "name": "transcript_purged_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "duration_ms": { + "name": "duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "usage_json": { + "name": "usage_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "execution_duration_ms": { + "name": "execution_duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "execution_usage_json": { + "name": "execution_usage_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "metric_run_id": { + "name": "metric_run_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "root_conversation_id": { + "name": "root_conversation_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "parent_turn_id": { + "name": "parent_turn_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "parent_event_seq": { + "name": "parent_event_seq", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "context_fork_seq": { + "name": "context_fork_seq", + "type": "integer", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "junior_conversations_last_activity_idx": { + "name": "junior_conversations_last_activity_idx", + "columns": [ + { + "expression": "last_activity_at", + "isExpression": false, + "asc": false, + "nulls": "last" + }, + { + "expression": "conversation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "junior_conversations_active_idx": { + "name": "junior_conversations_active_idx", + "columns": [ + { + "expression": "coalesce(\"execution_updated_at\", \"updated_at\")", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "conversation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"junior_conversations\".\"execution_status\" <> 'idle'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "junior_conversations_destination_activity_idx": { + "name": "junior_conversations_destination_activity_idx", + "columns": [ + { + "expression": "destination_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "last_activity_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "junior_conversations_actor_activity_idx": { + "name": "junior_conversations_actor_activity_idx", + "columns": [ + { + "expression": "actor_identity_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "last_activity_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "junior_conversations_origin_idx": { + "name": "junior_conversations_origin_idx", + "columns": [ + { + "expression": "origin_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "origin_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "last_activity_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "junior_conversations_parent_idx": { + "name": "junior_conversations_parent_idx", + "columns": [ + { + "expression": "parent_conversation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "junior_conversations_parent_event_idx": { + "name": "junior_conversations_parent_event_idx", + "columns": [ + { + "expression": "parent_conversation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "parent_event_seq", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "junior_conversations_root_idx": { + "name": "junior_conversations_root_idx", + "columns": [ + { + "expression": "root_conversation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "junior_conversations_destination_id_junior_destinations_id_fk": { + "name": "junior_conversations_destination_id_junior_destinations_id_fk", + "tableFrom": "junior_conversations", + "tableTo": "junior_destinations", + "columnsFrom": ["destination_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "junior_conversations_actor_identity_id_junior_identities_id_fk": { + "name": "junior_conversations_actor_identity_id_junior_identities_id_fk", + "tableFrom": "junior_conversations", + "tableTo": "junior_identities", + "columnsFrom": ["actor_identity_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "junior_conversations_creator_identity_id_junior_identities_id_fk": { + "name": "junior_conversations_creator_identity_id_junior_identities_id_fk", + "tableFrom": "junior_conversations", + "tableTo": "junior_identities", + "columnsFrom": ["creator_identity_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "junior_conversations_credential_subject_identity_id_junior_identities_id_fk": { + "name": "junior_conversations_credential_subject_identity_id_junior_identities_id_fk", + "tableFrom": "junior_conversations", + "tableTo": "junior_identities", + "columnsFrom": ["credential_subject_identity_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "junior_conversations_parent_conversation_id_junior_conversations_conversation_id_fk": { + "name": "junior_conversations_parent_conversation_id_junior_conversations_conversation_id_fk", + "tableFrom": "junior_conversations", + "tableTo": "junior_conversations", + "columnsFrom": ["parent_conversation_id"], + "columnsTo": ["conversation_id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "junior_conversations_root_conversation_id_junior_conversations_conversation_id_fk": { + "name": "junior_conversations_root_conversation_id_junior_conversations_conversation_id_fk", + "tableFrom": "junior_conversations", + "tableTo": "junior_conversations", + "columnsFrom": ["root_conversation_id"], + "columnsTo": ["conversation_id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "junior_conversations_not_own_parent_check": { + "name": "junior_conversations_not_own_parent_check", + "value": "\"junior_conversations\".\"parent_conversation_id\" is null or \"junior_conversations\".\"parent_conversation_id\" <> \"junior_conversations\".\"conversation_id\"" + }, + "junior_conversations_not_own_root_check": { + "name": "junior_conversations_not_own_root_check", + "value": "\"junior_conversations\".\"root_conversation_id\" is null or \"junior_conversations\".\"root_conversation_id\" <> \"junior_conversations\".\"conversation_id\"" + }, + "junior_conversations_context_fork_check": { + "name": "junior_conversations_context_fork_check", + "value": "\"junior_conversations\".\"context_fork_seq\" is null or (\"junior_conversations\".\"parent_event_seq\" is not null and \"junior_conversations\".\"context_fork_seq\" <= \"junior_conversations\".\"parent_event_seq\")" + }, + "junior_conversations_top_level_lineage_check": { + "name": "junior_conversations_top_level_lineage_check", + "value": "\"junior_conversations\".\"parent_conversation_id\" is not null or (\"junior_conversations\".\"root_conversation_id\" is null and \"junior_conversations\".\"parent_turn_id\" is null and \"junior_conversations\".\"parent_event_seq\" is null and \"junior_conversations\".\"context_fork_seq\" is null)" + }, + "junior_conversations_parent_correlation_check": { + "name": "junior_conversations_parent_correlation_check", + "value": "(\"junior_conversations\".\"parent_turn_id\" is null) = (\"junior_conversations\".\"parent_event_seq\" is null)" + }, + "junior_conversations_correlated_root_check": { + "name": "junior_conversations_correlated_root_check", + "value": "\"junior_conversations\".\"parent_turn_id\" is null or \"junior_conversations\".\"root_conversation_id\" is not null" + } + }, + "isRLSEnabled": false + }, + "public.junior_destinations": { + "name": "junior_destinations", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_tenant_id": { + "name": "provider_tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "provider_destination_id": { + "name": "provider_destination_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "parent_destination_id": { + "name": "parent_destination_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "visibility": { + "name": "visibility", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'unknown'" + }, + "metadata_json": { + "name": "metadata_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "junior_destinations_provider_destination_uidx": { + "name": "junior_destinations_provider_destination_uidx", + "columns": [ + { + "expression": "provider", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_tenant_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_destination_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "junior_destinations_provider_kind_idx": { + "name": "junior_destinations_provider_kind_idx", + "columns": [ + { + "expression": "provider", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.junior_identities": { + "name": "junior_identities", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_tenant_id": { + "name": "provider_tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "provider_subject_id": { + "name": "provider_subject_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "handle": { + "name": "handle", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "avatar_url": { + "name": "avatar_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata_json": { + "name": "metadata_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "email_normalized": { + "name": "email_normalized", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "email_verified": { + "name": "email_verified", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + } + }, + "indexes": { + "junior_identities_provider_subject_uidx": { + "name": "junior_identities_provider_subject_uidx", + "columns": [ + { + "expression": "provider", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_tenant_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_subject_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "junior_identities_user_idx": { + "name": "junior_identities_user_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "junior_identities_verified_email_idx": { + "name": "junior_identities_verified_email_idx", + "columns": [ + { + "expression": "email_normalized", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"junior_identities\".\"email_verified\" = true AND \"junior_identities\".\"email_normalized\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "junior_identities_kind_provider_idx": { + "name": "junior_identities_kind_provider_idx", + "columns": [ + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "junior_identities_user_id_junior_users_id_fk": { + "name": "junior_identities_user_id_junior_users_id_fk", + "tableFrom": "junior_identities", + "tableTo": "junior_users", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.junior_pending_deliveries": { + "name": "junior_pending_deliveries", + "schema": "", + "columns": { + "delivery_id": { + "name": "delivery_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "conversation_id": { + "name": "conversation_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "turn_id": { + "name": "turn_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "message_id": { + "name": "message_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "delivery_kind": { + "name": "delivery_kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "command_json": { + "name": "command_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "part_states_json": { + "name": "part_states_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "next_part_index": { + "name": "next_part_index", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "attempt_count": { + "name": "attempt_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "next_attempt_at": { + "name": "next_attempt_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "lease_owner": { + "name": "lease_owner", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lease_version": { + "name": "lease_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "lease_expires_at": { + "name": "lease_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_attempt_at": { + "name": "last_attempt_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "junior_pending_deliveries_conversation_turn_idx": { + "name": "junior_pending_deliveries_conversation_turn_idx", + "columns": [ + { + "expression": "conversation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "turn_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "junior_pending_deliveries_retry_idx": { + "name": "junior_pending_deliveries_retry_idx", + "columns": [ + { + "expression": "next_attempt_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "delivery_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "junior_pending_deliveries_conversation_id_fk": { + "name": "junior_pending_deliveries_conversation_id_fk", + "tableFrom": "junior_pending_deliveries", + "tableTo": "junior_conversations", + "columnsFrom": ["conversation_id"], + "columnsTo": ["conversation_id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "junior_pending_deliveries_cursor_check": { + "name": "junior_pending_deliveries_cursor_check", + "value": "\"junior_pending_deliveries\".\"next_part_index\" >= 0" + }, + "junior_pending_deliveries_attempt_count_check": { + "name": "junior_pending_deliveries_attempt_count_check", + "value": "\"junior_pending_deliveries\".\"attempt_count\" >= 0" + }, + "junior_pending_deliveries_lease_version_check": { + "name": "junior_pending_deliveries_lease_version_check", + "value": "\"junior_pending_deliveries\".\"lease_version\" >= 0" + } + }, + "isRLSEnabled": false + }, + "public.junior_users": { + "name": "junior_users", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "primary_email": { + "name": "primary_email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "primary_email_normalized": { + "name": "primary_email_normalized", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "junior_users_primary_email_normalized_uidx": { + "name": "junior_users_primary_email_normalized_uidx", + "columns": [ + { + "expression": "primary_email_normalized", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} diff --git a/packages/junior/migrations/meta/_journal.json b/packages/junior/migrations/meta/_journal.json index d5e259957..ff5f4f19a 100644 --- a/packages/junior/migrations/meta/_journal.json +++ b/packages/junior/migrations/meta/_journal.json @@ -50,6 +50,13 @@ "when": 1784089383071, "tag": "0005_visible_message_events", "breakpoints": true + }, + { + "idx": 7, + "version": "7", + "when": 1784127545103, + "tag": "0007_conversation_lineage", + "breakpoints": true } ] } diff --git a/packages/junior/src/chat/conversations/README.md b/packages/junior/src/chat/conversations/README.md index 0dd4d79b8..7b4461570 100644 --- a/packages/junior/src/chat/conversations/README.md +++ b/packages/junior/src/chat/conversations/README.md @@ -15,6 +15,9 @@ conversation events, compaction boundaries, search, retention, and legacy import second authority. - Context epochs identify replacement boundaries created by compaction or model handoff. +- Child rows carry immutable parent/root, parent-turn, and exact parent-event + correlation. Shared children additionally retain that parent sequence as + their context fork; isolated children retain no fork. - Provider payloads and old state-store mirrors are migration inputs, not canonical product records. @@ -48,6 +51,9 @@ must not become a dashboard or external API payload. - Restore state from durable events rather than a duplicate transcript cache. - Compaction replaces prior model context without rewriting visible history. - Imports and migrations are idempotent and preserve stable conversation IDs. +- Establish child lineage and its parent `subagent_started` reference through + the SQL-backed lineage service in one transaction. Retries must match the + original parent, root, turn, event, and history mode exactly. ## Visibility And Retention @@ -73,6 +79,8 @@ Follow `../../../../../policies/data-redaction.md` and stopped. Its fail-closed zero-gap verification is the cutover gate; start new workers only after it passes. - Purge and migration jobs operate in bounded batches and are safe to retry. +- The lineage backfill fills historical roots only. Missing historical turn, + event, and fork correlation remains null and therefore isolated. Representative coverage lives in `packages/junior/tests/integration/conversation-sql.test.ts` and the @@ -94,3 +102,9 @@ process death after destination acceptance and before the visible/session/terminal writes can still leave a started turn without a terminal event. The next delivery slice must add durable intent/receipt reconciliation for Slack before claiming crash-safe terminality. + +Subagent executions own separate child event streams. The parent records only +idempotent start/end references, and child events are never copied into the +parent. New child creation requires an existing parent turn and complete +lineage; only a metadata-bare row created by an earlier child event append may +be upgraded. Reparenting an existing conversation is rejected. diff --git a/packages/junior/src/chat/conversations/history.ts b/packages/junior/src/chat/conversations/history.ts index ecabe052c..7c35d73fd 100644 --- a/packages/junior/src/chat/conversations/history.ts +++ b/packages/junior/src/chat/conversations/history.ts @@ -276,6 +276,7 @@ const subagentStartedEventDataSchema = z parentToolCallId: z.string().min(1).optional(), reasoningLevel: z.string().min(1).optional(), childConversationId: z.string().min(1), + parentTurnId: z.string().min(1).optional(), historyMode: z.union([z.literal("isolated"), z.literal("shared")]), }) .strict(); @@ -284,6 +285,7 @@ const subagentEndedEventDataSchema = z .object({ type: z.literal("subagent_ended"), subagentInvocationId: z.string().min(1), + parentTurnId: z.string().min(1).optional(), outcome: z.union([ z.literal("success"), z.literal("error"), diff --git a/packages/junior/src/chat/conversations/sql/legacy-history-import.ts b/packages/junior/src/chat/conversations/sql/legacy-history-import.ts index 533efdfc1..92ab9b8db 100644 --- a/packages/junior/src/chat/conversations/sql/legacy-history-import.ts +++ b/packages/junior/src/chat/conversations/sql/legacy-history-import.ts @@ -269,7 +269,9 @@ export function convertLegacySessionLog(args: { ? { parentToolCallId: entry.parentToolCallId } : {}), childConversationId, - historyMode: "shared", + // The legacy transcript records no exact parent event fork. Keep + // it isolated rather than guessing a shared-context boundary. + historyMode: "isolated", }, entry.createdAtMs, ); @@ -483,7 +485,7 @@ export async function writeLegacyImport( args.child.events.length > 0 ? Math.max(...args.child.events.map((event) => event.createdAtMs)) : childCreatedAtMs; - await ensureChildConversationRow( + await insertLegacyAdvisorChildRow( executor, args.child.conversationId, args.conversationId, @@ -531,7 +533,7 @@ async function ensureConversationRow( }); } -async function ensureChildConversationRow( +async function insertLegacyAdvisorChildRow( executor: JuniorSqlDatabase, childConversationId: string, parentConversationId: string, @@ -544,6 +546,10 @@ async function ensureChildConversationRow( createdAt, lastActivityAt, ); + const rootConversationId = await resolveLegacyRootConversationId( + executor, + parentConversationId, + ); await executor .db() .insert(juniorConversations) @@ -551,6 +557,7 @@ async function ensureChildConversationRow( conversationId: childConversationId, schemaVersion: 1, parentConversationId, + rootConversationId, createdAt, lastActivityAt, updatedAt: lastActivityAt, @@ -559,10 +566,81 @@ async function ensureChildConversationRow( .onConflictDoUpdate({ target: juniorConversations.conversationId, set: { - parentConversationId: sql`coalesce(${juniorConversations.parentConversationId}, excluded.parent_conversation_id)`, createdAt: sql`least(${juniorConversations.createdAt}, excluded.created_at)`, lastActivityAt: sql`greatest(${juniorConversations.lastActivityAt}, excluded.last_activity_at)`, updatedAt: sql`greatest(${juniorConversations.updatedAt}, excluded.updated_at)`, }, }); + const rows = await executor + .db() + .select({ + parentConversationId: juniorConversations.parentConversationId, + rootConversationId: juniorConversations.rootConversationId, + parentTurnId: juniorConversations.parentTurnId, + parentEventSeq: juniorConversations.parentEventSeq, + contextForkSeq: juniorConversations.contextForkSeq, + }) + .from(juniorConversations) + .where(eq(juniorConversations.conversationId, childConversationId)); + const child = rows[0]; + if ( + !child || + child.parentConversationId !== parentConversationId || + child.rootConversationId !== rootConversationId || + child.parentTurnId !== null || + child.parentEventSeq !== null || + child.contextForkSeq !== null + ) { + throw new Error( + "Legacy advisor child conflicts with existing conversation lineage", + ); + } +} + +async function resolveLegacyRootConversationId( + executor: JuniorSqlDatabase, + conversationId: string, +): Promise { + let currentId = conversationId; + const seen = new Set(); + let declaredRootConversationId: string | undefined; + while (!seen.has(currentId)) { + seen.add(currentId); + const rows = await executor + .db() + .select({ + conversationId: juniorConversations.conversationId, + parentConversationId: juniorConversations.parentConversationId, + rootConversationId: juniorConversations.rootConversationId, + }) + .from(juniorConversations) + .where(eq(juniorConversations.conversationId, currentId)); + const row = rows[0]; + if (!row) { + throw new Error(`Legacy advisor parent is missing: ${currentId}`); + } + if ( + row.rootConversationId && + declaredRootConversationId && + row.rootConversationId !== declaredRootConversationId + ) { + throw new Error("Legacy advisor parent declares conflicting roots"); + } + if (row.rootConversationId) { + declaredRootConversationId = row.rootConversationId; + } + if (!row.parentConversationId) { + if ( + declaredRootConversationId && + declaredRootConversationId !== row.conversationId + ) { + throw new Error( + "Legacy advisor parent does not resolve to its declared root", + ); + } + return row.conversationId; + } + currentId = row.parentConversationId; + } + throw new Error("Legacy advisor parent lineage contains a cycle"); } diff --git a/packages/junior/src/chat/conversations/sql/store.ts b/packages/junior/src/chat/conversations/sql/store.ts index 1b1d125ca..b355efa4f 100644 --- a/packages/junior/src/chat/conversations/sql/store.ts +++ b/packages/junior/src/chat/conversations/sql/store.ts @@ -310,6 +310,23 @@ function conversationFromRow(readRow: ConversationReadRow): Conversation { lastActivityAtMs: requiredMsFromDate(row.lastActivityAt), updatedAtMs: requiredMsFromDate(row.updatedAt), execution, + ...(row.parentConversationId + ? { + lineage: { + parentConversationId: row.parentConversationId, + ...(row.rootConversationId + ? { rootConversationId: row.rootConversationId } + : {}), + ...(row.parentTurnId ? { parentTurnId: row.parentTurnId } : {}), + ...(row.parentEventSeq !== null + ? { parentEventSeq: row.parentEventSeq } + : {}), + ...(row.contextForkSeq !== null + ? { contextForkSeq: row.contextForkSeq } + : {}), + }, + } + : {}), ...(destination ? { destination } : {}), ...(actor ? { actor } : {}), ...(msFromDate(row.archivedAt) !== undefined @@ -607,46 +624,6 @@ export class SqlStore implements ConversationStore { }); } - async ensureChildConversation(args: { - conversationId: string; - parentConversationId: string; - nowMs?: number; - }): Promise { - const at = dateFromMs(args.nowMs ?? now()); - // The child FKs to its parent, so establish a bare parent row first for the - // rare case a step append has not created it yet (a no-op once it exists). - await this.executor - .db() - .insert(juniorConversations) - .values({ - conversationId: args.parentConversationId, - schemaVersion: 1, - createdAt: at, - lastActivityAt: at, - updatedAt: at, - executionStatus: "idle", - }) - .onConflictDoNothing({ target: juniorConversations.conversationId }); - await this.executor - .db() - .insert(juniorConversations) - .values({ - conversationId: args.conversationId, - schemaVersion: 1, - parentConversationId: args.parentConversationId, - createdAt: at, - lastActivityAt: at, - updatedAt: at, - executionStatus: "idle", - }) - .onConflictDoUpdate({ - target: juniorConversations.conversationId, - set: { - parentConversationId: sql`coalesce(${juniorConversations.parentConversationId}, excluded.parent_conversation_id)`, - }, - }); - } - /** Copy one conversation record and retained metrics into SQL during backfill. */ async backfillConversation( sourceConversation: Conversation, @@ -896,6 +873,12 @@ export class SqlStore implements ConversationStore { conversation.execution.lastEnqueuedAtMs === undefined ? null : dateFromMs(conversation.execution.lastEnqueuedAtMs), + parentConversationId: + conversation.lineage?.parentConversationId ?? null, + rootConversationId: conversation.lineage?.rootConversationId ?? null, + parentTurnId: conversation.lineage?.parentTurnId ?? null, + parentEventSeq: conversation.lineage?.parentEventSeq ?? null, + contextForkSeq: conversation.lineage?.contextForkSeq ?? null, }) .onConflictDoUpdate({ target: juniorConversations.conversationId, diff --git a/packages/junior/src/chat/conversations/state.ts b/packages/junior/src/chat/conversations/state.ts index 6a9169331..58b5455d2 100644 --- a/packages/junior/src/chat/conversations/state.ts +++ b/packages/junior/src/chat/conversations/state.ts @@ -16,9 +16,6 @@ export function createStateConversationStore( // Task-execution state has no destination records, so visibility is never // source-confirmed here and cross-context reads fail closed to private. getDestinationVisibility: async () => undefined, - // Child conversations are a SQL-only concept and never use this legacy - // metadata path. - ensureChildConversation: async () => undefined, recordActivity: (args) => recordTaskConversationActivity({ ...args, state }), recordExecution: (args) => diff --git a/packages/junior/src/chat/conversations/store.ts b/packages/junior/src/chat/conversations/store.ts index 10686403a..9f8c3798d 100644 --- a/packages/junior/src/chat/conversations/store.ts +++ b/packages/junior/src/chat/conversations/store.ts @@ -27,6 +27,15 @@ export interface ConversationExecution { updatedAtMs?: number; } +/** Immutable parent correlation for a child conversation. */ +export interface ConversationLineage { + contextForkSeq?: number; + parentConversationId: string; + parentEventSeq?: number; + parentTurnId?: string; + rootConversationId?: string; +} + export interface Conversation { archivedAtMs?: number; channelName?: string; @@ -35,6 +44,7 @@ export interface Conversation { destination?: Destination; execution: ConversationExecution; lastActivityAtMs: number; + lineage?: ConversationLineage; actor?: StoredSlackActor; schemaVersion: 1; source?: ConversationSource; @@ -71,19 +81,6 @@ export interface ConversationStore { /** Source-confirmed visibility from the current event's signal only. */ visibility?: ConversationPrivacy; }): Promise; - /** - * Establish a subagent child conversation row linked to its parent. - * - * Subagent histories live under their own child `conversation_id` with - * `parent_conversation_id` set; the child carries no destination and is - * excluded from top-level listings. Idempotent: it links a bare row a step - * append may have created first without clobbering it. - */ - ensureChildConversation(args: { - conversationId: string; - parentConversationId: string; - nowMs?: number; - }): Promise; /** Store task-execution metadata for long-term conversation history. */ recordExecution(args: { channelName?: string; diff --git a/packages/junior/src/chat/db.ts b/packages/junior/src/chat/db.ts index 2d11c9618..e649bde81 100644 --- a/packages/junior/src/chat/db.ts +++ b/packages/junior/src/chat/db.ts @@ -9,6 +9,7 @@ import { createSqlConversationSearchStore } from "@/chat/conversations/sql/searc import type { ConversationSearchStore } from "@/chat/conversations/search"; import type { JuniorDatabase, JuniorSqlExecutor } from "@/db/db"; import { createJuniorSqlExecutor } from "@/db/executor"; +import { SubagentLineageService } from "@/chat/services/subagent-lineage"; let current: | { @@ -19,6 +20,7 @@ let current: eventStore: ConversationEventStore; messageStore: ConversationMessageStore; searchStore: ConversationSearchStore; + subagentLineage: SubagentLineageService; } | undefined; @@ -59,6 +61,7 @@ export function getSqlExecutor(): JuniorSqlExecutor { eventStore: createSqlConversationEventStore(db), messageStore: createSqlConversationMessageStore(db), searchStore: createSqlConversationSearchStore(db), + subagentLineage: new SubagentLineageService(db), }; } return current.db; @@ -93,6 +96,12 @@ export function getConversationSearchStore(): ConversationSearchStore { return current!.searchStore; } +/** Return the SQL-backed immutable child-conversation lineage service. */ +export function getSubagentLineageService(): SubagentLineageService { + getSqlExecutor(); + return current!.subagentLineage; +} + /** Close the process SQL database when it has been opened. */ export async function closeDb(): Promise { const previous = current; diff --git a/packages/junior/src/chat/services/subagent-lineage.ts b/packages/junior/src/chat/services/subagent-lineage.ts new file mode 100644 index 000000000..a5143fb51 --- /dev/null +++ b/packages/junior/src/chat/services/subagent-lineage.ts @@ -0,0 +1,418 @@ +import { isDeepStrictEqual } from "node:util"; +import { eq } from "drizzle-orm"; +import type { JuniorSqlDatabase } from "@/db/db"; +import { juniorConversations } from "@/db/schema"; +import type { + ConversationEvent, + ConversationEventData, +} from "@/chat/conversations/history"; +import { createSqlConversationEventStore } from "@/chat/conversations/sql/history"; +import { withConversationEventLock } from "@/chat/conversations/sql/event-lock"; + +export type SubagentHistoryMode = "isolated" | "shared"; + +export interface StartSubagentReferenceInput { + childConversationId: string; + historyMode: SubagentHistoryMode; + modelId?: string; + nowMs?: number; + parentConversationId: string; + parentToolCallId?: string; + parentTurnId: string; + reasoningLevel?: string; + subagentInvocationId: string; + subagentKind: string; +} + +export interface SubagentLineage { + childConversationId: string; + contextForkSeq: number | null; + parentConversationId: string; + parentEventSeq: number; + parentTurnId: string; + rootConversationId: string; +} + +export interface FinishSubagentReferenceInput { + nowMs?: number; + outcome: "aborted" | "error" | "success"; + parentConversationId: string; + parentTurnId: string; + subagentInvocationId: string; +} + +/** Raised when a retry attempts to change an immutable subagent reference. */ +export class SubagentLineageConflictError extends Error { + constructor(message: string) { + super(message); + this.name = "SubagentLineageConflictError"; + } +} + +function startedKey(subagentInvocationId: string): string { + return `subagent:${subagentInvocationId}:started`; +} + +function endedKey(subagentInvocationId: string): string { + return `subagent:${subagentInvocationId}:ended`; +} + +function eventWithKey( + history: ConversationEvent[], + idempotencyKey: string, +): ConversationEvent | undefined { + return history.find((event) => event.idempotencyKey === idempotencyKey); +} + +function expectedStartData( + input: StartSubagentReferenceInput, +): Extract { + return { + type: "subagent_started", + subagentInvocationId: input.subagentInvocationId, + subagentKind: input.subagentKind, + ...(input.modelId ? { modelId: input.modelId } : {}), + ...(input.parentToolCallId + ? { parentToolCallId: input.parentToolCallId } + : {}), + ...(input.reasoningLevel ? { reasoningLevel: input.reasoningLevel } : {}), + childConversationId: input.childConversationId, + parentTurnId: input.parentTurnId, + historyMode: input.historyMode, + }; +} + +function expectedEndData( + input: FinishSubagentReferenceInput, +): Extract { + return { + type: "subagent_ended", + subagentInvocationId: input.subagentInvocationId, + parentTurnId: input.parentTurnId, + outcome: input.outcome, + }; +} + +function assertExactEvent( + event: ConversationEvent | undefined, + expected: ConversationEventData, + label: string, +): asserts event is ConversationEvent { + if (!event || !isDeepStrictEqual(event.data, expected)) { + throw new SubagentLineageConflictError( + `${label} conflicts with the persisted subagent reference`, + ); + } +} + +/** + * Own immutable SQL lineage and parent-stream references for subagent runs. + * Child execution events remain in the child conversation's event stream. + */ +export class SubagentLineageService { + constructor(private readonly executor: JuniorSqlDatabase) {} + + /** Append the parent start reference and establish the correlated child row atomically. */ + async start(input: StartSubagentReferenceInput): Promise { + if (input.childConversationId === input.parentConversationId) { + throw new SubagentLineageConflictError( + "A subagent conversation cannot be its own parent", + ); + } + const eventStore = createSqlConversationEventStore(this.executor); + const expected = expectedStartData(input); + + return await withConversationEventLock( + this.executor, + input.parentConversationId, + async () => + await this.executor.transaction(async () => { + const rootConversationId = await this.resolveRootConversationId( + input.parentConversationId, + ); + let history = await eventStore.loadHistory( + input.parentConversationId, + ); + if ( + !history.some( + (event) => + event.data.type === "turn_started" && + event.data.turnId === input.parentTurnId, + ) + ) { + throw new SubagentLineageConflictError( + "Subagent start does not match a parent turn", + ); + } + const existingInvocation = history.find( + (event) => + event.data.type === "subagent_started" && + event.data.subagentInvocationId === input.subagentInvocationId, + ); + const key = startedKey(input.subagentInvocationId); + let start = eventWithKey(history, key); + if (!start && existingInvocation) { + throw new SubagentLineageConflictError( + "Subagent invocation already has a different start reference", + ); + } + if (!start) { + await eventStore.append(input.parentConversationId, [ + { + data: expected, + idempotencyKey: key, + createdAtMs: input.nowMs ?? Date.now(), + }, + ]); + history = await eventStore.loadHistory(input.parentConversationId); + start = eventWithKey(history, key); + } + assertExactEvent(start, expected, "Subagent start retry"); + + const contextForkSeq = + input.historyMode === "shared" ? start.seq : null; + await this.establishChild({ + childConversationId: input.childConversationId, + contextForkSeq, + nowMs: input.nowMs ?? Date.now(), + parentConversationId: input.parentConversationId, + parentEventSeq: start.seq, + parentTurnId: input.parentTurnId, + rootConversationId, + }); + return { + childConversationId: input.childConversationId, + contextForkSeq, + parentConversationId: input.parentConversationId, + parentEventSeq: start.seq, + parentTurnId: input.parentTurnId, + rootConversationId, + }; + }), + ); + } + + /** Append the idempotent terminal reference after validating its exact start correlation. */ + async finish(input: FinishSubagentReferenceInput): Promise { + const eventStore = createSqlConversationEventStore(this.executor); + const expected = expectedEndData(input); + await withConversationEventLock( + this.executor, + input.parentConversationId, + async () => + await this.executor.transaction(async () => { + let history = await eventStore.loadHistory( + input.parentConversationId, + ); + const start = history.find( + (event) => + event.data.type === "subagent_started" && + event.data.subagentInvocationId === input.subagentInvocationId, + ); + if ( + !start || + start.data.type !== "subagent_started" || + start.data.parentTurnId !== input.parentTurnId + ) { + throw new SubagentLineageConflictError( + "Subagent finish does not match an exact parent start reference", + ); + } + await this.assertChildMatchesStart( + input.parentConversationId, + await this.resolveRootConversationId(input.parentConversationId), + start, + ); + + const key = endedKey(input.subagentInvocationId); + let end = eventWithKey(history, key); + if (!end) { + await eventStore.append(input.parentConversationId, [ + { + data: expected, + idempotencyKey: key, + createdAtMs: input.nowMs ?? Date.now(), + }, + ]); + history = await eventStore.loadHistory(input.parentConversationId); + end = eventWithKey(history, key); + } + assertExactEvent(end, expected, "Subagent finish retry"); + }), + ); + } + + private async establishChild(lineage: SubagentLineage & { nowMs: number }) { + const at = new Date(lineage.nowMs); + await this.executor + .db() + .insert(juniorConversations) + .values({ + conversationId: lineage.childConversationId, + parentConversationId: lineage.parentConversationId, + rootConversationId: lineage.rootConversationId, + parentTurnId: lineage.parentTurnId, + parentEventSeq: lineage.parentEventSeq, + contextForkSeq: lineage.contextForkSeq, + createdAt: at, + lastActivityAt: at, + updatedAt: at, + executionStatus: "idle", + }) + .onConflictDoNothing({ target: juniorConversations.conversationId }); + + const rows = await this.executor + .db() + .select() + .from(juniorConversations) + .where( + eq(juniorConversations.conversationId, lineage.childConversationId), + ) + .for("update"); + const child = rows[0]; + if (!child) { + throw new Error("Subagent child conversation was not created"); + } + const isBare = + child.parentConversationId === null && + child.rootConversationId === null && + child.parentTurnId === null && + child.parentEventSeq === null && + child.contextForkSeq === null && + child.transcriptPurgedAt === null && + child.source === null && + child.originType === null && + child.originId === null && + child.originRunId === null && + child.destinationId === null && + child.destination === null && + child.actorIdentityId === null && + child.creatorIdentityId === null && + child.credentialSubjectIdentityId === null && + child.actor === null && + child.channelName === null && + child.title === null && + child.executionUpdatedAt === null && + child.executionStatus === "idle" && + child.runId === null && + child.lastCheckpointAt === null && + child.lastEnqueuedAt === null && + child.durationMs === 0 && + child.usage === null && + child.executionDurationMs === 0 && + child.executionUsage === null && + child.metricRunId === null; + if (isBare) { + await this.executor + .db() + .update(juniorConversations) + .set({ + parentConversationId: lineage.parentConversationId, + rootConversationId: lineage.rootConversationId, + parentTurnId: lineage.parentTurnId, + parentEventSeq: lineage.parentEventSeq, + contextForkSeq: lineage.contextForkSeq, + }) + .where( + eq(juniorConversations.conversationId, lineage.childConversationId), + ); + return; + } + if ( + child.parentConversationId !== lineage.parentConversationId || + child.rootConversationId !== lineage.rootConversationId || + child.parentTurnId !== lineage.parentTurnId || + child.parentEventSeq !== lineage.parentEventSeq || + child.contextForkSeq !== lineage.contextForkSeq + ) { + throw new SubagentLineageConflictError( + "Subagent child conversation already has different lineage", + ); + } + } + + private async assertChildMatchesStart( + parentConversationId: string, + rootConversationId: string, + start: ConversationEvent, + ): Promise { + if (start.data.type !== "subagent_started") { + throw new SubagentLineageConflictError( + "Subagent start reference has an invalid event type", + ); + } + const rows = await this.executor + .db() + .select() + .from(juniorConversations) + .where( + eq(juniorConversations.conversationId, start.data.childConversationId), + ) + .for("update"); + const child = rows[0]; + const expectedFork = start.data.historyMode === "shared" ? start.seq : null; + if ( + !child || + child.parentConversationId !== parentConversationId || + child.rootConversationId !== rootConversationId || + child.parentTurnId !== start.data.parentTurnId || + child.parentEventSeq !== start.seq || + child.contextForkSeq !== expectedFork + ) { + throw new SubagentLineageConflictError( + "Subagent child lineage no longer matches its parent start reference", + ); + } + } + + private async resolveRootConversationId( + parentConversationId: string, + ): Promise { + let currentId = parentConversationId; + const seen = new Set(); + let declaredRootConversationId: string | undefined; + while (!seen.has(currentId)) { + seen.add(currentId); + const rows = await this.executor + .db() + .select({ + conversationId: juniorConversations.conversationId, + parentConversationId: juniorConversations.parentConversationId, + rootConversationId: juniorConversations.rootConversationId, + }) + .from(juniorConversations) + .where(eq(juniorConversations.conversationId, currentId)); + const row = rows[0]; + if (!row) { + throw new Error(`Parent conversation does not exist: ${currentId}`); + } + if ( + row.rootConversationId && + declaredRootConversationId && + row.rootConversationId !== declaredRootConversationId + ) { + throw new SubagentLineageConflictError( + "Conversation parent lineage declares conflicting roots", + ); + } + if (row.rootConversationId) { + declaredRootConversationId = row.rootConversationId; + } + if (!row.parentConversationId) { + if ( + declaredRootConversationId && + declaredRootConversationId !== row.conversationId + ) { + throw new SubagentLineageConflictError( + "Conversation parent lineage does not resolve to its declared root", + ); + } + return row.conversationId; + } + currentId = row.parentConversationId; + } + throw new SubagentLineageConflictError( + "Conversation parent lineage contains a cycle", + ); + } +} diff --git a/packages/junior/src/cli/upgrade.ts b/packages/junior/src/cli/upgrade.ts index 18decfe05..215a2a2f3 100644 --- a/packages/junior/src/cli/upgrade.ts +++ b/packages/junior/src/cli/upgrade.ts @@ -15,6 +15,7 @@ import { agentTurnSessionActorMigration } from "./upgrade/migrations/agent-turn- import { conversationUsageRepairMigration } from "./upgrade/migrations/conversation-usage"; import { conversationEventDataMigration } from "./upgrade/migrations/conversation-event-data"; import { conversationVisibleMessageEventsMigration } from "./upgrade/migrations/conversation-visible-message-events"; +import { conversationLineageMigration } from "./upgrade/migrations/conversation-lineage"; import type { MigrationContext, MigrationResult, @@ -32,6 +33,7 @@ const MIGRATIONS: UpgradeMigration[] = [ agentTurnSessionActorMigration, redisConversationStateMigration, coreSqlSchemaMigration, + conversationLineageMigration, sqlConversationMigration, conversationEventDataMigration, sqlConversationHistoryMigration, diff --git a/packages/junior/src/cli/upgrade/migrations/conversation-lineage.ts b/packages/junior/src/cli/upgrade/migrations/conversation-lineage.ts new file mode 100644 index 000000000..ed6359610 --- /dev/null +++ b/packages/junior/src/cli/upgrade/migrations/conversation-lineage.ts @@ -0,0 +1,130 @@ +import { and, asc, eq, isNotNull, isNull } from "drizzle-orm"; +import { getChatConfig } from "@/chat/config"; +import type { JuniorSqlExecutor } from "@/db/db"; +import { createJuniorSqlExecutor } from "@/db/executor"; +import { juniorConversations } from "@/db/schema"; +import type { MigrationContext, MigrationResult } from "../types"; + +const LINEAGE_BATCH_SIZE = 500; +const LINEAGE_BACKFILL_LOCK = "junior:upgrade:conversation-lineage"; + +async function resolveHistoricalRoot( + executor: JuniorSqlExecutor, + conversationId: string, +): Promise { + let currentId = conversationId; + const seen = new Set(); + let declaredRootConversationId: string | undefined; + while (!seen.has(currentId)) { + seen.add(currentId); + const rows = await executor + .db() + .select({ + conversationId: juniorConversations.conversationId, + parentConversationId: juniorConversations.parentConversationId, + rootConversationId: juniorConversations.rootConversationId, + }) + .from(juniorConversations) + .where(eq(juniorConversations.conversationId, currentId)); + const row = rows[0]; + if (!row) { + throw new Error( + `Conversation lineage references missing parent ${currentId}`, + ); + } + if ( + row.rootConversationId && + declaredRootConversationId && + row.rootConversationId !== declaredRootConversationId + ) { + throw new Error("Conversation lineage declares conflicting roots"); + } + if (row.rootConversationId) { + declaredRootConversationId = row.rootConversationId; + } + if (!row.parentConversationId) { + if ( + declaredRootConversationId && + declaredRootConversationId !== row.conversationId + ) { + throw new Error( + "Conversation lineage does not resolve to its declared root", + ); + } + return row.conversationId; + } + currentId = row.parentConversationId; + } + throw new Error(`Conversation lineage contains a cycle at ${currentId}`); +} + +/** Fill only historical root lineage; unknown fork/correlation stays null and isolated. */ +export async function migrateConversationLineage( + _context: MigrationContext, + options: { batchSize?: number; executor?: JuniorSqlExecutor } = {}, +): Promise { + let executor = options.executor; + let closeExecutor: (() => Promise) | undefined; + if (!executor) { + const { sql } = getChatConfig(); + executor = createJuniorSqlExecutor({ + connectionString: sql.databaseUrl, + driver: sql.driver, + }); + closeExecutor = () => executor!.close(); + } + const batchSize = Math.max( + 1, + Math.floor(options.batchSize ?? LINEAGE_BATCH_SIZE), + ); + let migrated = 0; + try { + while (true) { + const processed = await executor.withLock( + LINEAGE_BACKFILL_LOCK, + async () => { + const rows = await executor! + .db() + .select({ conversationId: juniorConversations.conversationId }) + .from(juniorConversations) + .where( + and( + isNotNull(juniorConversations.parentConversationId), + isNull(juniorConversations.rootConversationId), + ), + ) + .orderBy(asc(juniorConversations.conversationId)) + .limit(batchSize); + await executor!.transaction(async () => { + for (const row of rows) { + const rootConversationId = await resolveHistoricalRoot( + executor!, + row.conversationId, + ); + await executor! + .db() + .update(juniorConversations) + .set({ rootConversationId }) + .where( + eq(juniorConversations.conversationId, row.conversationId), + ); + } + }); + return rows.length; + }, + ); + if (processed === 0) { + break; + } + migrated += processed; + } + return { existing: 0, migrated, missing: 0, scanned: migrated }; + } finally { + await closeExecutor?.(); + } +} + +export const conversationLineageMigration = { + name: "backfill-conversation-lineage", + run: migrateConversationLineage, +}; diff --git a/packages/junior/src/db/schema/conversations.ts b/packages/junior/src/db/schema/conversations.ts index cbed75102..dcc01ee23 100644 --- a/packages/junior/src/db/schema/conversations.ts +++ b/packages/junior/src/db/schema/conversations.ts @@ -1,6 +1,7 @@ import { sql } from "drizzle-orm"; import { type AnyPgColumn, + check, index, integer, jsonb, @@ -65,6 +66,12 @@ export const juniorConversations = pgTable( executionUsage: jsonb("execution_usage_json").$type(), metricRunId: text("metric_run_id"), archivedAt: timestamptz("archived_at"), + rootConversationId: text("root_conversation_id").references( + (): AnyPgColumn => juniorConversations.conversationId, + ), + parentTurnId: text("parent_turn_id"), + parentEventSeq: integer("parent_event_seq"), + contextForkSeq: integer("context_fork_seq"), }, (table) => [ index("junior_conversations_last_activity_idx").on( @@ -92,5 +99,34 @@ export const juniorConversations = pgTable( table.lastActivityAt.desc(), ), index("junior_conversations_parent_idx").on(table.parentConversationId), + index("junior_conversations_parent_event_idx").on( + table.parentConversationId, + table.parentEventSeq, + ), + index("junior_conversations_root_idx").on(table.rootConversationId), + check( + "junior_conversations_not_own_parent_check", + sql`${table.parentConversationId} is null or ${table.parentConversationId} <> ${table.conversationId}`, + ), + check( + "junior_conversations_not_own_root_check", + sql`${table.rootConversationId} is null or ${table.rootConversationId} <> ${table.conversationId}`, + ), + check( + "junior_conversations_context_fork_check", + sql`${table.contextForkSeq} is null or (${table.parentEventSeq} is not null and ${table.contextForkSeq} <= ${table.parentEventSeq})`, + ), + check( + "junior_conversations_top_level_lineage_check", + sql`${table.parentConversationId} is not null or (${table.rootConversationId} is null and ${table.parentTurnId} is null and ${table.parentEventSeq} is null and ${table.contextForkSeq} is null)`, + ), + check( + "junior_conversations_parent_correlation_check", + sql`(${table.parentTurnId} is null) = (${table.parentEventSeq} is null)`, + ), + check( + "junior_conversations_correlated_root_check", + sql`${table.parentTurnId} is null or ${table.rootConversationId} is not null`, + ), ], ); diff --git a/packages/junior/tests/component/cli/upgrade-cli.test.ts b/packages/junior/tests/component/cli/upgrade-cli.test.ts index 7d6691e23..00affd590 100644 --- a/packages/junior/tests/component/cli/upgrade-cli.test.ts +++ b/packages/junior/tests/component/cli/upgrade-cli.test.ts @@ -49,7 +49,6 @@ const stateOnlyConversationStore: ConversationStore = { get: async () => undefined, getDestinationVisibility: async () => undefined, recordActivity: async () => {}, - ensureChildConversation: async () => {}, recordExecution: async () => {}, listByActivity: async () => [], }; diff --git a/packages/junior/tests/component/conversations/legacy-import.test.ts b/packages/junior/tests/component/conversations/legacy-import.test.ts index 49268647d..0af5702b1 100644 --- a/packages/junior/tests/component/conversations/legacy-import.test.ts +++ b/packages/junior/tests/component/conversations/legacy-import.test.ts @@ -135,6 +135,29 @@ describe("legacy conversation import", () => { await migrateSchema(fixture.sql); const eventStore = createSqlConversationEventStore(fixture.sql); const childId = `advisor:${CONVERSATION_ID}`; + const rootId = "legacy-root"; + const at = new Date(0); + await fixture.sql + .db() + .insert(juniorConversations) + .values([ + { + conversationId: rootId, + createdAt: at, + lastActivityAt: at, + updatedAt: at, + executionStatus: "idle", + }, + { + conversationId: CONVERSATION_ID, + parentConversationId: rootId, + rootConversationId: rootId, + createdAt: at, + lastActivityAt: at, + updatedAt: at, + executionStatus: "idle", + }, + ]); const entries: SessionLogEntry[] = [ { @@ -237,8 +260,13 @@ describe("legacy conversation import", () => { .db() .select({ conversationId: juniorConversations.conversationId, + contextForkSeq: juniorConversations.contextForkSeq, createdAt: juniorConversations.createdAt, lastActivityAt: juniorConversations.lastActivityAt, + parentConversationId: juniorConversations.parentConversationId, + parentEventSeq: juniorConversations.parentEventSeq, + parentTurnId: juniorConversations.parentTurnId, + rootConversationId: juniorConversations.rootConversationId, updatedAt: juniorConversations.updatedAt, }) .from(juniorConversations) @@ -257,8 +285,13 @@ describe("legacy conversation import", () => { }), expect.objectContaining({ conversationId: childId, + contextForkSeq: null, createdAt: new Date(960), lastActivityAt: new Date(961), + parentConversationId: CONVERSATION_ID, + parentEventSeq: null, + parentTurnId: null, + rootConversationId: rootId, updatedAt: new Date(961), }), ]), diff --git a/packages/junior/tests/component/services/subagent-lineage.test.ts b/packages/junior/tests/component/services/subagent-lineage.test.ts new file mode 100644 index 000000000..050496387 --- /dev/null +++ b/packages/junior/tests/component/services/subagent-lineage.test.ts @@ -0,0 +1,495 @@ +import { describe, expect, it } from "vitest"; +import type { PiMessage } from "@/chat/pi/messages"; +import { eq } from "drizzle-orm"; +import { migrateSchema } from "@/chat/conversations/sql/migrations"; +import { createSqlConversationEventStore } from "@/chat/conversations/sql/history"; +import { createSqlStore } from "@/chat/conversations/sql/store"; +import { resolveRootVisibility } from "@/chat/conversations/sql/purge"; +import { + SubagentLineageConflictError, + SubagentLineageService, +} from "@/chat/services/subagent-lineage"; +import { juniorConversations } from "@/db/schema"; +import { migrateConversationLineage } from "@/cli/upgrade/migrations/conversation-lineage"; +import { createLocalJuniorSqlFixture } from "../../fixtures/sql"; + +async function appendTurnStarted( + eventStore: ReturnType, + conversationId: string, + turnId: string, +) { + await eventStore.append(conversationId, [ + { + data: { + type: "turn_started", + turnId, + inputMessageIds: [`${turnId}:input`], + surface: "internal", + }, + idempotencyKey: `turn:${turnId}:started`, + createdAtMs: 1, + }, + ]); +} + +describe("subagent lineage service", () => { + it("records direct and nested references without copying child events", async () => { + const fixture = await createLocalJuniorSqlFixture(); + try { + await migrateSchema(fixture.sql); + const store = createSqlStore(fixture.sql); + const events = createSqlConversationEventStore(fixture.sql); + const lineage = new SubagentLineageService(fixture.sql); + await store.recordActivity({ + conversationId: "root", + destination: { + platform: "slack", + teamId: "T1", + channelId: "C1", + }, + source: "slack", + visibility: "private", + nowMs: 1, + }); + await appendTurnStarted(events, "root", "root-turn"); + + const child = await lineage.start({ + childConversationId: "child", + historyMode: "shared", + parentConversationId: "root", + parentTurnId: "root-turn", + subagentInvocationId: "child-call", + subagentKind: "task", + nowMs: 2, + }); + expect(child).toMatchObject({ + contextForkSeq: child.parentEventSeq, + rootConversationId: "root", + }); + await expect( + store.get({ conversationId: "child" }), + ).resolves.toMatchObject({ + lineage: { + contextForkSeq: child.parentEventSeq, + parentConversationId: "root", + parentEventSeq: child.parentEventSeq, + parentTurnId: "root-turn", + rootConversationId: "root", + }, + }); + await events.append("child", [ + { + data: { + type: "message", + message: { + role: "assistant", + content: [{ type: "text", text: "child-only" }], + timestamp: 3, + } as PiMessage, + }, + createdAtMs: 3, + }, + ]); + await appendTurnStarted(events, "child", "child-turn"); + const grandchild = await lineage.start({ + childConversationId: "grandchild", + historyMode: "isolated", + parentConversationId: "child", + parentTurnId: "child-turn", + subagentInvocationId: "grandchild-call", + subagentKind: "review", + nowMs: 4, + }); + expect(grandchild).toMatchObject({ + contextForkSeq: null, + rootConversationId: "root", + }); + await lineage.finish({ + parentConversationId: "child", + parentTurnId: "child-turn", + subagentInvocationId: "grandchild-call", + outcome: "success", + nowMs: 5, + }); + + const rootHistory = await events.loadHistory("root"); + const childHistory = await events.loadHistory("child"); + expect(rootHistory.map((event) => event.data.type)).toEqual([ + "turn_started", + "subagent_started", + ]); + expect(JSON.stringify(rootHistory)).not.toContain("child-only"); + expect(childHistory.map((event) => event.data.type)).toEqual([ + "message", + "turn_started", + "subagent_started", + "subagent_ended", + ]); + await expect( + resolveRootVisibility(fixture.sql, "grandchild"), + ).resolves.toEqual({ rootConversationId: "root", visibility: "private" }); + } finally { + await fixture.close(); + } + }); + + it("replays exact references and rejects mode, correlation, and reparenting conflicts", async () => { + const fixture = await createLocalJuniorSqlFixture(); + try { + await migrateSchema(fixture.sql); + const events = createSqlConversationEventStore(fixture.sql); + const lineage = new SubagentLineageService(fixture.sql); + await appendTurnStarted(events, "parent-a", "turn-a"); + await appendTurnStarted(events, "parent-b", "turn-b"); + await expect( + lineage.start({ + childConversationId: "orphaned-turn-child", + historyMode: "isolated", + parentConversationId: "parent-a", + parentTurnId: "missing-turn", + subagentInvocationId: "missing-turn-call", + subagentKind: "task", + }), + ).rejects.toBeInstanceOf(SubagentLineageConflictError); + const input = { + childConversationId: "child", + historyMode: "shared" as const, + parentConversationId: "parent-a", + parentTurnId: "turn-a", + subagentInvocationId: "call", + subagentKind: "task", + nowMs: 2, + }; + const first = await lineage.start(input); + await expect(lineage.start(input)).resolves.toEqual(first); + await lineage.finish({ + parentConversationId: "parent-a", + parentTurnId: "turn-a", + subagentInvocationId: "call", + outcome: "success", + nowMs: 3, + }); + await expect( + lineage.finish({ + parentConversationId: "parent-a", + parentTurnId: "turn-a", + subagentInvocationId: "call", + outcome: "success", + nowMs: 4, + }), + ).resolves.toBeUndefined(); + await expect( + lineage.start({ ...input, historyMode: "isolated" }), + ).rejects.toBeInstanceOf(SubagentLineageConflictError); + await expect( + lineage.start({ + ...input, + parentConversationId: "parent-b", + parentTurnId: "turn-b", + subagentInvocationId: "other-call", + }), + ).rejects.toBeInstanceOf(SubagentLineageConflictError); + expect( + (await events.loadHistory("parent-b")).map((event) => event.data.type), + ).toEqual(["turn_started"]); + expect( + (await events.loadHistory("parent-a")).filter( + (event) => event.data.type === "subagent_started", + ), + ).toHaveLength(1); + expect( + (await events.loadHistory("parent-a")).filter( + (event) => event.data.type === "subagent_ended", + ), + ).toHaveLength(1); + } finally { + await fixture.close(); + } + }); + + it("upgrades only metadata-bare children and rolls back a real-root conflict", async () => { + const fixture = await createLocalJuniorSqlFixture(); + try { + await migrateSchema(fixture.sql); + const store = createSqlStore(fixture.sql); + const events = createSqlConversationEventStore(fixture.sql); + const lineage = new SubagentLineageService(fixture.sql); + await appendTurnStarted(events, "parent", "turn"); + await events.append("bare-child", [ + { + data: { + type: "message", + message: { + role: "assistant", + content: [{ type: "text", text: "created before lineage" }], + timestamp: 1, + } as PiMessage, + }, + createdAtMs: 1, + }, + ]); + await expect( + lineage.start({ + childConversationId: "bare-child", + historyMode: "isolated", + parentConversationId: "parent", + parentTurnId: "turn", + subagentInvocationId: "bare-call", + subagentKind: "task", + nowMs: 2, + }), + ).resolves.toMatchObject({ rootConversationId: "parent" }); + + await store.recordActivity({ + conversationId: "real-root", + source: "local", + nowMs: 2, + }); + await expect( + lineage.start({ + childConversationId: "real-root", + historyMode: "isolated", + parentConversationId: "parent", + parentTurnId: "turn", + subagentInvocationId: "conflict-call", + subagentKind: "task", + nowMs: 3, + }), + ).rejects.toBeInstanceOf(SubagentLineageConflictError); + for (const [childConversationId, fields, invocationId] of [ + ["purged-root", { transcriptPurgedAt: new Date(3) }, "purged-call"], + [ + "execution-root", + { + executionStatus: "running" as const, + executionUpdatedAt: new Date(3), + lastCheckpointAt: new Date(3), + }, + "execution-call", + ], + ] as const) { + await events.append(childConversationId, [ + { + data: { + type: "message", + message: { + role: "assistant", + content: [{ type: "text", text: "existing root content" }], + timestamp: 2, + } as PiMessage, + }, + createdAtMs: 2, + }, + ]); + await fixture.sql + .db() + .update(juniorConversations) + .set(fields) + .where(eq(juniorConversations.conversationId, childConversationId)); + await expect( + lineage.start({ + childConversationId, + historyMode: "isolated", + parentConversationId: "parent", + parentTurnId: "turn", + subagentInvocationId: invocationId, + subagentKind: "task", + nowMs: 3, + }), + ).rejects.toBeInstanceOf(SubagentLineageConflictError); + } + expect( + (await events.loadHistory("parent")).filter( + (event) => + event.data.type === "subagent_started" && + event.data.subagentInvocationId === "conflict-call", + ), + ).toEqual([]); + } finally { + await fixture.close(); + } + }); + + it("atomically resolves different parents racing for one child", async () => { + const fixture = await createLocalJuniorSqlFixture(); + try { + await migrateSchema(fixture.sql); + const events = createSqlConversationEventStore(fixture.sql); + const lineage = new SubagentLineageService(fixture.sql); + await appendTurnStarted(events, "parent-a", "turn-a"); + await appendTurnStarted(events, "parent-b", "turn-b"); + const results = await Promise.allSettled([ + lineage.start({ + childConversationId: "contended-child", + historyMode: "isolated", + parentConversationId: "parent-a", + parentTurnId: "turn-a", + subagentInvocationId: "call-a", + subagentKind: "task", + }), + lineage.start({ + childConversationId: "contended-child", + historyMode: "isolated", + parentConversationId: "parent-b", + parentTurnId: "turn-b", + subagentInvocationId: "call-b", + subagentKind: "task", + }), + ]); + expect( + results.filter((result) => result.status === "fulfilled"), + ).toHaveLength(1); + expect( + results.filter((result) => result.status === "rejected"), + ).toHaveLength(1); + const childRows = await fixture.sql + .db() + .select() + .from(juniorConversations) + .where(eq(juniorConversations.conversationId, "contended-child")); + const winner = childRows[0]?.parentConversationId; + expect(winner === "parent-a" || winner === "parent-b").toBe(true); + for (const parent of ["parent-a", "parent-b"]) { + const starts = (await events.loadHistory(parent)).filter( + (event) => event.data.type === "subagent_started", + ); + expect(starts).toHaveLength(parent === winner ? 1 : 0); + } + } finally { + await fixture.close(); + } + }); + + it("backfills historical roots in bounded rerunnable batches without inventing forks", async () => { + const fixture = await createLocalJuniorSqlFixture(); + try { + await migrateSchema(fixture.sql); + const at = new Date(1); + await fixture.sql + .db() + .insert(juniorConversations) + .values([ + { + conversationId: "root", + createdAt: at, + lastActivityAt: at, + updatedAt: at, + executionStatus: "idle", + }, + { + conversationId: "child", + parentConversationId: "root", + createdAt: at, + lastActivityAt: at, + updatedAt: at, + executionStatus: "idle", + }, + { + conversationId: "grandchild", + parentConversationId: "child", + createdAt: at, + lastActivityAt: at, + updatedAt: at, + executionStatus: "idle", + }, + ]); + await expect( + migrateConversationLineage({} as never, { + batchSize: 1, + executor: fixture.sql, + }), + ).resolves.toMatchObject({ migrated: 2 }); + const rows = await fixture.sql + .db() + .select() + .from(juniorConversations) + .where(eq(juniorConversations.rootConversationId, "root")); + expect(rows).toHaveLength(2); + expect(rows).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + conversationId: "child", + contextForkSeq: null, + parentEventSeq: null, + parentTurnId: null, + }), + expect.objectContaining({ + conversationId: "grandchild", + contextForkSeq: null, + parentEventSeq: null, + parentTurnId: null, + }), + ]), + ); + await expect( + migrateConversationLineage({} as never, { executor: fixture.sql }), + ).resolves.toMatchObject({ migrated: 0 }); + } finally { + await fixture.close(); + } + }); + + it("rejects partial lineage while allowing historical uncorrelated children", async () => { + const fixture = await createLocalJuniorSqlFixture(); + try { + await migrateSchema(fixture.sql); + const at = new Date(1); + const base = { + createdAt: at, + lastActivityAt: at, + updatedAt: at, + executionStatus: "idle" as const, + }; + await fixture.sql + .db() + .insert(juniorConversations) + .values({ conversationId: "root", ...base }); + await expect( + fixture.sql + .db() + .insert(juniorConversations) + .values({ + conversationId: "poisoned-root", + rootConversationId: "root", + ...base, + }), + ).rejects.toThrow("Failed query"); + await expect( + fixture.sql + .db() + .insert(juniorConversations) + .values({ + conversationId: "half-correlated-child", + parentConversationId: "root", + rootConversationId: "root", + parentTurnId: "turn", + ...base, + }), + ).rejects.toThrow("Failed query"); + await expect( + fixture.sql + .db() + .insert(juniorConversations) + .values({ + conversationId: "rootless-correlated-child", + parentConversationId: "root", + parentTurnId: "turn", + parentEventSeq: 1, + ...base, + }), + ).rejects.toThrow("Failed query"); + await expect( + fixture.sql + .db() + .insert(juniorConversations) + .values({ + conversationId: "historical-child", + parentConversationId: "root", + ...base, + }), + ).resolves.toBeDefined(); + } finally { + await fixture.close(); + } + }); +}); diff --git a/packages/junior/tests/component/services/turn-session-record.test.ts b/packages/junior/tests/component/services/turn-session-record.test.ts index 7d5b34c0e..4385a21d8 100644 --- a/packages/junior/tests/component/services/turn-session-record.test.ts +++ b/packages/junior/tests/component/services/turn-session-record.test.ts @@ -32,7 +32,6 @@ function failingConversationStore(): ConversationStore { return { get: vi.fn(), getDestinationVisibility: vi.fn(async () => undefined), - ensureChildConversation: vi.fn(async () => undefined), recordActivity: vi.fn(async () => { throw new Error("conversation metadata unavailable"); }), diff --git a/packages/junior/tests/component/task-execution/conversation-work.test.ts b/packages/junior/tests/component/task-execution/conversation-work.test.ts index 4055fd7d8..6c1abb8a6 100644 --- a/packages/junior/tests/component/task-execution/conversation-work.test.ts +++ b/packages/junior/tests/component/task-execution/conversation-work.test.ts @@ -64,7 +64,6 @@ function failingMetadataStore(): ConversationStore { return { get: vi.fn(async () => undefined), getDestinationVisibility: vi.fn(async () => undefined), - ensureChildConversation: vi.fn(async () => undefined), recordActivity: vi.fn(), recordExecution: vi.fn(async () => { throw new Error("metadata unavailable"); @@ -77,7 +76,6 @@ function metadataEventsStore(events: string[]): ConversationStore { return { get: vi.fn(async () => undefined), getDestinationVisibility: vi.fn(async () => undefined), - ensureChildConversation: vi.fn(async () => undefined), recordActivity: vi.fn(), recordExecution: vi.fn(async () => { events.push("metadata"); diff --git a/packages/junior/tests/integration/api/conversations/stats.test.ts b/packages/junior/tests/integration/api/conversations/stats.test.ts index 30880c5e1..266b40361 100644 --- a/packages/junior/tests/integration/api/conversations/stats.test.ts +++ b/packages/junior/tests/integration/api/conversations/stats.test.ts @@ -2,6 +2,8 @@ import { describe, expect, test, vi } from "vitest"; import { readConversationStatsFromSql } from "@/api/conversations/stats.query"; import { migrateSchema } from "@/chat/conversations/sql/migrations"; import { createSqlStore } from "@/chat/conversations/sql/store"; +import { createSqlConversationEventStore } from "@/chat/conversations/sql/history"; +import { SubagentLineageService } from "@/chat/services/subagent-lineage"; import { juniorConversations } from "@/db/schema"; import { buildJuniorSqlConversation, @@ -120,9 +122,27 @@ describe("conversation stats API", () => { visibility: "public", nowMs: Date.parse("2026-02-01T10:00:00.000Z"), }); - await store.ensureChildConversation({ - conversationId: "advisor:child", + await createSqlConversationEventStore(fixture.sql).append( + "slack:C1:recent", + [ + { + data: { + type: "turn_started", + turnId: "turn-recent", + inputMessageIds: ["recent-input"], + surface: "slack", + }, + createdAtMs: Date.parse("2026-06-15T11:50:00.000Z"), + }, + ], + ); + await new SubagentLineageService(fixture.sql).start({ + childConversationId: "advisor:child", + historyMode: "isolated", parentConversationId: "slack:C1:recent", + parentTurnId: "turn-recent", + subagentInvocationId: "advisor-child", + subagentKind: "advisor", nowMs: Date.parse("2026-06-15T11:55:00.000Z"), }); diff --git a/packages/junior/tests/integration/dashboard-reporting.test.ts b/packages/junior/tests/integration/dashboard-reporting.test.ts index c19d038a0..d5fc0939d 100644 --- a/packages/junior/tests/integration/dashboard-reporting.test.ts +++ b/packages/junior/tests/integration/dashboard-reporting.test.ts @@ -3,6 +3,7 @@ import type { PiMessage } from "@/chat/pi/messages"; import { readConversationDetail } from "@/api/conversations/detail"; import { readConversationSubagent as readConversationSubagentTranscriptReport } from "@/api/conversations/subagent"; import { buildTurnFailureResponse } from "@/chat/logging"; +import { SubagentLineageService } from "@/chat/services/subagent-lineage"; vi.mock("@/chat/prompt", () => ({ buildSystemPrompt: vi.fn(() => "[system prompt]"), @@ -557,18 +558,38 @@ describe("dashboard reporting", () => { }); it("loads subagent transcript history from the child conversation", async () => { - const { getConversationEventStore, getConversationStore } = + const { getConversationEventStore, getSqlExecutor } = await import("@/chat/db"); const conversationId = "slack:C1:subagent-slices"; await confirmPublicSlackConversation(conversationId); const childConversationId = `task:${conversationId}`; - const conversationStore = getConversationStore(); const eventStore = getConversationEventStore(); + const lineage = new SubagentLineageService(getSqlExecutor()); - await conversationStore.ensureChildConversation({ - conversationId: childConversationId, + await eventStore.append(conversationId, [ + { + data: { + type: "turn_started", + turnId: "turn-subagent-slices", + inputMessageIds: ["subagent-slices-input"], + surface: "slack", + }, + createdAtMs: 1, + }, + ]); + + await lineage.start({ + childConversationId, + historyMode: "shared", + modelId: "openai/gpt-5.6-sol", parentConversationId: conversationId, + parentToolCallId: "task-plan", + parentTurnId: "turn-subagent-slices", + reasoningLevel: "high", + subagentInvocationId: "task-plan", + subagentKind: "task", + nowMs: 3, }); await eventStore.append(childConversationId, [ { @@ -641,34 +662,37 @@ describe("dashboard reporting", () => { }, ]); - // Repeated subagent calls share one child conversation, so both - // parent subagent markers name the same child history. - for (const subagentId of ["task-plan", "task-review"]) { - await eventStore.append(conversationId, [ - { - data: { - type: "subagent_started", - subagentInvocationId: subagentId, - subagentKind: "task", - parentToolCallId: subagentId, - childConversationId, - historyMode: "shared", - modelId: "openai/gpt-5.6-sol", - reasoningLevel: "high", - }, - createdAtMs: subagentId === "task-plan" ? 3 : 31, + await lineage.finish({ + parentConversationId: conversationId, + parentTurnId: "turn-subagent-slices", + subagentInvocationId: "task-plan", + outcome: "success", + nowMs: 25, + }); + // Preserve coverage for historical references that predate immutable + // per-invocation lineage and reused one advisor child transcript. + await eventStore.append(conversationId, [ + { + data: { + type: "subagent_started", + subagentInvocationId: "task-review", + subagentKind: "task", + parentToolCallId: "task-review", + childConversationId, + historyMode: "shared", + modelId: "openai/gpt-5.6-sol", + reasoningLevel: "high", }, - { - data: { - type: "subagent_ended", - subagentInvocationId: subagentId, - outcome: "success", - }, - createdAtMs: subagentId === "task-plan" ? 25 : 45, + createdAtMs: 31, + }, + { + data: { + type: "subagent_ended", + subagentInvocationId: "task-review", + outcome: "success", }, - ]); - } - await eventStore.append(conversationId, [ + createdAtMs: 45, + }, { data: { type: "subagent_started", @@ -732,19 +756,36 @@ describe("dashboard reporting", () => { }); it("redacts advisor subagent transcript history for private conversations", async () => { - const { getConversationEventStore, getConversationStore } = + const { getConversationEventStore, getSqlExecutor } = await import("@/chat/db"); const conversationId = "slack:D1:advisor-private"; const toolCallId = "advisor-private"; const privateAdvisorText = "private advisor question"; const childConversationId = `advisor:${conversationId}`; - const conversationStore = getConversationStore(); const eventStore = getConversationEventStore(); - await conversationStore.ensureChildConversation({ - conversationId: childConversationId, + await eventStore.append(conversationId, [ + { + data: { + type: "turn_started", + turnId: "turn-advisor-private", + inputMessageIds: ["advisor-private-input"], + surface: "slack", + }, + createdAtMs: 1, + }, + ]); + + await new SubagentLineageService(getSqlExecutor()).start({ + childConversationId, + historyMode: "shared", parentConversationId: conversationId, + parentToolCallId: toolCallId, + parentTurnId: "turn-advisor-private", + subagentInvocationId: toolCallId, + subagentKind: "advisor", + nowMs: 3, }); await eventStore.append(childConversationId, [ { @@ -759,27 +800,13 @@ describe("dashboard reporting", () => { createdAtMs: 10, }, ]); - await eventStore.append(conversationId, [ - { - data: { - type: "subagent_started", - subagentInvocationId: toolCallId, - subagentKind: "advisor", - parentToolCallId: toolCallId, - childConversationId, - historyMode: "shared", - }, - createdAtMs: 3, - }, - { - data: { - type: "subagent_ended", - subagentInvocationId: toolCallId, - outcome: "success", - }, - createdAtMs: 10, - }, - ]); + await new SubagentLineageService(getSqlExecutor()).finish({ + parentConversationId: conversationId, + parentTurnId: "turn-advisor-private", + subagentInvocationId: toolCallId, + outcome: "success", + nowMs: 10, + }); const transcript = await readConversationSubagentTranscriptReport( conversationId, diff --git a/packages/junior/tests/unit/conversations/legacy-history-import.test.ts b/packages/junior/tests/unit/conversations/legacy-history-import.test.ts index 83a853765..be10146c8 100644 --- a/packages/junior/tests/unit/conversations/legacy-history-import.test.ts +++ b/packages/junior/tests/unit/conversations/legacy-history-import.test.ts @@ -199,7 +199,7 @@ describe("convertLegacySessionLog", () => { subagentKind: "advisor", parentToolCallId: "call-1", childConversationId: `advisor:${CONVERSATION_ID}`, - historyMode: "shared", + historyMode: "isolated", }); expect(events[0]!.createdAtMs).toBe(50); expect(events[1]!.data).toEqual({ From 8237be06526896f6b4fe215c95c1127a447d48ab Mon Sep 17 00:00:00 2001 From: David Cramer Date: Wed, 15 Jul 2026 08:39:16 -0700 Subject: [PATCH 07/58] fix(migrations): Validate legacy adoption baseline Co-Authored-By: GPT-5 Codex --- packages/junior/migrations/README.md | 11 +- .../src/chat/conversations/sql/migrations.ts | 98 ++++- .../integration/conversation-sql.test.ts | 353 +++++++++++++----- 3 files changed, 361 insertions(+), 101 deletions(-) diff --git a/packages/junior/migrations/README.md b/packages/junior/migrations/README.md index dfea30191..e27b300b4 100644 --- a/packages/junior/migrations/README.md +++ b/packages/junior/migrations/README.md @@ -11,7 +11,16 @@ Drizzle ORM's migrator before any data backfills run. The `0000_initial.sql` baseline represents the schema already deployed by the pre-Drizzle Junior migration runner. During upgrade, existing installations adopt that baseline once; new installations execute it normally. All later -migrations are applied by Drizzle in journal order. +migrations are applied by Drizzle in journal order. Baseline adoption requires +the legacy `junior_agent_steps` base table and no +`junior_conversation_events` table. A post-cutover schema without its Drizzle +journal fails closed for operator repair instead of inferring completed +migrations from mutable table shape. Every expected legacy migration record +must retain its exact historical checksum; an ID alone cannot prove which SQL +ran. Legacy metrics adoption likewise requires either none or all four metric +columns, with the legacy metrics record agreeing with that physical state. The +later search index and `metric_run_id` column must also be absent because only +the Drizzle journal may prove those immutable migrations ran. `0004_conversation_events.sql` temporarily creates `junior_agent_steps` as an updatable 0.103.x compatibility view. It maps legacy `pi_message` reads and diff --git a/packages/junior/src/chat/conversations/sql/migrations.ts b/packages/junior/src/chat/conversations/sql/migrations.ts index 29c897b46..4f84fb5e4 100644 --- a/packages/junior/src/chat/conversations/sql/migrations.ts +++ b/packages/junior/src/chat/conversations/sql/migrations.ts @@ -13,6 +13,22 @@ const LEGACY_CORE_MIGRATION_IDS = [ "0005_conversation_transcripts", ] as const; const LEGACY_METRICS_MIGRATION_ID = "0006_conversation_metrics"; +// Pinned output of the pre-Drizzle runner's SHA-256(statement + NUL) algorithm. +const LEGACY_CORE_MIGRATION_CHECKSUMS = { + "0001_conversation_core": + "78fe050d8bec8ba18e2e3192497b3d8ad6b45fbb66ad4859377fb2202ed57651", + "0002_slack_destination_visibility_backfill": + "fb590a09fa51db471a748e3d7abb4137f521ee8df97f6e9ef5563121be98c394", + "0003_user_identities": + "67d9c9c26cbd76213614eb6d7a7cc7e2501fc20e92321eb5176a08ce39cd2efb", + "0004_actor_cutover": + "d41b8bfa66b8a88d69e84af38950025ba4c9be56341565cbe1411f0ca50c1dc2", + "0005_conversation_transcripts": + "add299d1b254e023f89b5993c417dd2248dc009e874efdeaf31ec0732e0d4fb4", + "0006_conversation_metrics": + "7c7ca5c9e11ed4b0e14737fd90d3348ea46e306c88fdf31199b7afb2a11c6a41", +} as const; +type LegacyCoreMigrationId = keyof typeof LEGACY_CORE_MIGRATION_CHECKSUMS; const MIGRATIONS_TABLE = "__drizzle_junior_core"; /** Resolve the packaged Drizzle migration directory in source or built output. */ @@ -44,8 +60,8 @@ SELECT } const migrations = readMigrationFiles({ migrationsFolder }); - const [metrics] = await executor.query<{ complete: boolean }>(` -SELECT count(*) = 4 AS complete + const [metrics] = await executor.query<{ columnCount: number }>(` +SELECT count(*)::integer AS "columnCount" FROM information_schema.columns WHERE table_schema = 'public' AND table_name = 'junior_conversations' @@ -60,22 +76,84 @@ WHERE table_schema = 'public' checksum: string; id: string; }>("SELECT id, checksum FROM junior_schema_migrations"); - const expectedIds = metrics?.complete + const legacyRecordsById = new Map( + legacyRecords.map((record) => [record.id, record.checksum]), + ); + const metricColumnCount = metrics?.columnCount ?? 0; + if (metricColumnCount !== 0 && metricColumnCount !== 4) { + throw new Error( + `Cannot adopt partial legacy metrics state: found ${metricColumnCount} of 4 required columns`, + ); + } + const metricsComplete = metricColumnCount === 4; + const hasMetricsRecord = legacyRecordsById.has(LEGACY_METRICS_MIGRATION_ID); + if (metricsComplete !== hasMetricsRecord) { + throw new Error( + "Cannot adopt legacy core migration state: legacy metrics migration record does not match physical metric columns", + ); + } + const expectedIds: readonly LegacyCoreMigrationId[] = metricsComplete ? [...LEGACY_CORE_MIGRATION_IDS, LEGACY_METRICS_MIGRATION_ID] : [...LEGACY_CORE_MIGRATION_IDS]; - const validIds = new Set( - legacyRecords - .filter((record) => record.checksum.trim().length > 0) - .map((record) => record.id), - ); - const missingIds = expectedIds.filter((id) => !validIds.has(id)); + const missingIds = expectedIds.filter((id) => !legacyRecordsById.has(id)); if (missingIds.length > 0) { throw new Error( `Cannot adopt partial legacy core migration state; missing: ${missingIds.join(", ")}`, ); } + const checksumMismatches = expectedIds.filter( + (id) => legacyRecordsById.get(id) !== LEGACY_CORE_MIGRATION_CHECKSUMS[id], + ); + if (checksumMismatches.length > 0) { + throw new Error( + `Cannot adopt legacy core migration state: checksum mismatch: ${checksumMismatches.join(", ")}`, + ); + } + + const [baseline] = await executor.query<{ + conversationEventsTable: string | null; + legacyAgentStepsTable: boolean; + metricRunIdColumn: boolean; + searchIndex: string | null; + }>(` +SELECT + to_regclass('public.junior_conversation_events')::text AS "conversationEventsTable", + to_regclass('public.junior_conversation_messages_search_idx')::text AS "searchIndex", + EXISTS ( + SELECT 1 + FROM information_schema.tables + WHERE table_schema = 'public' + AND table_name = 'junior_agent_steps' + AND table_type = 'BASE TABLE' + ) AS "legacyAgentStepsTable", + EXISTS ( + SELECT 1 + FROM information_schema.columns + WHERE table_schema = 'public' + AND table_name = 'junior_conversations' + AND column_name = 'metric_run_id' + ) AS "metricRunIdColumn" +`); + if (!baseline?.legacyAgentStepsTable || baseline.conversationEventsTable) { + throw new Error( + "Cannot adopt legacy core migration state: expected the pre-Drizzle junior_agent_steps table and no junior_conversation_events table", + ); + } + const postBaselineMarkers = [ + baseline.searchIndex + ? "junior_conversation_messages_search_idx" + : undefined, + baseline.metricRunIdColumn + ? "junior_conversations.metric_run_id" + : undefined, + ].filter((marker): marker is string => marker !== undefined); + if (postBaselineMarkers.length > 0) { + throw new Error( + `Cannot adopt legacy core migration state: post-baseline schema markers are already present: ${postBaselineMarkers.join(", ")}`, + ); + } - const migration = metrics?.complete ? migrations[1] : migrations[0]; + const migration = metricsComplete ? migrations[1] : migrations[0]; if (!migration) { throw new Error("No core Drizzle migrations were packaged"); } diff --git a/packages/junior/tests/integration/conversation-sql.test.ts b/packages/junior/tests/integration/conversation-sql.test.ts index aee638578..811fd91ca 100644 --- a/packages/junior/tests/integration/conversation-sql.test.ts +++ b/packages/junior/tests/integration/conversation-sql.test.ts @@ -1,6 +1,9 @@ +import { fileURLToPath } from "node:url"; import { getTableColumns, getTableName } from "drizzle-orm"; +import { readMigrationFiles } from "drizzle-orm/migrator"; import { describe, expect, it } from "vitest"; import { migrateSchema } from "@/chat/conversations/sql/migrations"; +import type { JuniorSqlMigrationExecutor } from "@/db/db"; import { createPostgresJuniorSqlExecutor } from "@/db/postgres"; import { juniorSqlSchema as schema } from "@/db/schema"; import { createSqlStore } from "@/chat/conversations/sql/store"; @@ -15,6 +18,87 @@ import { hasJuniorPostgresTestDatabase, } from "../fixtures/postgres/fixture"; +const coreMigrations = readMigrationFiles({ + migrationsFolder: fileURLToPath(new URL("../../migrations", import.meta.url)), +}); +const legacyCoreMigrationChecksums = { + "0001_conversation_core": + "78fe050d8bec8ba18e2e3192497b3d8ad6b45fbb66ad4859377fb2202ed57651", + "0002_slack_destination_visibility_backfill": + "fb590a09fa51db471a748e3d7abb4137f521ee8df97f6e9ef5563121be98c394", + "0003_user_identities": + "67d9c9c26cbd76213614eb6d7a7cc7e2501fc20e92321eb5176a08ce39cd2efb", + "0004_actor_cutover": + "d41b8bfa66b8a88d69e84af38950025ba4c9be56341565cbe1411f0ca50c1dc2", + "0005_conversation_transcripts": + "add299d1b254e023f89b5993c417dd2248dc009e874efdeaf31ec0732e0d4fb4", + "0006_conversation_metrics": + "7c7ca5c9e11ed4b0e14737fd90d3348ea46e306c88fdf31199b7afb2a11c6a41", +} as const; + +async function applyCoreMigration( + executor: JuniorSqlMigrationExecutor, + index: number, + options: { statementLimit?: number } = {}, +): Promise { + const migration = coreMigrations[index]; + if (!migration) { + throw new Error(`Missing core migration at index ${index}`); + } + await executor.transaction(async () => { + const statements = + options.statementLimit === undefined + ? migration.sql + : migration.sql.slice(0, options.statementLimit); + for (const statement of statements) { + if (statement.trim()) await executor.execute(statement); + } + }); +} + +async function seedLegacyCoreSchema( + executor: JuniorSqlMigrationExecutor, + options: { metrics?: boolean } = {}, +): Promise { + await applyCoreMigration(executor, 0); + if (options.metrics) await applyCoreMigration(executor, 1); +} + +async function recordLegacyCoreMigrations( + executor: JuniorSqlMigrationExecutor, + options: { metrics?: boolean } = {}, +): Promise { + await executor.execute(` +CREATE TABLE junior_schema_migrations ( + id TEXT PRIMARY KEY, + checksum TEXT NOT NULL, + applied_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP +) +`); + const records = Object.entries(legacyCoreMigrationChecksums).filter( + ([id]) => options.metrics || id !== "0006_conversation_metrics", + ); + for (const [id, checksum] of records) { + await executor.execute( + "INSERT INTO junior_schema_migrations (id, checksum) VALUES ($1, $2)", + [id, checksum], + ); + } +} + +async function expectNoDrizzleMigrationState( + executor: JuniorSqlMigrationExecutor, +): Promise { + const [state] = await executor.query<{ schemaExists: boolean }>(` +SELECT EXISTS ( + SELECT 1 + FROM pg_namespace + WHERE nspname = 'drizzle' +) AS "schemaExists" +`); + expect(state?.schemaExists).toBe(false); +} + describe("conversation SQL local mode", () => { it("creates migrated tables matching the Drizzle schema", async () => { const fixture = await createLocalJuniorSqlFixture(); @@ -44,10 +128,14 @@ ORDER BY table_name ASC, ordinal_position ASC const expected = new Map( Object.values(schema).map((table) => [ getTableName(table), - Object.values(getTableColumns(table)).map((column) => column.name), + Object.values(getTableColumns(table)) + .map((column) => column.name) + .sort(), ]), ); + for (const columns of actual.values()) columns.sort(); + expect(actual).toEqual(expected); expect(actual.has("junior_conversation_inbound_messages")).toBe(false); @@ -141,7 +229,7 @@ VALUES ('host-migration', 9999999999999) "SELECT count(*)::integer AS count FROM drizzle.__drizzle_junior_core", ); expect(host?.count).toBe(1); - expect(core?.count).toBe(5); + expect(core?.count).toBe(coreMigrations.length); } finally { await fixture.close(); } @@ -156,36 +244,8 @@ VALUES ('host-migration', 9999999999999) }); try { - await migrateSchema(fixture.sql); - await fixture.sql.execute(` -CREATE TABLE junior_schema_migrations ( - id TEXT PRIMARY KEY, - checksum TEXT NOT NULL, - applied_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP -) -`); - await fixture.sql.execute(` -INSERT INTO junior_schema_migrations (id, checksum) -VALUES - ('0001_conversation_core', 'legacy-checksum-1'), - ('0002_slack_destination_visibility_backfill', 'legacy-checksum-2'), - ('0003_user_identities', 'legacy-checksum-3'), - ('0004_actor_cutover', 'legacy-checksum-4'), - ('0005_conversation_transcripts', 'legacy-checksum-5') -`); - await fixture.sql.execute("DROP TABLE drizzle.__drizzle_junior_core"); - await fixture.sql.execute( - "DROP INDEX junior_conversation_messages_search_idx", - ); - await fixture.sql.execute(` -ALTER TABLE junior_conversations - DROP COLUMN archived_at, - DROP COLUMN duration_ms, - DROP COLUMN usage_json, - DROP COLUMN execution_duration_ms, - DROP COLUMN execution_usage_json, - DROP COLUMN metric_run_id -`); + await seedLegacyCoreSchema(fixture.sql); + await recordLegacyCoreMigrations(fixture.sql); await Promise.all([ fixture.sql.query("SELECT 1"), second.query("SELECT 1"), @@ -195,7 +255,7 @@ ALTER TABLE junior_conversations const [journal] = await fixture.sql.query<{ count: number }>( "SELECT count(*)::integer AS count FROM drizzle.__drizzle_junior_core", ); - expect(journal?.count).toBe(5); + expect(journal?.count).toBe(coreMigrations.length); } finally { await second.close(); await fixture.close(); @@ -263,7 +323,7 @@ WHERE conversation_id = $1 "SELECT count(*)::integer AS count FROM drizzle.__drizzle_junior_core", ); - expect(migrationRows?.count).toBe(5); + expect(migrationRows?.count).toBe(coreMigrations.length); expect(rows).toHaveLength(1); expect(rows[0]).toMatchObject({ conversation_id: "slack:C123:1718123456.000000", @@ -291,36 +351,8 @@ WHERE conversation_id = $1 const fixture = await createLocalJuniorSqlFixture(); try { - await migrateSchema(fixture.sql); - await fixture.sql.execute(` -CREATE TABLE junior_schema_migrations ( - id TEXT PRIMARY KEY, - checksum TEXT NOT NULL, - applied_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP -) -`); - await fixture.sql.execute(` -INSERT INTO junior_schema_migrations (id, checksum) -VALUES - ('0001_conversation_core', 'legacy-checksum-1'), - ('0002_slack_destination_visibility_backfill', 'legacy-checksum-2'), - ('0003_user_identities', 'legacy-checksum-3'), - ('0004_actor_cutover', 'legacy-checksum-4'), - ('0005_conversation_transcripts', 'legacy-checksum-5') -`); - await fixture.sql.execute("DROP SCHEMA drizzle CASCADE"); - await fixture.sql.execute( - "DROP INDEX junior_conversation_messages_search_idx", - ); - await fixture.sql.execute(` -ALTER TABLE junior_conversations - DROP COLUMN archived_at, - DROP COLUMN duration_ms, - DROP COLUMN usage_json, - DROP COLUMN execution_duration_ms, - DROP COLUMN execution_usage_json, - DROP COLUMN metric_run_id -`); + await seedLegacyCoreSchema(fixture.sql); + await recordLegacyCoreMigrations(fixture.sql); await migrateSchema(fixture.sql); @@ -347,41 +379,39 @@ ORDER BY column_name "execution_usage_json", "usage_json", ]); - expect(migrationRows?.count).toBe(5); + expect(migrationRows?.count).toBe(coreMigrations.length); } finally { await fixture.close(); } }); - it("adopts a fully migrated legacy schema without replaying metrics", async () => { + it("ignores unrelated records in the shared legacy migration journal", async () => { const fixture = await createLocalJuniorSqlFixture(); try { - await migrateSchema(fixture.sql); - await fixture.sql.execute(` -CREATE TABLE junior_schema_migrations ( - id TEXT PRIMARY KEY, - checksum TEXT NOT NULL, - applied_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP -) -`); - await fixture.sql.execute(` -INSERT INTO junior_schema_migrations (id, checksum) -VALUES - ('0001_conversation_core', 'legacy-checksum-1'), - ('0002_slack_destination_visibility_backfill', 'legacy-checksum-2'), - ('0003_user_identities', 'legacy-checksum-3'), - ('0004_actor_cutover', 'legacy-checksum-4'), - ('0005_conversation_transcripts', 'legacy-checksum-5'), - ('0006_conversation_metrics', 'legacy-checksum-6') -`); - await fixture.sql.execute("DROP SCHEMA drizzle CASCADE"); + await seedLegacyCoreSchema(fixture.sql); + await recordLegacyCoreMigrations(fixture.sql); await fixture.sql.execute( - "DROP INDEX junior_conversation_messages_search_idx", + "INSERT INTO junior_schema_migrations (id, checksum) VALUES ('plugin:example:0001', 'plugin-checksum')", ); - await fixture.sql.execute( - "ALTER TABLE junior_conversations DROP COLUMN archived_at", + + await migrateSchema(fixture.sql); + + const [migrationRows] = await fixture.sql.query<{ count: number }>( + "SELECT count(*)::integer AS count FROM drizzle.__drizzle_junior_core", ); + expect(migrationRows?.count).toBe(coreMigrations.length); + } finally { + await fixture.close(); + } + }); + + it("adopts a fully migrated legacy schema without replaying metrics", async () => { + const fixture = await createLocalJuniorSqlFixture(); + + try { + await seedLegacyCoreSchema(fixture.sql, { metrics: true }); + await recordLegacyCoreMigrations(fixture.sql, { metrics: true }); await migrateSchema(fixture.sql); @@ -404,7 +434,7 @@ WHERE table_schema = 'public' ) ORDER BY column_name `); - expect(migrationRows?.count).toBe(4); + expect(migrationRows?.count).toBe(coreMigrations.length - 1); expect(searchIndex?.exists).toBe(true); expect(metricColumns.map((row) => row.column_name)).toEqual([ "duration_ms", @@ -417,6 +447,145 @@ ORDER BY column_name } }); + it("rejects legacy adoption after the event-table cutover", async () => { + const fixture = await createLocalJuniorSqlFixture(); + + try { + await migrateSchema(fixture.sql); + await recordLegacyCoreMigrations(fixture.sql, { metrics: true }); + await fixture.sql.execute("DROP SCHEMA drizzle CASCADE"); + + await expect(migrateSchema(fixture.sql)).rejects.toThrow( + "Cannot adopt legacy core migration state: expected the pre-Drizzle junior_agent_steps table and no junior_conversation_events table", + ); + await expectNoDrizzleMigrationState(fixture.sql); + } finally { + await fixture.close(); + } + }); + + it("rejects partially applied legacy metric columns without mutation", async () => { + const fixture = await createLocalJuniorSqlFixture(); + + try { + await seedLegacyCoreSchema(fixture.sql); + await applyCoreMigration(fixture.sql, 1, { statementLimit: 1 }); + await recordLegacyCoreMigrations(fixture.sql); + + await expect(migrateSchema(fixture.sql)).rejects.toThrow( + "Cannot adopt partial legacy metrics state: found 1 of 4 required columns", + ); + await expectNoDrizzleMigrationState(fixture.sql); + } finally { + await fixture.close(); + } + }); + + it("rejects a legacy metrics record without metric columns", async () => { + const fixture = await createLocalJuniorSqlFixture(); + + try { + await seedLegacyCoreSchema(fixture.sql); + await recordLegacyCoreMigrations(fixture.sql, { metrics: true }); + + await expect(migrateSchema(fixture.sql)).rejects.toThrow( + "Cannot adopt legacy core migration state: legacy metrics migration record does not match physical metric columns", + ); + await expectNoDrizzleMigrationState(fixture.sql); + } finally { + await fixture.close(); + } + }); + + it("rejects complete metric columns without a legacy metrics record", async () => { + const fixture = await createLocalJuniorSqlFixture(); + + try { + await seedLegacyCoreSchema(fixture.sql, { metrics: true }); + await recordLegacyCoreMigrations(fixture.sql); + + await expect(migrateSchema(fixture.sql)).rejects.toThrow( + "Cannot adopt legacy core migration state: legacy metrics migration record does not match physical metric columns", + ); + await expectNoDrizzleMigrationState(fixture.sql); + } finally { + await fixture.close(); + } + }); + + it("rejects a changed legacy core checksum without mutation", async () => { + const fixture = await createLocalJuniorSqlFixture(); + + try { + await seedLegacyCoreSchema(fixture.sql); + await recordLegacyCoreMigrations(fixture.sql); + await fixture.sql.execute( + "UPDATE junior_schema_migrations SET checksum = 'changed' WHERE id = '0003_user_identities'", + ); + + await expect(migrateSchema(fixture.sql)).rejects.toThrow( + "Cannot adopt legacy core migration state: checksum mismatch: 0003_user_identities", + ); + await expectNoDrizzleMigrationState(fixture.sql); + } finally { + await fixture.close(); + } + }); + + it("rejects a changed legacy metrics checksum without mutation", async () => { + const fixture = await createLocalJuniorSqlFixture(); + + try { + await seedLegacyCoreSchema(fixture.sql, { metrics: true }); + await recordLegacyCoreMigrations(fixture.sql, { metrics: true }); + await fixture.sql.execute( + "UPDATE junior_schema_migrations SET checksum = 'changed' WHERE id = '0006_conversation_metrics'", + ); + + await expect(migrateSchema(fixture.sql)).rejects.toThrow( + "Cannot adopt legacy core migration state: checksum mismatch: 0006_conversation_metrics", + ); + await expectNoDrizzleMigrationState(fixture.sql); + } finally { + await fixture.close(); + } + }); + + it("rejects legacy adoption after immutable migration 0002 ran", async () => { + const fixture = await createLocalJuniorSqlFixture(); + + try { + await seedLegacyCoreSchema(fixture.sql, { metrics: true }); + await applyCoreMigration(fixture.sql, 2); + await recordLegacyCoreMigrations(fixture.sql, { metrics: true }); + + await expect(migrateSchema(fixture.sql)).rejects.toThrow( + "Cannot adopt legacy core migration state: post-baseline schema markers are already present: junior_conversation_messages_search_idx", + ); + await expectNoDrizzleMigrationState(fixture.sql); + } finally { + await fixture.close(); + } + }); + + it("rejects legacy adoption after immutable migration 0003 ran", async () => { + const fixture = await createLocalJuniorSqlFixture(); + + try { + await seedLegacyCoreSchema(fixture.sql, { metrics: true }); + await applyCoreMigration(fixture.sql, 2); + await applyCoreMigration(fixture.sql, 3); + await recordLegacyCoreMigrations(fixture.sql, { metrics: true }); + + await expect(migrateSchema(fixture.sql)).rejects.toThrow( + "Cannot adopt legacy core migration state: post-baseline schema markers are already present: junior_conversation_messages_search_idx, junior_conversations.metric_run_id", + ); + await expectNoDrizzleMigrationState(fixture.sql); + } finally { + await fixture.close(); + } + }); + it("rejects partial pre-Drizzle core migration state", async () => { const fixture = await createLocalJuniorSqlFixture(); @@ -430,12 +599,16 @@ CREATE TABLE junior_schema_migrations ( `); await fixture.sql.execute(` INSERT INTO junior_schema_migrations (id, checksum) -VALUES ('0001_conversation_core', 'legacy-checksum-1') +VALUES ( + '0001_conversation_core', + '78fe050d8bec8ba18e2e3192497b3d8ad6b45fbb66ad4859377fb2202ed57651' +) `); await expect(migrateSchema(fixture.sql)).rejects.toThrow( "Cannot adopt partial legacy core migration state", ); + await expectNoDrizzleMigrationState(fixture.sql); } finally { await fixture.close(); } From f2f9acb22792c972e1808d92605ca3b9e6f7d220 Mon Sep 17 00:00:00 2001 From: David Cramer Date: Wed, 15 Jul 2026 08:47:15 -0700 Subject: [PATCH 08/58] feat(conversations): Compose shared child context Co-Authored-By: GPT-5 Codex --- .../changes/conversation-event-log/tasks.md | 2 +- .../junior/src/chat/conversations/README.md | 12 + .../junior/src/chat/conversations/history.ts | 35 +- .../src/chat/conversations/projection.ts | 430 +++++++++++-- .../junior/src/chat/state/turn-session.ts | 13 +- .../shared-lineage-projection.test.ts | 581 ++++++++++++++++++ 6 files changed, 1025 insertions(+), 48 deletions(-) create mode 100644 packages/junior/tests/component/conversations/shared-lineage-projection.test.ts diff --git a/openspec/changes/conversation-event-log/tasks.md b/openspec/changes/conversation-event-log/tasks.md index e293c9b54..e18a7948b 100644 --- a/openspec/changes/conversation-event-log/tasks.md +++ b/openspec/changes/conversation-event-log/tasks.md @@ -48,7 +48,7 @@ - [x] Add parent/root conversation lineage and parent turn/event correlation. - [x] Record subagent start and finish references without copying child events. - [x] Represent shared context with an immutable parent sequence fork point. -- [ ] Verify isolated and shared child Pi projections and inherited privacy. +- [x] Verify isolated and shared child Pi projections and inherited privacy. ## 5. Cleanup And Verification diff --git a/packages/junior/src/chat/conversations/README.md b/packages/junior/src/chat/conversations/README.md index 7b4461570..6f6b20b5a 100644 --- a/packages/junior/src/chat/conversations/README.md +++ b/packages/junior/src/chat/conversations/README.md @@ -18,6 +18,11 @@ conversation events, compaction boundaries, search, retention, and legacy import - Child rows carry immutable parent/root, parent-turn, and exact parent-event correlation. Shared children additionally retain that parent sequence as their context fork; isolated children retain no fork. +- The Pi adapter recursively composes a shared child's pinned ancestor prefix + with only the child's current local epoch. Initial and inheriting rollback + markers preserve that relationship; compaction, handoff, and isolated epochs + are self-contained. Child sequence cursors count only child-local events, and + commits reject any mutation of the inherited prefix. - Provider payloads and old state-store mirrors are migration inputs, not canonical product records. @@ -54,6 +59,13 @@ must not become a dashboard or external API payload. - Establish child lineage and its parent `subagent_started` reference through the SQL-backed lineage service in one transaction. Retries must match the original parent, root, turn, event, and history mode exactly. +- Never copy inherited Pi messages into the child log. Resolve every restore + path through the lineage-aware Pi projection and bound each ancestor at its + immutable fork before reading the next descendant. +- Shared projection has an explicit maximum lineage depth. Parent event reads + currently load the parent's full retained history before selecting the fork; + a bounded store read remains a future optimization rather than a second + projection contract. ## Visibility And Retention diff --git a/packages/junior/src/chat/conversations/history.ts b/packages/junior/src/chat/conversations/history.ts index 7c35d73fd..70bc56a97 100644 --- a/packages/junior/src/chat/conversations/history.ts +++ b/packages/junior/src/chat/conversations/history.ts @@ -45,6 +45,7 @@ const contextEpochStartedEventDataSchema = z.union([ reason: z.literal("initial"), modelProfile: z.literal("standard"), modelId: z.string().min(1), + inheritsLineageContext: z.literal(true).optional(), }) .strict(), z @@ -58,7 +59,7 @@ const contextEpochStartedEventDataSchema = z.union([ z .object({ type: z.literal("context_epoch_started"), - reason: z.union([z.literal("compaction"), z.literal("rollback")]), + reason: z.literal("compaction"), // TODO(v0.104.0): Remove support for deployed compaction/rollback markers // without model bindings after those rows pass the retention horizon. modelProfile: z.undefined().optional(), @@ -68,11 +69,29 @@ const contextEpochStartedEventDataSchema = z.union([ z .object({ type: z.literal("context_epoch_started"), - reason: z.union([z.literal("compaction"), z.literal("rollback")]), + reason: z.literal("compaction"), modelProfile: modelProfileSchema, modelId: z.string().min(1), }) .strict(), + z + .object({ + type: z.literal("context_epoch_started"), + reason: z.literal("rollback"), + modelProfile: z.undefined().optional(), + modelId: z.undefined().optional(), + inheritsLineageContext: z.literal(true).optional(), + }) + .strict(), + z + .object({ + type: z.literal("context_epoch_started"), + reason: z.literal("rollback"), + modelProfile: modelProfileSchema, + modelId: z.string().min(1), + inheritsLineageContext: z.literal(true).optional(), + }) + .strict(), ]); const contextEpochMessageSchema = z @@ -91,6 +110,7 @@ export const contextEpochStartSchema = z.discriminatedUnion("reason", [ modelProfile: z.literal("standard"), modelId: z.string().min(1), messages: z.array(contextEpochMessageSchema), + inheritsLineageContext: z.literal(true).optional(), }) .strict(), z @@ -103,10 +123,19 @@ export const contextEpochStartSchema = z.discriminatedUnion("reason", [ .strict(), z .object({ - reason: z.union([z.literal("compaction"), z.literal("rollback")]), + reason: z.literal("compaction"), + modelProfile: modelProfileSchema, + modelId: z.string().min(1), + messages: z.array(contextEpochMessageSchema), + }) + .strict(), + z + .object({ + reason: z.literal("rollback"), modelProfile: modelProfileSchema, modelId: z.string().min(1), messages: z.array(contextEpochMessageSchema), + inheritsLineageContext: z.literal(true).optional(), }) .strict(), ]); diff --git a/packages/junior/src/chat/conversations/projection.ts b/packages/junior/src/chat/conversations/projection.ts index 9858025f1..2e322e5a6 100644 --- a/packages/junior/src/chat/conversations/projection.ts +++ b/packages/junior/src/chat/conversations/projection.ts @@ -14,8 +14,18 @@ import { import type { AuthorizationKind, ConversationEvent, + ConversationEventStore, } from "@/chat/conversations/history"; -import { getConversationEventStore } from "@/chat/db"; +import type { ConversationStore } from "@/chat/conversations/store"; +import { + getConversationEventStore, + getConversationStore, + getSqlExecutor, +} from "@/chat/db"; +import type { JuniorSqlDatabase } from "@/db/db"; +import { createSqlStore } from "@/chat/conversations/sql/store"; +import { createSqlConversationEventStore } from "@/chat/conversations/sql/history"; +import { withConversationEventLock } from "@/chat/conversations/sql/event-lock"; import { ensureConversationEventHistory, loadCurrentConversationEvents, @@ -117,22 +127,264 @@ interface ScopedConversation { conversationId: string; } +interface ComposedConversationProjection { + inheritsLineageContext: boolean; + inherited: PiConversationEventProjection; + local: PiConversationEventProjection; + projection: PiConversationProjection; +} + +/** Maximum number of shared parent edges composed into one Pi projection. */ +export const MAX_LINEAGE_PROJECTION_DEPTH = 32; + +/** Raised when durable child lineage cannot safely identify one pinned context. */ +export class ConversationLineageProjectionError extends Error { + constructor(message: string) { + super(message); + this.name = "ConversationLineageProjectionError"; + } +} + +function emptyEventProjection(): PiConversationEventProjection { + return { + messages: [], + provenance: [], + seqs: [], + modelProfile: "standard", + modelId: undefined, + }; +} + +function epochInheritsLineage(events: ConversationEvent[]): boolean { + const markers = events.filter( + (event) => event.data.type === "context_epoch_started", + ); + if (markers.length > 1) { + throw new ConversationLineageProjectionError( + "conversation epoch has conflicting context markers", + ); + } + if (markers.length === 0) { + return true; + } + const marker = markers[0]!; + return ( + marker.data.type === "context_epoch_started" && + (marker.data.reason === "initial" || marker.data.reason === "rollback") && + marker.data.inheritsLineageContext === true + ); +} + +function epochAtFork(args: { + childConversationId: string; + forkSeq: number; + history: ConversationEvent[]; +}): ConversationEvent[] { + const fork = args.history.find((event) => event.seq === args.forkSeq); + if (!fork) { + throw new ConversationLineageProjectionError( + `shared context fork is missing for ${args.childConversationId}`, + ); + } + return args.history.filter( + (event) => + event.seq <= args.forkSeq && event.contextEpoch === fork.contextEpoch, + ); +} + +async function loadComposedProjection(args: { + conversationId: string; + depth?: number; + localEvents: ConversationEvent[]; + eventStore: ConversationEventStore; + conversationStore: ConversationStore; + visited?: ReadonlySet; +}): Promise { + const local = projectConversationEvents(args.localEvents); + const conversation = await args.conversationStore.get({ + conversationId: args.conversationId, + }); + if (!conversation) { + const inherited = emptyEventProjection(); + return { + inheritsLineageContext: false, + inherited, + local, + projection: local, + }; + } + const lineage = conversation.lineage; + if (!lineage) { + const inherited = emptyEventProjection(); + return { + inheritsLineageContext: false, + inherited, + local, + projection: local, + }; + } + const { parentConversationId } = lineage; + const hasParentEvent = lineage.parentEventSeq !== undefined; + const hasParentTurn = lineage.parentTurnId !== undefined; + if ( + hasParentEvent !== hasParentTurn || + lineage.rootConversationId === undefined || + (lineage.contextForkSeq !== undefined && !hasParentEvent) + ) { + throw new ConversationLineageProjectionError( + `conversation lineage is incomplete for ${args.conversationId}`, + ); + } + const visited = new Set(args.visited ?? []); + if (visited.has(args.conversationId)) { + throw new ConversationLineageProjectionError( + `conversation lineage contains a cycle at ${args.conversationId}`, + ); + } + visited.add(args.conversationId); + if (visited.has(parentConversationId)) { + throw new ConversationLineageProjectionError( + `conversation lineage contains a cycle at ${parentConversationId}`, + ); + } + + const parent = await args.conversationStore.get({ + conversationId: parentConversationId, + }); + if (!parent) { + throw new ConversationLineageProjectionError( + `shared lineage parent is missing for ${args.conversationId}`, + ); + } + const expectedRoot = + parent.lineage === undefined + ? parent.conversationId + : parent.lineage.rootConversationId; + if (expectedRoot === undefined) { + throw new ConversationLineageProjectionError( + `conversation lineage parent is incomplete for ${args.conversationId}`, + ); + } + if (lineage.rootConversationId !== expectedRoot) { + throw new ConversationLineageProjectionError( + `shared lineage root conflicts for ${args.conversationId}`, + ); + } + if (!hasParentEvent || !hasParentTurn) { + const inherited = emptyEventProjection(); + return { + inheritsLineageContext: false, + inherited, + local, + projection: local, + }; + } + + // A correlated child can only be created from a canonical parent start + // event, so this read never needs the legacy importer. + const parentHistory = await args.eventStore.loadHistory(parentConversationId); + const parentEventSeq = lineage.parentEventSeq!; + const parentTurnId = lineage.parentTurnId!; + const fork = parentHistory.find((event) => event.seq === parentEventSeq); + if ( + !fork || + fork.data.type !== "subagent_started" || + fork.data.childConversationId !== args.conversationId || + fork.data.parentTurnId !== parentTurnId + ) { + throw new ConversationLineageProjectionError( + `conversation lineage reference conflicts for ${args.conversationId}`, + ); + } + const contextForkSeq = lineage.contextForkSeq; + if (contextForkSeq === undefined) { + if (fork.data.historyMode !== "isolated") { + throw new ConversationLineageProjectionError( + `isolated lineage reference conflicts for ${args.conversationId}`, + ); + } + const inherited = emptyEventProjection(); + return { + inheritsLineageContext: false, + inherited, + local, + projection: local, + }; + } + if (fork.data.historyMode !== "shared" || parentEventSeq !== contextForkSeq) { + throw new ConversationLineageProjectionError( + `shared lineage fork conflicts for ${args.conversationId}`, + ); + } + if (!epochInheritsLineage(args.localEvents)) { + const inherited = emptyEventProjection(); + return { + inheritsLineageContext: false, + inherited, + local, + projection: local, + }; + } + const depth = args.depth ?? 0; + if (depth >= MAX_LINEAGE_PROJECTION_DEPTH) { + throw new ConversationLineageProjectionError( + `conversation lineage exceeds maximum projection depth for ${args.conversationId}`, + ); + } + const parentProjection = await loadComposedProjection({ + conversationId: parentConversationId, + depth: depth + 1, + localEvents: epochAtFork({ + childConversationId: args.conversationId, + forkSeq: contextForkSeq, + history: parentHistory, + }), + eventStore: args.eventStore, + conversationStore: args.conversationStore, + visited, + }); + const inherited: PiConversationEventProjection = { + ...parentProjection.projection, + seqs: [...parentProjection.inherited.seqs, ...parentProjection.local.seqs], + }; + return { + inheritsLineageContext: true, + inherited, + local, + projection: { + messages: [...inherited.messages, ...local.messages], + provenance: [...inherited.provenance, ...local.provenance], + modelProfile: local.modelProfile, + modelId: local.modelId, + }, + }; +} + +async function loadCurrentComposedProjection( + args: ScopedConversation, +): Promise { + await ensureConversationEventHistory(args); + const eventStore = getConversationEventStore(); + return await loadComposedProjection({ + conversationId: args.conversationId, + localEvents: await eventStore.loadCurrentEpoch(args.conversationId), + eventStore, + conversationStore: getConversationStore(), + }); +} + /** Load the current-epoch Pi projection for a conversation. */ export async function loadProjection( args: ScopedConversation, ): Promise { - const events = await loadCurrentConversationEvents(args); - return projectConversationEvents(events).messages; + return (await loadCurrentComposedProjection(args)).projection.messages; } /** Load the current-epoch Pi projection with aligned per-message provenance. */ export async function loadConversationProjection( args: ScopedConversation, ): Promise { - const events = await loadCurrentConversationEvents(args); - const { messages, provenance, modelProfile, modelId } = - projectConversationEvents(events); - return { messages, provenance, modelProfile, modelId }; + return (await loadCurrentComposedProjection(args)).projection; } /** Open a standard initial epoch before a conversation's first model request. */ @@ -142,14 +394,14 @@ export async function openConversationProjection( await ensureConversationEventHistory(args); const eventStore = getConversationEventStore(); const events = await eventStore.loadCurrentEpoch(args.conversationId); - const projection = projectConversationEvents(events); - if ( - events.some( - (event) => - event.data.type === "context_epoch_started" || - event.data.type === "message", - ) - ) { + const composed = await loadComposedProjection({ + conversationId: args.conversationId, + localEvents: events, + eventStore, + conversationStore: getConversationStore(), + }); + const projection = composed.projection; + if (events.some((event) => event.data.type === "context_epoch_started")) { return projection; } // Host facts may predate the first model request. Keep them in epoch 0 and @@ -159,6 +411,9 @@ export async function openConversationProjection( modelProfile: "standard", modelId: args.modelId, messages: [], + ...(composed.inheritsLineageContext + ? { inheritsLineageContext: true as const } + : {}), }); return { messages: projection.messages, @@ -184,15 +439,31 @@ export async function loadTurnProjection(args: { conversationId: string; committedSeq: number; includeTail: boolean; -}): Promise { +}): Promise< + | (PiConversationProjection & { + /** Number of inherited messages before the child-local projection. */ + localMessageStartIndex: number; + /** Child-local message event sequences only. */ + seqs: number[]; + }) + | undefined +> { await ensureConversationEventHistory(args); const eventStore = getConversationEventStore(); // A record that committed no messages materializes the live projection, the // same way count-based records with a zero cursor did. if (args.committedSeq < 0) { - return projectConversationEvents( - await eventStore.loadCurrentEpoch(args.conversationId), - ); + const composed = await loadComposedProjection({ + conversationId: args.conversationId, + localEvents: await eventStore.loadCurrentEpoch(args.conversationId), + eventStore, + conversationStore: getConversationStore(), + }); + return { + ...composed.projection, + localMessageStartIndex: composed.inherited.messages.length, + seqs: composed.local.seqs, + }; } const history = await eventStore.loadHistory(args.conversationId); const committedEvent = history.find( @@ -204,9 +475,20 @@ export async function loadTurnProjection(args: { const epochEvents = history.filter( (event) => event.contextEpoch === committedEvent.contextEpoch, ); - return args.includeTail - ? projectConversationEvents(epochEvents) - : projectConversationEvents(epochEvents, { maxSeq: args.committedSeq }); + const localEvents = args.includeTail + ? epochEvents + : epochEvents.filter((event) => event.seq <= args.committedSeq); + const composed = await loadComposedProjection({ + conversationId: args.conversationId, + localEvents, + eventStore, + conversationStore: getConversationStore(), + }); + return { + ...composed.projection, + localMessageStartIndex: composed.inherited.messages.length, + seqs: composed.local.seqs, + }; } /** Load MCP providers durably connected in this conversation's current epoch. */ @@ -242,20 +524,75 @@ export async function commitMessages(args: { trailingMessageProvenance?: ConversationMessageProvenance[]; /** Default applied to the last new user message when no explicit array. */ newMessageProvenance?: ConversationMessageProvenance; + /** SQL authority for the atomic commit; defaults to the process executor. */ + executor?: JuniorSqlDatabase; }): Promise<{ committedSeq: number; + /** Index where child-local messages begin in the returned provenance. */ + localMessageStartIndex: number; + /** Child-local message event sequences only. */ messageSeqs: number[]; provenance: ConversationMessageProvenance[]; }> { - const eventStore = getConversationEventStore(); + const executor = args.executor ?? getSqlExecutor(); + return await withConversationEventLock( + executor, + args.conversationId, + async () => + executor.transaction( + async () => await commitMessagesLocked(args, executor), + ), + ); +} + +async function commitMessagesLocked( + args: Parameters[0], + executor: JuniorSqlDatabase, +): ReturnType { + const eventStore = createSqlConversationEventStore(executor); const currentEvents = await eventStore.loadCurrentEpoch(args.conversationId); - const existing = projectConversationEvents(currentEvents); - const matchingPrefix = countMatchingPrefix(existing.messages, args.messages); - const nextProvenance = resolveCommitProvenance({ - existing, - nextMessages: args.messages, + const composed = await loadComposedProjection({ + conversationId: args.conversationId, + localEvents: currentEvents, + eventStore, + conversationStore: createSqlStore(executor), + }); + const inheritedMessageCount = composed.inherited.messages.length; + if ( + countMatchingPrefix( + composed.inherited.messages, + args.messages.slice(0, inheritedMessageCount), + ) !== inheritedMessageCount + ) { + throw new ConversationLineageProjectionError( + `commit mutated inherited context for ${args.conversationId}`, + ); + } + if ( + args.provenance && + !isDeepStrictEqual( + args.provenance.slice(0, inheritedMessageCount), + composed.inherited.provenance, + ) + ) { + throw new ConversationLineageProjectionError( + `commit mutated inherited provenance for ${args.conversationId}`, + ); + } + const nextLocalMessages = args.messages.slice(inheritedMessageCount); + const matchingPrefix = countMatchingPrefix( + composed.local.messages, + nextLocalMessages, + ); + const nextLocalProvenance = resolveCommitProvenance({ + existing: composed.local, + nextMessages: nextLocalMessages, matchingPrefix, - ...(args.provenance ? { explicitProvenance: args.provenance } : {}), + ...(args.provenance + ? { + explicitProvenance: args.provenance.slice(inheritedMessageCount), + } + : {}), ...(args.trailingMessageProvenance ? { trailingMessageProvenance: args.trailingMessageProvenance } : {}), @@ -263,26 +600,37 @@ export async function commitMessages(args: { ? { newMessageProvenance: args.newMessageProvenance } : {}), }); - if (currentEvents.length === 0) { + const hasContextEpochMarker = currentEvents.some( + (event) => event.data.type === "context_epoch_started", + ); + if ( + currentEvents.length === 0 || + (!hasContextEpochMarker && + matchingPrefix === composed.local.messages.length) + ) { + const initialMessages = nextLocalMessages.slice(matchingPrefix); await eventStore.startEpoch(args.conversationId, { reason: "initial", modelProfile: "standard", modelId: args.modelId, - messages: args.messages.map((message, index) => ({ + messages: initialMessages.map((message, index) => ({ message, createdAtMs: messageTimestamp(message), - provenance: nextProvenance[index]!, + provenance: nextLocalProvenance[matchingPrefix + index]!, })), + ...(composed.inheritsLineageContext + ? { inheritsLineageContext: true as const } + : {}), }); - } else if (matchingPrefix === existing.messages.length) { - const newMessages = args.messages.slice(matchingPrefix); + } else if (matchingPrefix === composed.local.messages.length) { + const newMessages = nextLocalMessages.slice(matchingPrefix); await eventStore.append( args.conversationId, newMessages.map((message, index) => ({ data: { type: "message" as const, message, - provenance: nextProvenance[matchingPrefix + index]!, + provenance: nextLocalProvenance[matchingPrefix + index]!, }, createdAtMs: messageTimestamp(message), })), @@ -290,13 +638,16 @@ export async function commitMessages(args: { } else { await eventStore.startEpoch(args.conversationId, { reason: "rollback", - modelProfile: existing.modelProfile, + modelProfile: composed.local.modelProfile, modelId: args.modelId, - messages: args.messages.map((message, index) => ({ + messages: nextLocalMessages.map((message, index) => ({ message, createdAtMs: messageTimestamp(message), - provenance: nextProvenance[index]!, + provenance: nextLocalProvenance[index]!, })), + ...(composed.inheritsLineageContext + ? { inheritsLineageContext: true as const } + : {}), }); } const committed = projectConversationEvents( @@ -304,8 +655,9 @@ export async function commitMessages(args: { ); return { committedSeq: committed.seqs.at(-1) ?? -1, + localMessageStartIndex: inheritedMessageCount, messageSeqs: committed.seqs, - provenance: nextProvenance, + provenance: [...composed.inherited.provenance, ...nextLocalProvenance], }; } diff --git a/packages/junior/src/chat/state/turn-session.ts b/packages/junior/src/chat/state/turn-session.ts index 08ff78589..3a5ab32b8 100644 --- a/packages/junior/src/chat/state/turn-session.ts +++ b/packages/junior/src/chat/state/turn-session.ts @@ -375,7 +375,8 @@ export async function getAgentTurnSessionRecord( const turnStartMessageIndex = parsed.turnStartSeq === undefined ? undefined - : piProjection.seqs.filter((seq) => seq <= parsed.turnStartSeq!).length; + : piProjection.localMessageStartIndex + + piProjection.seqs.filter((seq) => seq <= parsed.turnStartSeq!).length; return materializeAgentTurnSessionRecord( parsed, @@ -614,15 +615,17 @@ export async function upsertAgentTurnSessionRecord(args: { const turnStartSeq = args.turnStartMessageIndex === undefined ? existingRecord?.turnStartSeq - : args.turnStartMessageIndex <= 0 + : args.turnStartMessageIndex <= commit.localMessageStartIndex ? -1 - : (commit.messageSeqs[args.turnStartMessageIndex - 1] ?? - commit.committedSeq); + : (commit.messageSeqs[ + args.turnStartMessageIndex - commit.localMessageStartIndex - 1 + ] ?? commit.committedSeq); const turnStartMessageIndex = args.turnStartMessageIndex ?? (turnStartSeq === undefined ? undefined - : commit.messageSeqs.filter((seq) => seq <= turnStartSeq).length); + : commit.localMessageStartIndex + + commit.messageSeqs.filter((seq) => seq <= turnStartSeq).length); return await setStoredRecord({ conversationStore: args.conversationStore, diff --git a/packages/junior/tests/component/conversations/shared-lineage-projection.test.ts b/packages/junior/tests/component/conversations/shared-lineage-projection.test.ts new file mode 100644 index 000000000..e594a6f42 --- /dev/null +++ b/packages/junior/tests/component/conversations/shared-lineage-projection.test.ts @@ -0,0 +1,581 @@ +import { eq } from "drizzle-orm"; +import { describe, expect, it } from "vitest"; +import { + commitMessages, + ConversationLineageProjectionError, + MAX_LINEAGE_PROJECTION_DEPTH, + loadConversationProjection, + loadProjection, + loadTurnProjection, + openConversationProjection, +} from "@/chat/conversations/projection"; +import { + getConversationEventStore, + getConversationStore, + getDb, + getSqlExecutor, + getSubagentLineageService, +} from "@/chat/db"; +import type { PiMessage } from "@/chat/pi/messages"; +import { juniorConversations } from "@/db/schema"; +import { resolveRootVisibility } from "@/chat/conversations/sql/purge"; +import { purgeConversation } from "@/chat/conversations/retention"; +import { + getAgentTurnSessionRecord, + upsertAgentTurnSessionRecord, +} from "@/chat/state/turn-session"; +import { migrateSchema } from "@/chat/conversations/sql/migrations"; +import { createSqlConversationEventStore } from "@/chat/conversations/sql/history"; +import { projectConversationEvents } from "@/chat/pi/conversation-events"; +import { createLocalJuniorSqlFixture } from "../../fixtures/sql"; + +const MODEL_ID = "openai/gpt-5.4"; + +function message(role: "assistant" | "user", text: string): PiMessage { + return { + role, + content: [{ type: "text", text }], + timestamp: 1, + } as PiMessage; +} + +async function appendTurn(conversationId: string, turnId: string) { + await getConversationEventStore().append(conversationId, [ + { + data: { + type: "turn_started", + turnId, + inputMessageIds: [`${turnId}:input`], + surface: "internal", + }, + idempotencyKey: `turn:${turnId}:started`, + createdAtMs: 2, + }, + ]); +} + +async function startChild(args: { + childConversationId: string; + historyMode: "isolated" | "shared"; + parentConversationId: string; + parentTurnId: string; +}) { + return await getSubagentLineageService().start({ + ...args, + subagentInvocationId: `${args.childConversationId}:invocation`, + subagentKind: "task", + nowMs: 3, + }); +} + +describe("shared child Pi projection", () => { + it("pins direct shared context and commits only the child-local suffix", async () => { + const parent = message("user", "parent before fork"); + await commitMessages({ + conversationId: "shared-parent", + modelId: MODEL_ID, + messages: [parent], + }); + await appendTurn("shared-parent", "parent-turn"); + await startChild({ + childConversationId: "shared-child", + historyMode: "shared", + parentConversationId: "shared-parent", + parentTurnId: "parent-turn", + }); + + await expect( + openConversationProjection({ + conversationId: "shared-child", + modelId: MODEL_ID, + }), + ).resolves.toMatchObject({ messages: [parent] }); + await expect( + loadProjection({ conversationId: "shared-child" }), + ).resolves.toEqual([parent]); + + const child = message("assistant", "child result"); + const commit = await commitMessages({ + conversationId: "shared-child", + modelId: MODEL_ID, + messages: [parent, child], + }); + expect(commit).toMatchObject({ + localMessageStartIndex: 1, + provenance: expect.any(Array), + }); + expect(commit.messageSeqs).toHaveLength(1); + await expect( + loadConversationProjection({ conversationId: "shared-child" }), + ).resolves.toMatchObject({ messages: [parent, child] }); + await expect( + loadTurnProjection({ + conversationId: "shared-child", + committedSeq: commit.committedSeq, + includeTail: false, + }), + ).resolves.toMatchObject({ + localMessageStartIndex: 1, + messages: [parent, child], + seqs: commit.messageSeqs, + }); + await upsertAgentTurnSessionRecord({ + conversationId: "shared-child", + modelId: MODEL_ID, + piMessages: [parent, child], + sessionId: "shared-child-session", + sliceId: 1, + state: "running", + turnStartMessageIndex: 1, + }); + await expect( + getAgentTurnSessionRecord("shared-child", "shared-child-session"), + ).resolves.toMatchObject({ + piMessages: [parent, child], + turnStartMessageIndex: 1, + }); + + const childHistory = + await getConversationEventStore().loadHistory("shared-child"); + expect( + childHistory.flatMap((event) => + event.data.type === "message" ? [event.data.message] : [], + ), + ).toEqual([child]); + expect(childHistory).toContainEqual( + expect.objectContaining({ + data: expect.objectContaining({ + type: "context_epoch_started", + reason: "initial", + inheritsLineageContext: true, + }), + }), + ); + + const regeneratedChild = message("assistant", "regenerated child result"); + await commitMessages({ + conversationId: "shared-child", + modelId: MODEL_ID, + messages: [parent, regeneratedChild], + }); + const rollbackEpoch = + await getConversationEventStore().loadCurrentEpoch("shared-child"); + expect(rollbackEpoch).toContainEqual( + expect.objectContaining({ + data: expect.objectContaining({ + type: "context_epoch_started", + reason: "rollback", + inheritsLineageContext: true, + }), + }), + ); + expect( + rollbackEpoch.flatMap((event) => + event.data.type === "message" ? [event.data.message] : [], + ), + ).toEqual([regeneratedChild]); + + const parentAfterFork = message("assistant", "parent after fork"); + await commitMessages({ + conversationId: "shared-parent", + modelId: MODEL_ID, + messages: [parent, parentAfterFork], + }); + await getConversationEventStore().startEpoch("shared-parent", { + reason: "compaction", + modelProfile: "standard", + modelId: MODEL_ID, + messages: [ + { + message: message("user", "later parent compaction"), + createdAtMs: 4, + }, + ], + }); + await expect( + loadProjection({ conversationId: "shared-child" }), + ).resolves.toEqual([parent, regeneratedChild]); + + const historyLength = ( + await getConversationEventStore().loadHistory("shared-child") + ).length; + await expect( + commitMessages({ + conversationId: "shared-child", + modelId: MODEL_ID, + messages: [message("user", "mutated parent"), regeneratedChild], + }), + ).rejects.toBeInstanceOf(ConversationLineageProjectionError); + expect( + await getConversationEventStore().loadHistory("shared-child"), + ).toHaveLength(historyLength); + + await getConversationEventStore().startEpoch("shared-child", { + reason: "compaction", + modelProfile: "standard", + modelId: MODEL_ID, + messages: [ + { + message: message("user", "self-contained child summary"), + createdAtMs: 5, + }, + ], + }); + await expect( + loadProjection({ conversationId: "shared-child" }), + ).resolves.toEqual([message("user", "self-contained child summary")]); + }); + + it("recursively composes nested shared children at each immutable fork", async () => { + const root = message("user", "root context"); + await commitMessages({ + conversationId: "nested-root", + modelId: MODEL_ID, + messages: [root], + }); + await appendTurn("nested-root", "root-turn"); + await startChild({ + childConversationId: "nested-child", + historyMode: "shared", + parentConversationId: "nested-root", + parentTurnId: "root-turn", + }); + const child = message("assistant", "child context"); + await commitMessages({ + conversationId: "nested-child", + modelId: MODEL_ID, + messages: [root, child], + }); + await appendTurn("nested-child", "child-turn"); + await startChild({ + childConversationId: "nested-grandchild", + historyMode: "shared", + parentConversationId: "nested-child", + parentTurnId: "child-turn", + }); + + await expect( + loadProjection({ conversationId: "nested-grandchild" }), + ).resolves.toEqual([root, child]); + const lateChild = message("user", "child after grandchild fork"); + await commitMessages({ + conversationId: "nested-child", + modelId: MODEL_ID, + messages: [root, child, lateChild], + }); + await expect( + loadProjection({ conversationId: "nested-grandchild" }), + ).resolves.toEqual([root, child]); + + const grandchild = message("assistant", "grandchild result"); + await commitMessages({ + conversationId: "nested-grandchild", + modelId: MODEL_ID, + messages: [root, child, grandchild], + }); + const grandchildMessages = ( + await getConversationEventStore().loadHistory("nested-grandchild") + ).flatMap((event) => + event.data.type === "message" ? [event.data.message] : [], + ); + expect(grandchildMessages).toEqual([grandchild]); + }); + + it("keeps isolated and historical null-fork children child-local", async () => { + const parent = message("user", "private parent context"); + await commitMessages({ + conversationId: "isolated-parent", + modelId: MODEL_ID, + messages: [parent], + }); + await appendTurn("isolated-parent", "isolated-parent-turn"); + await startChild({ + childConversationId: "isolated-child", + historyMode: "isolated", + parentConversationId: "isolated-parent", + parentTurnId: "isolated-parent-turn", + }); + await expect( + loadProjection({ conversationId: "isolated-child" }), + ).resolves.toEqual([]); + + const local = message("user", "isolated instruction"); + await commitMessages({ + conversationId: "isolated-child", + modelId: MODEL_ID, + messages: [local], + }); + await expect( + loadProjection({ conversationId: "isolated-child" }), + ).resolves.toEqual([local]); + expect( + await getConversationEventStore().loadHistory("isolated-child"), + ).not.toContainEqual( + expect.objectContaining({ + data: expect.objectContaining({ inheritsLineageContext: true }), + }), + ); + + const sibling = await startChild({ + childConversationId: "isolated-sibling", + historyMode: "isolated", + parentConversationId: "isolated-parent", + parentTurnId: "isolated-parent-turn", + }); + await getDb() + .update(juniorConversations) + .set({ parentEventSeq: sibling.parentEventSeq }) + .where(eq(juniorConversations.conversationId, "isolated-child")); + await expect( + loadProjection({ conversationId: "isolated-child" }), + ).rejects.toBeInstanceOf(ConversationLineageProjectionError); + await getDb() + .update(juniorConversations) + .set({ parentEventSeq: null, parentTurnId: null }) + .where(eq(juniorConversations.conversationId, "isolated-sibling")); + await expect( + loadProjection({ conversationId: "isolated-sibling" }), + ).resolves.toEqual([]); + }); + + it("fails closed when shared correlation no longer identifies its fork", async () => { + await commitMessages({ + conversationId: "conflict-parent", + modelId: MODEL_ID, + messages: [message("user", "parent")], + }); + await appendTurn("conflict-parent", "conflict-turn"); + const lineage = await startChild({ + childConversationId: "conflict-child", + historyMode: "shared", + parentConversationId: "conflict-parent", + parentTurnId: "conflict-turn", + }); + await getDb() + .update(juniorConversations) + .set({ contextForkSeq: lineage.parentEventSeq - 1 }) + .where(eq(juniorConversations.conversationId, "conflict-child")); + + await expect( + loadProjection({ conversationId: "conflict-child" }), + ).rejects.toBeInstanceOf(ConversationLineageProjectionError); + }); + + it("rejects a shared reference whose durable fork was cleared", async () => { + await appendTurn("lost-fork-parent", "lost-fork-turn"); + await startChild({ + childConversationId: "lost-fork-child", + historyMode: "shared", + parentConversationId: "lost-fork-parent", + parentTurnId: "lost-fork-turn", + }); + await getDb() + .update(juniorConversations) + .set({ contextForkSeq: null }) + .where(eq(juniorConversations.conversationId, "lost-fork-child")); + await expect( + loadProjection({ conversationId: "lost-fork-child" }), + ).rejects.toBeInstanceOf(ConversationLineageProjectionError); + }); + + it("fails closed beyond the bounded shared-lineage projection depth", async () => { + let parentConversationId = "depth-root"; + let parentTurnId = "depth-turn-0"; + await appendTurn(parentConversationId, parentTurnId); + for (let depth = 1; depth <= MAX_LINEAGE_PROJECTION_DEPTH + 1; depth += 1) { + const childConversationId = `depth-child-${depth}`; + await startChild({ + childConversationId, + historyMode: "shared", + parentConversationId, + parentTurnId, + }); + parentConversationId = childConversationId; + parentTurnId = `depth-turn-${depth}`; + await appendTurn(parentConversationId, parentTurnId); + } + await expect( + loadProjection({ conversationId: parentConversationId }), + ).rejects.toThrow("exceeds maximum projection depth"); + }); + + it("fails closed when correlated shared lineage forms a cycle", async () => { + await appendTurn("cycle-root", "cycle-root-turn"); + await appendTurn("cycle-a", "cycle-a-turn"); + await startChild({ + childConversationId: "cycle-b", + historyMode: "shared", + parentConversationId: "cycle-a", + parentTurnId: "cycle-a-turn", + }); + await appendTurn("cycle-b", "cycle-b-turn"); + await getConversationEventStore().append("cycle-b", [ + { + data: { + type: "subagent_started", + subagentInvocationId: "cycle-a:invocation", + subagentKind: "task", + childConversationId: "cycle-a", + parentTurnId: "cycle-b-turn", + historyMode: "shared", + }, + createdAtMs: 4, + }, + ]); + const cycleAReference = ( + await getConversationEventStore().loadHistory("cycle-b") + ).find( + (event) => + event.data.type === "subagent_started" && + event.data.childConversationId === "cycle-a", + )!; + await getDb() + .update(juniorConversations) + .set({ rootConversationId: "cycle-root" }) + .where(eq(juniorConversations.conversationId, "cycle-b")); + await getDb() + .update(juniorConversations) + .set({ + parentConversationId: "cycle-b", + rootConversationId: "cycle-root", + parentTurnId: "cycle-b-turn", + parentEventSeq: cycleAReference.seq, + contextForkSeq: cycleAReference.seq, + }) + .where(eq(juniorConversations.conversationId, "cycle-a")); + await expect(loadProjection({ conversationId: "cycle-b" })).rejects.toThrow( + "contains a cycle", + ); + }); + + it("serializes concurrent commits into individually reproducible boundaries", async () => { + const fixture = await createLocalJuniorSqlFixture(); + try { + await migrateSchema(fixture.sql); + const base = message("user", "base"); + await commitMessages({ + conversationId: "concurrent-commit", + executor: fixture.sql, + modelId: MODEL_ID, + messages: [base], + }); + const left = [base, message("assistant", "left")]; + const right = [base, message("assistant", "right")]; + const [leftCommit, rightCommit] = await Promise.all([ + commitMessages({ + conversationId: "concurrent-commit", + executor: fixture.sql, + modelId: MODEL_ID, + messages: left, + }), + commitMessages({ + conversationId: "concurrent-commit", + executor: fixture.sql, + modelId: MODEL_ID, + messages: right, + }), + ]); + const history = await createSqlConversationEventStore( + fixture.sql, + ).loadHistory("concurrent-commit"); + const projectionAt = (committedSeq: number) => { + const committed = history.find((event) => event.seq === committedSeq)!; + return projectConversationEvents( + history.filter( + (event) => + event.contextEpoch === committed.contextEpoch && + event.seq <= committedSeq, + ), + ); + }; + expect(projectionAt(leftCommit.committedSeq)).toMatchObject({ + messages: left, + provenance: leftCommit.provenance, + }); + expect(projectionAt(rightCommit.committedSeq)).toMatchObject({ + messages: right, + provenance: rightCommit.provenance, + }); + } finally { + await fixture.close(); + } + }); + + it("inherits a pre-fork compaction or handoff as a self-contained parent epoch", async () => { + for (const [reason, profile] of [ + ["compaction", "standard"], + ["handoff", "handoff"], + ] as const) { + const parentConversationId = `pre-fork-${reason}-parent`; + const childConversationId = `pre-fork-${reason}-child`; + await commitMessages({ + conversationId: parentConversationId, + modelId: MODEL_ID, + messages: [message("user", "discarded parent context")], + }); + const replacement = message("user", `${reason} replacement`); + await getConversationEventStore().startEpoch(parentConversationId, { + reason, + modelProfile: profile, + modelId: MODEL_ID, + messages: [{ message: replacement, createdAtMs: 2 }], + }); + const turnId = `pre-fork-${reason}-turn`; + await appendTurn(parentConversationId, turnId); + await startChild({ + childConversationId, + historyMode: "shared", + parentConversationId, + parentTurnId: turnId, + }); + await expect( + loadProjection({ conversationId: childConversationId }), + ).resolves.toEqual([replacement]); + } + }); + + it("rejects inherited provenance mutation and follows private-root purge", async () => { + await getConversationStore().recordActivity({ + conversationId: "private-root", + destination: { + platform: "slack", + teamId: "T123", + channelId: "C123", + }, + source: "slack", + visibility: "private", + nowMs: 1, + }); + const parent = message("user", "private inherited context"); + await commitMessages({ + conversationId: "private-root", + modelId: MODEL_ID, + messages: [parent], + }); + await appendTurn("private-root", "private-turn"); + await startChild({ + childConversationId: "private-child", + historyMode: "shared", + parentConversationId: "private-root", + parentTurnId: "private-turn", + }); + await expect( + commitMessages({ + conversationId: "private-child", + modelId: MODEL_ID, + messages: [parent], + provenance: [{ authority: "instruction" }], + }), + ).rejects.toThrow("mutated inherited provenance"); + await expect( + resolveRootVisibility(getSqlExecutor(), "private-child"), + ).resolves.toEqual({ + rootConversationId: "private-root", + visibility: "private", + }); + await purgeConversation(getSqlExecutor(), "private-root", { nowMs: 10 }); + await expect( + loadProjection({ conversationId: "private-child" }), + ).rejects.toThrow("conversation lineage reference conflicts"); + }); +}); From fede194d255a9fc6585ea2fd913ba9ad1408828d Mon Sep 17 00:00:00 2001 From: David Cramer Date: Wed, 15 Jul 2026 09:00:03 -0700 Subject: [PATCH 09/58] feat(conversations): Add safe reporting events Project canonical conversation history into a strict ordered API boundary. Keep visible content singular and omit internal payloads so reporting consumers can adapt one event stream safely. Co-Authored-By: GPT-5 Codex --- .../junior/src/api/conversations/events.ts | 163 +++++++ .../junior/src/api/conversations/schema.ts | 122 +++++ packages/junior/src/api/schema.ts | 4 + .../junior/src/chat/conversations/README.md | 14 +- .../unit/api/conversation-events.test.ts | 430 ++++++++++++++++++ 5 files changed, 729 insertions(+), 4 deletions(-) create mode 100644 packages/junior/src/api/conversations/events.ts create mode 100644 packages/junior/tests/unit/api/conversation-events.test.ts diff --git a/packages/junior/src/api/conversations/events.ts b/packages/junior/src/api/conversations/events.ts new file mode 100644 index 000000000..ca456c73c --- /dev/null +++ b/packages/junior/src/api/conversations/events.ts @@ -0,0 +1,163 @@ +import type { + ConversationEvent, + ConversationModelMessage, +} from "@/chat/conversations/history"; +import { + conversationReportEventSchema, + type ConversationReportEvent, + type ConversationReportEventData, +} from "./schema"; + +interface SubagentReference { + childConversationId: string; + historyMode: "isolated" | "shared"; + subagentKind: string; +} + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +function modelActivities( + message: ConversationModelMessage, +): Array<"thinking" | "tool_call" | "tool_result"> { + const record = message as Record; + const activities = new Set<"thinking" | "tool_call" | "tool_result">(); + if (record.role === "toolResult") activities.add("tool_result"); + + const content = record.content; + if (!Array.isArray(content)) return [...activities]; + for (const part of content) { + if (!isRecord(part)) continue; + if (part.type === "thinking") activities.add("thinking"); + if (part.type === "toolCall") activities.add("tool_call"); + if (part.type === "toolResult") activities.add("tool_result"); + } + return [...activities]; +} + +function reportEventData(args: { + canExposePayload: boolean; + data: ConversationEvent["data"]; +}): ConversationReportEventData | undefined { + const { data } = args; + switch (data.type) { + case "visible_message_recorded": + return { + type: "visible_message", + messageId: data.messageId, + role: data.role, + ...(args.canExposePayload + ? { text: data.text } + : { redacted: true as const }), + }; + case "visible_message_replied": + return { + type: "visible_message_replied", + messageId: data.messageId, + }; + case "message": { + const activities = modelActivities(data.message); + return activities.length > 0 + ? { type: "model_activity", activities } + : undefined; + } + case "tool_execution_started": + return { type: "tool_started", name: data.toolName }; + case "turn_started": + return { + type: "turn_lifecycle", + turnId: data.turnId, + state: "started", + }; + case "turn_completed": + return { + type: "turn_lifecycle", + turnId: data.turnId, + state: data.outcome === "success" ? "succeeded" : "no_reply", + }; + case "turn_failed": + return { + type: "turn_lifecycle", + turnId: data.turnId, + state: "failed", + }; + case "context_epoch_started": + if (data.reason === "compaction") return { type: "context_compacted" }; + if (data.reason === "handoff") return { type: "model_handoff" }; + return undefined; + case "delivery_intended": + return { + type: "delivery", + deliveryId: data.deliveryId, + state: "intended", + }; + case "delivery_accepted": + return { + type: "delivery", + deliveryId: data.deliveryId, + state: "accepted", + }; + case "delivery_failed": + return { + type: "delivery", + deliveryId: data.deliveryId, + state: "failed", + }; + default: + return undefined; + } +} + +/** + * Project canonical events into the ordered reporting boundary. + * + * `canExposePayload` must come from the authorized conversation-detail policy; + * this projection never derives visibility or authorization from event data. + */ +export function projectConversationReportEvents(args: { + canExposePayload: boolean; + events: ConversationEvent[]; +}): ConversationReportEvent[] { + const subagents = new Map(); + const projected: ConversationReportEvent[] = []; + + for (const event of args.events) { + let data: ConversationReportEventData | undefined; + if (event.data.type === "subagent_started") { + const reference: SubagentReference = { + childConversationId: event.data.childConversationId, + historyMode: event.data.historyMode, + subagentKind: event.data.subagentKind, + }; + subagents.set(event.data.subagentInvocationId, reference); + data = { type: "subagent_started", ...reference }; + } else if (event.data.type === "subagent_ended") { + const reference = subagents.get(event.data.subagentInvocationId); + if (reference) { + data = { + type: "subagent_ended", + ...reference, + outcome: event.data.outcome, + }; + } + } else { + data = reportEventData({ + canExposePayload: args.canExposePayload, + data: event.data, + }); + } + if (!data) continue; + + projected.push( + conversationReportEventSchema.parse({ + seq: event.seq, + contextEpoch: event.contextEpoch, + createdAt: new Date(event.createdAtMs).toISOString(), + data, + }), + ); + } + + return projected; +} diff --git a/packages/junior/src/api/conversations/schema.ts b/packages/junior/src/api/conversations/schema.ts index 01e848d95..efed9281b 100644 --- a/packages/junior/src/api/conversations/schema.ts +++ b/packages/junior/src/api/conversations/schema.ts @@ -145,6 +145,122 @@ export const conversationActivityReportSchema = z.discriminatedUnion("type", [ conversationSubagentActivityReportSchema, ]); +const conversationReportVisibleMessageEventDataSchema = z + .object({ + type: z.literal("visible_message"), + messageId: z.string().min(1), + role: z.enum(["assistant", "system", "user"]), + text: z.string().optional(), + redacted: z.literal(true).optional(), + }) + .strict() + .superRefine((data, context) => { + if ((data.text === undefined) === (data.redacted !== true)) { + context.addIssue({ + code: "custom", + message: "visible message content must be text or explicitly redacted", + }); + } + }); + +const conversationReportVisibleMessageRepliedEventDataSchema = z + .object({ + type: z.literal("visible_message_replied"), + messageId: z.string().min(1), + }) + .strict(); + +const conversationReportModelActivityEventDataSchema = z + .object({ + type: z.literal("model_activity"), + activities: z + .array(z.enum(["thinking", "tool_call", "tool_result"])) + .min(1), + }) + .strict() + .superRefine((data, context) => { + if (new Set(data.activities).size !== data.activities.length) { + context.addIssue({ + code: "custom", + message: "model activities must be unique", + }); + } + }); + +const conversationReportToolStartedEventDataSchema = z + .object({ + type: z.literal("tool_started"), + name: z.string().min(1), + }) + .strict(); + +const conversationReportTurnLifecycleEventDataSchema = z + .object({ + type: z.literal("turn_lifecycle"), + turnId: z.string().min(1), + state: z.enum(["started", "succeeded", "no_reply", "failed"]), + }) + .strict(); + +const conversationReportContextCompactedEventDataSchema = z + .object({ type: z.literal("context_compacted") }) + .strict(); + +const conversationReportModelHandoffEventDataSchema = z + .object({ type: z.literal("model_handoff") }) + .strict(); + +const conversationReportDeliveryEventDataSchema = z + .object({ + type: z.literal("delivery"), + deliveryId: z.string().min(1), + state: z.enum(["intended", "accepted", "failed"]), + }) + .strict(); + +const conversationReportSubagentStartedEventDataSchema = z + .object({ + type: z.literal("subagent_started"), + childConversationId: z.string().min(1), + subagentKind: z.string().min(1), + historyMode: z.enum(["isolated", "shared"]), + }) + .strict(); + +const conversationReportSubagentEndedEventDataSchema = z + .object({ + type: z.literal("subagent_ended"), + childConversationId: z.string().min(1), + subagentKind: z.string().min(1), + historyMode: z.enum(["isolated", "shared"]), + outcome: z.enum(["success", "error", "aborted"]), + }) + .strict(); + +/** Privacy-safe event variants owned by the conversation reporting API. */ +export const conversationReportEventDataSchema = z.discriminatedUnion("type", [ + conversationReportVisibleMessageEventDataSchema, + conversationReportVisibleMessageRepliedEventDataSchema, + conversationReportModelActivityEventDataSchema, + conversationReportToolStartedEventDataSchema, + conversationReportTurnLifecycleEventDataSchema, + conversationReportContextCompactedEventDataSchema, + conversationReportModelHandoffEventDataSchema, + conversationReportDeliveryEventDataSchema, + conversationReportSubagentStartedEventDataSchema, + conversationReportSubagentEndedEventDataSchema, +]); + +/** One ordered, privacy-safe canonical event projected for API consumers. */ +export const conversationReportEventSchema = z + .object({ + seq: z.number().int().nonnegative(), + contextEpoch: z.number().int().nonnegative(), + createdAt: z.string().datetime(), + data: conversationReportEventDataSchema, + }) + .strict(); + export const conversationContextEventSchema = z.discriminatedUnion("type", [ z .object({ @@ -289,6 +405,12 @@ export type ConversationToolActivityReport = z.infer< export type ConversationActivityReport = z.infer< typeof conversationActivityReportSchema >; +export type ConversationReportEventData = z.infer< + typeof conversationReportEventDataSchema +>; +export type ConversationReportEvent = z.infer< + typeof conversationReportEventSchema +>; export type ConversationDetailReport = z.infer< typeof conversationDetailReportSchema >; diff --git a/packages/junior/src/api/schema.ts b/packages/junior/src/api/schema.ts index c1b7a5da4..0b0f645d3 100644 --- a/packages/junior/src/api/schema.ts +++ b/packages/junior/src/api/schema.ts @@ -3,6 +3,8 @@ export type { DailyConversationActivity } from "./activity"; export { conversationDetailReportSchema, conversationFeedSchema, + conversationReportEventDataSchema, + conversationReportEventSchema, conversationStatsReportSchema, conversationSubagentTranscriptReportSchema, } from "./conversations/schema"; @@ -14,6 +16,8 @@ export type { ConversationCost, ConversationDetailReport, ConversationFeed, + ConversationReportEvent, + ConversationReportEventData, ConversationReportStatus, ConversationMetricDay, ConversationStatsItem, diff --git a/packages/junior/src/chat/conversations/README.md b/packages/junior/src/chat/conversations/README.md index 6f6b20b5a..952a994a7 100644 --- a/packages/junior/src/chat/conversations/README.md +++ b/packages/junior/src/chat/conversations/README.md @@ -42,6 +42,13 @@ Reporting APIs must project events into an authorized, redacted product contract. Raw `ConversationEventData` is an internal persistence boundary and must not become a dashboard or external API payload. +The reporting API owns a strict ordered event projection with only safe product +fields. It preserves canonical sequence order, sources display text only from +visible-message events, reduces model messages to content-free activity, and +omits provider receipts, delivery commands, failure codes, authorization +identifiers, arbitrary metadata, and persistence-envelope fields. The current +detail response and dashboard have not yet cut over to this projection. + ## Write Rules - Persist user input before agent execution. @@ -98,10 +105,9 @@ Representative coverage lives in `packages/junior/tests/integration/conversation-sql.test.ts` and the conversation storage component tests. -The local runtime is the first lifecycle-event writer, and current detail -reporting reduces `turn_failed` to one privacy-safe error marker. Slack, -dispatch, delivery-attempt events, and the ordered safe event API remain -follow-up cutovers. +The local runtime writes lifecycle events, and detail reporting reduces +`turn_failed` to one privacy-safe error marker. Slack delivery, dispatch, and +continuation recovery remain follow-up work. The structural failure marker never exposes failure code or event ID. An independently delivered fallback remains ordinary visible content, so a public diff --git a/packages/junior/tests/unit/api/conversation-events.test.ts b/packages/junior/tests/unit/api/conversation-events.test.ts new file mode 100644 index 000000000..d6d4adf12 --- /dev/null +++ b/packages/junior/tests/unit/api/conversation-events.test.ts @@ -0,0 +1,430 @@ +import { describe, expect, it } from "vitest"; +import { projectConversationReportEvents } from "@/api/conversations/events"; +import { conversationReportEventSchema } from "@/api/conversations/schema"; +import { + conversationEventSchema, + type ConversationEvent, + type ConversationEventData, + type ConversationModelMessage, +} from "@/chat/conversations/history"; + +function event( + seq: number, + data: ConversationEventData, + createdAtMs = seq * 1_000, + contextEpoch = 0, +): ConversationEvent { + return conversationEventSchema.parse({ + schemaVersion: 1, + seq, + contextEpoch, + idempotencyKey: `private-idempotency-${seq}`, + createdAtMs, + data, + }); +} + +describe("conversation report event projection", () => { + it("keeps canonical sequence order and sources display text only from visible messages", () => { + const events = [ + event( + 10, + { + type: "visible_message_recorded", + messageId: "visible-1", + role: "assistant", + text: "one user-facing answer", + }, + 30_000, + ), + event( + 11, + { + type: "message", + message: { + role: "assistant", + content: [ + { type: "text", text: "one user-facing answer" }, + { type: "thinking", thinking: "private chain of thought" }, + { + type: "toolCall", + name: "search", + arguments: { query: "private query" }, + }, + ], + } as ConversationModelMessage, + }, + 10_000, + ), + event( + 12, + { + type: "tool_execution_started", + toolCallId: "private-tool-call-id", + toolName: "search", + args: { query: "private query" }, + }, + 5_000, + ), + event( + 13, + { type: "visible_message_replied", messageId: "visible-1" }, + 1_000, + ), + ]; + const projected = projectConversationReportEvents({ + canExposePayload: true, + events, + }); + + expect(projected).toEqual([ + { + seq: 10, + contextEpoch: 0, + createdAt: "1970-01-01T00:00:30.000Z", + data: { + type: "visible_message", + messageId: "visible-1", + role: "assistant", + text: "one user-facing answer", + }, + }, + { + seq: 11, + contextEpoch: 0, + createdAt: "1970-01-01T00:00:10.000Z", + data: { + type: "model_activity", + activities: ["thinking", "tool_call"], + }, + }, + { + seq: 12, + contextEpoch: 0, + createdAt: "1970-01-01T00:00:05.000Z", + data: { type: "tool_started", name: "search" }, + }, + { + seq: 13, + contextEpoch: 0, + createdAt: "1970-01-01T00:00:01.000Z", + data: { + type: "visible_message_replied", + messageId: "visible-1", + }, + }, + ]); + expect(projected.map(({ seq }) => seq)).toEqual([10, 11, 12, 13]); + expect( + projected.filter( + ({ seq, data }) => seq === 11 && data.type === "model_activity", + ), + ).toHaveLength(1); + expect( + JSON.stringify(projected).match(/one user-facing answer/g), + ).toHaveLength(1); + }); + + it("keeps projected prefixes byte-equivalent when later facts arrive", () => { + const events = [ + event(1, { + type: "visible_message_recorded", + messageId: "visible-1", + role: "user", + text: "question", + }), + event(2, { + type: "visible_message_replied", + messageId: "visible-1", + }), + ]; + const prefix = projectConversationReportEvents({ + canExposePayload: true, + events: events.slice(0, 1), + }); + const complete = projectConversationReportEvents({ + canExposePayload: true, + events, + }); + + expect(JSON.stringify(prefix)).toBe(JSON.stringify(complete.slice(0, 1))); + }); + + it("redacts private content and strips every internal persistence or payload field", () => { + const eventId = "0123456789abcdef0123456789abcdef"; + const projected = projectConversationReportEvents({ + canExposePayload: false, + events: [ + event(1, { + type: "visible_message_recorded", + messageId: "visible-private", + role: "user", + text: "private visible text", + authorIdentityId: "private-actor-id", + meta: { + arbitraryMeta: "private arbitrary metadata", + authorizationId: "private-authorization-id", + }, + }), + event(2, { + type: "message", + message: { + role: "toolResult", + name: "private-tool-name", + toolCallId: "private-tool-call-id", + isError: true, + content: [{ type: "text", text: "private tool result" }], + errorMessage: "private provider error", + } as ConversationModelMessage, + }), + event(3, { + type: "tool_execution_started", + toolCallId: "private-tool-call-id", + toolName: "safe_tool_name", + args: { token: "private tool argument" }, + }), + event(4, { + type: "turn_failed", + turnId: "turn-1", + failureCode: "model_execution_failed", + eventId, + }), + event(5, { + type: "delivery_intended", + deliveryId: "delivery:1", + correlation: { + kind: "authorization", + authorizationId: "private-authorization-id", + }, + messageId: "visible-private", + deliveryKind: "assistant_reply", + provider: "slack", + partCount: 2, + }), + event(6, { + type: "delivery_accepted", + deliveryId: "delivery:1", + providerMessageIds: ["123.456"], + }), + event(7, { + type: "delivery_failed", + deliveryId: "delivery:2", + failureCode: "provider_rejected", + }), + event(8, { + type: "authorization_requested", + kind: "mcp", + provider: "private-provider", + actorId: "private-actor-id", + authorizationId: "private-authorization-id", + delivery: "private_link_sent", + }), + event(9, { + type: "authorization_completed", + kind: "mcp", + provider: "private-provider", + actorId: "private-actor-id", + authorizationId: "private-authorization-id", + }), + event(10, { + type: "visible_message_metadata_updated", + messageId: "visible-private", + meta: { privateUpdate: "private metadata update" }, + }), + ], + }); + + expect(projected[0]?.data).toEqual({ + type: "visible_message", + messageId: "visible-private", + role: "user", + redacted: true, + }); + expect(projected[1]?.data).toEqual({ + type: "model_activity", + activities: ["tool_result"], + }); + const serialized = JSON.stringify(projected); + for (const forbidden of [ + "schemaVersion", + "idempotencyKey", + "createdAtMs", + "private visible text", + "private-actor-id", + "private arbitrary metadata", + "private-authorization-id", + "private-tool-name", + "private-tool-call-id", + "private tool result", + "private provider error", + "private tool argument", + "model_execution_failed", + eventId, + "providerMessageIds", + "123.456", + "provider_rejected", + "private-provider", + "private metadata update", + "correlation", + "deliveryKind", + "partCount", + "actorId", + "authorizationId", + "eventId", + "failureCode", + "args", + "content", + "meta", + ]) { + expect(serialized).not.toContain(forbidden); + } + }); + + it("emits only safe structural lifecycle, context, delivery, and child references", () => { + const projected = projectConversationReportEvents({ + canExposePayload: true, + events: [ + event(1, { + type: "turn_started", + turnId: "turn-1", + inputMessageIds: ["private-input-id"], + surface: "slack", + }), + event(2, { + type: "context_epoch_started", + reason: "compaction", + modelProfile: "standard", + modelId: "private-model-id", + }), + event(3, { + type: "context_epoch_started", + reason: "handoff", + modelProfile: "fast", + modelId: "private-handoff-model-id", + }), + event(4, { + type: "context_epoch_started", + reason: "rollback", + modelProfile: "standard", + modelId: "private-rollback-model-id", + }), + event(5, { + type: "delivery_intended", + deliveryId: "delivery:1", + correlation: { kind: "turn", turnId: "turn-1" }, + messageId: "message-1", + deliveryKind: "assistant_reply", + provider: "slack", + partCount: 1, + }), + event(6, { + type: "delivery_accepted", + deliveryId: "delivery:1", + providerMessageIds: ["123.456"], + }), + event(7, { + type: "subagent_started", + subagentInvocationId: "private-invocation-id", + subagentKind: "advisor", + modelId: "private-child-model-id", + parentToolCallId: "private-parent-tool-id", + reasoningLevel: "private-reasoning-level", + childConversationId: "child-conversation-1", + parentTurnId: "turn-1", + historyMode: "shared", + }), + event(8, { + type: "subagent_ended", + subagentInvocationId: "private-invocation-id", + parentTurnId: "turn-1", + outcome: "error", + errorCode: "private-child-error-code", + }), + event(9, { + type: "subagent_ended", + subagentInvocationId: "orphan-private-invocation-id", + outcome: "aborted", + errorCode: "orphan-private-error-code", + }), + event(10, { + type: "turn_completed", + turnId: "turn-2", + outcome: "no_reply", + }), + ], + }); + + expect(projected.map(({ seq }) => seq)).toEqual([1, 2, 3, 5, 6, 7, 8, 10]); + expect(projected.map(({ data }) => data)).toEqual([ + { type: "turn_lifecycle", turnId: "turn-1", state: "started" }, + { type: "context_compacted" }, + { type: "model_handoff" }, + { type: "delivery", deliveryId: "delivery:1", state: "intended" }, + { type: "delivery", deliveryId: "delivery:1", state: "accepted" }, + { + type: "subagent_started", + childConversationId: "child-conversation-1", + subagentKind: "advisor", + historyMode: "shared", + }, + { + type: "subagent_ended", + childConversationId: "child-conversation-1", + subagentKind: "advisor", + historyMode: "shared", + outcome: "error", + }, + { type: "turn_lifecycle", turnId: "turn-2", state: "no_reply" }, + ]); + const serialized = JSON.stringify(projected); + for (const forbidden of [ + "private-input-id", + "private-model-id", + "private-handoff-model-id", + "private-rollback-model-id", + "123.456", + "private-invocation-id", + "private-child-model-id", + "private-parent-tool-id", + "private-reasoning-level", + "private-child-error-code", + "orphan-private-invocation-id", + "orphan-private-error-code", + ]) { + expect(serialized).not.toContain(forbidden); + } + }); + + it("owns a strict runtime boundary for envelope and data fields", () => { + const valid = { + seq: 1, + contextEpoch: 0, + createdAt: "2026-07-15T12:00:00.000Z", + data: { + type: "turn_lifecycle", + turnId: "turn-1", + state: "failed", + }, + }; + + expect(conversationReportEventSchema.safeParse(valid).success).toBe(true); + expect( + conversationReportEventSchema.safeParse({ + ...valid, + schemaVersion: 1, + }).success, + ).toBe(false); + expect( + conversationReportEventSchema.safeParse({ + ...valid, + data: { ...valid.data, failureCode: "private-failure-code" }, + }).success, + ).toBe(false); + expect( + conversationReportEventSchema.safeParse({ + ...valid, + data: { type: "visible_message", messageId: "m1", role: "user" }, + }).success, + ).toBe(false); + }); +}); From 697ee2915a50dbde8d728dbd4c1bf2031a15fdfb Mon Sep 17 00:00:00 2001 From: David Cramer Date: Wed, 15 Jul 2026 09:59:53 -0700 Subject: [PATCH 10/58] feat(conversations): Cut reporting over to event history Make canonical conversation events the only detail and dashboard history authority. Remove transcript and subagent compatibility APIs, inherit child privacy from the locked root, and make retention purge whole trees without append races. Co-Authored-By: GPT-5 Codex --- packages/junior-dashboard/README.md | 6 +- .../junior-dashboard/e2e/dashboard.spec.ts | 23 +- packages/junior-dashboard/src/app.ts | 18 +- packages/junior-dashboard/src/client/api.ts | 31 - .../components/ConversationTranscript.tsx | 45 +- .../client/components/ExecutionSignature.tsx | 35 - .../components/SubagentTranscriptDrawer.tsx | 208 +- .../src/client/components/Transcript.tsx | 2 +- .../components/TranscriptContextEventView.tsx | 136 +- .../components/TranscriptSubagentView.tsx | 13 +- .../client/components/TranscriptToolView.tsx | 2 +- .../components/transcriptBottomPinning.ts | 2 +- .../components/transcriptRenderModel.ts | 11 +- .../client/components/transcriptSearch.tsx | 20 +- .../src/client/eventTranscript.ts | 140 + .../junior-dashboard/src/client/format.ts | 19 +- .../src/client/markdownExport.ts | 69 +- .../src/client/pages/ConversationPage.tsx | 7 +- .../src/client/subagentTranscript.ts | 51 - .../src/client/transcriptActivity.ts | 358 --- packages/junior-dashboard/src/client/types.ts | 69 +- .../src/mock-conversations.ts | 1908 ++++------- .../src/mock-release-conversation.ts | 592 ---- .../src/mock-reporting/activity.ts | 90 - .../src/mock-reporting/time.ts | 6 - .../src/mock-reporting/transcript.ts | 152 - .../tests/dashboard-mock-routes.test.ts | 485 +-- .../tests/dashboard-routes.test.ts | 1 - .../junior-dashboard/tests/format.test.ts | 780 +---- .../tests/markdownExport.test.ts | 505 +-- .../tests/telemetry-components.test.tsx | 2424 ++++---------- .../tests/transcriptBottomPinning.test.ts | 55 +- .../tests/transcriptContextEvent.test.tsx | 38 +- .../tests/transcriptRenderModel.test.ts | 847 +---- .../junior/src/api/conversations/activity.ts | 170 - .../api/conversations/detail-projection.ts | 541 +--- .../src/api/conversations/detail.query.ts | 72 +- .../src/api/conversations/list.query.ts | 200 ++ .../junior/src/api/conversations/schema.ts | 224 +- .../src/api/conversations/transcript.ts | 431 --- packages/junior/src/api/schema.ts | 17 +- .../junior/src/chat/conversations/README.md | 11 +- .../src/chat/conversations/retention.ts | 9 +- .../src/chat/conversations/sql/privacy.ts | 160 + .../src/chat/conversations/sql/purge.ts | 216 +- .../conversation-transcripts-sql.test.ts | 7 +- .../component/conversations/retention.test.ts | 36 + .../shared-lineage-projection.test.ts | 2 +- .../services/subagent-lineage.test.ts | 2 +- .../junior/tests/integration/api/rest.test.ts | 13 +- .../integration/dashboard-reporting.test.ts | 2853 ++++------------- .../integration/reporting-support.test.ts | 188 ++ .../runtime/agent-run-provider-retry.test.ts | 24 +- .../unit/api/conversation-events.test.ts | 128 +- .../tests/unit/conversations/privacy.test.ts | 167 + 55 files changed, 4054 insertions(+), 10565 deletions(-) delete mode 100644 packages/junior-dashboard/src/client/components/ExecutionSignature.tsx create mode 100644 packages/junior-dashboard/src/client/eventTranscript.ts delete mode 100644 packages/junior-dashboard/src/client/subagentTranscript.ts delete mode 100644 packages/junior-dashboard/src/client/transcriptActivity.ts delete mode 100644 packages/junior-dashboard/src/mock-release-conversation.ts delete mode 100644 packages/junior-dashboard/src/mock-reporting/activity.ts delete mode 100644 packages/junior-dashboard/src/mock-reporting/time.ts delete mode 100644 packages/junior-dashboard/src/mock-reporting/transcript.ts delete mode 100644 packages/junior/src/api/conversations/activity.ts create mode 100644 packages/junior/src/api/conversations/list.query.ts delete mode 100644 packages/junior/src/api/conversations/transcript.ts create mode 100644 packages/junior/src/chat/conversations/sql/privacy.ts create mode 100644 packages/junior/tests/integration/reporting-support.test.ts create mode 100644 packages/junior/tests/unit/conversations/privacy.test.ts diff --git a/packages/junior-dashboard/README.md b/packages/junior-dashboard/README.md index a1bbe3397..697a7fab2 100644 --- a/packages/junior-dashboard/README.md +++ b/packages/junior-dashboard/README.md @@ -11,9 +11,9 @@ state. - Better Auth owns authentication; dashboard routes fail closed when identity or required configuration is missing. - API schemas under `src/api/` define the client/server boundary. -- Reporting projections expose normalized visible messages, agent activity, - artifacts, and tool summaries rather than raw provider payloads or runtime - state. +- Conversation detail exposes one ordered, privacy-safe event log. The + dashboard owns the event-to-view reduction and never merges a duplicate + transcript, activity stream, or provider runtime state. - Private conversation access requires authenticated authorization at the server boundary. Client-side route hiding is not authorization. - The package remains stateless apart from normal auth/session infrastructure; diff --git a/packages/junior-dashboard/e2e/dashboard.spec.ts b/packages/junior-dashboard/e2e/dashboard.spec.ts index 2970997ad..985a71203 100644 --- a/packages/junior-dashboard/e2e/dashboard.spec.ts +++ b/packages/junior-dashboard/e2e/dashboard.spec.ts @@ -315,17 +315,20 @@ test("scrolls long conversation and transcript panes independently", async ({ ...conversations[0], displayTitle: "Long transcript", generatedAt, - transcript: Array.from({ length: 60 }, (_, index) => ({ - parts: [ - { - text: `Transcript message ${index + 1} with enough content to occupy a visible row.`, - type: "text", - }, - ], - role: index % 2 === 0 ? "user" : "assistant", - timestamp: Date.parse(generatedAt) + index * 1_000, + eventHistory: { status: "available" }, + events: Array.from({ length: 60 }, (_, index) => ({ + contextEpoch: 0, + createdAt: new Date( + Date.parse(generatedAt) + index * 1_000, + ).toISOString(), + data: { + type: "visible_message", + messageId: `message-${index + 1}`, + role: index % 2 === 0 ? "user" : "assistant", + text: `Transcript message ${index + 1} with enough content to occupy a visible row.`, + }, + seq: index, })), - transcriptAvailable: true, }, }); }); diff --git a/packages/junior-dashboard/src/app.ts b/packages/junior-dashboard/src/app.ts index 38288d3db..cce7fc1d1 100644 --- a/packages/junior-dashboard/src/app.ts +++ b/packages/junior-dashboard/src/app.ts @@ -8,14 +8,12 @@ import { conversationFeedSchema, conversationParamsSchema, conversationStatsReportSchema, - conversationSubagentTranscriptReportSchema, locationDetailReportSchema, locationDirectoryReportSchema, locationParamsSchema, personParamsSchema, actorDirectoryReportSchema, actorProfileReportSchema, - subagentParamsSchema, } from "@sentry/junior/api/schema"; import { initSentry } from "@sentry/junior/instrumentation"; import type { @@ -41,7 +39,6 @@ import { readMockConversationDetail, readMockConversationFeed, readMockConversationStats, - readMockConversationSubagent, readMockLocationDetail, readMockLocationDirectory, readMockPeopleDirectory, @@ -282,8 +279,8 @@ function isAuthorized( return Boolean( session.user.emailVerified && - emailDomain && - allowedDomains.includes(emailDomain), + emailDomain && + allowedDomains.includes(emailDomain), ); } @@ -720,17 +717,6 @@ export function createDashboardApp( ? Response.json(conversationDetailReportSchema.parse(report)) : Response.json({ error: "Conversation not found." }, { status: 404 }); }); - app.get("/api/conversations/:conversationId/subagents/:subagentId", (c) => { - const { conversationId, subagentId } = subagentParamsSchema.parse( - c.req.param(), - ); - const report = conversationSubagentTranscriptReportSchema.parse( - readMockConversationSubagent(conversationId, subagentId), - ); - return report.unavailableReason === "not_found" - ? Response.json(report, { status: 404 }) - : Response.json(report); - }); } app.route("/", createJuniorApi()); app.get("/api/config", () => { diff --git a/packages/junior-dashboard/src/client/api.ts b/packages/junior-dashboard/src/client/api.ts index b751901e8..78fab379c 100644 --- a/packages/junior-dashboard/src/client/api.ts +++ b/packages/junior-dashboard/src/client/api.ts @@ -1,14 +1,12 @@ import { QueryClient, useMutation, useQuery } from "@tanstack/react-query"; import type { ZodType } from "zod"; import type { ConversationDetailReport } from "@sentry/junior/api/schema"; -import type { ConversationSubagentTranscriptReport } from "@sentry/junior/api/schema"; import type { ActorProfileReport } from "@sentry/junior/api/schema"; import type { LocationDetailReport } from "@sentry/junior/api/schema"; import { conversationDetailReportSchema, conversationFeedSchema, conversationStatsReportSchema, - conversationSubagentTranscriptReportSchema, } from "@sentry/junior/api/schema"; import { actorDirectoryReportSchema, @@ -251,32 +249,3 @@ export function readConversationData( `/api/conversations/${encodeURIComponent(conversationId)}`, ); } - -/** Fetch one child-agent transcript for the conversation detail drawer. */ -export function useConversationSubagentTranscriptData( - params: - | { - conversationId: string; - subagentId: string; - } - | undefined, -) { - return useQuery({ - enabled: Boolean(params), - queryKey: [ - "conversation-subagent", - params?.conversationId, - params?.subagentId, - ], - queryFn: async (): Promise => { - const active = params!; - return await read( - conversationSubagentTranscriptReportSchema, - `/api/conversations/${encodeURIComponent( - active.conversationId, - )}/subagents/${encodeURIComponent(active.subagentId)}`, - ); - }, - retry: false, - }); -} diff --git a/packages/junior-dashboard/src/client/components/ConversationTranscript.tsx b/packages/junior-dashboard/src/client/components/ConversationTranscript.tsx index a0760ae31..8d9ac65e7 100644 --- a/packages/junior-dashboard/src/client/components/ConversationTranscript.tsx +++ b/packages/junior-dashboard/src/client/components/ConversationTranscript.tsx @@ -30,14 +30,13 @@ import { visualStatusForSummary, } from "../format"; import { cn } from "../styles"; -import { conversationTranscriptMessages } from "../transcriptActivity"; +import { conversationTranscriptMessages } from "../eventTranscript"; import type { ConversationTranscript, TranscriptViewMessage, TranscriptViewPart, TranscriptViewSubagentPart, } from "../types"; -import { ExecutionSignature } from "./ExecutionSignature"; import { StatusBadge } from "./StatusBadge"; import { ToolFrame, toolFrameClass } from "./ToolFrame"; import { @@ -248,14 +247,15 @@ function SegmentEvents(props: { return (
- {props.conversation.transcriptAvailable ? ( + {props.conversation.eventHistory.status === "available" ? ( - ) : props.conversation.transcriptRedacted && messages.length > 0 ? ( + ) : props.conversation.eventHistory.status === "redacted" && + messages.length > 0 ? ( + onOpenTranscript={(part: TranscriptViewSubagentPart) => props.onOpenSubagentTranscript?.({ part, conversation: props.conversation, @@ -428,11 +428,12 @@ function TranscriptEntryList(props: { } function TranscriptFailureView(props: { - outcome: "error" | "aborted"; + outcome: "error" | "aborted" | "delivery_failed"; timestamp?: number; }) { const timestamp = formatMessageTimestamp(props.timestamp); - const isError = props.outcome === "error"; + const isError = props.outcome !== "aborted"; + const deliveryFailed = props.outcome === "delivery_failed"; return (
- {isError ? "Agent response failed" : "Agent response stopped"} + {deliveryFailed + ? "Message delivery failed" + : isError + ? "Agent response failed" + : "Agent response stopped"}
- {isError - ? "The model response ended before Junior could complete this turn." - : "The model response was stopped before Junior could complete this turn."} + {deliveryFailed + ? "Junior could not deliver this message to its destination." + : isError + ? "The model response ended before Junior could complete this turn." + : "The model response was stopped before Junior could complete this turn."}
{timestamp ? ( @@ -573,7 +580,7 @@ function RedactedTranscriptView(props: { kind="subagent" > + onOpenTranscript={(part: TranscriptViewSubagentPart) => props.onOpenSubagentTranscript?.({ part, conversation: props.conversation, @@ -707,7 +714,7 @@ function RedactedToolView(props: { ? formatMs(props.resultTimestamp - props.timestamp) : undefined; const missingResultLabel = - props.call?.status === "running" ? "running" : "missing result"; + props.call?.status === "running" ? "running" : "started"; const meta = [ duration, props.result ? undefined : missingResultLabel, @@ -757,18 +764,6 @@ function transcriptMeta( const toolSummary = summarizeToolCalls(conversation); const turnSummary = summarizeTurns(conversation); const items: Array = [ - conversation.modelId || conversation.reasoningLevel - ? { - content: ( - - ), - key: "execution", - } - : undefined, duration !== "none" ? { content: ( diff --git a/packages/junior-dashboard/src/client/components/ExecutionSignature.tsx b/packages/junior-dashboard/src/client/components/ExecutionSignature.tsx deleted file mode 100644 index 2d320366e..000000000 --- a/packages/junior-dashboard/src/client/components/ExecutionSignature.tsx +++ /dev/null @@ -1,35 +0,0 @@ -import { cn } from "../styles"; - -/** Show the model execution settings without competing with transcript status. */ -export function ExecutionSignature(props: { - className?: string; - modelId?: string; - reasoningLevel?: string; -}) { - const modelId = props.modelId?.trim(); - const reasoningLevel = props.reasoningLevel?.trim(); - if (!modelId && !reasoningLevel) return null; - const modelName = modelId?.split("/").at(-1) ?? modelId; - - return ( - - {modelName} - {reasoningLevel ? ( - - {modelName ? " " : null}({reasoningLevel}) - - ) : null} - - ); -} diff --git a/packages/junior-dashboard/src/client/components/SubagentTranscriptDrawer.tsx b/packages/junior-dashboard/src/client/components/SubagentTranscriptDrawer.tsx index b35e0612f..67b32597e 100644 --- a/packages/junior-dashboard/src/client/components/SubagentTranscriptDrawer.tsx +++ b/packages/junior-dashboard/src/client/components/SubagentTranscriptDrawer.tsx @@ -1,53 +1,34 @@ import { useEffect } from "react"; import { Bot, ExternalLink, X } from "lucide-react"; -import type { ConversationSubagentTranscriptReport } from "@sentry/junior/api/schema"; +import { Link } from "react-router"; -import { useConversationSubagentTranscriptData } from "../api"; -import { formatMessageTimestamp, formatMs } from "../format"; -import { buildSubagentMarkdown } from "../markdownExport"; -import { cn } from "../styles"; -import { - subagentDurationMs, - subagentConversationTranscript, -} from "../subagentTranscript"; -import type { - ConversationTranscript, - TranscriptViewSubagentPart, -} from "../types"; +import { useConversationData } from "../api"; +import { conversationPath, formatMessageTimestamp } from "../format"; +import { buildConversationMarkdown } from "../markdownExport"; +import type { TranscriptViewSubagentPart } from "../types"; import { Button } from "./Button"; import { CopyMarkdownButton } from "./CopyMarkdownButton"; -import { ExecutionSignature } from "./ExecutionSignature"; import { Transcript } from "./Transcript"; import { TranscriptLoading } from "./TranscriptLoading"; import { transcriptEmptyClass } from "./transcriptStyles"; export interface SubagentTranscriptTarget { conversationId: string; - conversation: ConversationTranscript; part: TranscriptViewSubagentPart; } -/** Show a lazily loaded child-agent transcript without leaving the parent conversation. */ +/** Show a child conversation through the ordinary authorized detail API. */ export function SubagentTranscriptDrawer(props: { onClose: () => void; target: SubagentTranscriptTarget | undefined; }) { - const query = useConversationSubagentTranscriptData( - props.target - ? { - conversationId: props.target.conversationId, - subagentId: props.target.part.id, - } - : undefined, - ); + const query = useConversationData(props.target?.conversationId); useEffect(() => { if (!props.target) return undefined; const onKeyDown = (event: KeyboardEvent) => { - if (event.key === "Escape") { - props.onClose(); - } + if (event.key === "Escape") props.onClose(); }; window.addEventListener("keydown", onKeyDown); return () => window.removeEventListener("keydown", onKeyDown); @@ -55,17 +36,11 @@ export function SubagentTranscriptDrawer(props: { if (!props.target) return null; - const report = query.data; - const visible = report ?? subagentFallback(props.target); - const label = visible.subagentKind; - const duration = subagentDuration(visible); - const transcriptTurn = report - ? subagentConversationTranscript(props.target.conversationId, report) - : undefined; + const detail = query.data; + const label = detail?.displayTitle || props.target.part.subagentKind; const meta = [ - statusLabel(visible), - duration, - formatMessageTimestamp(Date.parse(visible.createdAt)), + detail?.status ?? statusLabel(props.target.part), + detail ? formatMessageTimestamp(Date.parse(detail.startedAt)) : undefined, ].filter(isString); return ( @@ -90,24 +65,27 @@ export function SubagentTranscriptDrawer(props: { {label}
- - - {meta.length > 0 ? ( -
- {meta.join(" · ")} -
- ) : null} +
+ + {props.target.conversationId} + + + Open conversation +
+
+ {meta.join(" · ")} +
buildSubagentMarkdown(report, transcriptTurn) - : undefined + detail ? () => buildConversationMarkdown(detail) : undefined } />
@@ -141,124 +117,16 @@ export function SubagentTranscriptDrawer(props: { ); } -function DrawerConversationIdentity(props: { - report: ConversationSubagentTranscriptReport; -}) { - if ( - !props.report.subagentConversationId && - !props.report.subagentSentryConversationUrl - ) { - return null; - } - - return ( -
- {props.report.subagentConversationId ? ( - - - Conversation ID - - - {props.report.subagentConversationId - .split(":") - .map((segment, index, segments) => ( - - {segment} - {index < segments.length - 1 ? ( - <> - : - - ) : null} - - ))} - - - ) : null} - {props.report.subagentSentryConversationUrl ? ( - - View in Sentry - - ) : null} -
- ); -} - -function subagentFallback( - target: SubagentTranscriptTarget, -): ConversationSubagentTranscriptReport { - return { - type: "subagent", - createdAt: new Date(target.conversation.startedAt).toISOString(), - id: target.part.id, - status: target.part.status, - subagentKind: target.part.subagentKind, - transcript: [], - transcriptAvailable: false, - ...(target.part.endedAt ? { endedAt: target.part.endedAt } : {}), - ...(target.part.modelId ? { modelId: target.part.modelId } : {}), - ...(target.part.outcome ? { outcome: target.part.outcome } : {}), - ...(target.part.parentToolCallId - ? { parentToolCallId: target.part.parentToolCallId } - : {}), - ...(target.part.reasoningLevel - ? { reasoningLevel: target.part.reasoningLevel } - : {}), - }; -} - -function subagentDuration( - report: Pick, -): string | undefined { - const durationMs = subagentDurationMs(report); - return durationMs === undefined ? undefined : formatMs(durationMs); -} - -function statusLabel( - report: ConversationSubagentTranscriptReport, -): string | undefined { - if (report.outcome === "error") return "error"; - if (report.outcome === "aborted") return "aborted"; - if (report.status === "error" || report.status === "aborted") { - return report.status; - } - return undefined; -} - -function subagentUnavailableLabel( - report: ConversationSubagentTranscriptReport | undefined, -): string { - if (report?.transcriptRedacted) { - return "Transcript hidden because this conversation is not public."; - } - - if (report?.unavailableReason === "missing_transcript_range") { - return "This subagent was recorded before per-invocation transcripts were available."; - } - - if (report?.unavailableReason === "missing_transcript_ref") { - return "The referenced subagent transcript is no longer available."; - } - - return "No subagent transcript is available."; +function statusLabel(part: TranscriptViewSubagentPart): string { + return part.outcome ?? part.status; } function DrawerEmptyState(props: { children: string; - tone?: "error" | "muted"; + tone?: "default" | "error"; }) { return ( -
+
{props.children}
); diff --git a/packages/junior-dashboard/src/client/components/Transcript.tsx b/packages/junior-dashboard/src/client/components/Transcript.tsx index 45e3384bd..6f3d148a4 100644 --- a/packages/junior-dashboard/src/client/components/Transcript.tsx +++ b/packages/junior-dashboard/src/client/components/Transcript.tsx @@ -30,7 +30,7 @@ export function Transcript(props: { const [view, setView] = useState("rich"); const [search, setSearch] = useState(""); - const redacted = Boolean(props.transcript?.transcriptRedacted); + const redacted = props.transcript?.eventHistory.status === "redacted"; const bottomPinning = usePinnedTranscriptBottom({ enabled: props.live ?? false, version: transcriptBottomVersion(props.transcript), diff --git a/packages/junior-dashboard/src/client/components/TranscriptContextEventView.tsx b/packages/junior-dashboard/src/client/components/TranscriptContextEventView.tsx index bc013867c..2465a5203 100644 --- a/packages/junior-dashboard/src/client/components/TranscriptContextEventView.tsx +++ b/packages/junior-dashboard/src/client/components/TranscriptContextEventView.tsx @@ -1,47 +1,27 @@ -import { ArrowRight } from "lucide-react"; - import { formatMessageTimestamp } from "../format"; import type { TranscriptViewContextEventPart } from "../types"; -import { TranscriptMarkdown } from "./TranscriptMarkdown"; -import { HighlightText, useTranscriptSearch } from "./transcriptSearch"; - -function modelLabel(modelId: string): string { - return modelId.split("/").at(-1) ?? modelId; -} - -function ModelLabel(props: { modelId: string }) { - return ( - - {modelLabel(props.modelId)} - - ); -} -/** Render an inline context change without exposing storage epochs to operators. */ +/** Render a structural context change from the privacy-safe event API. */ export function TranscriptContextEventView(props: { part: TranscriptViewContextEventPart; timestamp?: number; }) { - const event = props.part.event; - if (event.type === "model_handoff") { - return ; - } - return ; -} - -function ModelHandoffView(props: { - event: Extract< - TranscriptViewContextEventPart["event"], - { type: "model_handoff" } - >; - timestamp?: number; -}) { - const { active: searchActive } = useTranscriptSearch(); - const header = ( -
+ const handoff = props.part.event.type === "model_handoff"; + return ( +
- - Model handoff + + {handoff ? "Model handoff" : "Context compacted"} {typeof props.timestamp === "number" ? ( @@ -49,86 +29,10 @@ function ModelHandoffView(props: { ) : null}
-
- {props.event.fromModelId ? ( - - ) : ( - Previous model - )} -
-
- ); - const className = - "min-w-0 rounded-lg border border-sky-300/10 bg-sky-300/[0.035] first:mt-1"; - - if (!props.event.message) { - return
{header}
; - } - - return ( -
- - {header} - - 
-        
-
-
- ); -} - -function ContextCompactedView(props: { - event: Extract< - TranscriptViewContextEventPart["event"], - { type: "context_compacted" } - >; - timestamp?: number; -}) { - const { active: searchActive } = useTranscriptSearch(); - - return ( -
-
-
- - Context compacted - - {typeof props.timestamp === "number" ? ( - - {formatMessageTimestamp(props.timestamp)} - - ) : null} -
- - {props.event.modelId ? ( -
- Continuing with -
- ) : ( -
- Earlier context was summarized for the next turn. -
- )} - - {props.event.summary ? ( -
- - View summary - -
- {searchActive ? ( - - ) : ( - - )} -
-
- ) : null} +
+ {handoff + ? "Execution continued with a different model." + : "Earlier context was summarized for the next turn."}
); diff --git a/packages/junior-dashboard/src/client/components/TranscriptSubagentView.tsx b/packages/junior-dashboard/src/client/components/TranscriptSubagentView.tsx index aa9e4156d..4e79e88a5 100644 --- a/packages/junior-dashboard/src/client/components/TranscriptSubagentView.tsx +++ b/packages/junior-dashboard/src/client/components/TranscriptSubagentView.tsx @@ -1,4 +1,4 @@ -import { formatMessageTimestamp, formatMs } from "../format"; +import { formatMessageTimestamp } from "../format"; import type { TranscriptViewSubagentPart } from "../types"; import { ToolFrame } from "./ToolFrame"; import { HighlightText } from "./transcriptSearch"; @@ -10,20 +10,9 @@ export function TranscriptSubagentView(props: { timestamp?: number; }) { const label = props.part.subagentKind; - const endedAt = props.part.endedAt - ? Date.parse(props.part.endedAt) - : undefined; - const duration = - typeof props.timestamp === "number" && - typeof endedAt === "number" && - Number.isFinite(endedAt) && - endedAt >= props.timestamp - ? formatMs(endedAt - props.timestamp) - : undefined; const status = statusLabel(props.part); const meta = [ status, - duration, typeof props.timestamp === "number" ? formatMessageTimestamp(props.timestamp) : undefined, diff --git a/packages/junior-dashboard/src/client/components/TranscriptToolView.tsx b/packages/junior-dashboard/src/client/components/TranscriptToolView.tsx index 04a513a82..1c63b2183 100644 --- a/packages/junior-dashboard/src/client/components/TranscriptToolView.tsx +++ b/packages/junior-dashboard/src/client/components/TranscriptToolView.tsx @@ -40,7 +40,7 @@ export function TranscriptToolView(props: { ? formatMs(props.resultTimestamp - props.timestamp) : undefined; const missingResultLabel = - props.call?.status === "running" ? "running" : "missing result"; + props.call?.status === "running" ? "running" : "started"; const meta = [ duration, props.result ? formatBytes(outputBytes) : undefined, diff --git a/packages/junior-dashboard/src/client/components/transcriptBottomPinning.ts b/packages/junior-dashboard/src/client/components/transcriptBottomPinning.ts index d875ca70c..075050547 100644 --- a/packages/junior-dashboard/src/client/components/transcriptBottomPinning.ts +++ b/packages/junior-dashboard/src/client/components/transcriptBottomPinning.ts @@ -10,7 +10,7 @@ import { } from "react"; import type { ConversationTranscript, TranscriptViewPart } from "../types"; -import { conversationTranscriptMessages } from "../transcriptActivity"; +import { conversationTranscriptMessages } from "../eventTranscript"; const BOTTOM_PROXIMITY_PX = 96; const USER_SCROLL_DELTA_PX = 2; diff --git a/packages/junior-dashboard/src/client/components/transcriptRenderModel.ts b/packages/junior-dashboard/src/client/components/transcriptRenderModel.ts index cc7b6380a..b9748146d 100644 --- a/packages/junior-dashboard/src/client/components/transcriptRenderModel.ts +++ b/packages/junior-dashboard/src/client/components/transcriptRenderModel.ts @@ -36,7 +36,7 @@ export type RenderedThinkingEntry = { export type RenderedFailureEntry = { kind: "failure"; - outcome: "error" | "aborted"; + outcome: "error" | "aborted" | "delivery_failed"; timestamp?: number; }; @@ -300,7 +300,6 @@ export function messageRawText(message: TranscriptViewMessage): string { stringifyPartValue({ id: part.id, outcome: part.outcome, - parentToolCallId: part.parentToolCallId, status: part.status, }), ] @@ -310,12 +309,8 @@ export function messageRawText(message: TranscriptViewMessage): string { if (part.type === "context_event") { const event = part.event; return event.type === "model_handoff" - ? ["model handoff", event.fromModelId, event.toModelId, event.message] - .filter(isString) - .join("\n") - : ["context compacted", event.modelId, event.summary] - .filter(isString) - .join("\n"); + ? "model handoff" + : "context compacted"; } return stringifyPartValue(part.output ?? part.input ?? part.text ?? part); }) diff --git a/packages/junior-dashboard/src/client/components/transcriptSearch.tsx b/packages/junior-dashboard/src/client/components/transcriptSearch.tsx index 1aec26a1c..16556c469 100644 --- a/packages/junior-dashboard/src/client/components/transcriptSearch.tsx +++ b/packages/junior-dashboard/src/client/components/transcriptSearch.tsx @@ -128,9 +128,11 @@ export function entryMatchesSearch( if (entry.kind === "failure") { return textContains( - entry.outcome === "error" - ? "agent response failed error" - : "agent response stopped aborted", + entry.outcome === "delivery_failed" + ? "message delivery failed" + : entry.outcome === "error" + ? "agent response failed error" + : "agent response stopped aborted", normalizedQuery, ); } @@ -154,8 +156,7 @@ export function entryMatchesSearch( textContains(entry.part.subagentKind, normalizedQuery) || textContains(entry.part.id, normalizedQuery) || textContains(entry.part.status, normalizedQuery) || - textContains(entry.part.outcome, normalizedQuery) || - textContains(entry.part.parentToolCallId, normalizedQuery) + textContains(entry.part.outcome, normalizedQuery) ); } @@ -166,13 +167,8 @@ export function entryMatchesSearch( if (entry.kind === "context") { const event = entry.part.event; return event.type === "model_handoff" - ? textContains("model handoff", normalizedQuery) || - textContains(event.fromModelId, normalizedQuery) || - textContains(event.toModelId, normalizedQuery) || - textContains(event.message, normalizedQuery) - : textContains("context compacted", normalizedQuery) || - textContains(event.modelId, normalizedQuery) || - textContains(event.summary, normalizedQuery); + ? textContains("model handoff", normalizedQuery) + : textContains("context compacted", normalizedQuery); } return false; diff --git a/packages/junior-dashboard/src/client/eventTranscript.ts b/packages/junior-dashboard/src/client/eventTranscript.ts new file mode 100644 index 000000000..09444dfe0 --- /dev/null +++ b/packages/junior-dashboard/src/client/eventTranscript.ts @@ -0,0 +1,140 @@ +import type { ConversationReportEvent } from "@sentry/junior/api/schema"; + +import type { + ConversationTranscript, + TranscriptViewMessage, + TranscriptViewPart, + TranscriptViewSubagentPart, +} from "./types"; + +function eventTimestamp(event: ConversationReportEvent): number { + return Date.parse(event.createdAt); +} + +function eventMessage( + event: ConversationReportEvent, + role: TranscriptViewMessage["role"], + parts: TranscriptViewPart[], +): TranscriptViewMessage { + return { role, timestamp: eventTimestamp(event), parts }; +} + +function subagentOutcomes( + events: ConversationReportEvent[], +): Map< + string, + Extract< + ConversationReportEvent["data"], + { type: "subagent_ended" } + >["outcome"] +> { + const outcomes = new Map< + string, + Extract< + ConversationReportEvent["data"], + { type: "subagent_ended" } + >["outcome"] + >(); + for (const event of events) { + if (event.data.type === "subagent_ended") { + outcomes.set(event.data.childConversationId, event.data.outcome); + } + } + return outcomes; +} + +function subagentPart( + data: Extract, + outcome: "aborted" | "error" | "success" | undefined, +): TranscriptViewSubagentPart { + return { + type: "subagent", + id: data.childConversationId, + childConversationId: data.childConversationId, + subagentKind: data.subagentKind, + status: + outcome === "error" || outcome === "aborted" + ? outcome + : outcome === "success" + ? "completed" + : "running", + ...(outcome ? { outcome } : {}), + }; +} + +/** Reduce the ordered reporting event API into dashboard-only transcript rows. */ +export function conversationTranscriptMessages( + conversation: ConversationTranscript, +): TranscriptViewMessage[] { + const outcomes = subagentOutcomes(conversation.events); + const messages: TranscriptViewMessage[] = []; + + // API sequence is the only ordering authority. Do not sort by timestamps: + // producers may preserve ingestion order while clocks are skewed. + for (const event of conversation.events) { + const data = event.data; + if (data.type === "visible_message") { + messages.push( + eventMessage(event, data.role, [ + data.redacted + ? { type: "text", redacted: true } + : { type: "text", text: data.text }, + ]), + ); + continue; + } + + if (data.type === "tool_started") { + messages.push( + eventMessage(event, "tool", [ + // Reporting intentionally has no completion state. This is a neutral + // structural start row, not a claim that the tool is still running. + { type: "tool_call", name: data.name }, + ]), + ); + continue; + } + + if (data.type === "subagent_started") { + messages.push( + eventMessage(event, "tool", [ + subagentPart(data, outcomes.get(data.childConversationId)), + ]), + ); + continue; + } + + if (data.type === "context_compacted" || data.type === "model_handoff") { + messages.push( + eventMessage(event, "system", [ + { + type: "context_event", + event: { type: data.type, createdAt: event.createdAt }, + }, + ]), + ); + continue; + } + + if (data.type === "turn_lifecycle" && data.state === "failed") { + messages.push({ + role: "assistant", + outcome: "error", + parts: [], + timestamp: eventTimestamp(event), + }); + continue; + } + + if (data.type === "delivery" && data.state === "failed") { + messages.push({ + role: "system", + outcome: "delivery_failed", + parts: [], + timestamp: eventTimestamp(event), + }); + } + } + + return messages; +} diff --git a/packages/junior-dashboard/src/client/format.ts b/packages/junior-dashboard/src/client/format.ts index d5cc6df5f..7ed321d28 100644 --- a/packages/junior-dashboard/src/client/format.ts +++ b/packages/junior-dashboard/src/client/format.ts @@ -17,7 +17,7 @@ import type { } from "./types"; import { sameToolInvocation } from "./toolInvocations"; import { formatDuration } from "./components/Duration"; -import { conversationTranscriptMessages } from "./transcriptActivity"; +import { conversationTranscriptMessages } from "./eventTranscript"; let dashboardTimeZone = "America/Los_Angeles"; @@ -183,12 +183,6 @@ function transcriptSource(conversation: ConversationTranscript) { return conversationTranscriptMessages(conversation); } -function rawTranscriptSource(conversation: ConversationTranscript) { - return conversation.transcriptAvailable - ? conversation.transcript - : (conversation.transcriptMetadata ?? []); -} - /** Normalized role category for transcript messages. */ export type TranscriptRoleKind = | "assistant" @@ -230,11 +224,7 @@ function isConversationMessage( export function conversationMessageCount( conversation: ConversationTranscript, ): number { - const source = rawTranscriptSource(conversation); - if (source.length > 0) { - return source.filter(isConversationMessage).length; - } - return conversation.transcriptMessageCount ?? 0; + return transcriptSource(conversation).filter(isConversationMessage).length; } export type ToolCallSummaryItem = { @@ -661,9 +651,12 @@ export function visualStatusForConversation( export function unavailableTranscriptLabel( conversation: ConversationTranscript, ): string { - if (conversation.transcriptRedacted) { + if (conversation.eventHistory.status === "redacted") { return "Transcript hidden because this conversation is not public."; } + if (conversation.eventHistory.status === "expired") { + return "Transcript expired for this conversation."; + } const status = visualStatusForSummary(conversation); if (status === "active") { return "Transcript pending while this conversation is active."; diff --git a/packages/junior-dashboard/src/client/markdownExport.ts b/packages/junior-dashboard/src/client/markdownExport.ts index 58cb8f0d5..9765053e4 100644 --- a/packages/junior-dashboard/src/client/markdownExport.ts +++ b/packages/junior-dashboard/src/client/markdownExport.ts @@ -13,9 +13,7 @@ import { groupTranscriptMessages, messageRawText, } from "./components/transcriptRenderModel"; -import { conversationTranscriptMessages } from "./transcriptActivity"; -import type { ConversationDetailReport } from "@sentry/junior/api/schema"; -import type { ConversationSubagentTranscriptReport } from "@sentry/junior/api/schema"; +import { conversationTranscriptMessages } from "./eventTranscript"; import type { Conversation, ConversationTranscript, @@ -27,7 +25,7 @@ import type { /** Build a clipboard Markdown transcript from the already-authorized dashboard report. */ export function buildConversationMarkdown( - detail: ConversationDetailReport, + detail: ConversationTranscript, conversation?: Conversation, ): string { const lines: string[] = []; @@ -55,43 +53,21 @@ export function buildConversationMarkdown( return finishMarkdown(lines); } -/** Build Markdown for one child-agent transcript using the shared formatter. */ -export function buildSubagentMarkdown( - report: ConversationSubagentTranscriptReport, - conversationTranscript: ConversationTranscript, -): string { - const lines: string[] = [`# ${headingText(report.subagentKind)}`, ""]; - addMetaLine(lines, "Subagent ID", inlineCode(report.id)); - addMetaLine(lines, "Conversation ID", report.subagentConversationId); - addMetaLine(lines, "Created", report.createdAt); - addMetaLine(lines, "Status", report.outcome ?? report.status); - addMetaLine( - lines, - "Duration", - formatMs(conversationTranscript.cumulativeDurationMs), - ); - addMetaLine( - lines, - "Sentry conversation", - report.subagentSentryConversationUrl, - ); - lines.push("", "## Transcript"); - appendConversationTranscript(lines, conversationTranscript); - return finishMarkdown(lines); -} - function appendConversationTranscript( lines: string[], conversationTranscript: ConversationTranscript, ): void { const transcript = conversationTranscriptMessages(conversationTranscript); - if (conversationTranscript.transcriptAvailable) { + if (conversationTranscript.eventHistory.status === "available") { appendTranscriptMessages(lines, conversationTranscript, transcript, false); return; } - if (conversationTranscript.transcriptRedacted && transcript.length) { + if ( + conversationTranscript.eventHistory.status === "redacted" && + transcript.length + ) { lines.push( "", "Transcript hidden because this conversation is not public.", @@ -187,21 +163,25 @@ function appendTranscriptMessages( function appendFailure( lines: string[], conversationTranscript: ConversationTranscript, - outcome: "error" | "aborted", + outcome: "error" | "aborted" | "delivery_failed", timestamp: number | undefined, ): void { lines.push( "", - outcome === "error" - ? "### Agent response failed" - : "### Agent response stopped", + outcome === "delivery_failed" + ? "### Message delivery failed" + : outcome === "error" + ? "### Agent response failed" + : "### Agent response stopped", ); addEventMeta(lines, conversationTranscript, timestamp); lines.push( "", - outcome === "error" - ? "The model response ended before Junior could complete this turn." - : "The model response was stopped before Junior could complete this turn.", + outcome === "delivery_failed" + ? "Junior could not deliver this message to its destination." + : outcome === "error" + ? "The model response ended before Junior could complete this turn." + : "The model response was stopped before Junior could complete this turn.", ); } @@ -219,14 +199,6 @@ function appendContextEvent( : "### Context compacted", ); addEventMeta(lines, conversationTranscript, timestamp); - if (event.type === "model_handoff") { - addMetaLine(lines, "From model", event.fromModelId); - addMetaLine(lines, "To model", event.toModelId); - } else { - addMetaLine(lines, "Model", event.modelId); - } - const body = event.type === "model_handoff" ? event.message : event.summary; - if (body) lines.push("", body); } function appendMessage( @@ -275,7 +247,6 @@ function appendSubagent( lines.push("", `### Subagent: ${headingText(part.subagentKind)}`); addEventMeta(lines, conversationTranscript, timestamp); addMetaLine(lines, "Status", part.outcome ?? part.status); - addMetaLine(lines, "Parent tool call", part.parentToolCallId); } function appendTool( @@ -336,7 +307,7 @@ function appendToolHeader( addMetaLine( lines, "Result", - call?.status === "running" ? "running" : "missing", + call?.status === "running" ? "running" : "started", ); } } @@ -356,7 +327,7 @@ function addEventMeta( } function conversationTitle( - detail: ConversationDetailReport, + detail: ConversationTranscript, conversation: Conversation | undefined, ): string { const title = detail.displayTitle.trim(); diff --git a/packages/junior-dashboard/src/client/pages/ConversationPage.tsx b/packages/junior-dashboard/src/client/pages/ConversationPage.tsx index d1a4d4c6a..6aac44ae7 100644 --- a/packages/junior-dashboard/src/client/pages/ConversationPage.tsx +++ b/packages/junior-dashboard/src/client/pages/ConversationPage.tsx @@ -137,9 +137,10 @@ export function ConversationPage(props: { /> } live={conversationIsLive(visualStatus, detail.data)} - onOpenSubagentTranscript={({ part, conversation }) => { - if (!conversationId) return; - setSubagentTarget({ conversation, conversationId, part }); + onOpenSubagentTranscript={({ part }) => { + const childConversationId = part.childConversationId; + if (!childConversationId) return; + setSubagentTarget({ conversationId: childConversationId, part }); }} transcript={detail.data} /> diff --git a/packages/junior-dashboard/src/client/subagentTranscript.ts b/packages/junior-dashboard/src/client/subagentTranscript.ts deleted file mode 100644 index 231dd5286..000000000 --- a/packages/junior-dashboard/src/client/subagentTranscript.ts +++ /dev/null @@ -1,51 +0,0 @@ -import type { ConversationSubagentTranscriptReport } from "@sentry/junior/api/schema"; -import type { ConversationTranscript } from "./types"; - -/** Project a child-agent report through the shared transcript renderer. */ -export function subagentConversationTranscript( - conversationId: string, - report: ConversationSubagentTranscriptReport, -): ConversationTranscript { - const status = - report.status === "running" - ? "active" - : report.status === "error" || report.status === "aborted" - ? "failed" - : "completed"; - - return { - conversationId, - assistantLabel: report.subagentKind, - cumulativeDurationMs: subagentDurationMs(report) ?? 0, - displayTitle: report.subagentKind, - lastProgressAt: report.endedAt ?? report.createdAt, - lastSeenAt: report.endedAt ?? report.createdAt, - startedAt: report.createdAt, - status, - surface: "internal", - transcript: report.transcript, - transcriptAvailable: report.transcriptAvailable, - ...(report.transcriptMessageCount !== undefined - ? { transcriptMessageCount: report.transcriptMessageCount } - : {}), - ...(report.transcriptRedacted - ? { transcriptRedacted: report.transcriptRedacted } - : {}), - ...(report.transcriptRedactionReason - ? { transcriptRedactionReason: report.transcriptRedactionReason } - : {}), - }; -} - -/** Calculate a completed child-agent duration when timestamps are valid. */ -export function subagentDurationMs( - report: Pick, -): number | undefined { - if (!report.endedAt) return undefined; - const startedAt = Date.parse(report.createdAt); - const endedAt = Date.parse(report.endedAt); - if (!Number.isFinite(startedAt) || !Number.isFinite(endedAt)) { - return undefined; - } - return endedAt >= startedAt ? endedAt - startedAt : undefined; -} diff --git a/packages/junior-dashboard/src/client/transcriptActivity.ts b/packages/junior-dashboard/src/client/transcriptActivity.ts deleted file mode 100644 index 9c77b4312..000000000 --- a/packages/junior-dashboard/src/client/transcriptActivity.ts +++ /dev/null @@ -1,358 +0,0 @@ -import { sameToolInvocation } from "./toolInvocations"; -import type { ConversationActivityReport } from "@sentry/junior/api/schema"; -import type { - ConversationTranscript, - TranscriptViewMessage, - TranscriptViewPart, - TranscriptViewSubagentPart, - TranscriptViewToolCallPart, -} from "./types"; - -type ToolActivity = Extract< - ConversationActivityReport, - { type: "tool_execution" } ->; - -type SubagentActivity = Extract< - ConversationActivityReport, - { type: "subagent" } ->; - -type IndexedMessage = { - message: TranscriptViewMessage; - order: number; -}; - -function activityTimestamp(value: string | undefined): number | undefined { - if (!value) return undefined; - const timestamp = Date.parse(value); - return Number.isFinite(timestamp) ? timestamp : undefined; -} - -function isToolCall( - part: TranscriptViewPart, -): part is TranscriptViewToolCallPart { - return part.type === "tool_call"; -} - -function isToolResult(part: TranscriptViewPart): boolean { - return part.type === "tool_result"; -} - -function partMatchesToolActivity( - part: TranscriptViewPart, - activity: ToolActivity, -): boolean { - return sameToolInvocation(part, { - id: activity.toolCallId, - name: activity.toolName, - }); -} - -function toolCallPart( - activity: ToolActivity, - existing?: TranscriptViewToolCallPart, -): TranscriptViewPart { - const input = existing?.input ?? activity.args; - const part: TranscriptViewPart = { - type: "tool_call", - id: activity.toolCallId, - name: activity.toolName, - status: activity.status, - }; - if (activity.redacted) { - part.redacted = true; - if (activity.inputKeys) part.inputKeys = activity.inputKeys; - if (activity.inputSizeBytes !== undefined) { - part.inputSizeBytes = activity.inputSizeBytes; - } - if (activity.inputSizeChars !== undefined) { - part.inputSizeChars = activity.inputSizeChars; - } - if (activity.inputType) part.inputType = activity.inputType; - return part; - } - if (input !== undefined) part.input = input; - return part; -} - -function subagentPart(activity: SubagentActivity): TranscriptViewSubagentPart { - return { - type: "subagent", - id: activity.id, - ...(activity.modelId ? { modelId: activity.modelId } : {}), - subagentKind: activity.subagentKind, - status: activity.status, - ...(activity.outcome ? { outcome: activity.outcome } : {}), - ...(activity.parentToolCallId - ? { parentToolCallId: activity.parentToolCallId } - : {}), - ...(activity.reasoningLevel - ? { reasoningLevel: activity.reasoningLevel } - : {}), - ...(activity.endedAt ? { endedAt: activity.endedAt } : {}), - ...(activity.transcriptAvailable !== undefined - ? { transcriptAvailable: activity.transcriptAvailable } - : {}), - }; -} - -function toolActivities(conversation: ConversationTranscript): ToolActivity[] { - return (conversation.activity ?? []).filter( - (activity): activity is ToolActivity => activity.type === "tool_execution", - ); -} - -function orphanSubagentActivities( - conversation: ConversationTranscript, -): SubagentActivity[] { - return (conversation.activity ?? []).filter( - (activity): activity is SubagentActivity => activity.type === "subagent", - ); -} - -function activityMessage( - timestamp: number | undefined, - part: TranscriptViewPart, -): TranscriptViewMessage { - return { - role: "tool", - ...(timestamp !== undefined ? { timestamp } : {}), - parts: [part], - }; -} - -function upgradeToolCalls( - messages: TranscriptViewMessage[], - activities: ToolActivity[], -): { - messages: TranscriptViewMessage[]; - usedToolCallIds: Set; -} { - const usedToolCallIds = new Set(); - if (activities.length === 0) { - return { messages, usedToolCallIds }; - } - - const upgraded = messages.map((message) => { - let changed = false; - const parts = message.parts.map((part) => { - if (!isToolCall(part)) return part; - - const activity = activities.find( - (candidate) => - !usedToolCallIds.has(candidate.toolCallId) && - partMatchesToolActivity(part, candidate), - ); - if (!activity) return part; - - usedToolCallIds.add(activity.toolCallId); - changed = true; - return toolCallPart(activity, part); - }); - - return changed ? { ...message, parts } : message; - }); - - return { messages: upgraded, usedToolCallIds }; -} - -function syntheticMessages( - activities: ToolActivity[], - orphanSubagents: SubagentActivity[], - usedToolCallIds: Set, -): IndexedMessage[] { - const messages: IndexedMessage[] = []; - const emittedSubagentKeys = new Set(); - const emittedToolCallIds = new Set(usedToolCallIds); - let order = 0; - - for (const activity of activities) { - if (!emittedToolCallIds.has(activity.toolCallId)) { - messages.push({ - message: activityMessage( - activityTimestamp(activity.createdAt), - toolCallPart(activity), - ), - order: order + 0.1, - }); - emittedToolCallIds.add(activity.toolCallId); - } - - for (const subagent of activity.subagents) { - const key = subagentActivityKey(subagent); - if (emittedSubagentKeys.has(key)) continue; - - messages.push({ - message: activityMessage( - activityTimestamp(subagent.createdAt), - subagentPart(subagent), - ), - order: order + 0.2, - }); - emittedSubagentKeys.add(key); - order += 1; - } - order += 1; - } - - for (const subagent of orphanSubagents) { - const key = subagentActivityKey(subagent); - if (emittedSubagentKeys.has(key)) continue; - - messages.push({ - message: activityMessage( - activityTimestamp(subagent.createdAt), - subagentPart(subagent), - ), - order: order + 0.2, - }); - emittedSubagentKeys.add(key); - order += 1; - } - - return messages; -} - -function subagentActivityKey(activity: SubagentActivity): string { - return [ - activity.parentToolCallId ?? "", - activity.id, - activity.subagentKind, - ].join("\u0000"); -} - -function messageTimestamp(message: TranscriptViewMessage): number | undefined { - return typeof message.timestamp === "number" && - Number.isFinite(message.timestamp) - ? message.timestamp - : undefined; -} - -function findMatchingToolResultIndex( - messages: TranscriptViewMessage[], - part: TranscriptViewToolCallPart, -): number | undefined { - const index = messages.findIndex((message) => - message.parts.some( - (candidate) => - isToolResult(candidate) && sameToolInvocation(candidate, part), - ), - ); - return index >= 0 ? index : undefined; -} - -function findParentToolCallIndex( - messages: TranscriptViewMessage[], - parentToolCallId: string, -): number | undefined { - const index = messages.findIndex((message) => - message.parts.some( - (candidate) => isToolCall(candidate) && candidate.id === parentToolCallId, - ), - ); - return index >= 0 ? index : undefined; -} - -function isSubagentForParent( - message: TranscriptViewMessage, - parentToolCallId: string, -): boolean { - return message.parts.some( - (candidate) => - candidate.type === "subagent" && - candidate.parentToolCallId === parentToolCallId, - ); -} - -function subagentInsertionIndex( - messages: TranscriptViewMessage[], - parentToolCallId: string, -): number | undefined { - const parentIndex = findParentToolCallIndex(messages, parentToolCallId); - if (parentIndex === undefined) return undefined; - - let index = parentIndex + 1; - while ( - index < messages.length && - isSubagentForParent(messages[index]!, parentToolCallId) - ) { - index += 1; - } - return index; -} - -function syntheticInsertionIndex( - messages: TranscriptViewMessage[], - message: TranscriptViewMessage, -): number { - const part = message.parts[0]; - if (part && isToolCall(part)) { - const resultIndex = findMatchingToolResultIndex(messages, part); - if (resultIndex !== undefined) return resultIndex; - } - - if (part?.type === "subagent" && part.parentToolCallId) { - const index = subagentInsertionIndex(messages, part.parentToolCallId); - if (index !== undefined) return index; - } - - const timestamp = messageTimestamp(message); - if (timestamp === undefined) return messages.length; - - const index = messages.findIndex((candidate) => { - const candidateTimestamp = messageTimestamp(candidate); - return candidateTimestamp !== undefined && candidateTimestamp > timestamp; - }); - return index >= 0 ? index : messages.length; -} - -function mergeMessages( - messages: TranscriptViewMessage[], - synthetic: IndexedMessage[], -): TranscriptViewMessage[] { - const merged = [...messages]; - for (const entry of synthetic) { - merged.splice( - syntheticInsertionIndex(merged, entry.message), - 0, - entry.message, - ); - } - return merged; -} - -/** Return the transcript rows that dashboard views should render for a conversation. */ -export function conversationTranscriptMessages( - conversation: ConversationTranscript, -): TranscriptViewMessage[] { - const source = conversation.transcriptAvailable - ? conversation.transcript - : (conversation.transcriptMetadata ?? []); - const messagesWithContext: TranscriptViewMessage[] = [...source]; - (conversation.contextEvents ?? []).forEach((event, offset) => { - messagesWithContext.splice( - event.transcriptIndex + offset, - 0, - activityMessage(activityTimestamp(event.createdAt), { - type: "context_event", - event, - }), - ); - }); - const activities = toolActivities(conversation); - const orphanSubagents = orphanSubagentActivities(conversation); - if (activities.length === 0 && orphanSubagents.length === 0) { - return messagesWithContext; - } - - const { messages, usedToolCallIds } = upgradeToolCalls( - messagesWithContext, - activities, - ); - - return mergeMessages( - messages, - syntheticMessages(activities, orphanSubagents, usedToolCallIds), - ); -} diff --git a/packages/junior-dashboard/src/client/types.ts b/packages/junior-dashboard/src/client/types.ts index e4ceec022..b7265ee25 100644 --- a/packages/junior-dashboard/src/client/types.ts +++ b/packages/junior-dashboard/src/client/types.ts @@ -8,23 +8,38 @@ import type { ConversationStatsReport, ConversationSummaryReport, } from "@sentry/junior/api/schema"; -import type { - ConversationActivityStatus, - ConversationContextEvent, - ConversationDetailReport, - TranscriptMessage, - TranscriptPart, -} from "@sentry/junior/api/schema"; +import type { ConversationDetailReport } from "@sentry/junior/api/schema"; import type { DashboardConfig, DashboardIdentity } from "../api/schema"; -// Dashboard view transcript parts merge reporting transcript payloads with -// lifecycle activity rows; the backend reporting transcript contract is unchanged. -type TranscriptViewReportingPart = TranscriptPart & { - endedAt?: never; +export type TranscriptViewStatus = + | "aborted" + | "completed" + | "error" + | "running" + | "success"; + +type TranscriptViewReportingPart = { + bytes?: number; + chars?: number; + id?: string; + input?: unknown; + inputKeys?: string[]; + inputSizeBytes?: number; + inputSizeChars?: number; + inputType?: string; + name?: string; + output?: unknown; + outputKeys?: string[]; + outputSizeBytes?: number; + outputSizeChars?: number; + outputType?: string; outcome?: never; - parentToolCallId?: never; - status?: ConversationActivityStatus; + redacted?: boolean; + sourceType?: string; + status?: TranscriptViewStatus; subagentKind?: never; + text?: string; + type: "text" | "thinking" | "tool_call" | "tool_result" | "unknown"; }; export type TranscriptViewToolCallPart = TranscriptViewReportingPart & { @@ -34,14 +49,13 @@ export type TranscriptViewToolCallPart = TranscriptViewReportingPart & { export type TranscriptViewSubagentPart = { bytes?: never; chars?: never; - endedAt?: string; + childConversationId?: string; id: string; input?: never; inputKeys?: never; inputSizeBytes?: never; inputSizeChars?: never; inputType?: never; - modelId?: string; name?: never; outcome?: "success" | "error" | "aborted"; output?: never; @@ -49,12 +63,9 @@ export type TranscriptViewSubagentPart = { outputSizeBytes?: never; outputSizeChars?: never; outputType?: never; - parentToolCallId?: string; - reasoningLevel?: string; redacted?: boolean; - status: ConversationActivityStatus; + status: TranscriptViewStatus; subagentKind: string; - transcriptAvailable?: boolean; text?: never; type: "subagent"; }; @@ -62,15 +73,16 @@ export type TranscriptViewSubagentPart = { export type TranscriptViewContextEventPart = { bytes?: never; chars?: never; - endedAt?: never; - event: ConversationContextEvent; + event: { + createdAt: string; + type: "context_compacted" | "model_handoff"; + }; id?: never; input?: never; inputKeys?: never; inputSizeBytes?: never; inputSizeChars?: never; inputType?: never; - modelId?: never; name?: never; outcome?: never; output?: never; @@ -78,13 +90,10 @@ export type TranscriptViewContextEventPart = { outputSizeBytes?: never; outputSizeChars?: never; outputType?: never; - parentToolCallId?: never; - reasoningLevel?: never; redacted?: never; status?: never; subagentKind?: never; text?: never; - transcriptAvailable?: never; type: "context_event"; }; @@ -94,14 +103,14 @@ export type TranscriptViewPart = | TranscriptViewSubagentPart | TranscriptViewToolCallPart; -export type TranscriptViewMessage = Omit & { +export type TranscriptViewMessage = { + outcome?: "error" | "aborted" | "delivery_failed"; parts: TranscriptViewPart[]; + role: "assistant" | "system" | "tool" | "toolResult" | "unknown" | "user"; + timestamp?: number; }; -export type ConversationTranscript = Omit< - ConversationDetailReport, - "generatedAt" | "sentryConversationUrl" -> & { +export type ConversationTranscript = ConversationDetailReport & { assistantLabel?: string; }; diff --git a/packages/junior-dashboard/src/mock-conversations.ts b/packages/junior-dashboard/src/mock-conversations.ts index 46037e241..68ea5e94a 100644 --- a/packages/junior-dashboard/src/mock-conversations.ts +++ b/packages/junior-dashboard/src/mock-conversations.ts @@ -1,51 +1,35 @@ import type { - ConversationStatsItem, - ConversationStatsReport, -} from "@sentry/junior/api/schema"; -import type { - ActorActivityDayReport, ActorDirectoryReport, + ActorActivityDayReport, ActorIdentity, ActorProfileReport, ActorSummaryReport, + ConversationDetailReport, ConversationFeed, + ConversationReportEvent, + ConversationReportEventData, + ConversationStatsItem, + ConversationStatsReport, ConversationSummaryReport, - ConversationUsage, - PeopleActivityDayReport, -} from "@sentry/junior/api/schema"; -import type { - ConversationDetailReport, - TranscriptMessage, -} from "@sentry/junior/api/schema"; -import type { - DailyConversationActivity, + LocationDetailReport, LocationActorSummaryReport, LocationActivityDayReport, - LocationDetailReport, LocationDirectoryReport, LocationSummaryReport, + PeopleActivityDayReport, } from "@sentry/junior/api/schema"; -import type { ConversationSubagentTranscriptReport } from "@sentry/junior/api/schema"; - -import { longReleaseConversation } from "./mock-release-conversation"; -import { - mockSubagentActivity, - mockToolActivity, -} from "./mock-reporting/activity"; -import { - mockToolCallPart, - mockToolResultPart, - mockTranscriptMessage, -} from "./mock-reporting/transcript"; -const INCIDENT_CONVERSATION_ID = "slack:CQA123:1770000000.000100"; const ACTIVE_CONVERSATION_ID = "slack:CQA123:1770003600.000200"; +const INCIDENT_CONVERSATION_ID = "slack:CQA123:1770000000.000100"; const PRIVATE_CONVERSATION_ID = "slack:DQA123:1770007200.000300"; const SANDBOX_CONVERSATION_ID = "slack:CQA999:1770010800.000400"; const FAILED_CONVERSATION_ID = "slack:CQA777:1770014400.000500"; +const LONG_CONVERSATION_ID = "slack:CQA456:1770021600.000600"; const SCHEDULER_CONVERSATION_ID = "scheduler:daily-ops-digest"; export const DASHBOARD_QA_CONVERSATION_ID = "internal:dashboard-qa"; -const DASHBOARD_QA_ADVISOR_CONVERSATION_ID = `junior:${DASHBOARD_QA_CONVERSATION_ID}:advisor_session`; +const DASHBOARD_QA_PLAN_ID = "junior:internal:dashboard-qa:advisor-plan"; +const DASHBOARD_QA_REVIEW_ID = "junior:internal:dashboard-qa:advisor-review"; +const RECENT_CONVERSATION_STATS_WINDOW_MS = 90 * 24 * 60 * 60 * 1000; const PEOPLE_ACTIVITY_DAYS = 90; const PEOPLE_PROFILE_ACTIVITY_DAYS = 365; const PUBLIC_MOCK_CHANNEL_IDS = new Set([ @@ -63,885 +47,391 @@ function sentryConversationUrl(conversationId: string): string { return `https://sentry.example.com/organizations/acme/explore/conversations/${encodeURIComponent(conversationId)}/`; } -function sentryTraceUrl(traceId: string): string { - return `https://sentry.example.com/performance/trace/${traceId}/`; +function reportEvent( + seq: number, + createdAt: string, + data: ConversationReportEventData, + contextEpoch = 0, +): ConversationReportEvent { + return { seq, contextEpoch, createdAt, data }; } -function summaryFromConversation( - conversation: ConversationDetailReport, -): ConversationSummaryReport { +type DetailOptions = Omit< + ConversationDetailReport, + | "cumulativeDurationMs" + | "displayTitle" + | "eventHistory" + | "events" + | "generatedAt" + | "lastProgressAt" + | "lastSeenAt" + | "startedAt" + | "status" + | "surface" +> & { + cumulativeDurationMs?: number; + displayTitle: string; + eventHistory?: ConversationDetailReport["eventHistory"]; + events: ConversationReportEvent[]; + lastProgressAt?: string; + lastSeenAt?: string; + startedAt: string; + status?: ConversationDetailReport["status"]; + surface?: ConversationDetailReport["surface"]; +}; + +function detail( + nowMs: number, + options: DetailOptions, +): ConversationDetailReport { return { - displayTitle: conversation.displayTitle, - cumulativeDurationMs: conversation.cumulativeDurationMs, - conversationId: conversation.conversationId, - status: conversation.status, - startedAt: conversation.startedAt, - lastSeenAt: conversation.lastSeenAt, - lastProgressAt: conversation.lastProgressAt, - surface: conversation.surface, - ...(conversation.cumulativeUsage - ? { cumulativeUsage: conversation.cumulativeUsage } - : {}), - ...(conversation.actorIdentity - ? { actorIdentity: conversation.actorIdentity } - : {}), - ...(conversation.channel ? { channel: conversation.channel } : {}), - ...(conversation.channelName - ? { channelName: conversation.channelName } - : {}), - ...(conversation.channelNameRedacted !== undefined - ? { channelNameRedacted: conversation.channelNameRedacted } - : {}), - ...(conversation.channel && - PUBLIC_MOCK_CHANNEL_IDS.has(conversation.channel) - ? { locationId: mockLocationId(conversation.channel) } - : {}), - ...(conversation.sentryTraceUrl - ? { sentryTraceUrl: conversation.sentryTraceUrl } - : {}), - ...(conversation.traceId ? { traceId: conversation.traceId } : {}), + ...options, + cumulativeDurationMs: options.cumulativeDurationMs ?? 0, + eventHistory: options.eventHistory ?? { status: "available" }, + generatedAt: iso(nowMs), + lastProgressAt: options.lastProgressAt ?? options.startedAt, + lastSeenAt: options.lastSeenAt ?? options.startedAt, + status: options.status ?? "completed", + surface: options.surface ?? "internal", }; } -function publicIncidentConversation(nowMs: number): ConversationDetailReport { - const traceId = "5f2c7f7df83e4a37a03c9d4a14f4c991"; - const secondStartedAt = iso(nowMs, -44 * 60_000); - - return { - conversationId: INCIDENT_CONVERSATION_ID, - displayTitle: "Checkout latency triage", - generatedAt: iso(nowMs), - sentryConversationUrl: sentryConversationUrl(INCIDENT_CONVERSATION_ID), - status: "completed", - startedAt: secondStartedAt, - lastProgressAt: iso(nowMs, -42 * 60_000), - lastSeenAt: iso(nowMs, -41 * 60_000), - cumulativeDurationMs: 206_000, - cumulativeUsage: { - cachedInputTokens: 3100, - inputTokens: 5200, - outputTokens: 950, - reasoningTokens: 420, - totalTokens: 9250, - cost: { - input: 0.0156, - output: 0.0114, - cacheRead: 0.0062, - total: 0.0332, - }, - }, +function activeConversation(nowMs: number): ConversationDetailReport { + const startedAt = iso(nowMs, -6 * 60_000); + return detail(nowMs, { + conversationId: ACTIVE_CONVERSATION_ID, + displayTitle: "Investigate checkout latency", + startedAt, + lastProgressAt: iso(nowMs, -20_000), + lastSeenAt: iso(nowMs, -10_000), + status: "active", surface: "slack", - actorIdentity: { - email: "morgan@sentry.io", - fullName: "Morgan Lee", - slackUserId: "UQA222", - slackUserName: "morgan", - }, channel: "CQA123", channelName: "proj-checkout", - sentryTraceUrl: sentryTraceUrl(traceId), - traceId, - transcriptAvailable: true, - transcriptMessageCount: 3, - transcript: [ - { + actorIdentity: actor("morgan@sentry.io", "Morgan Lee", "morgan"), + cumulativeDurationMs: 31_000, + cumulativeUsage: usage(0.041), + sentryConversationUrl: sentryConversationUrl(ACTIVE_CONVERSATION_ID), + events: [ + reportEvent(0, startedAt, { + type: "visible_message", + messageId: "active-user", role: "user", - timestamp: Date.parse(secondStartedAt), - parts: [ - { - type: "text", - text: "Can you draft the rollback note with the exact evidence?", - }, - ], - }, - { + text: "Find the slow checkout requests from the last deployment.", + }), + reportEvent(1, iso(Date.parse(startedAt), 5_000), { + type: "turn_lifecycle", + turnId: "active-turn", + state: "started", + }), + reportEvent(2, iso(Date.parse(startedAt), 8_000), { + type: "tool_started", + name: "datacat.search_logs", + }), + ], + }); +} + +function dashboardQaConversation(nowMs: number): ConversationDetailReport { + const startedAt = iso(nowMs, -11 * 60_000); + return detail(nowMs, { + conversationId: DASHBOARD_QA_CONVERSATION_ID, + displayTitle: "Dashboard QA edge cases", + startedAt, + lastSeenAt: iso(nowMs, -8 * 60_000), + lastProgressAt: iso(nowMs, -8 * 60_000), + actorIdentity: actor("morgan@sentry.io", "Morgan Lee", "morgan"), + cumulativeDurationMs: 98_000, + events: [ + reportEvent(0, startedAt, { + type: "visible_message", + messageId: "qa-user", + role: "user", + text: "Review the dashboard plan before editing.", + }), + reportEvent(1, iso(Date.parse(startedAt), 2_000), { + type: "tool_started", + name: "advisor", + }), + reportEvent(2, iso(Date.parse(startedAt), 3_000), { + type: "subagent_started", + childConversationId: DASHBOARD_QA_PLAN_ID, + subagentKind: "advisor", + historyMode: "shared", + }), + reportEvent(3, iso(Date.parse(startedAt), 20_000), { + type: "subagent_ended", + childConversationId: DASHBOARD_QA_PLAN_ID, + subagentKind: "advisor", + historyMode: "shared", + outcome: "success", + }), + reportEvent(4, iso(Date.parse(startedAt), 25_000), { + type: "subagent_started", + childConversationId: DASHBOARD_QA_REVIEW_ID, + subagentKind: "advisor", + historyMode: "shared", + }), + reportEvent(5, iso(Date.parse(startedAt), 44_000), { + type: "subagent_ended", + childConversationId: DASHBOARD_QA_REVIEW_ID, + subagentKind: "advisor", + historyMode: "shared", + outcome: "success", + }), + reportEvent(6, iso(Date.parse(startedAt), 50_000), { + type: "context_compacted", + }), + reportEvent(7, iso(Date.parse(startedAt), 55_000), { + type: "visible_message", + messageId: "qa-assistant", + role: "assistant", + text: "The canonical event rendering looks sound.", + }), + ], + }); +} + +function advisorConversation( + nowMs: number, + conversationId: string, + text: string, +): ConversationDetailReport { + const startedAt = iso(nowMs, -10 * 60_000); + return detail(nowMs, { + conversationId, + displayTitle: "Advisor review", + startedAt, + lastSeenAt: iso(Date.parse(startedAt), 18_000), + lastProgressAt: iso(Date.parse(startedAt), 18_000), + cumulativeDurationMs: 18_000, + events: [ + reportEvent(0, startedAt, { + type: "visible_message", + messageId: `${conversationId}:input`, + role: "user", + text, + }), + reportEvent(1, iso(Date.parse(startedAt), 18_000), { + type: "visible_message", + messageId: `${conversationId}:output`, role: "assistant", - timestamp: Date.parse(secondStartedAt) + 12_000, - parts: [ - { - id: "toolu_mock_issue_lookup", - name: "sentry.get_issue", - input: { - issue: "CHECKOUT-9B2", - project: "checkout-api", - }, - type: "tool_call", - }, - { - id: "toolu_mock_release_lookup", - name: "vercel.get_deployment", - input: { - deployment: "payments-v42", - team: "acme", - }, - type: "tool_call", - }, - ], + text: "Review complete; no blocking issues found.", + }), + ], + }); +} + +function longConversation(nowMs: number): ConversationDetailReport { + const startedAt = iso(nowMs, -92 * 60_000); + const events: ConversationReportEvent[] = [ + reportEvent(0, startedAt, { + type: "visible_message", + messageId: "release-user", + role: "user", + text: "Release the package, update the example app, and open a PR.", + }), + ]; + for (let index = 0; index < 12; index += 1) { + events.push( + reportEvent( + index + 1, + iso(Date.parse(startedAt), 2_000 + index * 4_000), + { + type: "tool_started", + name: "bash", + }, + ), + ); + } + events.push( + reportEvent(13, iso(Date.parse(startedAt), 53_000), { + type: "context_compacted", + }), + reportEvent( + 14, + iso(Date.parse(startedAt), 90_000), + { + type: "model_handoff", }, + 1, + ), + reportEvent( + 15, + iso(Date.parse(startedAt), 166_000), { + type: "visible_message", + messageId: "release-assistant", role: "assistant", - timestamp: Date.parse(secondStartedAt) + 188_000, - parts: [ - { - type: "text", - text: [ - "Rollback note:", - "", - "```md", - "Rolling back payments-v42. Evidence: checkout p95 rose from 740ms to 2.3s, traces isolate the regression to stripe.payment_intents.create, and CHECKOUT-9B2 began after the deployment window.", - "```", - "", - "Keep the rollback scoped to payments; frontend traffic and auth spans look stable.", - ].join("\n"), - }, - ], + text: "Released the package and opened the update pull request.", }, - ], - }; + 1, + ), + ); + return detail(nowMs, { + conversationId: LONG_CONVERSATION_ID, + displayTitle: "Package release and self-update", + startedAt, + lastSeenAt: iso(nowMs, -81 * 60_000), + lastProgressAt: iso(nowMs, -81 * 60_000), + surface: "slack", + channel: "CQA456", + channelName: "proj-release", + actorIdentity: actor(undefined, "Jordan Blake", "jordan"), + cumulativeDurationMs: 552_761, + cumulativeUsage: usage(0.18), + events, + }); } -function activeConversation(nowMs: number): ConversationDetailReport { - const startedAt = iso(nowMs, -6 * 60_000); - - return { - conversationId: ACTIVE_CONVERSATION_ID, - displayTitle: "Deploy rollout watch", - generatedAt: iso(nowMs), - sentryConversationUrl: sentryConversationUrl(ACTIVE_CONVERSATION_ID), - status: "active", +function incidentConversation(nowMs: number): ConversationDetailReport { + const startedAt = iso(nowMs, -44 * 60_000); + return detail(nowMs, { + conversationId: INCIDENT_CONVERSATION_ID, + displayTitle: "Checkout latency triage", startedAt, - lastProgressAt: iso(nowMs, -18_000), - lastSeenAt: iso(nowMs, -12_000), - cumulativeDurationMs: 348_000, - cumulativeUsage: { - inputTokens: 7800, - outputTokens: 620, - totalTokens: 8420, - }, + lastSeenAt: iso(nowMs, -41 * 60_000), + lastProgressAt: iso(nowMs, -42 * 60_000), surface: "slack", - actorIdentity: { - email: "sam@sentry.io", - fullName: "Sam Rivera", - slackUserId: "UQA333", - slackUserName: "sam", - }, channel: "CQA123", channelName: "proj-checkout", - transcriptAvailable: true, - transcriptMessageCount: 2, - transcript: [ - { + actorIdentity: actor("morgan@sentry.io", "Morgan Lee", "morgan"), + cumulativeDurationMs: 206_000, + cumulativeUsage: usage(0.0332), + events: [ + reportEvent(0, startedAt, { + type: "visible_message", + messageId: "incident-user", role: "user", - timestamp: Date.parse(startedAt), - parts: [ - { - type: "text", - text: "Watch the rollout for the next few minutes and call out anything that looks unsafe.", - }, - ], - }, - { + text: "Draft the rollback note with the exact evidence.", + }), + reportEvent(1, iso(Date.parse(startedAt), 12_000), { + type: "tool_started", + name: "sentry.get_issue", + }), + reportEvent(2, iso(Date.parse(startedAt), 35_000), { + type: "visible_message", + messageId: "incident-assistant", role: "assistant", - timestamp: Date.parse(startedAt) + 41_000, - parts: [ - { - type: "thinking", - output: - "Keep the user updated only if the rollout crosses the agreed error-budget threshold.", - }, - { - id: "toolu_mock_datacat_rollout", - name: "datacat.search_logs", - input: { - query: "service:checkout-api env:prod rollout:v42", - window: "15m", - }, - type: "tool_call", - }, - ], - }, + text: "The regression started with payments-v42; rollback is recommended.", + }), ], - }; + }); } function privateConversation(nowMs: number): ConversationDetailReport { const startedAt = iso(nowMs, -24 * 60_000); - - return { + return detail(nowMs, { conversationId: PRIVATE_CONVERSATION_ID, displayTitle: "Direct Message", - generatedAt: iso(nowMs), - status: "completed", startedAt, - lastProgressAt: iso(nowMs, -23 * 60_000), - lastSeenAt: iso(nowMs, -22 * 60_000), - cumulativeDurationMs: 94_000, - cumulativeUsage: { - inputTokens: 3100, - outputTokens: 440, - totalTokens: 3540, - }, + lastSeenAt: iso(nowMs, -21 * 60_000), + lastProgressAt: iso(nowMs, -21 * 60_000), surface: "slack", - actorIdentity: { - email: "private-user@sentry.io", - slackUserId: "UQA444", - slackUserName: "private-user", - }, channel: "DQA123", - channelName: "Direct Message", + channelName: "Private conversation", channelNameRedacted: true, - transcriptAvailable: false, - transcriptMessageCount: 4, - transcriptMetadata: redactedPrivateTranscript(Date.parse(startedAt)), - transcriptRedacted: true, - transcriptRedactionReason: "non_public_conversation", - transcript: [], - }; -} - -function redactedPrivateTranscript(startedAtMs: number): TranscriptMessage[] { - return [ - { - role: "user", - timestamp: startedAtMs, - parts: [ - { - bytes: 174, - chars: 172, - redacted: true, - type: "text", - }, - ], - }, - { - role: "assistant", - timestamp: startedAtMs + 18_000, - parts: [ - { - outputKeys: ["strategy", "risk"], - outputSizeBytes: 188, - outputSizeChars: 188, - outputType: "object", - redacted: true, - type: "thinking", - }, - ], - }, - { - role: "assistant", - timestamp: startedAtMs + 29_000, - parts: [ - { - id: "toolu_mock_private_thread", - inputKeys: ["channel", "ts"], - inputSizeBytes: 58, - inputSizeChars: 58, - inputType: "object", - name: "slack.fetch_thread", - redacted: true, - type: "tool_call", - }, - ], - }, - { - role: "toolResult", - timestamp: startedAtMs + 47_000, - parts: [ - { - id: "toolu_mock_private_thread", - name: "slack.fetch_thread", - outputKeys: ["messages"], - outputSizeBytes: 962, - outputSizeChars: 950, - outputType: "object", - redacted: true, - type: "tool_result", - }, - ], - }, - ]; -} - -function sandboxConversation(nowMs: number): ConversationDetailReport { - const startedAt = iso(nowMs, -18 * 60_000); - - return { - conversationId: SANDBOX_CONVERSATION_ID, - displayTitle: "Sandbox QA run", - generatedAt: iso(nowMs), - sentryConversationUrl: sentryConversationUrl(SANDBOX_CONVERSATION_ID), - status: "active", - startedAt, - lastProgressAt: iso(nowMs, -11 * 60_000), - lastSeenAt: iso(nowMs, -10 * 60_000), - cumulativeDurationMs: 480_000, - cumulativeUsage: { - inputTokens: 11_200, - outputTokens: 800, - totalTokens: 12_000, - }, - surface: "slack", - actorIdentity: { - email: "dana@sentry.io", - fullName: "Dana Chen", - slackUserId: "UQA555", - slackUserName: "dana", - }, - channel: "CQA999", - channelName: "quality", - transcriptAvailable: true, - transcriptMessageCount: 3, - transcript: [ - { + actorIdentity: actor("avery@sentry.io", "Avery Chen", "avery"), + cumulativeDurationMs: 42_000, + eventHistory: { status: "redacted", reason: "non_public_conversation" }, + events: [ + reportEvent(0, startedAt, { + type: "visible_message", + messageId: "private-user", role: "user", - timestamp: Date.parse(startedAt), - parts: [ - { - type: "text", - text: "Run the checkout smoke test in the sandbox and tell me if the redirect loop still reproduces.", - }, - ], - }, - { - role: "assistant", - timestamp: Date.parse(startedAt) + 35_000, - parts: [ - { - id: "toolu_mock_sandbox_run", - name: "sandbox.run_command", - input: { - args: ["pnpm", "test", "checkout-smoke"], - cwd: "/repo", - timeoutMs: 600000, - }, - type: "tool_call", - }, - ], - }, - { + redacted: true, + }), + reportEvent(1, iso(Date.parse(startedAt), 10_000), { + type: "tool_started", + name: "sentry.search", + }), + reportEvent(2, iso(Date.parse(startedAt), 30_000), { + type: "visible_message", + messageId: "private-assistant", role: "assistant", - timestamp: Date.parse(startedAt) + 2 * 60_000, - parts: [ - { - type: "text", - text: "The sandbox command started. I am waiting on the browser trace before calling the result.", - }, - ], - }, + redacted: true, + }), ], - }; + }); } function failedConversation(nowMs: number): ConversationDetailReport { - const traceId = "29bbf789f14b469cb4f6ed830a47224d"; const startedAt = iso(nowMs, -36 * 60_000); - - return { + return detail(nowMs, { conversationId: FAILED_CONVERSATION_ID, - displayTitle: "OAuth callback investigation", - generatedAt: iso(nowMs), - sentryConversationUrl: sentryConversationUrl(FAILED_CONVERSATION_ID), - status: "failed", + displayTitle: "Deployment investigation failed", startedAt, - lastProgressAt: iso(nowMs, -35 * 60_000), lastSeenAt: iso(nowMs, -35 * 60_000), - cumulativeDurationMs: 72_000, - cumulativeUsage: { - inputTokens: 4500, - outputTokens: 390, - totalTokens: 4890, - }, + lastProgressAt: iso(nowMs, -35 * 60_000), + status: "failed", surface: "slack", - actorIdentity: { - email: "riley@sentry.io", - fullName: "Riley Patel", - slackUserId: "UQA666", - slackUserName: "riley", - }, channel: "CQA777", - channelName: "platform-auth", - sentryTraceUrl: sentryTraceUrl(traceId), - traceId, - transcriptAvailable: true, - transcriptMessageCount: 3, - transcript: [ - { + channelName: "proj-incidents", + actorIdentity: actor("riley@sentry.io", "Riley Park", "riley"), + cumulativeDurationMs: 19_000, + events: [ + reportEvent(0, startedAt, { + type: "visible_message", + messageId: "failed-user", role: "user", - timestamp: Date.parse(startedAt), - parts: [ - { - type: "text", - text: "Why are new Notion OAuth callbacks failing in staging?", - }, - ], - }, - { - role: "assistant", - timestamp: Date.parse(startedAt) + 15_000, - parts: [ - { - id: "toolu_mock_oauth_logs", - name: "sentry.search_errors", - input: { - environment: "staging", - query: "OAuth callback Notion status:500", - }, - type: "tool_call", - }, - ], - }, - { - role: "toolResult", - timestamp: Date.parse(startedAt) + 53_000, - parts: [ - { - id: "toolu_mock_oauth_logs", - name: "sentry.search_errors", - output: { - error: - "Provider token exchange failed: invalid_client for staging callback origin", - traceId, - }, - type: "tool_result", - }, - ], - }, - { - role: "assistant", - outcome: "error", - timestamp: Date.parse(startedAt) + 72_000, - parts: [], - }, + text: "Check the deployment failure.", + }), + reportEvent(1, iso(Date.parse(startedAt), 1_000), { + type: "turn_lifecycle", + turnId: "failed-turn", + state: "started", + }), + reportEvent(2, iso(Date.parse(startedAt), 19_000), { + type: "turn_lifecycle", + turnId: "failed-turn", + state: "failed", + }), ], - }; + }); } -function schedulerConversation(nowMs: number): ConversationDetailReport { +function simpleConversation( + nowMs: number, + options: { + conversationId: string; + displayTitle: string; + surface: "internal" | "scheduler" | "slack"; + channel?: string; + }, +): ConversationDetailReport { const startedAt = iso(nowMs, -2 * 60 * 60_000); - - return { - conversationId: SCHEDULER_CONVERSATION_ID, - displayTitle: "Daily operations digest", - generatedAt: iso(nowMs), - status: "completed", + return detail(nowMs, { + ...options, startedAt, - lastProgressAt: iso(nowMs, -119 * 60_000), - lastSeenAt: iso(nowMs, -118 * 60_000), - cumulativeDurationMs: 115_000, - cumulativeUsage: { - inputTokens: 6200, - outputTokens: 760, - totalTokens: 6960, - }, - surface: "scheduler", - transcriptAvailable: true, - transcriptMessageCount: 2, - transcript: [ - { - role: "user", - timestamp: Date.parse(startedAt), - parts: [ - { - type: "text", - text: "Scheduled task: summarize overnight production risk for the checkout team.", - }, - ], - }, - { - role: "assistant", - timestamp: Date.parse(startedAt) + 109_000, - parts: [ - { - type: "text", - text: "Overnight risk stayed low. One staging OAuth regression is still open; checkout production latency returned to baseline after the payments rollback.", - }, - ], - }, - ], - }; -} - -function dashboardQaConversation(nowMs: number): ConversationDetailReport { - const conversationId = DASHBOARD_QA_CONVERSATION_ID; - const displayTitle = "Dashboard QA edge cases"; - const activityStartedAt = iso(nowMs, -11 * 60_000); - const transcriptStartedAt = iso(nowMs, -10 * 60_000); - const runningToolId = "toolu_mock_dashboard_running"; - const invertedToolId = "toolu_mock_dashboard_inverted"; - const advisorPlanToolId = "toolu_mock_dashboard_advisor_plan"; - const advisorReviewToolId = "toolu_mock_dashboard_advisor_review"; - const readFileToolId = "toolu_mock_dashboard_read_file"; - const editFileToolId = "toolu_mock_dashboard_edit_file"; - - return { - conversationId, - displayTitle, - generatedAt: iso(nowMs), - status: "completed", - startedAt: activityStartedAt, - lastProgressAt: iso(nowMs, -5 * 60_000), - lastSeenAt: iso(nowMs, -5 * 60_000), - cumulativeDurationMs: 180_000, - surface: "internal", - transcriptAvailable: true, - transcriptMessageCount: 12, - transcript: [ - mockTranscriptMessage({ - role: "assistant", - timestamp: Date.parse(transcriptStartedAt) + 2_000, - parts: [ - mockToolCallPart({ - id: invertedToolId, - name: "mock.inverted_timestamp_tool", - input: { order: "call before result" }, - }), - ], - }), - mockTranscriptMessage({ - role: "toolResult", - timestamp: Date.parse(transcriptStartedAt) + 1_000, - parts: [ - mockToolResultPart({ - id: invertedToolId, - name: "mock.inverted_timestamp_tool", - output: { ok: true }, - }), - ], - }), - mockTranscriptMessage({ - role: "user", - timestamp: nowMs - 8 * 60_000, - parts: [ - { - type: "text", - text: "Add a people profile page to the dashboard and make conversation emails link to the profile. Be careful with privacy and keep the UI practical.", - }, - ], - }), - mockTranscriptMessage({ - role: "assistant", - timestamp: nowMs - 8 * 60_000 + 4_000, - parts: [ - mockToolCallPart({ - id: advisorPlanToolId, - name: "advisor", - input: { - question: - "Review the dashboard plan before editing. Focus on whether actor email can be trusted, what profile metrics are useful, and what UI risks to avoid.", - }, - }), - ], - }), - mockTranscriptMessage({ - role: "toolResult", - timestamp: nowMs - 8 * 60_000 + 35_000, - parts: [ - mockToolResultPart({ - id: advisorPlanToolId, - name: "advisor", - output: { - verdict: "proceed", - summary: - "Use trusted actorIdentity.email, keep metrics to conversations/runtime/tokens, and make profile activity scannable before adding heavier analytics.", - }, - }), - ], - }), - mockTranscriptMessage({ + lastSeenAt: iso(nowMs, -110 * 60_000), + lastProgressAt: iso(nowMs, -110 * 60_000), + actorIdentity: actor("ops@sentry.io", "Ops Bot", "ops"), + cumulativeDurationMs: 12_000, + events: [ + reportEvent(0, startedAt, { + type: "visible_message", + messageId: `${options.conversationId}:message`, role: "assistant", - timestamp: nowMs - 7 * 60_000 + 5_000, - parts: [ - mockToolCallPart({ - id: readFileToolId, - name: "readFile", - input: { - path: "packages/junior-dashboard/src/client/pages/ConversationPage.tsx", - }, - }), - mockToolCallPart({ - id: editFileToolId, - name: "editFile", - input: { - path: "packages/junior-dashboard/src/client/pages/people/PeoplePage.tsx", - operations: [ - { - action: "insert", - anchor: "ProfileMetrics", - lines: 42, - }, - { - action: "replace", - anchor: "RecentConversationList", - lines: 18, - }, - ], - summary: - "Add actor activity grid, recent conversations, and email profile links.", - }, - }), - ], - }), - mockTranscriptMessage({ - role: "toolResult", - timestamp: nowMs - 6 * 60_000 + 10_000, - parts: [ - mockToolResultPart({ - id: readFileToolId, - name: "readFile", - output: { - lines: 260, - result: "Conversation detail component inspected.", - imports: [ - "Transcript", - "ConversationStats", - "ConversationIdentity", - ], - risks: { - auth: "dashboard routes remain authenticated", - privacy: - "actor emails are trusted from normalized reporting identity", - }, - }, - }), - mockToolResultPart({ - id: editFileToolId, - name: "editFile", - output: { - filesChanged: [ - { - path: "packages/junior-dashboard/src/client/pages/people/PeoplePage.tsx", - added: 216, - removed: 0, - }, - { - path: "packages/junior-dashboard/src/client/components/ConversationSummary.tsx", - added: 18, - removed: 4, - }, - ], - checks: { - typecheck: "passed", - visualQa: "needs browser review", - }, - notes: - "The profile page uses a contribution-style activity grid, compact stat row, and searchable recent conversations. Keep the grid cell size stable so long month labels do not shift the layout.", - }, - }), - ], - }), - mockTranscriptMessage({ - role: "assistant", - timestamp: nowMs - 6 * 60_000 + 20_000, - parts: [ - mockToolCallPart({ - id: advisorReviewToolId, - name: "advisor", - input: { - question: - "Review the implementation after the first advisor pass. Check whether the UI is too noisy and whether any data shape assumptions are weak.", - }, - }), - ], - }), - mockTranscriptMessage({ - role: "toolResult", - timestamp: nowMs - 5 * 60_000 + 20_000, - parts: [ - mockToolResultPart({ - id: advisorReviewToolId, - name: "advisor", - output: { - verdict: "revise", - summary: - "Remove low-signal attention widgets, add list search/filtering, and verify the activity grid fills the available width.", - }, - }), - ], - }), - mockTranscriptMessage({ - role: "assistant", - timestamp: nowMs - 5 * 60_000 + 30_000, - parts: [ - { - type: "text", - text: "Implemented the people profile route, linked actor emails, and tightened the dashboard widgets based on the advisor review.", - }, - ], + text: "Scheduled operation completed successfully.", }), ], - activity: [ - mockToolActivity({ - id: runningToolId, - toolCallId: runningToolId, - toolName: "mock.dashboard_running_tool", - createdAt: iso(nowMs, -10 * 60_000), - status: "running", - args: { query: "activity-only edge case" }, - }), - mockToolActivity({ - id: invertedToolId, - toolCallId: invertedToolId, - toolName: "mock.inverted_timestamp_tool", - createdAt: transcriptStartedAt, - status: "completed", - }), - mockToolActivity({ - id: advisorPlanToolId, - toolCallId: advisorPlanToolId, - toolName: "advisor", - createdAt: iso(nowMs, -8 * 60_000 + 4_000), - status: "completed", - args: { - question: - "Review the dashboard plan before editing. Focus on whether actor email can be trusted, what profile metrics are useful, and what UI risks to avoid.", - }, - subagents: [ - mockSubagentActivity({ - id: advisorPlanToolId, - modelId: "openai/gpt-5.6-sol", - parentToolCallId: advisorPlanToolId, - reasoningLevel: "high", - subagentKind: "advisor", - createdAt: iso(nowMs, -8 * 60_000 + 6_000), - endedAt: iso(nowMs, -8 * 60_000 + 35_000), - status: "completed", - outcome: "success", - transcriptAvailable: true, - }), - ], - }), - mockToolActivity({ - id: readFileToolId, - toolCallId: readFileToolId, - toolName: "readFile", - createdAt: iso(nowMs, -7 * 60_000 + 5_000), - status: "completed", - args: { - path: "packages/junior-dashboard/src/client/pages/ConversationPage.tsx", - }, - }), - mockToolActivity({ - id: editFileToolId, - toolCallId: editFileToolId, - toolName: "editFile", - createdAt: iso(nowMs, -7 * 60_000 + 20_000), - status: "completed", - args: { - path: "packages/junior-dashboard/src/client/pages/people/PeoplePage.tsx", - }, - }), - mockToolActivity({ - id: advisorReviewToolId, - toolCallId: advisorReviewToolId, - toolName: "advisor", - createdAt: iso(nowMs, -6 * 60_000 + 20_000), - status: "completed", - args: { - question: - "Review the implementation after the first advisor pass. Check whether the UI is too noisy and whether any data shape assumptions are weak.", - }, - subagents: [ - mockSubagentActivity({ - id: advisorReviewToolId, - modelId: "openai/gpt-5.6-sol", - parentToolCallId: advisorReviewToolId, - reasoningLevel: "high", - subagentKind: "advisor", - createdAt: iso(nowMs, -6 * 60_000 + 25_000), - endedAt: iso(nowMs, -5 * 60_000 + 20_000), - status: "completed", - outcome: "success", - transcriptAvailable: true, - }), - ], - }), - ], - }; + }); } -function dashboardQaAdvisorTranscript( - nowMs: number, - subagentId: string, -): ConversationSubagentTranscriptReport | undefined { - const createdAt = - subagentId === "toolu_mock_dashboard_advisor_plan" - ? iso(nowMs, -8 * 60_000 + 6_000) - : subagentId === "toolu_mock_dashboard_advisor_review" - ? iso(nowMs, -6 * 60_000 + 25_000) - : undefined; - const endedAt = - subagentId === "toolu_mock_dashboard_advisor_plan" - ? iso(nowMs, -8 * 60_000 + 35_000) - : subagentId === "toolu_mock_dashboard_advisor_review" - ? iso(nowMs, -5 * 60_000 + 20_000) - : undefined; - if (!createdAt || !endedAt) return undefined; - - const sharedAdvisorSession: TranscriptMessage[] = [ - mockTranscriptMessage({ - role: "user", - timestamp: Date.parse(createdAt), - parts: [ - { - type: "text", - text: "Review the dashboard plan before editing. Focus on whether actor email can be trusted, what profile metrics are useful, and what UI risks to avoid.", - }, - ], - }), - mockTranscriptMessage({ - role: "assistant", - timestamp: Date.parse(createdAt) + 23_000, - parts: [ - { - type: "text", - text: "Actor identity email is a reasonable profile key because reporting already normalizes trusted identities. Keep the first cut narrow: total conversations, runtime, token volume, recent conversations, and a contribution-style activity grid. Avoid attention widgets until there is an explicit operator workflow.", - }, - ], - }), - mockTranscriptMessage({ - role: "user", - timestamp: Date.parse(iso(nowMs, -6 * 60_000 + 25_000)), - parts: [ - { - type: "text", - text: "Review the implementation after the first advisor pass. Check whether the UI is too noisy and whether any data shape assumptions are weak.", - }, - ], - }), - mockTranscriptMessage({ - role: "assistant", - timestamp: Date.parse(endedAt), - parts: [ - { - type: "text", - text: "The implementation is directionally right, but it should be more aggressive about removing weak dashboard widgets. Conversation and profile search are more useful than top-N summaries. The activity grid should use smaller fixed cells and fill the row without sparse-looking gaps.", - }, - ], - }), - ]; - const slice = - subagentId === "toolu_mock_dashboard_advisor_plan" - ? sharedAdvisorSession.slice(0, 2) - : sharedAdvisorSession.slice(0, 4); +function actor( + email: string | undefined, + fullName: string, + slackUserName: string, +): ActorIdentity { + return { ...(email ? { email } : {}), fullName, slackUserName }; +} +function usage(cost: number) { return { - type: "subagent", - createdAt, - endedAt, - id: subagentId, - modelId: "openai/gpt-5.6-sol", - outcome: "success", - parentToolCallId: subagentId, - reasoningLevel: "high", - status: "success", - subagentConversationId: DASHBOARD_QA_ADVISOR_CONVERSATION_ID, - subagentKind: "advisor", - subagentSentryConversationUrl: sentryConversationUrl( - DASHBOARD_QA_ADVISOR_CONVERSATION_ID, - ), - transcript: slice, - transcriptAvailable: true, - transcriptMessageCount: 2, + inputTokens: 1_200, + outputTokens: 420, + cachedInputTokens: 300, + cost: { input: cost * 0.4, output: cost * 0.6, total: cost }, }; } @@ -949,28 +439,45 @@ function mockConversations(nowMs: number): ConversationDetailReport[] { return [ activeConversation(nowMs), dashboardQaConversation(nowMs), - longReleaseConversation(nowMs), - publicIncidentConversation(nowMs), + advisorConversation( + nowMs, + DASHBOARD_QA_PLAN_ID, + "Review the dashboard plan before editing.", + ), + advisorConversation( + nowMs, + DASHBOARD_QA_REVIEW_ID, + "Review the implementation after the first advisor pass.", + ), + longConversation(nowMs), + incidentConversation(nowMs), privateConversation(nowMs), failedConversation(nowMs), - sandboxConversation(nowMs), - schedulerConversation(nowMs), - ].map((conversation) => ({ - modelId: "openai/gpt-5.6-sol", - reasoningLevel: "high", - ...conversation, - })); + simpleConversation(nowMs, { + conversationId: SANDBOX_CONVERSATION_ID, + displayTitle: "Sandbox validation", + surface: "slack", + channel: "CQA999", + }), + simpleConversation(nowMs, { + conversationId: SCHEDULER_CONVERSATION_ID, + displayTitle: "Daily operations digest", + surface: "scheduler", + }), + ]; } -function mockConversationMap( - nowMs: number, -): Map { - return new Map( - mockConversations(nowMs).map((conversation) => [ - conversation.conversationId, - conversation, - ]), - ); +function summaryFromConversation( + conversation: ConversationDetailReport, +): ConversationSummaryReport { + const { + eventHistory: _eventHistory, + events: _events, + generatedAt: _generatedAt, + sentryConversationUrl: _sentryConversationUrl, + ...summary + } = conversation; + return summary; } function mockConversationFeed(nowMs: number): ConversationFeed { @@ -981,90 +488,204 @@ function mockConversationFeed(nowMs: number): ConversationFeed { }; } +function summaryTokenTotal(summary: ConversationSummaryReport): number { + const usage = summary.cumulativeUsage; + return ( + (usage?.inputTokens ?? 0) + + (usage?.outputTokens ?? 0) + + (usage?.cachedInputTokens ?? 0) + + (usage?.cacheCreationTokens ?? 0) + ); +} + +function statsItem(label: string): ConversationStatsItem { + return { active: 0, conversations: 0, durationMs: 0, failed: 0, label }; +} + +function addSummary( + item: ConversationStatsItem, + summary: ConversationSummaryReport, +) { + item.active += summary.status === "active" ? 1 : 0; + item.conversations += 1; + item.durationMs += summary.cumulativeDurationMs; + item.failed += summary.status === "failed" ? 1 : 0; + const tokens = summaryTokenTotal(summary); + if (tokens) item.tokens = (item.tokens ?? 0) + tokens; + const cost = summary.cumulativeUsage?.cost?.total; + if (cost !== undefined) item.costUsd = (item.costUsd ?? 0) + cost; +} + +function locationLabel(summary: ConversationSummaryReport): string { + if (summary.channel?.startsWith("C")) { + return summary.channelName ? `#${summary.channelName}` : "Public Channel"; + } + if (summary.channel?.startsWith("D")) return "Direct Message"; + return summary.surface === "scheduler" ? "Scheduler" : "Internal"; +} + +function actorLabel(identity: ActorIdentity | undefined): string { + return ( + identity?.email ?? + identity?.fullName ?? + identity?.slackUserName ?? + "Unknown" + ); +} + +function windowBounds(nowMs: number) { + return { + windowEnd: iso(nowMs), + windowStart: iso(nowMs - RECENT_CONVERSATION_STATS_WINDOW_MS), + }; +} + +/** Return the explicit canonical-event visual-QA feed, optionally scoped by actor. */ +export function readMockConversationFeed( + actorEmail?: string, +): ConversationFeed { + const feed = mockConversationFeed(Date.now()); + if (!actorEmail) return feed; + return { + ...feed, + conversations: feed.conversations.filter( + (conversation) => + conversation.actorIdentity?.email?.toLowerCase() === + actorEmail.toLowerCase(), + ), + }; +} + +/** Return one canonical-event visual-QA conversation detail fixture. */ +export function readMockConversationDetail( + conversationId: string, +): ConversationDetailReport | undefined { + const conversation = mockConversations(Date.now()).find( + (candidate) => candidate.conversationId === conversationId, + ); + if (!conversation) return undefined; + return conversation.channel && + PUBLIC_MOCK_CHANNEL_IDS.has(conversation.channel) + ? { ...conversation, locationId: `mock:${conversation.channel}` } + : conversation; +} + +/** Build mock dashboard stats from canonical-event mock conversations. */ +export function readMockConversationStats(): ConversationStatsReport { + const nowMs = Date.now(); + const summaries = mockConversationFeed(nowMs).conversations; + const total = statsItem("All conversations"); + const actorItems = new Map(); + const locationItems = new Map(); + for (const summary of summaries) { + addSummary(total, summary); + const actorName = actorLabel(summary.actorIdentity); + const actorItem = actorItems.get(actorName) ?? statsItem(actorName); + addSummary(actorItem, summary); + actorItems.set(actorName, actorItem); + const place = locationLabel(summary); + const locationItem = locationItems.get(place) ?? statsItem(place); + addSummary(locationItem, summary); + locationItems.set(place, locationItem); + } + return { + active: total.active, + actors: [...actorItems.values()], + conversations: total.conversations, + costUsd: total.costUsd, + durationMs: total.durationMs, + failed: total.failed, + generatedAt: iso(nowMs), + locations: [...locationItems.values()], + source: "conversation_index", + tokens: total.tokens, + ...windowBounds(nowMs), + }; +} + function mockPeopleActivityDays( nowMs: number, - conversations: ConversationSummaryReport[], + summaries: ConversationSummaryReport[], ): PeopleActivityDayReport[] { - const sparse = new Map< + const byDate = new Map< string, { actors: Set; conversations: number } >(); - for (const conversation of conversations) { - const email = conversation.actorIdentity?.email?.trim().toLowerCase(); + for (const summary of summaries) { + const email = summary.actorIdentity?.email?.toLowerCase(); if (!email) continue; - const date = conversation.lastSeenAt.slice(0, 10); - const day = sparse.get(date) ?? { actors: new Set(), conversations: 0 }; + const date = summary.lastSeenAt.slice(0, 10); + const day = byDate.get(date) ?? { + actors: new Set(), + conversations: 0, + }; day.actors.add(email); day.conversations += 1; - sparse.set(date, day); + byDate.set(date, day); } + return activityDates(nowMs, PEOPLE_ACTIVITY_DAYS).map((date) => ({ + activePeople: byDate.get(date)?.actors.size ?? 0, + conversations: byDate.get(date)?.conversations ?? 0, + date, + })); +} +function activityDates(nowMs: number, days = PEOPLE_ACTIVITY_DAYS): string[] { const end = new Date(nowMs); end.setUTCHours(0, 0, 0, 0); - const start = new Date(end); - start.setUTCDate(start.getUTCDate() - (PEOPLE_ACTIVITY_DAYS - 1)); - const days: PeopleActivityDayReport[] = []; - for ( - const cursor = new Date(start); - cursor.getTime() <= end.getTime(); - cursor.setUTCDate(cursor.getUTCDate() + 1) - ) { - const date = cursor.toISOString().slice(0, 10); - const day = sparse.get(date); - days.push({ - activePeople: day?.actors.size ?? 0, - conversations: day?.conversations ?? 0, - date, - }); - } - return days; + return Array.from({ length: days }, (_, index) => { + const date = new Date(end); + date.setUTCDate(date.getUTCDate() - (days - 1 - index)); + return date.toISOString().slice(0, 10); + }); } -/** Build mock People analytics from the explicit visual-QA conversation feed. */ +/** Build mock People analytics from canonical-event mock conversations. */ export function readMockPeopleDirectory(): ActorDirectoryReport { const nowMs = Date.now(); - const conversations = mockConversationFeed(nowMs).conversations; - const people = new Map }>(); - for (const conversation of conversations) { - const actor = conversation.actorIdentity; - const email = actor?.email?.trim().toLowerCase(); - if (!actor || !email) continue; - const existing = people.get(email) ?? { + const summaries = mockConversationFeed(nowMs).conversations; + const byEmail = new Map< + string, + ActorSummaryReport & { dates: Set } + >(); + for (const summary of summaries) { + const identity = summary.actorIdentity; + const email = identity?.email?.toLowerCase(); + if (!identity || !email) continue; + const existing = byEmail.get(email) ?? { active: 0, activeDays: 0, + actor: { ...identity, email }, conversations: 0, dates: new Set(), durationMs: 0, failed: 0, - firstSeenAt: conversation.startedAt, - lastSeenAt: conversation.lastSeenAt, - actor: { ...actor, email }, + firstSeenAt: summary.startedAt, + lastSeenAt: summary.lastSeenAt, }; - existing.active += conversation.status === "active" ? 1 : 0; + existing.active += summary.status === "active" ? 1 : 0; existing.conversations += 1; - existing.dates.add(conversation.lastSeenAt.slice(0, 10)); + existing.dates.add(summary.lastSeenAt.slice(0, 10)); existing.activeDays = existing.dates.size; - existing.durationMs += conversation.cumulativeDurationMs; - existing.failed += conversation.status === "failed" ? 1 : 0; + existing.durationMs += summary.cumulativeDurationMs; + existing.failed += summary.status === "failed" ? 1 : 0; existing.firstSeenAt = - Date.parse(conversation.startedAt) < Date.parse(existing.firstSeenAt) - ? conversation.startedAt + Date.parse(summary.startedAt) < Date.parse(existing.firstSeenAt) + ? summary.startedAt : existing.firstSeenAt; existing.lastSeenAt = - Date.parse(conversation.lastSeenAt) > Date.parse(existing.lastSeenAt) - ? conversation.lastSeenAt + Date.parse(summary.lastSeenAt) > Date.parse(existing.lastSeenAt) + ? summary.lastSeenAt : existing.lastSeenAt; - const tokens = usageTokenTotal(conversation.cumulativeUsage); - if (tokens !== undefined) { - existing.tokens = (existing.tokens ?? 0) + tokens; - } - people.set(email, existing); + const tokens = summaryTokenTotal(summary); + if (tokens) existing.tokens = (existing.tokens ?? 0) + tokens; + byEmail.set(email, existing); } - const activityDays = mockPeopleActivityDays(nowMs, conversations); + const activityDays = mockPeopleActivityDays(nowMs, summaries); return { activityDays, generatedAt: iso(nowMs), - people: [...people.values()] + people: [...byEmail.values()] .map(({ dates: _dates, ...person }) => person) .sort( (left, right) => @@ -1076,20 +697,17 @@ export function readMockPeopleDirectory(): ActorDirectoryReport { }; } -/** Build one mock People profile from the explicit visual-QA conversation feed. */ +/** Build one mock People profile from canonical-event mock conversations. */ export function readMockPeopleProfile( email: string, ): ActorProfileReport | undefined { const nowMs = Date.now(); - const normalizedEmail = email.trim().toLowerCase(); - const conversations = mockConversationFeed(nowMs).conversations.filter( - (conversation) => - conversation.actorIdentity?.email?.trim().toLowerCase() === - normalizedEmail, + const normalized = email.toLowerCase(); + const summaries = mockConversationFeed(nowMs).conversations.filter( + (summary) => summary.actorIdentity?.email?.toLowerCase() === normalized, ); - const actor = conversations[0]?.actorIdentity; - if (!actor?.email) return undefined; - + const identity = summaries[0]?.actorIdentity; + if (!identity) return undefined; const totals: ActorProfileReport["totals"] = { active: 0, activeDays: 0, @@ -1101,16 +719,15 @@ export function readMockPeopleProfile( const sparseDays = new Map(); const locations = new Map(); const surfaces = new Map(); - - for (const conversation of conversations) { - totals.active += conversation.status === "active" ? 1 : 0; + for (const summary of summaries) { + totals.active += summary.status === "active" ? 1 : 0; totals.conversations += 1; - totals.durationMs += conversation.cumulativeDurationMs; - totals.failed += conversation.status === "failed" ? 1 : 0; - const tokens = usageTokenTotal(conversation.cumulativeUsage); - if (tokens !== undefined) totals.tokens = (totals.tokens ?? 0) + tokens; + totals.durationMs += summary.cumulativeDurationMs; + totals.failed += summary.status === "failed" ? 1 : 0; + const tokens = summaryTokenTotal(summary); + if (tokens) totals.tokens = (totals.tokens ?? 0) + tokens; - const date = conversation.lastSeenAt.slice(0, 10); + const date = summary.lastSeenAt.slice(0, 10); activeDates.add(date); const day = sparseDays.get(date) ?? { active: 0, @@ -1119,40 +736,29 @@ export function readMockPeopleProfile( durationMs: 0, failed: 0, }; - day.active += conversation.status === "active" ? 1 : 0; + day.active += summary.status === "active" ? 1 : 0; day.conversations += 1; - day.durationMs += conversation.cumulativeDurationMs; - day.failed += conversation.status === "failed" ? 1 : 0; - if (tokens !== undefined) day.tokens = (day.tokens ?? 0) + tokens; + day.durationMs += summary.cumulativeDurationMs; + day.failed += summary.status === "failed" ? 1 : 0; + if (tokens) day.tokens = (day.tokens ?? 0) + tokens; sparseDays.set(date, day); - const place = locationLabel(conversation); - const location = locations.get(place) ?? emptyStatsItem(place); - addConversationStats(location, conversation); + const place = locationLabel(summary); + const location = locations.get(place) ?? statsItem(place); + addSummary(location, summary); locations.set(place, location); const surfaceLabel = - conversation.surface === "api" + summary.surface === "api" ? "API" - : `${conversation.surface.charAt(0).toUpperCase()}${conversation.surface.slice(1)}`; - const surface = surfaces.get(surfaceLabel) ?? emptyStatsItem(surfaceLabel); - addConversationStats(surface, conversation); + : `${summary.surface.charAt(0).toUpperCase()}${summary.surface.slice(1)}`; + const surface = surfaces.get(surfaceLabel) ?? statsItem(surfaceLabel); + addSummary(surface, summary); surfaces.set(surfaceLabel, surface); } totals.activeDays = activeDates.size; - - const activityDays: ActorActivityDayReport[] = []; - const end = new Date(nowMs); - end.setUTCHours(0, 0, 0, 0); - for ( - let offset = PEOPLE_PROFILE_ACTIVITY_DAYS - 1; - offset >= 0; - offset -= 1 - ) { - const cursor = new Date(end); - cursor.setUTCDate(cursor.getUTCDate() - offset); - const date = cursor.toISOString().slice(0, 10); - activityDays.push( + const activityDays = activityDates(nowMs, PEOPLE_PROFILE_ACTIVITY_DAYS).map( + (date): ActorActivityDayReport => sparseDays.get(date) ?? { active: 0, conversations: 0, @@ -1160,16 +766,14 @@ export function readMockPeopleProfile( durationMs: 0, failed: 0, }, - ); - } - + ); return { activityDays, generatedAt: iso(nowMs), - locations: statsItems(locations).map( + locations: [...locations.values()].map( ({ costUsd: _costUsd, ...item }) => item, ), - recentConversations: conversations.map( + recentConversations: summaries.map( ({ cumulativeUsage: _usage, sentryTraceUrl: _url, @@ -1177,9 +781,9 @@ export function readMockPeopleProfile( ...item }) => item, ), - actor: { ...actor, email: normalizedEmail }, + actor: { ...identity, email: normalized }, source: "conversation_index", - surfaces: statsItems(surfaces).map( + surfaces: [...surfaces.values()].map( ({ costUsd: _costUsd, ...item }) => item, ), totals, @@ -1298,62 +902,69 @@ function publicMockLocation( }; } -/** Aggregate the typed mock feed into the location directory wire report. */ -function mockLocationDirectory(nowMs: number): LocationDirectoryReport { +function publicLocation( + summaries: ConversationSummaryReport[], + channel: string, +): LocationSummaryReport { + const matching = summaries.filter((summary) => summary.channel === channel); + const item = statsItem(locationLabel(matching[0]!)); + matching.forEach((summary) => addSummary(item, summary)); + return { + ...item, + firstSeenAt: matching.at(-1)!.startedAt, + id: `mock:${channel}`, + kind: "channel", + lastSeenAt: matching[0]!.lastSeenAt, + provider: "slack", + providerDestinationId: channel, + visibility: "public", + }; +} + +/** Build the mock public-location directory from canonical-event summaries. */ +export function readMockLocationDirectory(): LocationDirectoryReport { + const nowMs = Date.now(); const summaries = mockConversationFeed(nowMs).conversations; - const locations = new Map(); - const privateActivity = emptyStatsItem("Private activity"); + const channels = [ + ...new Set( + summaries + .map((summary) => summary.channel) + .filter((channel): channel is string => + Boolean(channel && PUBLIC_MOCK_CHANNEL_IDS.has(channel)), + ), + ), + ]; + const privateActivity = statsItem("Private activity"); const sparseActivity = new Map(); - - for (const conversation of summaries) { - const initial = publicMockLocation(conversation); - const date = conversation.lastSeenAt.slice(0, 10); + for (const summary of summaries) { + const isPublic = Boolean( + summary.channel && PUBLIC_MOCK_CHANNEL_IDS.has(summary.channel), + ); + const date = summary.lastSeenAt.slice(0, 10); const day = sparseActivity.get(date) ?? { date, privateConversations: 0, publicConversations: 0, }; - if (initial) day.publicConversations += 1; - else day.privateConversations += 1; - sparseActivity.set(date, day); - if (!initial) { - addConversationStats(privateActivity, conversation); - continue; - } - const location = locations.get(initial.id) ?? initial; - addConversationStats(location, conversation); - if (Date.parse(conversation.startedAt) < Date.parse(location.firstSeenAt)) { - location.firstSeenAt = conversation.startedAt; - } - if (Date.parse(conversation.lastSeenAt) > Date.parse(location.lastSeenAt)) { - location.lastSeenAt = conversation.lastSeenAt; + if (isPublic) day.publicConversations += 1; + else { + day.privateConversations += 1; + addSummary(privateActivity, summary); } - locations.set(location.id, location); + sparseActivity.set(date, day); } - - const activityDays: LocationActivityDayReport[] = []; - const end = new Date(nowMs); - end.setUTCHours(0, 0, 0, 0); - for (let offset = 89; offset >= 0; offset -= 1) { - const cursor = new Date(end); - cursor.setUTCDate(cursor.getUTCDate() - offset); - const date = cursor.toISOString().slice(0, 10); - activityDays.push( + const activityDays = activityDates(nowMs).map( + (date): LocationActivityDayReport => sparseActivity.get(date) ?? { date, privateConversations: 0, publicConversations: 0, }, - ); - } - + ); return { activityDays, generatedAt: iso(nowMs), - locations: [...locations.values()].sort( - (left, right) => - Date.parse(right.lastSeenAt) - Date.parse(left.lastSeenAt), - ), + locations: channels.map((channel) => publicLocation(summaries, channel)), privateActivity, source: "conversation_index", windowEnd: `${activityDays.at(-1)!.date}T00:00:00.000Z`, @@ -1361,276 +972,71 @@ function mockLocationDirectory(nowMs: number): LocationDirectoryReport { }; } -/** Fill the fixed 90-day activity series used by mock location detail. */ -function mockLocationActivityDays( - nowMs: number, - conversations: ConversationSummaryReport[], -): DailyConversationActivity[] { - const days = new Map(); - for (const conversation of conversations) { - const date = conversation.lastSeenAt.slice(0, 10); - const day = days.get(date) ?? { +/** Build one mock public-location detail from canonical-event summaries. */ +export function readMockLocationDetail( + locationId: string, +): LocationDetailReport | undefined { + const nowMs = Date.now(); + const directory = readMockLocationDirectory(); + const location = directory.locations.find((item) => item.id === locationId); + if (!location) return undefined; + const recentConversations = mockConversationFeed(nowMs).conversations.filter( + (summary) => summary.channel === location.providerDestinationId, + ); + const actorItems = new Map(); + const sparseDays = new Map< + string, + LocationDetailReport["activityDays"][number] + >(); + for (const summary of recentConversations) { + const identity = summary.actorIdentity; + const actorKey = identity?.email ?? identity?.slackUserId; + if (identity && actorKey) { + const actorItem = actorItems.get(actorKey) ?? { + ...statsItem(actorLabel(identity)), + actor: identity, + }; + addSummary(actorItem, summary); + actorItems.set(actorKey, actorItem); + } + const date = summary.lastSeenAt.slice(0, 10); + const day = sparseDays.get(date) ?? { active: 0, conversations: 0, date, durationMs: 0, failed: 0, }; + day.active += summary.status === "active" ? 1 : 0; day.conversations += 1; - day.durationMs += conversation.cumulativeDurationMs; - day.active += conversation.status === "active" ? 1 : 0; - day.failed += conversation.status === "failed" ? 1 : 0; - const tokens = usageTokenTotal(conversation.cumulativeUsage); - if (tokens !== undefined) day.tokens = (day.tokens ?? 0) + tokens; - days.set(date, day); + day.durationMs += summary.cumulativeDurationMs; + day.failed += summary.status === "failed" ? 1 : 0; + const tokens = summaryTokenTotal(summary); + if (tokens) day.tokens = (day.tokens ?? 0) + tokens; + sparseDays.set(date, day); } - const result: DailyConversationActivity[] = []; - const end = new Date(nowMs); - end.setUTCHours(0, 0, 0, 0); - for (let offset = 89; offset >= 0; offset -= 1) { - const cursor = new Date(end); - cursor.setUTCDate(cursor.getUTCDate() - offset); - const date = cursor.toISOString().slice(0, 10); - result.push( - days.get(date) ?? { + const activityDays = activityDates(nowMs).map( + (date): LocationDetailReport["activityDays"][number] => + sparseDays.get(date) ?? { active: 0, conversations: 0, date, durationMs: 0, failed: 0, }, - ); - } - return result; -} - -/** Build the typed mock location directory used by dashboard visual QA. */ -export function readMockLocationDirectory(): LocationDirectoryReport { - return mockLocationDirectory(Date.now()); -} - -/** Build one typed public-location detail report for dashboard visual QA. */ -export function readMockLocationDetail( - locationId: string, -): LocationDetailReport | undefined { - const nowMs = Date.now(); - const location = mockLocationDirectory(nowMs).locations.find( - (item) => item.id === locationId, - ); - if (!location) return undefined; - const conversations = mockConversationFeed(nowMs).conversations.filter( - (conversation) => - conversation.channel === location.providerDestinationId && - PUBLIC_MOCK_CHANNEL_IDS.has(conversation.channel ?? ""), ); - const actors = new Map(); - for (const conversation of conversations) { - const actor = conversation.actorIdentity; - const key = actor?.email ?? actor?.slackUserId; - if (!actor || !key) continue; - const label = actorLabel(actor) ?? "Unknown"; - const item = actors.get(key) ?? { - ...emptyStatsItem(label), - actor, - }; - addConversationStats(item, conversation); - actors.set(key, item); - } - const activityDays = mockLocationActivityDays(nowMs, conversations); return { ...location, activityDays, - actors: [...actors.values()].sort( + actors: [...actorItems.values()].sort( (left, right) => right.conversations - left.conversations || left.label.localeCompare(right.label), ), generatedAt: iso(nowMs), - recentConversations: conversations, + recentConversations, source: "conversation_index", - windowEnd: `${activityDays.at(-1)?.date}T00:00:00.000Z`, - windowStart: `${activityDays[0]?.date}T00:00:00.000Z`, - }; -} - -function usageTokenTotal(usage: ConversationUsage | undefined) { - if (!usage) return undefined; - const components = [ - usage.inputTokens, - usage.outputTokens, - usage.cachedInputTokens, - usage.cacheCreationTokens, - ].reduce((sum, value) => { - const count = - typeof value === "number" && Number.isFinite(value) - ? Math.max(0, Math.floor(value)) - : undefined; - return count === undefined ? sum : (sum ?? 0) + count; - }, undefined); - if (components !== undefined) { - return components; - } - return typeof usage.totalTokens === "number" && - Number.isFinite(usage.totalTokens) - ? Math.max(0, Math.floor(usage.totalTokens)) - : undefined; -} - -function usageCostTotal(usage: ConversationUsage | undefined) { - if (!usage?.cost) return undefined; - if ( - typeof usage.cost.total === "number" && - Number.isFinite(usage.cost.total) - ) { - return Math.max(0, usage.cost.total); - } - return [ - usage.cost.input, - usage.cost.output, - usage.cost.cacheRead, - usage.cost.cacheWrite, - ].reduce((sum, value) => { - const amount = - typeof value === "number" && Number.isFinite(value) - ? Math.max(0, value) - : undefined; - return amount === undefined ? sum : (sum ?? 0) + amount; - }, undefined); -} - -function addUsd(left: number | undefined, right: number): number { - return Math.round(((left ?? 0) + right) * 1e12) / 1e12; -} - -function addTokenTotal( - total: number | undefined, - tokens: number | undefined, -): number | undefined { - return tokens === undefined ? total : (total ?? 0) + tokens; -} - -function actorLabel(actor: ActorIdentity | undefined): string | undefined { - const email = actor?.email?.trim() || undefined; - const fullName = actor?.fullName?.trim() || undefined; - const slackUserName = actor?.slackUserName?.trim() || undefined; - return email ?? fullName ?? slackUserName ?? actor?.slackUserId; -} - -function locationLabel(conversation: ConversationSummaryReport): string { - const channelId = conversation.channel; - const name = conversation.channelName?.replace(/^#/, ""); - if (channelId?.startsWith("D")) { - return "Direct Message"; - } - if (channelId?.startsWith("C")) { - return name ? `#${name}` : "Public Channel"; - } - if (channelId?.startsWith("G")) { - if (name?.startsWith("mpdm-")) return "Group DM"; - return "Private Channel"; - } - return conversation.surface === "scheduler" - ? "Scheduler" - : conversation.surface === "api" - ? "API" - : conversation.surface === "internal" - ? "Internal" - : (name ?? channelId ?? "Unknown"); -} - -function emptyStatsItem(label: string): ConversationStatsItem { - return { - active: 0, - conversations: 0, - durationMs: 0, - failed: 0, - label, - }; -} - -function addItemTokens( - item: ConversationStatsItem, - tokens: number | undefined, -): void { - if (tokens !== undefined) { - item.tokens = (item.tokens ?? 0) + tokens; - } -} - -function addItemCost( - item: ConversationStatsItem, - costUsd: number | undefined, -): void { - if (costUsd !== undefined) { - item.costUsd = addUsd(item.costUsd, costUsd); - } -} - -function addConversationStats( - item: ConversationStatsItem, - conversation: ConversationSummaryReport, -): void { - item.conversations += 1; - item.durationMs += conversation.cumulativeDurationMs; - item.active += conversation.status === "active" ? 1 : 0; - item.failed += conversation.status === "failed" ? 1 : 0; - addItemTokens(item, usageTokenTotal(conversation.cumulativeUsage)); - addItemCost(item, usageCostTotal(conversation.cumulativeUsage)); -} - -function statsItems(map: Map) { - return [...map.values()].sort( - (left, right) => - right.conversations - left.conversations || - right.durationMs - left.durationMs || - left.label.localeCompare(right.label), - ); -} - -/** Return the explicit visual-QA conversation feed, optionally scoped by actor. */ -export function readMockConversationFeed( - actorEmail?: string, -): ConversationFeed { - const feed = mockConversationFeed(Date.now()); - if (!actorEmail) return feed; - return { - ...feed, - conversations: feed.conversations.filter( - (conversation) => - conversation.actorIdentity?.email?.toLowerCase() === actorEmail, - ), - }; -} - -/** Return one explicit visual-QA conversation detail fixture. */ -export function readMockConversationDetail( - conversationId: string, -): ConversationDetailReport | undefined { - const conversation = mockConversationMap(Date.now()).get(conversationId); - if (!conversation) return undefined; - return conversation.channel && - PUBLIC_MOCK_CHANNEL_IDS.has(conversation.channel) - ? { ...conversation, locationId: mockLocationId(conversation.channel) } - : conversation; -} - -/** Return one explicit visual-QA subagent transcript fixture. */ -export function readMockConversationSubagent( - conversationId: string, - subagentId: string, -): ConversationSubagentTranscriptReport { - if (conversationId === DASHBOARD_QA_CONVERSATION_ID) { - const report = dashboardQaAdvisorTranscript(Date.now(), subagentId); - if (report) return report; - } - return { - type: "subagent", - createdAt: new Date(0).toISOString(), - id: subagentId, - status: "error", - subagentKind: "unknown", - transcript: [], - transcriptAvailable: false, - unavailableReason: "not_found", + windowEnd: `${activityDays.at(-1)!.date}T00:00:00.000Z`, + windowStart: `${activityDays[0]!.date}T00:00:00.000Z`, }; } diff --git a/packages/junior-dashboard/src/mock-release-conversation.ts b/packages/junior-dashboard/src/mock-release-conversation.ts deleted file mode 100644 index 7d1f15f57..000000000 --- a/packages/junior-dashboard/src/mock-release-conversation.ts +++ /dev/null @@ -1,592 +0,0 @@ -import type { - ConversationDetailReport, - TranscriptMessage, -} from "@sentry/junior/api/schema"; - -const LONG_CONVERSATION_ID = "slack:CQA456:1770021600.000600"; - -function iso(nowMs: number, offsetMs = 0): string { - return new Date(nowMs + offsetMs).toISOString(); -} - -function sentryConversationUrl(conversationId: string): string { - return `https://sentry.example.com/organizations/acme/explore/conversations/${encodeURIComponent(conversationId)}/`; -} - -function sentryTraceUrl(traceId: string): string { - return `https://sentry.example.com/performance/trace/${traceId}/`; -} - -function mockSystemPrompt(): string { - return [ - "You are Junior, a Slack-native helper for engineering workflows.", - "", - "# Operating Contract", - "", - "- Lead with the answer, then support it with evidence.", - "- Use available tools for repository, build, and deployment checks.", - "- Keep progress updates short when the work takes multiple steps.", - "- Do not expose secrets, private channel contents, or raw provider credentials.", - "- Verify package changes with install, check, typecheck, and build where possible.", - "- When a check fails, preserve the exact failure in the final summary.", - "", - "# Repository Defaults", - "", - "- Default application repo: acme/junior-demo.", - "- Default package namespace: @acme/junior.", - "- Open pull requests as drafts unless the actor asks for ready review.", - "", - "# Slack Output", - "", - "- Be concise.", - "- Prefer bullets for multi-step engineering results.", - "- Include the PR link when one was created.", - ].join("\n"); -} - -function bashOutput( - command: string, - stdout: string, - options?: { - durationMs?: number; - exitCode?: number; - stderr?: string; - timedOut?: boolean; - }, -): string { - const exitCode = options?.exitCode ?? 0; - return JSON.stringify({ - ok: exitCode === 0 && !options?.timedOut, - command, - cwd: "/vercel/sandbox", - exit_code: exitCode, - signal: null, - timed_out: options?.timedOut ?? false, - stdout, - stderr: options?.stderr ?? "", - stdout_truncated: false, - stderr_truncated: false, - }); -} - -function toolCall( - startedAtMs: number, - offsetMs: number, - id: string, - name: string, - input: unknown, -): TranscriptMessage { - return { - role: "assistant", - timestamp: startedAtMs + offsetMs, - parts: [ - { - id, - input, - name, - type: "tool_call", - }, - ], - }; -} - -function toolResult( - startedAtMs: number, - offsetMs: number, - id: string, - name: string, - output: unknown, -): TranscriptMessage { - return { - role: "toolResult", - timestamp: startedAtMs + offsetMs, - parts: [ - { - id, - name, - output, - type: "tool_result", - }, - ], - }; -} - -function bashPair( - startedAtMs: number, - index: number, - offsetMs: number, - command: string, - stdout: string, - options?: { - durationMs?: number; - exitCode?: number; - stderr?: string; - timedOut?: boolean; - }, -): TranscriptMessage[] { - const id = `toolu_mock_release_bash_${index}`; - const durationMs = options?.durationMs ?? 900 + ((index * 977) % 4200); - return [ - toolCall(startedAtMs, offsetMs, id, "bash", { - command, - ...(options?.timedOut ? { timeoutMs: 120_000 } : {}), - }), - toolResult( - startedAtMs, - offsetMs + durationMs, - id, - "bash", - bashOutput(command, stdout, options), - ), - ]; -} - -function progressPair( - startedAtMs: number, - index: number, - offsetMs: number, - message: string, -): TranscriptMessage[] { - const id = `toolu_mock_release_progress_${index}`; - return [ - toolCall(startedAtMs, offsetMs, id, "reportProgress", { message }), - toolResult(startedAtMs, offsetMs + 500, id, "reportProgress", "ok"), - ]; -} - -function releaseConversationTranscriptOne( - startedAtMs: number, -): TranscriptMessage[] { - return [ - { - role: "system", - timestamp: startedAtMs, - parts: [{ type: "text", text: mockSystemPrompt() }], - }, - { - role: "user", - timestamp: startedAtMs + 1_000, - parts: [ - { - type: "text", - text: "minor bump junior", - }, - { - type: "text", - text: "please publish the package bump and open a self-update PR in the app repo once it is ready.", - }, - ], - }, - { - role: "assistant", - timestamp: startedAtMs + 6_000, - parts: [ - { - type: "thinking", - output: - "This is release work, so I need to inspect package versions first, publish, then update the app configuration repo.", - }, - { - id: "toolu_mock_release_skill_0", - input: { skill: "release-package" }, - name: "loadSkill", - type: "tool_call", - }, - ], - }, - toolResult( - startedAtMs, - 7_100, - "toolu_mock_release_skill_0", - "loadSkill", - [ - "Loaded skill: release-package", - "", - "Verify workspace package versions, run release checks, and publish with the package manager.", - ].join("\n"), - ), - ...progressPair(startedAtMs, 0, 9_000, "Checking package versions"), - ...bashPair( - startedAtMs, - 0, - 12_000, - "pnpm view @acme/junior version", - "0.62.0\n", - ), - ...bashPair( - startedAtMs, - 1, - 16_000, - "pnpm version --recursive minor --no-git-tag-version", - [ - "@acme/junior 0.63.0", - "@acme/junior-dashboard 0.63.0", - "@acme/junior-github 0.63.0", - "@acme/junior-scheduler 0.63.0", - ].join("\n") + "\n", - { durationMs: 2_700 }, - ), - ...progressPair(startedAtMs, 1, 22_000, "Running release checks"), - ...bashPair( - startedAtMs, - 2, - 26_000, - "pnpm release:check", - [ - "Checking release package manifests...", - "✓ package versions aligned", - "✓ changelog entries present", - "✓ dist tags ready", - ].join("\n") + "\n", - { durationMs: 4_800 }, - ), - ...bashPair( - startedAtMs, - 3, - 34_000, - "pnpm publish -r --tag latest", - [ - "Publishing packages:", - "+ @acme/junior@0.63.0", - "+ @acme/junior-dashboard@0.63.0", - "+ @acme/junior-github@0.63.0", - "+ @acme/junior-scheduler@0.63.0", - "Done in 12.4s.", - ].join("\n") + "\n", - { durationMs: 10_900 }, - ), - { - role: "assistant", - timestamp: startedAtMs + 46_000, - parts: [ - { - type: "text", - text: "published `@acme/junior@0.63.0`. moving on to the self-update PR.", - }, - ], - }, - ]; -} - -function releaseConversationTranscriptTwo( - startedAtMs: number, -): TranscriptMessage[] { - return [ - { - role: "user", - timestamp: startedAtMs + 1_000, - parts: [ - { - type: "text", - text: "continue with the self-update app PR for the 0.63.0 package set.", - }, - ], - }, - { - role: "assistant", - timestamp: startedAtMs + 7_000, - parts: [ - { - type: "thinking", - output: - "I need to clone the app repo, update every @acme/junior package to the same version, run checks, and create a draft PR.", - }, - { - id: "toolu_mock_release_skill_1", - input: { skill: "self-update" }, - name: "loadSkill", - type: "tool_call", - }, - ], - }, - toolResult( - startedAtMs, - 8_000, - "toolu_mock_release_skill_1", - "loadSkill", - [ - "Loaded skill: self-update", - "", - "Use repository-native commands.", - "Check package manifests and lockfiles before editing.", - "Prefer a draft PR when publishing dependency bumps.", - ].join("\n"), - ), - ...progressPair(startedAtMs, 2, 12_000, "Cloning app repo"), - ...bashPair( - startedAtMs, - 4, - 15_000, - "git clone https://github.com/acme/junior-demo.git /tmp/junior-demo", - "Cloning into '/tmp/junior-demo'...\n", - { durationMs: 3_300 }, - ), - ...bashPair( - startedAtMs, - 5, - 19_000, - "cd /tmp/junior-demo && jq '.dependencies | keys[]' package.json", - [ - '"@acme/junior"', - '"@acme/junior-agent-browser"', - '"@acme/junior-dashboard"', - '"@acme/junior-github"', - '"@acme/junior-scheduler"', - ].join("\n") + "\n", - ), - ...bashPair( - startedAtMs, - 6, - 24_000, - "cd /tmp/junior-demo && git checkout -b build/update-junior-0.63.0", - "Switched to a new branch 'build/update-junior-0.63.0'\n", - ), - ...progressPair(startedAtMs, 3, 28_000, "Updating dependencies"), - ...bashPair( - startedAtMs, - 7, - 32_000, - "cd /tmp/junior-demo && pnpm add -E @acme/junior@0.63.0 @acme/junior-dashboard@0.63.0 @acme/junior-github@0.63.0 @acme/junior-scheduler@0.63.0", - [ - "Progress: resolved 1, reused 0, downloaded 0, added 0", - "Progress: resolved 63, reused 0, downloaded 48, added 0", - "Progress: resolved 184, reused 0, downloaded 139, added 0", - "Progress: resolved 312, reused 0, downloaded 240, added 0", - "Progress: resolved 628, reused 0, downloaded 552, added 0", - "Packages: +673", - "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++", - "dependencies:", - "+ @acme/junior 0.63.0", - "+ @acme/junior-dashboard 0.63.0", - "+ @acme/junior-github 0.63.0", - "+ @acme/junior-scheduler 0.63.0", - "", - "Ignored build scripts: native-compression@1.0.0, protobufjs@7.5.5.", - "Done in 16.8s using pnpm v10.34.1", - ].join("\n") + "\n", - { durationMs: 16_800, timedOut: false }, - ), - { - role: "assistant", - timestamp: startedAtMs + 58_000, - parts: [ - { - type: "thinking", - output: - "The package install completed and updated the lockfile. Next I should run the app-specific registration check before broader checks.", - }, - { - type: "text", - text: "dependencies updated; checking app package registration now.", - }, - ], - }, - ...bashPair( - startedAtMs, - 8, - 64_000, - "cd /tmp/junior-demo && node scripts/check-plugin-packages.mjs", - "✓ All @acme/junior-* plugin packages are registered in nitro.config.ts\n", - { durationMs: 2_300 }, - ), - ...bashPair( - startedAtMs, - 9, - 69_000, - "cd /tmp/junior-demo && git diff --name-only", - "package.json\npnpm-lock.yaml\n", - ), - ...progressPair(startedAtMs, 4, 73_000, "Running checks"), - ...bashPair( - startedAtMs, - 10, - 78_000, - "cd /tmp/junior-demo && pnpm check", - [ - "> @acme/junior-demo@0.1.0 check /tmp/junior-demo", - "> node scripts/check-plugin-packages.mjs && junior check", - "", - "✓ app files", - "✓ plugin github-repo", - "✓ packaged plugin agent-browser (@acme/junior-agent-browser)", - "✓ packaged plugin github (@acme/junior-github)", - "✓ packaged plugin scheduler (@acme/junior-scheduler)", - "✓ Validation passed (7 plugin manifests, 14 skill directories checked).", - ].join("\n") + "\n", - { durationMs: 9_400 }, - ), - ...bashPair( - startedAtMs, - 11, - 93_000, - "cd /tmp/junior-demo && pnpm typecheck", - "> @acme/junior-demo@0.1.0 typecheck /tmp/junior-demo\n> tsc --noEmit\n\n", - { durationMs: 4_100 }, - ), - { - role: "assistant", - timestamp: startedAtMs + 104_000, - parts: [ - { - type: "thinking", - output: - "Check and typecheck are green. Build may need runtime secrets; if so, capture the exact failure and continue with a draft PR.", - }, - { - type: "text", - text: "check + typecheck are green. trying build next.", - }, - ], - }, - ...bashPair( - startedAtMs, - 12, - 110_000, - "cd /tmp/junior-demo && pnpm build", - [ - "> @acme/junior-demo@0.1.0 build /tmp/junior-demo", - "> nitro build", - "", - "Loaded plugins (2): github-repo, devinfra-docs", - "Resolving sandbox snapshot profile...", - "junior command failed: CACHE_URL is required for durable thread state", - ].join("\n") + "\n", - { durationMs: 7_300, exitCode: 1 }, - ), - { - role: "assistant", - timestamp: startedAtMs + 126_000, - parts: [ - { - type: "thinking", - output: - "The build failure is an environment constraint, not a dependency failure. I should include it in the PR body and commit only the package files.", - }, - { - type: "text", - text: "build hit the expected sandbox env wall: `CACHE_URL is required for durable thread state`. committing the package update with that called out.", - }, - ], - }, - ...bashPair( - startedAtMs, - 13, - 132_000, - 'cd /tmp/junior-demo && git add package.json pnpm-lock.yaml && git commit -m "build(deps): update junior packages to 0.63.0"', - "[build/update-junior-0.63.0 885bf7f] build(deps): update junior packages to 0.63.0\n 2 files changed, 68 insertions(+), 67 deletions(-)\n", - { durationMs: 2_600 }, - ), - ...bashPair( - startedAtMs, - 14, - 142_000, - "cd /tmp/junior-demo && git push -u origin build/update-junior-0.63.0", - [ - "remote:", - "remote: Create a pull request for 'build/update-junior-0.63.0' on GitHub by visiting:", - "remote: https://github.com/acme/junior-demo/pull/new/build/update-junior-0.63.0", - "To https://github.com/acme/junior-demo.git", - " * [new branch] build/update-junior-0.63.0 -> build/update-junior-0.63.0", - "branch 'build/update-junior-0.63.0' set up to track 'origin/build/update-junior-0.63.0'.", - ].join("\n") + "\n", - { durationMs: 4_400 }, - ), - ...bashPair( - startedAtMs, - 15, - 154_000, - 'cd /tmp/junior-demo && gh pr create --draft --title "build(deps): update junior packages to 0.63.0"', - "https://github.com/acme/junior-demo/pull/42\n", - { durationMs: 3_100 }, - ), - ...progressPair(startedAtMs, 5, 160_000, "Draft PR created"), - { - role: "assistant", - timestamp: startedAtMs + 166_000, - parts: [ - { - type: "text", - text: [ - "done. full recap:", - "", - "- **released** `@acme/junior@0.63.0`", - "- **self-update PR** opened at https://github.com/acme/junior-demo/pull/42", - "- `pnpm check` and `pnpm typecheck` passed", - "- `pnpm build` failed on missing `CACHE_URL`, which is a sandbox environment constraint", - ].join("\n"), - }, - ], - }, - ]; -} - -/** Build a long sanitized release/update transcript for dashboard visual QA. */ -export function longReleaseConversation( - nowMs: number, -): ConversationDetailReport { - const traceId = "7a4f12c9e3d84901b6c7d8e9f0123456"; - const firstStartedAt = iso(nowMs, -92 * 60_000); - const secondStartedAt = iso(nowMs, -90 * 60_000); - const firstTranscript = releaseConversationTranscriptOne( - Date.parse(firstStartedAt), - ); - const secondTranscript = releaseConversationTranscriptTwo( - Date.parse(secondStartedAt), - ); - const handoffTranscriptIndex = - firstTranscript.length + - secondTranscript.findIndex( - (message) => - (message.timestamp ?? 0) > Date.parse(secondStartedAt) + 129_000, - ); - - return { - conversationId: LONG_CONVERSATION_ID, - displayTitle: "Package release and self-update", - generatedAt: iso(nowMs), - sentryConversationUrl: sentryConversationUrl(LONG_CONVERSATION_ID), - status: "completed", - startedAt: firstStartedAt, - lastProgressAt: iso(nowMs, -81 * 60_000), - lastSeenAt: iso(nowMs, -81 * 60_000), - cumulativeDurationMs: 552_761, - cumulativeUsage: { - cachedInputTokens: 1_266_200, - cacheCreationTokens: 21_129, - inputTokens: 43, - outputTokens: 5765, - }, - surface: "slack", - actorIdentity: { - fullName: "Jordan Blake", - slackUserId: "UQA777", - slackUserName: "jordan", - }, - channel: "CQA456", - channelName: "proj-release", - sentryTraceUrl: sentryTraceUrl(traceId), - traceId, - contextEvents: [ - { - type: "context_compacted", - createdAt: iso(Date.parse(firstStartedAt), 49_000), - modelId: "openai/gpt-5.4", - summary: - "The @acme/junior 0.63.0 package set was published after release checks passed. The remaining request is to update the junior-demo app, verify every package stays aligned, and open a draft pull request.", - transcriptIndex: firstTranscript.length, - }, - { - type: "model_handoff", - createdAt: iso(Date.parse(secondStartedAt), 129_000), - fromModelId: "openai/gpt-5.4", - toModelId: "openai/gpt-5.6-sol", - message: - "Model handoff checkpoint. Continue the outstanding request now using this summary as the complete prior context:\nFinish the self-update change after the build failed only because CACHE_URL is unavailable in the sandbox. Check and typecheck passed; commit package.json and pnpm-lock.yaml, push the branch, and open a draft PR that records the build limitation.", - transcriptIndex: handoffTranscriptIndex, - }, - ], - transcriptAvailable: true, - transcriptMessageCount: firstTranscript.length + secondTranscript.length, - transcript: [...firstTranscript, ...secondTranscript], - }; -} diff --git a/packages/junior-dashboard/src/mock-reporting/activity.ts b/packages/junior-dashboard/src/mock-reporting/activity.ts deleted file mode 100644 index b801df7b7..000000000 --- a/packages/junior-dashboard/src/mock-reporting/activity.ts +++ /dev/null @@ -1,90 +0,0 @@ -import type { - ConversationActivityStatus, - ConversationSubagentActivityReport, - ConversationToolActivityReport, -} from "@sentry/junior/api/schema"; - -import { mockIso } from "./time"; - -export type MockSubagentActivityOptions = { - createdAt?: string; - endedAt?: string; - id?: string; - modelId?: string; - outcome?: "success" | "error" | "aborted"; - parentToolCallId?: string; - reasoningLevel?: string; - status?: ConversationActivityStatus; - subagentKind?: string; - transcriptAvailable?: boolean; -}; - -/** Build a subagent activity record constrained to the reporting API shape. */ -export function mockSubagentActivity( - options: MockSubagentActivityOptions = {}, -): ConversationSubagentActivityReport { - return { - type: "subagent", - createdAt: options.createdAt ?? mockIso(), - id: options.id ?? "mock-subagent", - ...(options.modelId !== undefined ? { modelId: options.modelId } : {}), - status: options.status ?? "running", - subagentKind: options.subagentKind ?? "advisor", - ...(options.endedAt !== undefined ? { endedAt: options.endedAt } : {}), - ...(options.outcome !== undefined ? { outcome: options.outcome } : {}), - ...(options.parentToolCallId !== undefined - ? { parentToolCallId: options.parentToolCallId } - : {}), - ...(options.reasoningLevel !== undefined - ? { reasoningLevel: options.reasoningLevel } - : {}), - ...(options.transcriptAvailable !== undefined - ? { transcriptAvailable: options.transcriptAvailable } - : {}), - } satisfies ConversationSubagentActivityReport; -} - -export type MockToolActivityOptions = { - args?: unknown; - createdAt?: string; - id?: string; - inputKeys?: string[]; - inputSizeBytes?: number; - inputSizeChars?: number; - inputType?: string; - redacted?: boolean; - status?: ConversationActivityStatus; - subagents?: ConversationSubagentActivityReport[]; - toolCallId?: string; - toolName?: string; -}; - -/** Build a tool activity record constrained to the reporting API shape. */ -export function mockToolActivity( - options: MockToolActivityOptions = {}, -): ConversationToolActivityReport { - const toolCallId = options.toolCallId ?? "toolu_mock_activity"; - return { - type: "tool_execution", - createdAt: options.createdAt ?? mockIso(), - id: options.id ?? toolCallId, - status: options.status ?? "running", - subagents: options.subagents ?? [], - toolCallId, - toolName: options.toolName ?? "mock.tool", - ...(options.args !== undefined ? { args: options.args } : {}), - ...(options.inputKeys !== undefined - ? { inputKeys: options.inputKeys } - : {}), - ...(options.inputSizeBytes !== undefined - ? { inputSizeBytes: options.inputSizeBytes } - : {}), - ...(options.inputSizeChars !== undefined - ? { inputSizeChars: options.inputSizeChars } - : {}), - ...(options.inputType !== undefined - ? { inputType: options.inputType } - : {}), - ...(options.redacted !== undefined ? { redacted: options.redacted } : {}), - } satisfies ConversationToolActivityReport; -} diff --git a/packages/junior-dashboard/src/mock-reporting/time.ts b/packages/junior-dashboard/src/mock-reporting/time.ts deleted file mode 100644 index bebc5bf31..000000000 --- a/packages/junior-dashboard/src/mock-reporting/time.ts +++ /dev/null @@ -1,6 +0,0 @@ -export const defaultMockTimeMs = Date.parse("2026-01-01T00:00:00.000Z"); - -/** Return a stable ISO timestamp for deterministic mock reporting records. */ -export function mockIso(timeMs = defaultMockTimeMs): string { - return new Date(timeMs).toISOString(); -} diff --git a/packages/junior-dashboard/src/mock-reporting/transcript.ts b/packages/junior-dashboard/src/mock-reporting/transcript.ts deleted file mode 100644 index dba32de31..000000000 --- a/packages/junior-dashboard/src/mock-reporting/transcript.ts +++ /dev/null @@ -1,152 +0,0 @@ -import type { - TranscriptMessage, - TranscriptPart, - TranscriptRole, -} from "@sentry/junior/api/schema"; - -export type MockTextPartOptions = { - bytes?: number; - chars?: number; - redacted?: boolean; - sourceType?: string; - text?: string; -}; - -/** Build a transcript text part constrained to the reporting API shape. */ -export function mockTextPart( - options: MockTextPartOptions = {}, -): TranscriptPart { - return { - type: "text", - text: options.text ?? "Mock transcript text", - ...(options.bytes !== undefined ? { bytes: options.bytes } : {}), - ...(options.chars !== undefined ? { chars: options.chars } : {}), - ...(options.redacted !== undefined ? { redacted: options.redacted } : {}), - ...(options.sourceType !== undefined - ? { sourceType: options.sourceType } - : {}), - } satisfies TranscriptPart; -} - -export type MockThinkingPartOptions = { - output?: unknown; - outputKeys?: string[]; - outputSizeBytes?: number; - outputSizeChars?: number; - outputType?: string; - redacted?: boolean; -}; - -/** Build a transcript thinking part constrained to the reporting API shape. */ -export function mockThinkingPart( - options: MockThinkingPartOptions = {}, -): TranscriptPart { - return { - type: "thinking", - output: options.output ?? "Inspect the mock reporting state.", - ...(options.outputKeys !== undefined - ? { outputKeys: options.outputKeys } - : {}), - ...(options.outputSizeBytes !== undefined - ? { outputSizeBytes: options.outputSizeBytes } - : {}), - ...(options.outputSizeChars !== undefined - ? { outputSizeChars: options.outputSizeChars } - : {}), - ...(options.outputType !== undefined - ? { outputType: options.outputType } - : {}), - ...(options.redacted !== undefined ? { redacted: options.redacted } : {}), - } satisfies TranscriptPart; -} - -export type MockToolCallPartOptions = { - id?: string; - input?: unknown; - inputKeys?: string[]; - inputSizeBytes?: number; - inputSizeChars?: number; - inputType?: string; - name?: string; - redacted?: boolean; -}; - -/** Build a transcript tool-call part constrained to the reporting API shape. */ -export function mockToolCallPart( - options: MockToolCallPartOptions = {}, -): TranscriptPart { - return { - type: "tool_call", - id: options.id ?? "toolu_mock_call", - name: options.name ?? "mock.tool", - ...(options.input !== undefined ? { input: options.input } : {}), - ...(options.inputKeys !== undefined - ? { inputKeys: options.inputKeys } - : {}), - ...(options.inputSizeBytes !== undefined - ? { inputSizeBytes: options.inputSizeBytes } - : {}), - ...(options.inputSizeChars !== undefined - ? { inputSizeChars: options.inputSizeChars } - : {}), - ...(options.inputType !== undefined - ? { inputType: options.inputType } - : {}), - ...(options.redacted !== undefined ? { redacted: options.redacted } : {}), - } satisfies TranscriptPart; -} - -export type MockToolResultPartOptions = { - id?: string; - name?: string; - output?: unknown; - outputKeys?: string[]; - outputSizeBytes?: number; - outputSizeChars?: number; - outputType?: string; - redacted?: boolean; -}; - -/** Build a transcript tool-result part constrained to the reporting API shape. */ -export function mockToolResultPart( - options: MockToolResultPartOptions = {}, -): TranscriptPart { - return { - type: "tool_result", - id: options.id ?? "toolu_mock_call", - name: options.name ?? "mock.tool", - ...(options.output !== undefined ? { output: options.output } : {}), - ...(options.outputKeys !== undefined - ? { outputKeys: options.outputKeys } - : {}), - ...(options.outputSizeBytes !== undefined - ? { outputSizeBytes: options.outputSizeBytes } - : {}), - ...(options.outputSizeChars !== undefined - ? { outputSizeChars: options.outputSizeChars } - : {}), - ...(options.outputType !== undefined - ? { outputType: options.outputType } - : {}), - ...(options.redacted !== undefined ? { redacted: options.redacted } : {}), - } satisfies TranscriptPart; -} - -export type MockTranscriptMessageOptions = { - parts?: TranscriptPart[]; - role?: TranscriptRole; - timestamp?: number; -}; - -/** Build a transcript message constrained to the reporting API shape. */ -export function mockTranscriptMessage( - options: MockTranscriptMessageOptions = {}, -): TranscriptMessage { - return { - role: options.role ?? "assistant", - parts: options.parts ?? [mockTextPart()], - ...(options.timestamp !== undefined - ? { timestamp: options.timestamp } - : {}), - } satisfies TranscriptMessage; -} diff --git a/packages/junior-dashboard/tests/dashboard-mock-routes.test.ts b/packages/junior-dashboard/tests/dashboard-mock-routes.test.ts index 391d05a81..467e3d915 100644 --- a/packages/junior-dashboard/tests/dashboard-mock-routes.test.ts +++ b/packages/junior-dashboard/tests/dashboard-mock-routes.test.ts @@ -1,20 +1,16 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { actorDirectoryReportSchema, - conversationSubagentTranscriptReportSchema, - type ConversationSubagentTranscriptReport, + conversationDetailReportSchema, } from "@sentry/junior/api/schema"; + import { createDashboardApp } from "../src/app"; import { DASHBOARD_QA_CONVERSATION_ID } from "../src/mock-conversations"; -describe("dashboard mock conversation routes", () => { - afterEach(() => { - vi.useRealTimers(); - }); +describe("dashboard canonical-event mock routes", () => { + afterEach(() => vi.useRealTimers()); - it("overlays mock conversations for local dashboard visual QA", async () => { - // Pin time to match the hardcoded conversation dates in the mock reporting fixture. - // Without this, recentConversationGroups filters out conversations older than 90 days. + it("serves canonical detail, directory, and aggregate reports", async () => { vi.useFakeTimers({ now: new Date("2026-05-30T00:00:00.000Z") }); const app = createDashboardApp({ authRequired: false, @@ -26,422 +22,199 @@ describe("dashboard mock conversation routes", () => { new Request("http://localhost/api/conversations"), ); expect(conversations.status).toBe(200); - const conversationBody = (await conversations.json()) as { + const body = (await conversations.json()) as { conversations: Array<{ - activity?: unknown; + actorIdentity?: { email?: string }; conversationId: string; cumulativeDurationMs: number; }>; + source: string; }; - expect(conversationBody.conversations[0]?.conversationId).toBe( + expect(body.source).toBe("conversation_index"); + expect(body.conversations[0]?.conversationId).toBe( "slack:CQA123:1770003600.000200", ); - const personalConversations = await app.fetch( + expect(body.conversations.map((item) => item.conversationId)).toContain( + DASHBOARD_QA_CONVERSATION_ID, + ); + expect(body.conversations[0]).not.toHaveProperty("events"); + + const personal = await app.fetch( new Request( "http://localhost/api/conversations?actorEmail=morgan%40sentry.io", ), ); - expect(personalConversations.status).toBe(200); - const personalBody = (await personalConversations.json()) as { - conversations: Array<{ - actorIdentity?: { email?: string }; - conversationId: string; - }>; - }; - expect(personalBody.conversations.length).toBeGreaterThan(0); + const personalBody = (await personal.json()) as typeof body; expect( personalBody.conversations.every( - (conversation) => - conversation.actorIdentity?.email === "morgan@sentry.io", + (item) => item.actorIdentity?.email === "morgan@sentry.io", ), ).toBe(true); - expect( - conversationBody.conversations.map( - (conversation) => conversation.conversationId, - ), - ).toContain("slack:CQA456:1770021600.000600"); - expect( - conversationBody.conversations.map( - (conversation) => conversation.conversationId, - ), - ).toContain(DASHBOARD_QA_CONVERSATION_ID); - const qaConversationSummary = conversationBody.conversations.find( - (conversation) => - conversation.conversationId === DASHBOARD_QA_CONVERSATION_ID, - ); - expect(qaConversationSummary).toBeDefined(); - expect(qaConversationSummary).not.toHaveProperty("activity"); - const conversationStats = await app.fetch( + + const stats = await app.fetch( new Request("http://localhost/api/conversations/stats"), ); - expect(conversationStats.status).toBe(200); - const statsBody = (await conversationStats.json()) as { + const statsBody = (await stats.json()) as { conversations: number; costUsd?: number; durationMs: number; windowEnd: string; windowStart: string; }; - expect(statsBody).toMatchObject({ - conversations: new Set( - conversationBody.conversations.map( - (conversation) => conversation.conversationId, - ), - ).size, - }); - const rawDurationMs = conversationBody.conversations.reduce( - (sum, conversation) => sum + conversation.cumulativeDurationMs, - 0, + expect(statsBody.conversations).toBe(body.conversations.length); + expect(statsBody.durationMs).toBe( + body.conversations.reduce( + (sum, conversation) => sum + conversation.cumulativeDurationMs, + 0, + ), ); - expect(statsBody.durationMs).toBe(rawDurationMs); expect(statsBody.costUsd).toBeGreaterThan(0); expect( Date.parse(statsBody.windowEnd) - Date.parse(statsBody.windowStart), ).toBe(89 * 24 * 60 * 60 * 1000); + const people = await app.fetch(new Request("http://localhost/api/people")); + const peopleBody = actorDirectoryReportSchema.parse(await people.json()); + expect(peopleBody.people.length).toBeGreaterThan(0); + expect(peopleBody.activityDays).toHaveLength(90); + const profileResponse = await app.fetch( + new Request( + `http://localhost/api/people/${encodeURIComponent( + peopleBody.people[0]!.actor.email, + )}`, + ), + ); + const profile = (await profileResponse.json()) as { + activityDays: unknown[]; + locations: unknown[]; + surfaces: unknown[]; + }; + expect(profile.activityDays).toHaveLength(90); + expect(profile.locations.length).toBeGreaterThan(0); + expect(profile.surfaces.length).toBeGreaterThan(0); + const locations = await app.fetch( new Request("http://localhost/api/locations"), ); - expect(locations.status).toBe(200); const locationBody = (await locations.json()) as { locations: Array<{ id: string; label: string }>; privateActivity: { conversations: number }; }; - expect(locationBody.locations.length).toBeGreaterThan(0); - expect(locationBody.locations.map((location) => location.label)).toContain( + expect(locationBody.locations.map((item) => item.label)).toContain( "#proj-checkout", ); expect(locationBody.privateActivity.conversations).toBeGreaterThan(0); - const locationDetail = await app.fetch( + const locationResponse = await app.fetch( new Request( - `http://localhost/api/locations/${encodeURIComponent(locationBody.locations[0]!.id)}`, + `http://localhost/api/locations/${encodeURIComponent( + locationBody.locations[0]!.id, + )}`, ), ); - expect(locationDetail.status).toBe(200); - await expect(locationDetail.json()).resolves.toMatchObject({ - visibility: "public", - activityDays: expect.any(Array), - recentConversations: expect.any(Array), - }); + const location = (await locationResponse.json()) as { + activityDays: unknown[]; + actors: unknown[]; + }; + expect(location.activityDays).toHaveLength(90); + expect(location.actors.length).toBeGreaterThan(0); + }); - const people = await app.fetch(new Request("http://localhost/api/people")); - expect(people.status).toBe(200); - const peopleBody = actorDirectoryReportSchema.parse(await people.json()); - const actorEmail = peopleBody.people[0]!.actor.email; - const personProfile = await app.fetch( - new Request( - `http://localhost/api/people/${encodeURIComponent(actorEmail)}`, - ), - ); - expect(personProfile.status).toBe(200); - await expect(personProfile.json()).resolves.toMatchObject({ - activityDays: expect.any(Array), - actor: { email: actorEmail }, - recentConversations: expect.any(Array), + it("serves direct canonical event fixtures for every dashboard state", async () => { + const app = createDashboardApp({ + authRequired: false, + allowedGoogleDomains: [], + mockConversations: true, }); - const activeConversation = await app.fetch( - new Request( - "http://localhost/api/conversations/slack%3ACQA123%3A1770003600.000200", - ), - ); - expect(activeConversation.status).toBe(200); - const activeConversationBody = (await activeConversation.json()) as { - transcript: Array<{ - parts: Array<{ name?: string }>; - }>; + const readDetail = async (conversationId: string) => { + const response = await app.fetch( + new Request( + `http://localhost/api/conversations/${encodeURIComponent(conversationId)}`, + ), + ); + expect(response.status).toBe(200); + return conversationDetailReportSchema.parse(await response.json()); }; - expect( - activeConversationBody.transcript - .flatMap((message) => message.parts) - .map((part) => part.name) - .filter(Boolean), - ).toContain("datacat.search_logs"); - const failedConversation = await app.fetch( - new Request( - "http://localhost/api/conversations/slack%3ACQA777%3A1770014400.000500", + const active = await readDetail("slack:CQA123:1770003600.000200"); + expect( + active.events.flatMap((event) => + event.data.type === "tool_started" ? [event.data.name] : [], ), - ); - expect(failedConversation.status).toBe(200); - const failedConversationBody = (await failedConversation.json()) as { - transcript: Array<{ - outcome?: string; - parts: unknown[]; - role: string; - }>; - transcriptMessageCount?: number; - }; - expect(failedConversationBody.transcript.at(-1)).toEqual( - expect.objectContaining({ - role: "assistant", - outcome: "error", - parts: [], - }), - ); - expect(failedConversationBody.transcriptMessageCount).toBe(3); + ).toContain("datacat.search_logs"); - const qaConversation = await app.fetch( - new Request( - `http://localhost/api/conversations/${encodeURIComponent( - DASHBOARD_QA_CONVERSATION_ID, - )}`, - ), - ); - expect(qaConversation.status).toBe(200); - const qaConversationBody = (await qaConversation.json()) as { - activity?: Array<{ - status?: string; - subagents?: Array<{ - parentToolCallId?: string; - status?: string; - subagentKind?: string; - type: string; - }>; - toolCallId?: string; - toolName?: string; - type: string; - }>; - conversationId: string; - transcript: Array<{ - parts: Array<{ id?: string; name?: string; type: string }>; - timestamp?: number; - }>; - transcriptMessageCount?: number; - }; - expect(qaConversationBody.activity?.[0]).toMatchObject({ - type: "tool_execution", - status: "running", - toolName: "mock.dashboard_running_tool", - }); - const invertedMessages = qaConversationBody.transcript.filter((message) => - message.parts.some( - (part) => part.name === "mock.inverted_timestamp_tool", - ), - ); - expect(invertedMessages[0]?.parts[0]).toMatchObject({ - type: "tool_call", - name: "mock.inverted_timestamp_tool", + const failed = await readDetail("slack:CQA777:1770014400.000500"); + expect(failed.events.at(-1)?.data).toMatchObject({ + type: "turn_lifecycle", + state: "failed", }); - expect(invertedMessages[1]?.parts[0]).toMatchObject({ - type: "tool_result", - name: "mock.inverted_timestamp_tool", - }); - expect(invertedMessages[1]?.timestamp).toBeLessThan( - invertedMessages[0]?.timestamp ?? 0, + + const privateConversation = await readDetail( + "slack:DQA123:1770007200.000300", ); - expect(qaConversationBody.conversationId).toBe("internal:dashboard-qa"); - expect( - qaConversationBody.transcript - .flatMap((message) => message.parts) - .filter((part) => part.name === "advisor") - .map((part) => part.type), - ).toEqual(["tool_call", "tool_result", "tool_call", "tool_result"]); - expect( - qaConversationBody.activity?.find( - (activity) => - activity.toolCallId === "toolu_mock_dashboard_advisor_plan", - ), - ).toMatchObject({ - type: "tool_execution", - status: "completed", - toolCallId: "toolu_mock_dashboard_advisor_plan", - toolName: "advisor", - subagents: [ - { - type: "subagent", - status: "completed", - subagentKind: "advisor", - parentToolCallId: "toolu_mock_dashboard_advisor_plan", - transcriptAvailable: true, - }, - ], + expect(privateConversation.eventHistory).toEqual({ + status: "redacted", + reason: "non_public_conversation", }); - expect( - qaConversationBody.activity?.find( - (activity) => - activity.toolCallId === "toolu_mock_dashboard_advisor_review", - ), - ).toMatchObject({ - type: "tool_execution", - status: "completed", - toolCallId: "toolu_mock_dashboard_advisor_review", - toolName: "advisor", - subagents: [ - { - type: "subagent", - status: "completed", - subagentKind: "advisor", - parentToolCallId: "toolu_mock_dashboard_advisor_review", - transcriptAvailable: true, - }, - ], + expect(privateConversation.events[0]?.data).toMatchObject({ + type: "visible_message", + redacted: true, }); - const firstAdvisorTranscript = await app.fetch( - new Request( - `http://localhost/api/conversations/${encodeURIComponent( - DASHBOARD_QA_CONVERSATION_ID, - )}/subagents/toolu_mock_dashboard_advisor_plan`, - ), - ); - expect(firstAdvisorTranscript.status).toBe(200); - const firstAdvisorBody = - (await firstAdvisorTranscript.json()) as ConversationSubagentTranscriptReport; - expect(firstAdvisorBody.subagentConversationId).toBe( - "junior:internal:dashboard-qa:advisor_session", - ); - expect(firstAdvisorBody.subagentSentryConversationUrl).toContain( - encodeURIComponent("junior:internal:dashboard-qa:advisor_session"), - ); - expect(firstAdvisorBody.transcriptAvailable).toBe(true); - expect(JSON.stringify(firstAdvisorBody.transcript)).toContain( - "Review the dashboard plan before editing", - ); - expect(JSON.stringify(firstAdvisorBody.transcript)).not.toContain( - "Review the implementation after the first advisor pass", - ); - - const secondAdvisorTranscript = await app.fetch( - new Request( - `http://localhost/api/conversations/${encodeURIComponent( - DASHBOARD_QA_CONVERSATION_ID, - )}/subagents/toolu_mock_dashboard_advisor_review`, - ), - ); - expect(secondAdvisorTranscript.status).toBe(200); - const secondAdvisorBody = - (await secondAdvisorTranscript.json()) as ConversationSubagentTranscriptReport; - expect(JSON.stringify(secondAdvisorBody.transcript)).toContain( - "Review the dashboard plan before editing", - ); - expect(JSON.stringify(secondAdvisorBody.transcript)).toContain( - "Review the implementation after the first advisor pass", - ); - - const longConversation = await app.fetch( - new Request( - "http://localhost/api/conversations/slack%3ACQA456%3A1770021600.000600", - ), - ); - expect(longConversation.status).toBe(200); - const longConversationBody = (await longConversation.json()) as { - contextEvents?: Array<{ summary?: string; type: string }>; - transcript: Array<{ - role: string; - parts: Array<{ id?: string; name?: string; type: string }>; - timestamp?: number; - }>; - transcriptMessageCount?: number; - }; - const longConversationParts = longConversationBody.transcript.flatMap( - (message) => message.parts, - ); - const systemMessages = longConversationBody.transcript.filter( - (message) => message.role === "system", - ); - const bashCallTimes = new Map(); - const bashDurations = longConversationBody.transcript.flatMap((message) => - message.parts.flatMap((part) => { - if (part.name !== "bash" || !part.id || !message.timestamp) { - return []; - } - if (part.type === "tool_call") { - bashCallTimes.set(part.id, message.timestamp); - return []; - } - const startedAt = bashCallTimes.get(part.id); - return startedAt === undefined ? [] : [message.timestamp - startedAt]; - }), - ); - expect(systemMessages).toHaveLength(1); - expect(longConversationBody.contextEvents).toEqual([ - expect.objectContaining({ type: "context_compacted" }), - expect.objectContaining({ type: "model_handoff" }), - ]); - expect(longConversationBody.transcriptMessageCount).toBe( - longConversationBody.transcript.length, + const long = await readDetail("slack:CQA456:1770021600.000600"); + expect(long.events.map((event) => event.data.type)).toEqual( + expect.arrayContaining(["context_compacted", "model_handoff"]), ); expect( - longConversationParts.filter((part) => part.name === "bash").length, - ).toBeGreaterThan(20); - expect(new Set(bashDurations).size).toBeGreaterThan(8); - expect(Math.max(...bashDurations)).toBeGreaterThan(10_000); - expect(longConversationParts.some((part) => part.type === "thinking")).toBe( - true, - ); - - const conversation = await app.fetch( - new Request( - "http://localhost/api/conversations/slack%3ADQA123%3A1770007200.000300", + long.events.filter( + (event) => + event.data.type === "tool_started" && event.data.name === "bash", ), - ); - expect(conversation.status).toBe(200); - const redactedConversationBody = (await conversation.json()) as { - transcriptAvailable: boolean; - transcriptMetadata?: Array<{ role: string }>; - transcriptRedacted?: boolean; - }; - expect(redactedConversationBody).toMatchObject({ - conversationId: "slack:DQA123:1770007200.000300", - transcriptAvailable: false, - transcriptRedacted: true, - transcript: [], - }); - expect(redactedConversationBody.transcriptMetadata?.[0]?.role).toBe("user"); + ).toHaveLength(12); }); - it("serves explicit mock conversation data", async () => { + it("loads canonical child conversations through the ordinary detail route", async () => { const app = createDashboardApp({ authRequired: false, allowedGoogleDomains: [], mockConversations: true, }); - - const response = await app.fetch( - new Request("http://localhost/api/conversations"), + const parentResponse = await app.fetch( + new Request( + `http://localhost/api/conversations/${encodeURIComponent( + DASHBOARD_QA_CONVERSATION_ID, + )}`, + ), ); - - expect(response.status).toBe(200); - const body = (await response.json()) as { - conversations: Array<{ conversationId: string; status: string }>; - source: string; - }; - expect(body.source).toBe("conversation_index"); - expect(body.conversations[0]).toMatchObject({ - conversationId: "slack:CQA123:1770003600.000200", - status: "active", - }); - const stats = await app.fetch( - new Request("http://localhost/api/conversations/stats"), + const parent = conversationDetailReportSchema.parse( + await parentResponse.json(), ); - expect(stats.status).toBe(200); - expect(await stats.json()).toMatchObject({ - conversations: expect.any(Number), - }); - }); - - it("returns the canonical subagent not-found response", async () => { - const app = createDashboardApp({ - authRequired: false, - allowedGoogleDomains: [], - mockConversations: true, - }); + const childIds = parent.events.flatMap((event) => + event.data.type === "subagent_started" + ? [event.data.childConversationId] + : [], + ); + expect(childIds).toHaveLength(2); - const response = await app.fetch( + for (const childId of childIds) { + const response = await app.fetch( + new Request( + `http://localhost/api/conversations/${encodeURIComponent(childId)}`, + ), + ); + expect(response.status).toBe(200); + const child = conversationDetailReportSchema.parse(await response.json()); + expect(child.conversationId).toBe(childId); + expect(child.events.length).toBeGreaterThan(0); + } + + const removedRoute = await app.fetch( new Request( - "http://localhost/api/conversations/missing/subagents/missing-child", + `http://localhost/api/conversations/${encodeURIComponent( + DASHBOARD_QA_CONVERSATION_ID, + )}/subagents/legacy`, ), ); - - expect(response.status).toBe(404); - expect( - conversationSubagentTranscriptReportSchema.parse(await response.json()), - ).toMatchObject({ - id: "missing-child", - transcript: [], - transcriptAvailable: false, - unavailableReason: "not_found", - }); + expect(removedRoute.status).toBe(404); }); }); diff --git a/packages/junior-dashboard/tests/dashboard-routes.test.ts b/packages/junior-dashboard/tests/dashboard-routes.test.ts index 9d0d875d4..941ca22e4 100644 --- a/packages/junior-dashboard/tests/dashboard-routes.test.ts +++ b/packages/junior-dashboard/tests/dashboard-routes.test.ts @@ -310,7 +310,6 @@ describe("dashboard routes", () => { "/api/locations/destination-1", "/api/plugin-reports", "/api/conversations/slack%3AC1%3A123", - "/api/conversations/slack%3AC1%3A123/subagents/advisor-call", "/api/config", "/api/me", ]) { diff --git a/packages/junior-dashboard/tests/format.test.ts b/packages/junior-dashboard/tests/format.test.ts index eb1978885..9aabdee8e 100644 --- a/packages/junior-dashboard/tests/format.test.ts +++ b/packages/junior-dashboard/tests/format.test.ts @@ -1,14 +1,20 @@ import { afterEach, describe, expect, it, vi } from "vitest"; -import type { ConversationSummaryReport } from "@sentry/junior/api/schema"; -import type { ConversationDetailReport } from "@sentry/junior/api/schema"; +import type { + ConversationDetailReport, + ConversationReportEvent, + ConversationReportEventData, + ConversationSummaryReport, +} from "@sentry/junior/api/schema"; import { + actorLabel, buildConversations, canRenderStructuredMarkup, + conversationActorLabel, conversationDisplayTitle, conversationFromDetail, conversationIdentityMeta, - conversationActorLabel, + conversationMessageCount, filterConversationList, formatCompactNumber, formatConversationDuration, @@ -17,23 +23,30 @@ import { formatTranscriptDuration, formatUsageTotal, parseMarkdownBlocks, - actorLabel, slackLocationLabel, summarizeMessages, - summarizeTurns, - summarizeCost, summarizeToolCalls, - summarizeUsage, - conversationMessageCount, + summarizeTurns, } from "../src/client/format"; import { formatDuration } from "../src/client/components/Duration"; import type { ConversationTranscript } from "../src/client/types"; -afterEach(() => { - vi.useRealTimers(); -}); +afterEach(() => vi.useRealTimers()); + +function event( + seq: number, + data: ConversationReportEventData, +): ConversationReportEvent { + return { + seq, + contextEpoch: 0, + createdAt: `2026-01-01T00:00:${String(seq).padStart(2, "0")}.000Z`, + data, + }; +} function transcript( + events: ConversationReportEvent[] = [], overrides: Partial = {}, ): ConversationTranscript { const startedAt = "2026-01-01T00:00:00.000Z"; @@ -41,18 +54,19 @@ function transcript( conversationId: "conversation-1", cumulativeDurationMs: 0, displayTitle: "Conversation", + eventHistory: { status: "available" }, + events, + generatedAt: startedAt, lastProgressAt: startedAt, lastSeenAt: startedAt, startedAt, status: "completed", surface: "internal", - transcript: [], - transcriptAvailable: true, ...overrides, }; } -describe("dashboard token formatting", () => { +describe("dashboard conversation formatting", () => { it("scales large values through billions and trillions", () => { expect(formatCompactNumber(1_912_000_000)).toBe("1.9b"); expect(formatCompactNumber(2_100_000_000_000)).toBe("2.1t"); @@ -68,26 +82,6 @@ describe("dashboard token formatting", () => { totalTokens: 999, }), ).toBe("80 tokens"); - }); - - it("formats cumulative estimated conversation cost", () => { - const usage = { - cost: { - input: 0.005, - output: 0.007, - cacheRead: 0.0001, - total: 0.0121, - }, - }; - - expect(summarizeCost(usage)).toEqual({ - input: 0.005, - output: 0.007, - cacheRead: 0.0001, - cacheWrite: undefined, - total: 0.0121, - }); - expect(formatCostTotal(usage)).toBe("$0.01"); expect(formatCostTotal({ cost: { total: 1.999 } })).toBe("$2.00"); }); @@ -102,671 +96,201 @@ describe("dashboard token formatting", () => { it("formats cumulative conversation runtime", () => { expect(formatRuntime(3_500)).toBe("3.5s"); expect(formatRuntime(0)).toBe(""); + expect(formatTranscriptDuration({ cumulativeDurationMs: 7_000 })).toBe( + "7.0s", + ); }); - it("formats transcript duration from cumulative execution time", () => { - expect( - formatTranscriptDuration({ + it("formats conversation duration from cumulative execution time", () => { + const [conversation] = buildConversations([ + { + conversationId: "slack:C1:123", cumulativeDurationMs: 7_000, - }), - ).toBe("7.0s"); + displayTitle: "Conversation", + lastProgressAt: "2026-06-01T10:02:29.000Z", + lastSeenAt: "2026-06-01T10:02:29.000Z", + startedAt: "2026-06-01T10:00:00.000Z", + status: "completed", + surface: "slack", + }, + ]); + + expect(formatConversationDuration(conversation!)).toBe("7.0s"); }); - it("counts conversational transcript messages instead of tool events", () => { - const conversation = transcript({ - transcript: [ - { - role: "user", - parts: [{ type: "text", text: "run the search" }], - }, - { - role: "assistant", - parts: [{ type: "thinking", output: "I should search first" }], - }, - { - role: "assistant", - parts: [{ type: "tool_call", name: "search", input: {} }], - }, - { - role: "toolResult", - parts: [{ type: "tool_result", name: "search", output: [] }], - }, - { - role: "assistant", - parts: [{ type: "text", text: "done" }], - }, - ], - }); + it("omits conversation runtime when no execution time is recorded", () => { + const [conversation] = buildConversations([ + { + conversationId: "slack:C1:123", + cumulativeDurationMs: 0, + displayTitle: "Conversation", + lastProgressAt: "2026-06-01T10:02:29.000Z", + lastSeenAt: "not-a-date", + startedAt: "2026-06-01T10:00:00.000Z", + status: "completed", + surface: "slack", + }, + ]); - expect(conversationMessageCount(conversation)).toBe(2); + expect(formatConversationDuration(conversation!)).toBe("none"); }); - it("summarizes tooltip metrics from visible transcripts", () => { - const conversation = transcript({ - actorIdentity: { fullName: "alice" }, - transcript: [ - { + it("summarizes visible canonical messages and structural tools", () => { + const conversation = transcript( + [ + event(0, { + type: "visible_message", + messageId: "user", role: "user", - parts: [{ type: "text", text: "run search" }], - }, - { - role: "assistant", - timestamp: 1_000, - parts: [{ type: "tool_call", id: "call-1", name: "search" }], - }, - { - role: "toolResult", - timestamp: 2_500, - parts: [{ type: "tool_result", id: "call-1", name: "search" }], - }, - { + text: "run search", + }), + event(1, { type: "tool_started", name: "search" }), + event(2, { + type: "visible_message", + messageId: "assistant", role: "assistant", - parts: [{ type: "text", text: "done" }], - }, + text: "done", + }), ], - }); + { actorIdentity: { fullName: "Alice" } }, + ); + expect(conversationMessageCount(conversation)).toBe(2); expect(summarizeToolCalls(conversation)).toEqual({ - items: [{ count: 1, name: "search", totalDurationMs: 1_500 }], + items: [{ count: 1, name: "search" }], total: 1, }); expect(summarizeMessages(conversation)).toEqual({ items: [ - { author: "alice", bytes: 10 }, + { author: "Alice", bytes: 10 }, { author: "Junior", bytes: 4 }, ], total: 2, }); expect(summarizeTurns(conversation)).toEqual({ - items: [{ author: "alice", bytes: 10 }], - total: 1, - }); - expect( - summarizeUsage({ - cachedInputTokens: 2, - inputTokens: 3, - outputTokens: 5, - reasoningTokens: 10, - }), - ).toMatchObject({ - cachedInputTokens: 2, - inputTokens: 3, - outputTokens: 5, - reasoningTokens: 10, - totalTokens: 10, - }); - }); - - it("does not count assistant-only transcripts as actor turns", () => { - const conversation = transcript({ - transcript: [ - { - role: "assistant", - parts: [{ type: "text", text: "proactive update" }], - }, - ], - }); - - expect(summarizeTurns(conversation)).toBeUndefined(); - }); - - it("counts activity-only tool calls in tool summaries", () => { - const conversation = { - conversationId: "conversation-activity", - cumulativeDurationMs: 0, - displayTitle: "Activity", - lastProgressAt: "2026-01-01T00:00:01.000Z", - lastSeenAt: "2026-01-01T00:00:01.000Z", - startedAt: "2026-01-01T00:00:00.000Z", - status: "active", - surface: "internal", - transcriptAvailable: true, - transcript: [], - activity: [ - { - type: "tool_execution", - id: "call-activity", - toolCallId: "call-activity", - toolName: "advisor", - createdAt: "2026-01-01T00:00:01.000Z", - status: "running", - subagents: [], - }, - ], - } satisfies ConversationTranscript; - - expect(summarizeToolCalls(conversation)).toEqual({ - items: [{ count: 1, name: "advisor" }], + items: [{ author: "Alice", bytes: 10 }], total: 1, }); }); - it("uses transcript message count when only activity rows are visible", () => { - const conversation = { - conversationId: "conversation-activity", - cumulativeDurationMs: 0, - displayTitle: "Activity", - lastProgressAt: "2026-01-01T00:00:01.000Z", - lastSeenAt: "2026-01-01T00:00:01.000Z", - startedAt: "2026-01-01T00:00:00.000Z", - status: "active", - surface: "internal", - transcriptAvailable: true, - transcript: [], - transcriptMessageCount: 3, - activity: [ - { - type: "tool_execution", - id: "call-activity", - toolCallId: "call-activity", - toolName: "advisor", - createdAt: "2026-01-01T00:00:01.000Z", - status: "running", - subagents: [], - }, - ], - } satisfies ConversationTranscript; - - expect(conversationMessageCount(conversation)).toBe(3); - expect(summarizeToolCalls(conversation)).toEqual({ - items: [{ count: 1, name: "advisor" }], - total: 1, - }); - }); - - it("does not match id-bearing tool calls to name-only results", () => { - const conversation = transcript({ - transcript: [ - { - role: "assistant", - timestamp: 1_000, - parts: [{ type: "tool_call", id: "call-1", name: "search" }], - }, - { - role: "assistant", - timestamp: 1_200, - parts: [{ type: "tool_call", id: "call-2", name: "search" }], - }, - { - role: "toolResult", - timestamp: 1_800, - parts: [{ type: "tool_result", name: "search" }], - }, - ], - }); - - expect(summarizeToolCalls(conversation)).toEqual({ - items: [{ count: 2, name: "search" }], - total: 2, - }); - }); - - it("does not infer tool durations for unnamed calls and results", () => { - const conversation = transcript({ - transcript: [ - { - role: "assistant", - timestamp: 1_000, - parts: [{ type: "tool_call" }], - }, - { - role: "toolResult", - timestamp: 2_000, - parts: [{ type: "tool_result" }], - }, - ], - }); - - expect(summarizeToolCalls(conversation)).toEqual({ - items: [{ count: 1, name: "unknown" }], - total: 1, - }); - }); - - it("uses the API-supplied displayTitle directly", () => { - const conversations: ConversationSummaryReport[] = [ - { - channel: "C1", - conversationId: "slack:C1:123", - cumulativeDurationMs: 0, - displayTitle: "Public Channel", - lastProgressAt: "2026-06-01T10:05:00.000Z", - lastSeenAt: "2026-06-01T10:05:00.000Z", - actorIdentity: { - slackUserId: "U1", - slackUserName: "Alice Reviewer", - }, - startedAt: "2026-06-01T10:00:00.000Z", - status: "completed", - surface: "slack", - }, - ]; - const [conversation] = buildConversations(conversations); - - expect(conversationDisplayTitle(conversation)).toBe("Public Channel"); - expect(conversationIdentityMeta(conversation, conversation?.id)).toBe( - "Alice Reviewer · slack:C1:123", - ); - }); - - it("does not render a fake identity line before route data exists", () => { - expect(conversationIdentityMeta(undefined, undefined)).toBe(""); - }); - - it("keeps Slack display names with spaces as actor labels", () => { - expect( - actorLabel({ slackUserId: "U1", slackUserName: "Alice Reviewer" }), - ).toBe("Alice Reviewer"); - }); - - it("uses the displayTitle from the most recent turn", () => { - const [conversation] = buildConversations([ - { - conversationId: "slack:C1:123", - cumulativeDurationMs: 0, - displayTitle: "Older title", - lastProgressAt: "2026-06-01T10:05:00.000Z", - lastSeenAt: "2026-06-01T10:05:00.000Z", - startedAt: "2026-06-01T10:00:00.000Z", - status: "completed", - surface: "slack", - }, - { - conversationId: "slack:C1:123", - cumulativeDurationMs: 0, - displayTitle: "Newer title", - lastProgressAt: "2026-06-01T11:05:00.000Z", - lastSeenAt: "2026-06-01T11:05:00.000Z", - startedAt: "2026-06-01T11:00:00.000Z", - status: "completed", - surface: "slack", - }, + it("does not count assistant-only history as actor turns", () => { + const conversation = transcript([ + event(0, { + type: "visible_message", + messageId: "assistant", + role: "assistant", + text: "proactive update", + }), ]); - - expect(conversationDisplayTitle(conversation)).toBe("Newer title"); + expect(summarizeTurns(conversation)).toBeUndefined(); }); - it("builds permalink header metadata from conversation detail reports", () => { - const conversation = conversationFromDetail({ + it("uses canonical detail metadata without carrying absent fields", () => { + const detail = transcript([], { channel: "C1", channelName: "proj-alpha", conversationId: "slack:C1:123", - cumulativeDurationMs: 0, displayTitle: "Detail Title", - generatedAt: "2026-06-01T11:06:00.000Z", - lastProgressAt: "2026-06-01T11:05:00.000Z", - lastSeenAt: "2026-06-01T11:05:00.000Z", actorIdentity: { email: "alice@example.com" }, - sentryConversationUrl: "https://sentry.example/conversations/123", - startedAt: "2026-06-01T10:00:00.000Z", - status: "completed", - surface: "slack", - transcriptAvailable: true, - transcript: [], - } satisfies ConversationDetailReport); - + }); + const conversation = conversationFromDetail(detail); expect(conversationDisplayTitle(conversation)).toBe("Detail Title"); - expect(conversation?.channelName).toBe("proj-alpha"); expect(conversationIdentityMeta(conversation, conversation?.id)).toBe( "alice@example.com · slack:C1:123", ); }); - it("does not carry missing SQL fields across conversation rows", () => { + it("uses the most recent API-supplied display title", () => { + const base = { + conversationId: "slack:C1:123", + cumulativeDurationMs: 0, + lastProgressAt: "2026-06-01T10:05:00.000Z", + startedAt: "2026-06-01T10:00:00.000Z", + status: "completed" as const, + surface: "slack" as const, + }; const [conversation] = buildConversations([ { - channel: "C1", - channelName: "proj-alpha", - conversationId: "slack:C1:123", - cumulativeDurationMs: 0, - displayTitle: "#proj-alpha", - lastProgressAt: "2026-06-01T10:05:00.000Z", + ...base, + displayTitle: "Older", lastSeenAt: "2026-06-01T10:05:00.000Z", - startedAt: "2026-06-01T10:00:00.000Z", - status: "completed", - surface: "slack", }, { - channel: "C1", - conversationId: "slack:C1:123", - cumulativeDurationMs: 0, - displayTitle: "Public Channel", - lastProgressAt: "2026-06-01T11:05:00.000Z", + ...base, + displayTitle: "Newer", lastSeenAt: "2026-06-01T11:05:00.000Z", - startedAt: "2026-06-01T11:00:00.000Z", - status: "completed", - surface: "slack", }, ]); - - expect(conversation?.channelName).toBeUndefined(); - expect(conversationDisplayTitle(conversation)).toBe("Public Channel"); + expect(conversationDisplayTitle(conversation)).toBe("Newer"); }); - it("keeps actor labels even when the title matches", () => { - const conversations: ConversationSummaryReport[] = [ + it("filters conversation rows by text and source", () => { + const summaries: ConversationSummaryReport[] = [ { - channel: "C1", - channelName: "alice", - conversationId: "slack:C1:123", - displayTitle: "Alice", + conversationId: "slack:C1:1", cumulativeDurationMs: 0, - lastProgressAt: "2026-06-01T10:05:00.000Z", - lastSeenAt: "2026-06-01T10:05:00.000Z", - actorIdentity: { - fullName: "alice", - slackUserId: "U1", - }, - startedAt: "2026-06-01T10:00:00.000Z", - status: "completed", + displayTitle: "Checkout incident", + lastProgressAt: "2026-01-01T00:00:00.000Z", + lastSeenAt: "2026-01-01T00:00:00.000Z", + startedAt: "2026-01-01T00:00:00.000Z", + status: "failed", surface: "slack", }, - ]; - const [conversation] = buildConversations(conversations); - - expect(conversationActorLabel(conversation)).toBe("alice"); - expect(conversationIdentityMeta(conversation, conversation?.id)).toBe( - "alice · slack:C1:123", - ); - }); - - it("filters conversation rows by text, source, and actor", () => { - const conversations = buildConversations([ { - channel: "C1", - channelName: "proj-checkout", - locationId: "destination-1", - conversationId: "slack:C1:123", + conversationId: "scheduler:1", cumulativeDurationMs: 0, - displayTitle: "Checkout latency triage", - lastProgressAt: "2026-06-01T10:05:00.000Z", - lastSeenAt: "2026-06-01T10:05:00.000Z", - actorIdentity: { - email: "morgan@example.com", - fullName: "Morgan Lee", - }, - startedAt: "2026-06-01T10:00:00.000Z", + displayTitle: "Daily digest", + lastProgressAt: "2026-01-01T00:00:00.000Z", + lastSeenAt: "2026-01-01T00:00:00.000Z", + startedAt: "2026-01-01T00:00:00.000Z", status: "completed", - surface: "slack", + surface: "scheduler", }, - { - conversationId: "internal:memory:456", - cumulativeDurationMs: 0, - displayTitle: "Memory cleanup", - lastProgressAt: "2026-06-01T11:05:00.000Z", - lastSeenAt: "2026-06-01T11:05:00.000Z", - actorIdentity: { fullName: "Casey" }, - startedAt: "2026-06-01T11:00:00.000Z", - status: "completed", - surface: "internal", - }, - ]); - - expect( - filterConversationList(conversations, { - query: "checkout", - actor: "morgan@example.com", - location: "destination-1", - source: "slack", - }).map((conversation) => conversation.id), - ).toEqual(["slack:C1:123"]); - expect( - filterConversationList(conversations, { - query: "checkout", - actor: "Casey", - location: "destination-1", - source: "slack", - }), - ).toEqual([]); - }); - - it("formats cumulative runtime instead of the conversation wall time", () => { - const [conversation] = buildConversations([ - { - conversationId: "slack:C1:123", - cumulativeDurationMs: 7_000, - displayTitle: "Conversation", - lastProgressAt: "2026-06-01T10:02:29.000Z", - lastSeenAt: "2026-06-01T10:02:29.000Z", - startedAt: "2026-06-01T10:00:00.000Z", - status: "completed", - surface: "slack", - }, - ]); - - expect(formatConversationDuration(conversation!)).toBe("7.0s"); + ]; + const rows = buildConversations(summaries); + expect(filterConversationList(rows, { query: "checkout" })).toHaveLength(1); + expect(filterConversationList(rows, { source: "scheduler" })).toHaveLength( + 1, + ); }); - it("omits conversation runtime when no execution time is recorded", () => { - const [conversation] = buildConversations([ - { - conversationId: "slack:C1:123", + it("formats actor and Slack labels", () => { + expect(actorLabel({ slackUserName: "Alice Reviewer" })).toBe( + "Alice Reviewer", + ); + expect( + conversationActorLabel({ + id: "1", cumulativeDurationMs: 0, - displayTitle: "Conversation", - lastProgressAt: "2026-06-01T10:02:29.000Z", - lastSeenAt: "not-a-date", - startedAt: "2026-06-01T10:00:00.000Z", + displayTitle: "x", + lastProgressAt: "x", + lastSeenAt: "x", + startedAt: "x", status: "completed", surface: "slack", - }, - ]); - - expect(formatConversationDuration(conversation!)).toBe("none"); - }); -}); - -describe("parseMarkdownBlocks prose language detection", () => { - describe("default mode (detectLanguage — for user/system messages)", () => { - it("detects XML-looking prose as xml", () => { - const [block] = parseMarkdownBlocks("bar"); - expect(block?.language).toBe("xml"); - expect(block?.fenced).toBe(false); - }); - - it("detects HTML-looking prose as xml or html (collapsible)", () => { - const [block] = parseMarkdownBlocks("
Hello
"); - expect(["xml", "html"]).toContain(block?.language); - }); - - it("detects mixed prose + block-level XML as xml (system prompt pattern)", () => { - const text = [ - "You are a Slack-based helper assistant.", - "", - "", - "Your Slack username is `junior`.", - "", - "", - "", - "## core identity", - "- you are junior", - "", - ].join("\n"); - const [block] = parseMarkdownBlocks(text); - expect(block?.language).toBe("xml"); - }); - - it("does not detect an unclosed block tag as xml", () => { - const text = [ - "Here is an example:", - "", - "
", - "## heading", - "- bullet", - ].join("\n"); - expect(parseMarkdownBlocks(text)[0]?.language).not.toBe("xml"); - }); - - it("keeps normal markdown without XML blocks as markdown", () => { - const text = ["Intro", "", "## heading", "- bullet"].join("\n"); - expect(parseMarkdownBlocks(text)[0]?.language).toBe("markdown"); - }); - - it("detects valid JSON prose as json", () => { - const [block] = parseMarkdownBlocks('{"a":1}'); - expect(block?.language).toBe("json"); - expect(block?.fenced).toBe(false); - }); - - it("marks prose blocks as not fenced", () => { - const blocks = parseMarkdownBlocks("some prose text"); - expect(blocks[0]?.fenced).toBe(false); - }); - - it("marks explicit fenced blocks as fenced", () => { - const blocks = parseMarkdownBlocks("before\n```xml\n\n```\nafter"); - expect(blocks[1]?.language).toBe("xml"); - expect(blocks[1]?.fenced).toBe(true); - }); - }); - - describe("outputOnly: true mode (detectOutputLanguage — for assistant messages)", () => { - it("treats XML-looking prose as markdown, never auto-detects XML", () => { - const [block] = parseMarkdownBlocks("bar", { - outputOnly: true, - }); - expect(block?.language).toBe("markdown"); - expect(block?.fenced).toBe(false); - }); - - it("treats HTML-looking prose as markdown", () => { - const [block] = parseMarkdownBlocks("
Hello
", { - outputOnly: true, - }); - expect(block?.language).toBe("markdown"); - }); - - it("treats TypeScript-looking prose as markdown", () => { - const [block] = parseMarkdownBlocks("const value = 1;", { - outputOnly: true, - }); - expect(block?.language).toBe("markdown"); - }); - - it("treats shell-looking prose as markdown", () => { - const [block] = parseMarkdownBlocks("npm install", { outputOnly: true }); - expect(block?.language).toBe("markdown"); - }); - - it("detects valid JSON prose as json and pretty-prints it", () => { - const [block] = parseMarkdownBlocks('{"a":1}', { outputOnly: true }); - expect(block?.language).toBe("json"); - expect(block?.code).toBe('{\n "a": 1\n}'); - expect(block?.fenced).toBe(false); - }); - - it("keeps prose blocks as markdown even when fenced XML is present", () => { - const blocks = parseMarkdownBlocks("before\n```xml\n\n```\nafter", { - outputOnly: true, - }); - expect(blocks[0]?.language).toBe("markdown"); - expect(blocks[0]?.fenced).toBe(false); - expect(blocks[2]?.language).toBe("markdown"); - expect(blocks[2]?.fenced).toBe(false); - }); - - it("still detects fenced xml blocks as xml", () => { - const blocks = parseMarkdownBlocks("before\n```xml\n\n```\nafter", { - outputOnly: true, - }); - expect(blocks[1]?.language).toBe("xml"); - expect(blocks[1]?.fenced).toBe(true); - }); - }); -}); - -describe("canRenderStructuredMarkup", () => { - it("returns true for xml blocks regardless of fenced status", () => { - expect( - canRenderStructuredMarkup({ - code: "", - language: "xml", - fenced: false, + actorIdentity: { email: "alice@example.com" }, }), - ).toBe(true); + ).toBe("alice@example.com"); expect( - canRenderStructuredMarkup({ - code: "", - language: "xml", - fenced: true, + slackLocationLabel({ + channel: "C1", + channelName: "proj-alpha", + channelNameRedacted: false, }), - ).toBe(true); - expect(canRenderStructuredMarkup({ code: "", language: "xml" })).toBe( - true, - ); + ).toBe("#proj-alpha (C1)"); }); +}); - it("returns true for html blocks", () => { - expect( - canRenderStructuredMarkup({ - code: "
", - language: "html", - fenced: true, - }), - ).toBe(true); +describe("structured transcript markup", () => { + it("detects fenced and inline structured blocks", () => { + expect(parseMarkdownBlocks('```json\n{"ok":true}\n```')[0]).toMatchObject({ + language: "json", + }); expect( - canRenderStructuredMarkup({ code: "
", language: "html" }), + canRenderStructuredMarkup({ code: "", language: "xml" }), ).toBe(true); - }); - - it("returns false for non-xml/html blocks", () => { expect( - canRenderStructuredMarkup({ - code: "const x = 1", - language: "typescript", - fenced: true, - }), - ).toBe(false); - }); - - it("returns false for markdown blocks", () => { - expect( - canRenderStructuredMarkup({ code: "some text", language: "markdown" }), + canRenderStructuredMarkup({ code: "plain", language: "markdown" }), ).toBe(false); }); - - // The guard against assistant prose misclassification is now at the - // parseMarkdownBlocks level (outputOnly option), not canRenderStructuredMarkup. - it("relies on caller to pass outputOnly:true for assistant prose", () => { - // With outputOnly:true, XML-looking assistant prose stays as markdown - const [block] = parseMarkdownBlocks("bar", { outputOnly: true }); - expect(block?.language).toBe("markdown"); - expect(canRenderStructuredMarkup(block!)).toBe(false); - }); -}); - -describe("slackLocationLabel redacted labels", () => { - it("returns redacted type labels verbatim when the report marks them redacted", () => { - expect( - slackLocationLabel({ - channel: "C123", - channelName: "Private Conversation", - channelNameRedacted: true, - }), - ).toBe("Private Conversation (C123)"); - expect( - slackLocationLabel( - { - channel: "C123", - channelName: "Private Conversation", - channelNameRedacted: true, - }, - { includeId: false }, - ), - ).toBe("Private Conversation"); - }); - - it("still formats real channel names with a # prefix", () => { - expect( - slackLocationLabel( - { channel: "C123", channelName: "proj-alpha" }, - { includeId: false }, - ), - ).toBe("#proj-alpha"); - expect( - slackLocationLabel( - { channel: "C123", channelName: "Private Conversation" }, - { includeId: false }, - ), - ).toBe("#Private Conversation"); - }); }); diff --git a/packages/junior-dashboard/tests/markdownExport.test.ts b/packages/junior-dashboard/tests/markdownExport.test.ts index b28036646..eccb62f6d 100644 --- a/packages/junior-dashboard/tests/markdownExport.test.ts +++ b/packages/junior-dashboard/tests/markdownExport.test.ts @@ -1,382 +1,179 @@ import { describe, expect, it } from "vitest"; -import type { ConversationDetailReport } from "@sentry/junior/api/schema"; -import type { ConversationSubagentTranscriptReport } from "@sentry/junior/api/schema"; - -import { - buildConversationMarkdown, - buildSubagentMarkdown, -} from "../src/client/markdownExport"; -import { subagentConversationTranscript } from "../src/client/subagentTranscript"; -import type { Conversation } from "../src/client/types"; - -describe("dashboard markdown export", () => { - it("serializes child-agent transcripts with shared formatting", () => { - const report = { - type: "subagent", - createdAt: "2026-01-01T00:00:00.000Z", - endedAt: "2026-01-01T00:00:02.000Z", - id: "advisor-call", - outcome: "success", - status: "success", - subagentConversationId: "junior:conversation-1:advisor_session", - subagentKind: "advisor", - subagentSentryConversationUrl: - "https://sentry.example/explore/conversations/advisor", - transcript: [ - { +import type { + ConversationReportEvent, + ConversationReportEventData, +} from "@sentry/junior/api/schema"; + +import { buildConversationMarkdown } from "../src/client/markdownExport"; +import type { ConversationTranscript } from "../src/client/types"; + +function event( + seq: number, + data: ConversationReportEventData, +): ConversationReportEvent { + return { + seq, + contextEpoch: 0, + createdAt: `2026-01-01T00:00:${String(seq).padStart(2, "0")}.000Z`, + data, + }; +} + +function conversation( + events: ConversationReportEvent[], + overrides: Partial = {}, +): ConversationTranscript { + return { + actorIdentity: { email: "alice@example.com" }, + channel: "C1", + channelName: "proj-alpha", + conversationId: "conversation-1", + cumulativeDurationMs: 3_000, + displayTitle: "Canonical conversation", + eventHistory: { status: "available" }, + events, + generatedAt: "2026-01-01T00:01:00.000Z", + lastProgressAt: "2026-01-01T00:00:10.000Z", + lastSeenAt: "2026-01-01T00:00:10.000Z", + startedAt: "2026-01-01T00:00:00.000Z", + status: "completed", + surface: "slack", + ...overrides, + }; +} + +describe("dashboard canonical-event Markdown export", () => { + it("exports visible messages once in API order", () => { + const markdown = buildConversationMarkdown( + conversation([ + event(0, { + type: "visible_message", + messageId: "user-1", role: "user", - timestamp: Date.parse("2026-01-01T00:00:00.000Z"), - parts: [{ type: "text", text: "Review the implementation." }], - }, - { + text: "please investigate", + }), + event(1, { + type: "model_activity", + activities: ["thinking"], + }), + event(2, { + type: "visible_message", + messageId: "assistant-1", role: "assistant", - timestamp: Date.parse("2026-01-01T00:00:02.000Z"), - parts: [{ type: "text", text: "The implementation is sound." }], - }, - ], - transcriptAvailable: true, - } satisfies ConversationSubagentTranscriptReport; - const turn = subagentConversationTranscript("conversation-1", report); - - const markdown = buildSubagentMarkdown(report, turn); - - expect(markdown).toContain("# advisor"); - expect(markdown).toContain("- Subagent ID: `advisor-call`"); - expect(markdown).toContain( - "- Conversation ID: junior:conversation-1:advisor_session", + text: "investigation complete", + }), + ]), ); - expect(markdown).toContain("- Duration: 2.0s"); - expect(markdown).toContain("### User"); - expect(markdown).toContain("Review the implementation."); - expect(markdown).toContain("### advisor"); - expect(markdown).toContain("The implementation is sound."); - }); - it("serializes visible conversation transcripts as Markdown", () => { - const startedAt = "2026-01-01T00:00:00.000Z"; - const detail = { - conversationId: "slack:C1:222", - cumulativeDurationMs: 0, - displayTitle: "Copy button discussion", - generatedAt: "2026-01-01T00:00:08.000Z", - channel: "C1", - channelName: "eng", - lastProgressAt: "2026-01-01T00:00:07.000Z", - lastSeenAt: "2026-01-01T00:00:07.000Z", - actorIdentity: { fullName: "Alice" }, - startedAt, - status: "completed", - surface: "slack", - contextEvents: [ - { - type: "context_compacted", - createdAt: "2026-01-01T00:00:01.500Z", - modelId: "openai/gpt-5.4", - summary: "Earlier investigation was summarized.", - transcriptIndex: 1, - }, - { - type: "model_handoff", - createdAt: "2026-01-01T00:00:04.000Z", - fromModelId: "openai/gpt-5.4", - toModelId: "openai/gpt-5.6-sol", - message: "Continue with the implementation evidence.", - transcriptIndex: 3, - }, - ], - transcriptAvailable: true, - transcript: [ - { - role: "user", - timestamp: Date.parse(startedAt) + 1_000, - parts: [ - { - type: "text", - text: " copy this conversation \n", - }, - ], - }, - { - role: "assistant", - timestamp: Date.parse(startedAt) + 2_000, - parts: [ - { - type: "thinking", - output: "Need a deterministic export. \n", - }, - { - type: "tool_call", - id: "call-1", - name: "search", - input: { query: "copy markdown" }, - }, - ], - }, - { - role: "toolResult", - timestamp: Date.parse(startedAt) + 3_500, - parts: [ - { - type: "tool_result", - id: "call-1", - name: "search", - output: { ok: true }, - }, - ], - }, - { - role: "assistant", - timestamp: Date.parse(startedAt) + 5_000, - parts: [ - { - type: "text", - text: "## Done\n\n\n\nCopied as Markdown.", - }, - ], - }, - ], - } satisfies ConversationDetailReport; - - const markdown = buildConversationMarkdown(detail); + expect(markdown).toContain("# Canonical conversation"); + expect(markdown).toContain("### alice@example.com"); + expect(markdown).toContain("please investigate"); + expect(markdown).toContain("### Junior"); + expect(markdown.match(/investigation complete/g)).toHaveLength(1); + }); - expect(markdown).toContain("# Copy button discussion"); - expect(markdown).toContain("- Conversation ID: `slack:C1:222`"); - expect(markdown).toContain("- Actor: Alice"); - expect(markdown).toContain("- Location: #eng (C1)"); - expect(markdown).toContain("## Transcript"); - expect(markdown).toContain("### Context compacted"); - expect(markdown).toContain("- Model: openai/gpt-5.4"); - expect(markdown).toContain("Earlier investigation was summarized."); - expect(markdown).toContain("### Model handoff"); - expect(markdown).toContain("- From model: openai/gpt-5.4"); - expect(markdown).toContain("- To model: openai/gpt-5.6-sol"); - expect(markdown).toContain("Continue with the implementation evidence."); - expect(markdown.indexOf("### Context compacted")).toBeLessThan( - markdown.indexOf("### Model handoff"), + it("exports structural tool, context, subagent, and failure rows", () => { + const markdown = buildConversationMarkdown( + conversation([ + event(0, { type: "tool_started", name: "search" }), + event(1, { + type: "subagent_started", + childConversationId: "child-1", + subagentKind: "advisor", + historyMode: "shared", + }), + event(2, { type: "context_compacted" }), + event(3, { + type: "turn_lifecycle", + turnId: "turn-1", + state: "failed", + }), + ]), ); - expect(markdown).not.toContain("## Turn"); - expect(markdown).not.toContain("- Turns:"); - expect(markdown).not.toContain("- Turn ID:"); - expect(markdown).toContain("### Alice"); - expect(markdown).toContain(" copy this conversation \n"); - expect(markdown).toContain("### Thinking"); - expect(markdown).toContain("Need a deterministic export. \n"); + expect(markdown).toContain("### Tool: search"); - expect(markdown).toContain('"query": "copy markdown"'); - expect(markdown).toContain("## Done\n\n\n\nCopied as Markdown."); + expect(markdown).toContain("- Result: started"); + expect(markdown).toContain("### Subagent: advisor"); + expect(markdown).toContain("### Context compacted"); + expect(markdown).toContain("### Agent response failed"); + expect(markdown).not.toContain("missing"); + expect(markdown).not.toContain("Result: running"); }); - it("exports terminal assistant outcomes with safe copy", () => { - const startedAt = "2026-01-01T00:00:00.000Z"; - const detail = { - conversationId: "slack:C1:failed", - cumulativeDurationMs: 0, - displayTitle: "Failed responses", - generatedAt: "2026-01-01T00:00:03.000Z", - lastProgressAt: "2026-01-01T00:00:02.000Z", - lastSeenAt: "2026-01-01T00:00:02.000Z", - startedAt, - status: "completed", - surface: "slack", - transcriptAvailable: true, - transcript: [ - { - role: "assistant", - outcome: "error", - timestamp: Date.parse(startedAt) + 1_000, - parts: [], - }, - { - role: "assistant", - outcome: "aborted", - timestamp: Date.parse(startedAt) + 2_000, - parts: [], - }, - ], - } satisfies ConversationDetailReport; - - const markdown = buildConversationMarkdown(detail); - - expect(markdown).toContain("### Agent response failed"); - expect(markdown).toContain( - "The model response ended before Junior could complete this turn.", + it("exports failed delivery history without mislabeling accepted delivery", () => { + const markdown = buildConversationMarkdown( + conversation([ + event(0, { + type: "delivery", + deliveryId: "delivery-1", + state: "accepted", + }), + event(1, { + type: "delivery", + deliveryId: "delivery-2", + state: "failed", + }), + ]), ); - expect(markdown).toContain("### Agent response stopped"); + + expect(markdown).toContain("### Message delivery failed"); expect(markdown).toContain( - "The model response was stopped before Junior could complete this turn.", + "Junior could not deliver this message to its destination.", ); + expect(markdown).not.toContain("delivery-1"); + expect(markdown).not.toContain("Agent response failed"); }); - it("prefers the freshly loaded detail title over a stale list row title", () => { - const generatedAt = "2026-01-01T00:00:08.000Z"; - const detail = { - conversationId: "slack:C1:222", - cumulativeDurationMs: 0, - displayTitle: "Fresh async title", - generatedAt, - lastProgressAt: generatedAt, - lastSeenAt: generatedAt, - startedAt: generatedAt, - status: "completed", - surface: "slack", - transcript: [], - transcriptAvailable: false, - } satisfies ConversationDetailReport; - const conversation = { - channel: "C1", - channelName: "eng", - cumulativeDurationMs: 0, - displayTitle: "Public Channel", - id: "slack:C1:222", - lastProgressAt: generatedAt, - lastSeenAt: generatedAt, - startedAt: "2026-01-01T00:00:00.000Z", - status: "completed", - surface: "slack", - } satisfies Conversation; - - const markdown = buildConversationMarkdown(detail, conversation); - - expect(markdown).toContain("# Fresh async title"); - expect(markdown).not.toContain("# Public Channel"); - }); - - it("exports running tool and subagent activity from derived transcript rows", () => { - const detail = { - conversationId: "conversation-activity", - displayTitle: "Activity transcript", - generatedAt: "2026-01-01T00:00:08.000Z", - cumulativeDurationMs: 0, - lastProgressAt: "2026-01-01T00:00:02.000Z", - lastSeenAt: "2026-01-01T00:00:02.000Z", - startedAt: "2026-01-01T00:00:00.000Z", - status: "active", - surface: "internal", - transcriptAvailable: true, - transcript: [], - activity: [ - { - type: "tool_execution", - id: "advisor-call", - toolCallId: "advisor-call", - toolName: "advisor", - createdAt: "2026-01-01T00:00:01.000Z", - status: "running", - subagents: [ - { - type: "subagent", - id: "advisor-call", - subagentKind: "advisor", - parentToolCallId: "advisor-call", - createdAt: "2026-01-01T00:00:02.000Z", - status: "running", - }, - ], + it("labels redacted structural tool starts neutrally", () => { + const markdown = buildConversationMarkdown( + conversation([event(0, { type: "tool_started", name: "search" })], { + eventHistory: { + status: "redacted", + reason: "non_public_conversation", }, - ], - } satisfies ConversationDetailReport; - - const markdown = buildConversationMarkdown(detail); + }), + ); - expect(markdown).toContain("### Tool: advisor"); - expect(markdown).toContain("- Result: running"); - expect(markdown).toContain("### Subagent: advisor"); - expect(markdown).toContain("- Status: running"); - expect(markdown).toContain("- Parent tool call: advisor-call"); + expect(markdown).toContain("### Tool: search"); + expect(markdown).toContain("- Result: started"); + expect(markdown).not.toContain("missing result"); }); - it("exports only safe redaction metadata for private transcripts", () => { - const detail = { - conversationId: "slack:D1:222", - displayTitle: "Direct Message", - generatedAt: "2026-01-01T00:00:08.000Z", - channel: "D1", - channelName: "Direct Message", - cumulativeDurationMs: 7_000, - lastProgressAt: "2026-01-01T00:00:07.000Z", - lastSeenAt: "2026-01-01T00:00:07.000Z", - actorIdentity: { email: "alice@example.com" }, - startedAt: "2026-01-01T00:00:00.000Z", - status: "completed", - surface: "slack", - transcriptAvailable: false, - transcriptRedacted: true, - transcriptRedactionReason: "non_public_conversation", - transcript: [], - transcriptMetadata: [ - { - role: "user", - timestamp: 1_767_225_601_000, - parts: [ - { - bytes: 24, - chars: 24, - redacted: true, - text: "private question", - type: "text", - }, - ], - }, - { - role: "assistant", - timestamp: 1_767_225_602_000, - parts: [ - { - bytes: 22, - chars: 22, - redacted: true, - text: "private answer", - type: "text", - }, - { - id: "call-1", - input: { query: "private search value" }, - inputKeys: ["query"], - inputSizeBytes: 42, - inputType: "object", - name: "search", - redacted: true, - type: "tool_call", - }, - ], - }, + it("exports only safe placeholders for redacted event history", () => { + const markdown = buildConversationMarkdown( + conversation( + [ + event(0, { + type: "visible_message", + messageId: "private-user", + role: "user", + redacted: true, + }), + ], { - role: "toolResult", - timestamp: 1_767_225_603_000, - parts: [ - { - id: "call-1", - name: "search", - output: "private tool result", - outputSizeBytes: 19, - outputType: "string", - redacted: true, - type: "tool_result", - }, - ], + eventHistory: { + status: "redacted", + reason: "non_public_conversation", + }, }, - ], - } satisfies ConversationDetailReport; - - const markdown = buildConversationMarkdown(detail); + ), + ); - expect(markdown).toContain("# Direct Message"); - expect(markdown).not.toContain("## Turn"); - expect(markdown).not.toContain("- Turn ID:"); expect(markdown).toContain( "Transcript hidden because this conversation is not public.", ); - expect(markdown).toContain(" - 24 chars - 24 bytes"); - expect(markdown).toContain(" - 22 chars - 22 bytes"); - expect(markdown).toContain( - " - tool_call - name: `search` - input: object - input keys: query", - ); - expect(markdown).toContain( - " - tool_result - name: `search` - output: string", + expect(markdown).toContain(""); + }); + + it("explains expired event history", () => { + const markdown = buildConversationMarkdown( + conversation([], { + eventHistory: { + status: "expired", + expiredAt: "2026-01-01T00:00:00.000Z", + }, + }), ); - expect(markdown).not.toContain("private question"); - expect(markdown).not.toContain("private answer"); - expect(markdown).not.toContain("private search value"); - expect(markdown).not.toContain("private tool result"); + expect(markdown).toContain("Transcript expired for this conversation."); }); }); diff --git a/packages/junior-dashboard/tests/telemetry-components.test.tsx b/packages/junior-dashboard/tests/telemetry-components.test.tsx index 2040c605b..d994e9888 100644 --- a/packages/junior-dashboard/tests/telemetry-components.test.tsx +++ b/packages/junior-dashboard/tests/telemetry-components.test.tsx @@ -1,59 +1,77 @@ import { renderToStaticMarkup } from "react-dom/server"; import { QueryClientProvider } from "@tanstack/react-query"; import { MemoryRouter, Route, Routes } from "react-router"; -import { afterEach, describe, expect, it, vi } from "vitest"; +import { afterEach, describe, expect, it } from "vitest"; import type { - ConversationFeed, - ConversationSummaryReport, -} from "@sentry/junior/api/schema"; -import type { ConversationDetailReport } from "@sentry/junior/api/schema"; -import type { - ActorDirectoryReport, ActorProfileReport, + ConversationReportEvent, + ConversationReportEventData, + ConversationSummaryReport, + LocationDetailReport, + LocationDirectoryReport, } from "@sentry/junior/api/schema"; -import type { LocationDirectoryReport } from "@sentry/junior/api/schema"; -import type { LocationDetailReport } from "@sentry/junior/api/schema"; +import { client } from "../src/client/api"; import { HighlightedCode } from "../src/client/code"; -import { ToolCallsMetric } from "../src/client/components/TelemetryMetrics"; import { Button } from "../src/client/components/Button"; +import { ConversationTranscriptView } from "../src/client/components/ConversationTranscript"; import { PluginReports } from "../src/client/components/PluginReports"; -import { StatusBadge } from "../src/client/components/StatusBadge"; -import { CardHeader } from "../src/client/components/layout/CardHeader"; import { SubagentTranscriptDrawer, type SubagentTranscriptTarget, } from "../src/client/components/SubagentTranscriptDrawer"; -import { ToolValueInspector } from "../src/client/components/ToolValueInspector"; import { TranscriptHeader } from "../src/client/components/TranscriptHeader"; -import { TranscriptSubagentView } from "../src/client/components/TranscriptSubagentView"; import { TranscriptToolView } from "../src/client/components/TranscriptToolView"; -import { ConversationTranscriptView } from "../src/client/components/ConversationTranscript"; +import { ToolValueInspector } from "../src/client/components/ToolValueInspector"; import { TranscriptSearchProvider } from "../src/client/components/transcriptSearch"; -import { client } from "../src/client/api"; -import { ContributionGrid } from "../src/client/components/ContributionGrid"; import { ConversationPage } from "../src/client/pages/ConversationPage"; -import { ConversationWorkspace } from "../src/client/pages/ConversationWorkspace"; -import { PeoplePageContent } from "../src/client/pages/people/PeoplePage"; -import { PeopleDirectory } from "../src/client/pages/people/PeopleDirectory"; -import { Profile } from "../src/client/pages/people/PersonProfilePage"; -import { - LocationDetailPage, - LocationDetailPageContent, -} from "../src/client/pages/locations/LocationDetailPage"; +import { LocationDetailPageContent } from "../src/client/pages/locations/LocationDetailPage"; import { LocationsPageContent } from "../src/client/pages/locations/LocationsPage"; -import { SkillInventory } from "../src/client/pages/system/SkillInventory"; +import { Profile } from "../src/client/pages/people/PersonProfilePage"; import { SystemPage } from "../src/client/pages/system/SystemPage"; import type { ConversationTranscript, SystemData } from "../src/client/types"; -afterEach(() => { - client.clear(); - vi.useRealTimers(); -}); +afterEach(() => client.clear()); + +function event( + seq: number, + data: ConversationReportEventData, + createdAt = `2026-01-01T00:00:${String(seq).padStart(2, "0")}.000Z`, +): ConversationReportEvent { + return { seq, contextEpoch: 0, createdAt, data }; +} + +function conversation( + events: ConversationReportEvent[], + overrides: Partial = {}, +): ConversationTranscript { + return { + conversationId: "conversation-1", + cumulativeDurationMs: 3_000, + displayTitle: "Conversation", + eventHistory: { status: "available" }, + events, + generatedAt: "2026-01-01T00:01:00.000Z", + lastProgressAt: "2026-01-01T00:00:10.000Z", + lastSeenAt: "2026-01-01T00:00:10.000Z", + startedAt: "2026-01-01T00:00:00.000Z", + status: "completed", + surface: "internal", + ...overrides, + }; +} + +function renderTranscript(detail: ConversationTranscript): string { + return renderToStaticMarkup( + + + + + , + ); +} -function dashboardData( - conversationSummaries: ConversationSummaryReport[], -): SystemData & { conversations: ConversationFeed } { +function systemData(): SystemData { return { config: { allowedEmailCount: 0, @@ -68,29 +86,16 @@ function dashboardData( conversationStats: { active: 0, actors: [], - conversations: conversationSummaries.length, - durationMs: conversationSummaries.reduce( - (sum, conversation) => sum + conversation.cumulativeDurationMs, - 0, - ), - failed: conversationSummaries.filter( - (conversation) => conversation.status === "failed", - ).length, + conversations: 2, + costUsd: 1.25, + durationMs: 2_000, + failed: 0, generatedAt: "2026-01-01T00:00:00.000Z", - metricDays: [ - { - costUsd: 4.56, - date: "2026-01-01", - durationMs: 12_000, - tokens: 12_345, - }, - ], locations: [], source: "conversation_index", - tokens: 12_345, - costUsd: 4.56, + tokens: 1_200, windowEnd: "2026-01-01T00:00:00.000Z", - windowStart: "2025-12-25T00:00:00.000Z", + windowStart: "2025-10-03T00:00:00.000Z", }, conversationStatsError: false, conversationStatsLoading: false, @@ -102,300 +107,211 @@ function dashboardData( pluginReportsError: false, pluginReportsLoading: false, plugins: [], - conversations: { - generatedAt: "2026-01-01T00:00:00.000Z", - conversations: conversationSummaries, - source: "conversation_index", - }, skills: [], }; } -function renderConversationPage(data: { - conversations: ConversationFeed; -}): string { - return renderToStaticMarkup( - - - - - } - path="/conversations/:conversationId" - /> - - - , - ); -} +describe("dashboard canonical-event components", () => { + it("keeps shared buttons out of form-submit mode", () => { + expect(renderToStaticMarkup()).toContain( + 'type="button"', + ); + }); -function toolRunTurn( - toolCount: number, - finalMessage = false, -): ConversationTranscript { - return { - conversationId: "conversation-1", - lastProgressAt: "2026-01-01T00:00:10.000Z", - lastSeenAt: "2026-01-01T00:00:10.000Z", - startedAt: "2026-01-01T00:00:00.000Z", - status: "completed", - surface: "slack", - displayTitle: "Conversation", - transcript: [ - ...Array.from({ length: toolCount }, (_, index) => ({ - role: "assistant", - timestamp: Date.parse("2026-01-01T00:00:10.000Z") + index, - parts: [ + it("exposes pressed state for transcript view controls", () => { + const html = renderToStaticMarkup( + {}} />, + ); + expect(html.match(/aria-pressed="true"/g) ?? []).toHaveLength(1); + expect(html.match(/aria-pressed="false"/g) ?? []).toHaveLength(1); + }); + + it("renders visible messages once and keeps API order over timestamps", () => { + const html = renderTranscript( + conversation([ + event( + 0, + { + type: "visible_message", + messageId: "first", + role: "user", + text: "first by sequence", + }, + "2026-01-01T00:00:09.000Z", + ), + event(1, { type: "model_activity", activities: ["thinking"] }), + event( + 2, { - id: `call-${index}`, - name: `tool-${index}`, - type: "tool_call" as const, + type: "visible_message", + messageId: "second", + role: "assistant", + text: "second by sequence", }, + "2026-01-01T00:00:01.000Z", + ), + ]), + ); + expect(html.indexOf("first by sequence")).toBeLessThan( + html.indexOf("second by sequence"), + ); + expect(html.match(/second by sequence/g)).toHaveLength(1); + }); + + it("renders redacted visible events without exposing text", () => { + const html = renderTranscript( + conversation( + [ + event(0, { + type: "visible_message", + messageId: "private", + role: "user", + redacted: true, + }), ], - })), - ...(finalMessage - ? [ - { - role: "assistant" as const, - timestamp: Date.parse("2026-01-01T00:00:11.000Z"), - parts: [{ type: "text" as const, text: "done" }], - }, - ] - : []), - ], - transcriptAvailable: true, - } as ConversationTranscript; -} - -describe("dashboard telemetry components", () => { - it("keeps card supporting text on the shared readable contrast tier", () => { - const html = renderToStaticMarkup( - , + { + eventHistory: { + status: "redacted", + reason: "non_public_conversation", + }, + }, + ), ); - - expect(html).toContain("text-white/50"); - expect(html).toContain("text-white/55"); - expect(html).not.toContain("text-white/30"); - expect(html).not.toContain("text-white/35"); + expect(html).toContain("<redacted>"); }); - it("keeps shared command buttons out of form-submit mode", () => { - const html = renderToStaticMarkup(); - const iconHtml = renderToStaticMarkup( - "); - expect(html).toContain("Peak daily active"); - expect(html).toContain("Avery Example"); - expect(html).not.toContain("@avery"); - expect(html).not.toContain("last 1 day ago"); + expect(html).toContain("Location telemetry refresh failed"); + expect(html).toContain("Investigate checkout"); + expect(html).toContain("avery@example.com"); }); - it("renders a stable skeleton while directory sorting catches up", () => { - const html = renderToStaticMarkup( + it("keeps plugin loading, failure, and stale data states distinct", () => { + const loading = systemData(); + loading.pluginReportsLoading = true; + loading.plugins = [{ name: "github" }]; + const loadingHtml = renderToStaticMarkup( - {}} - onSortChange={() => {}} - people={[]} - query="" - sort="runtime" - totalPeople={2} - /> + , ); + expect(loadingHtml).toContain("Loading plugin stats."); - expect(html).toContain('aria-label="Loading sorted results"'); - }); - - it("keeps completed status badges quiet unless explicitly requested", () => { - expect(renderToStaticMarkup()).toBe(""); - expect( - renderToStaticMarkup(), - ).toContain("completed"); - expect( - renderToStaticMarkup(), - ).toContain("checking"); - expect(renderToStaticMarkup()).toContain( - "error", + const stale = systemData(); + stale.pluginReportsError = true; + stale.pluginReports!.reports = [ + { + metrics: [{ label: "active", value: "1" }], + pluginName: "scheduler", + title: "Scheduler", + }, + ]; + const staleHtml = renderToStaticMarkup( + + + , ); + expect(staleHtml).toContain("Plugin stats failed to load."); + expect(staleHtml).toContain("Scheduler"); }); - it("keeps the Sentry trace link in transcript headers", () => { - const turn = { - conversationId: "conversation-1", - cumulativeDurationMs: 0, - lastProgressAt: "2026-01-01T00:00:00.000Z", - lastSeenAt: "2026-01-01T00:00:00.000Z", - sentryTraceUrl: "https://sentry.example/trace/abc", - startedAt: "2026-01-01T00:00:00.000Z", - status: "completed", - surface: "slack", - displayTitle: "Conversation", - transcript: [], - transcriptAvailable: true, - } as ConversationTranscript; - - const html = renderToStaticMarkup( - , - ); - - expect(html).toContain("View in Sentry"); - expect(html).toContain("https://sentry.example/trace/abc"); - }); - - it("renders terminal assistant outcomes as distinct safe callouts", () => { - const turn = { - conversationId: "conversation-1", - cumulativeDurationMs: 0, - lastProgressAt: "2026-01-01T00:00:02.000Z", - lastSeenAt: "2026-01-01T00:00:02.000Z", - startedAt: "2026-01-01T00:00:00.000Z", - status: "completed", - surface: "slack", - displayTitle: "Conversation", - transcript: [ - { - role: "assistant", - outcome: "error", - timestamp: 1_000, - parts: [], - }, - { - role: "assistant", - outcome: "aborted", - timestamp: 2_000, - parts: [], - }, - ], - transcriptAvailable: true, - } as ConversationTranscript; - - const html = renderToStaticMarkup( - , - ); - - expect(html).toContain('data-transcript-failure="error"'); - expect(html).toContain('data-transcript-failure="aborted"'); - expect(html).toContain("Agent response failed"); - expect(html).toContain("Agent response stopped"); - }); - - it("renders terminal assistant outcomes from redacted transcript metadata", () => { - const turn = { - conversationId: "conversation-private", - cumulativeDurationMs: 0, - lastProgressAt: "2026-01-01T00:00:01.000Z", - lastSeenAt: "2026-01-01T00:00:01.000Z", - startedAt: "2026-01-01T00:00:00.000Z", - status: "completed", - surface: "slack", - displayTitle: "Private conversation", - transcript: [], - transcriptAvailable: false, - transcriptMetadata: [ - { - role: "assistant", - outcome: "error", - timestamp: 1_000, - parts: [], - }, - ], - transcriptRedacted: true, - transcriptRedactionReason: "non_public_conversation", - } as ConversationTranscript; - - const html = renderToStaticMarkup( - , - ); - - expect(html).toContain('data-transcript-failure="error"'); - expect(html).toContain("Agent response failed"); - }); - - it("removes residual grid row gap from collapsed system prompts", () => { - const turn = { - conversationId: "conversation-1", - cumulativeDurationMs: 0, - lastProgressAt: "2026-01-01T00:00:00.000Z", - lastSeenAt: "2026-01-01T00:00:00.000Z", - startedAt: "2026-01-01T00:00:00.000Z", - status: "completed", - surface: "slack", - displayTitle: "Conversation", - transcript: [ - { - role: "system", - parts: [{ type: "text", text: "System prompt" }], - }, - ], - transcriptAvailable: true, - } as ConversationTranscript; - - const html = renderToStaticMarkup( - - - , - ); - - expect(html).toContain("gap-y-0"); - expect(html).toContain("flex min-w-0 items-center justify-between gap-3"); - expect(html).toContain( - 'font-mono leading-none text-[0.78rem] text-[#888]">13b', - ); - }); - - it("keeps message timestamps in a shared heading row without elapsed offsets", () => { - const turn = { - conversationId: "conversation-1", - lastProgressAt: "2026-01-01T00:00:10.000Z", - lastSeenAt: "2026-01-01T00:00:10.000Z", - startedAt: "2026-01-01T00:00:00.000Z", - status: "completed", - surface: "slack", - displayTitle: "Conversation", - transcript: [ - { - role: "user", - timestamp: Date.parse("2026-01-01T00:00:10.000Z"), - parts: [{ type: "text", text: "Can you check this?" }], - }, - ], - transcriptAvailable: true, - } as ConversationTranscript; - - const html = renderToStaticMarkup( - - - , - ); - - expect(html).toContain("flex min-w-0 items-center justify-between gap-3"); - expect(html).toContain("font-mono leading-none text-[0.78rem] text-[#888]"); - expect(html).toContain("flex flex-col items-center pt-1.5"); - expect(html).not.toContain("+10s"); - expect(html).not.toContain("· +"); - expect(html).not.toContain("items-baseline gap-2 text-[0.88rem]"); - }); - - it("renders safe markdown links as transcript anchors", () => { - const turn = { - conversationId: "conversation-1", - lastProgressAt: "2026-01-01T00:00:10.000Z", - lastSeenAt: "2026-01-01T00:00:10.000Z", - startedAt: "2026-01-01T00:00:00.000Z", - status: "completed", - surface: "slack", - displayTitle: "Conversation", - transcript: [ - { - role: "assistant", - timestamp: Date.parse("2026-01-01T00:00:10.000Z"), - parts: [ - { - type: "text", - text: "See [the trace](https://sentry.example/trace/abc), [wiki](https://en.wikipedia.org/wiki/Foo_(bar)), https://docs.example/Foo_(bar)., https://., https://after-invalid.example/ok, [broken [real](https://nested.example/ok), [local](/api/me), and [bad](javascript:alert).", - }, - ], - }, - ], - transcriptAvailable: true, - } as ConversationTranscript; - - const html = renderToStaticMarkup( - - - , - ); - - expect(html).toContain('href="https://sentry.example/trace/abc"'); - expect(html).toContain('target="_blank"'); - expect(html).toContain('rel="noreferrer"'); - expect(html).toContain(">the trace"); - expect(html).toContain('href="https://en.wikipedia.org/wiki/Foo_(bar)"'); - expect(html).toContain(">wiki"); - expect(html).toContain('href="https://docs.example/Foo_(bar)"'); - expect(html).toContain(">https://docs.example/Foo_(bar)."); - expect(html).toContain("https://."); - expect(html).toContain('href="https://after-invalid.example/ok"'); - expect(html).toContain(">https://after-invalid.example/ok"); - expect(html).toContain("[broken "); - expect(html).toContain('href="https://nested.example/ok"'); - expect(html).toContain(">real"); - expect(html).not.toContain(">broken [real"); - expect(html).toContain("[local](/api/me)"); - expect(html).toContain("[bad](javascript:alert)"); - expect(html).not.toContain('href="/api/me"'); - expect(html).not.toContain('href="javascript:alert"'); - }); - - it("renders cached highlighted markdown links", () => { - const text = - "## Trace summary\n- [cached trace](https://cached.example/trace)."; - client.setQueryData( - ["highlight", "markdown", text, "transcript-markdown"], - '
## Trace summary\n- cached trace.
', - ); - const turn = { - conversationId: "conversation-1", - lastProgressAt: "2026-01-01T00:00:10.000Z", - lastSeenAt: "2026-01-01T00:00:10.000Z", - startedAt: "2026-01-01T00:00:00.000Z", - status: "completed", - surface: "slack", - displayTitle: "Conversation", - transcript: [ - { - role: "assistant", - timestamp: Date.parse("2026-01-01T00:00:10.000Z"), - parts: [{ type: "text", text }], - }, - ], - transcriptAvailable: true, - } as ConversationTranscript; - - const html = renderToStaticMarkup( - - - , - ); - - expect(html).toContain('data-cached="yes"'); - expect(html).toContain('href="https://cached.example/trace"'); - expect(html).toContain(">cached trace"); - expect(html).not.toContain("[cached trace]"); - }); - - it("omits empty tool-call summaries", () => { - expect( - renderToStaticMarkup( - , - ), - ).toBe(""); - }); - - it("omits the conversation tool-call metric slot when the loaded detail has no tool calls", () => { - const session = { - conversationId: "conversation-1", - cumulativeDurationMs: 0, - lastProgressAt: "2026-01-01T00:00:00.000Z", - lastSeenAt: "not-a-date", - startedAt: "2026-01-01T00:00:00.000Z", - status: "completed", - surface: "internal", - displayTitle: "Conversation", - } satisfies ConversationSummaryReport; - const detail = { - ...session, - generatedAt: "2026-01-01T00:00:00.000Z", - transcript: [], - transcriptAvailable: true, - } satisfies ConversationDetailReport; - client.setQueryData(["conversation", "conversation-1"], detail); - - const html = renderConversationPage(dashboardData([session])); - - expect(html).not.toContain("turn"); - expect(html).not.toContain("tool call"); - }); - - it("counts actor turns and omits the redundant started header item", () => { - const session = { - conversationId: "conversation-1", - cumulativeDurationMs: 60_000, - lastProgressAt: "2026-01-01T00:01:00.000Z", - lastSeenAt: "2026-01-01T00:01:00.000Z", - startedAt: "2026-01-01T00:00:00.000Z", - status: "completed", - surface: "internal", - displayTitle: "Conversation", - } satisfies ConversationSummaryReport; - const detail = { - ...session, - generatedAt: "2026-01-01T00:01:00.000Z", - transcript: [ - { role: "user", parts: [{ type: "text", text: "first" }] }, - { role: "assistant", parts: [{ type: "text", text: "done" }] }, - { role: "user", parts: [{ type: "text", text: "second" }] }, - { role: "assistant", parts: [{ type: "text", text: "done" }] }, - ], - transcriptAvailable: true, - } satisfies ConversationDetailReport; - client.setQueryData(["conversation", "conversation-1"], detail); - - const html = renderConversationPage(dashboardData([session])); - const header = html.slice(0, html.indexOf('aria-label="Transcript view"')); - const transcript = html.slice(html.indexOf('aria-label="Transcript view"')); - - expect(header).toContain("2 turns"); - expect(header).not.toContain("4 messages"); - expect(header).not.toContain("started Jan"); - expect(transcript).toContain("2 turns"); - expect(transcript).not.toContain("4 messages"); - }); - - it("shows the conversation model and thinking level in the transcript header", () => { - const session = { - conversationId: "conversation-1", - cumulativeDurationMs: 0, - lastProgressAt: "2026-01-01T00:00:00.000Z", - lastSeenAt: "2026-01-01T00:00:00.000Z", - startedAt: "2026-01-01T00:00:00.000Z", - status: "completed", - surface: "internal", - displayTitle: "Conversation", - } satisfies ConversationSummaryReport; - const detail = { - ...session, - generatedAt: "2026-01-01T00:00:00.000Z", - modelId: "openai/gpt-5.6-sol", - reasoningLevel: "high", - transcript: [], - transcriptAvailable: true, - } satisfies ConversationDetailReport; - client.setQueryData(["conversation", "conversation-1"], detail); - - const html = renderConversationPage(dashboardData([session])); - const transcriptStart = html.indexOf('aria-label="Transcript view"'); - const detailHeader = html.slice(0, transcriptStart); - const transcript = html.slice(transcriptStart); - - expect(detailHeader).not.toContain("gpt-5.6-sol"); - expect(transcript).toContain( - 'aria-label="Execution settings: openai/gpt-5.6-sol, high"', - ); - expect(transcript).toContain("break-all font-mono"); - expect(transcript).toContain("gpt-5.6-sol"); - expect(transcript).toContain("(high)"); - }); - - it("renders execution activity inside the transcript", () => { - const session = { - conversationId: "conversation-1", - cumulativeDurationMs: 0, - lastProgressAt: "2026-01-01T00:00:00.000Z", - lastSeenAt: "2026-01-01T00:00:00.000Z", - startedAt: "2026-01-01T00:00:00.000Z", - status: "completed", - surface: "internal", - displayTitle: "Conversation", - } satisfies ConversationSummaryReport; - const detail = { - ...session, - generatedAt: "2026-01-01T00:00:00.000Z", - activity: [ - { - type: "tool_execution", - id: "advisor-call-1", - toolCallId: "advisor-call-1", - toolName: "advisor", - createdAt: "2026-01-01T00:00:01.000Z", - status: "running", - subagents: [ - { - type: "subagent", - id: "advisor-call-1", - subagentKind: "advisor", - parentToolCallId: "advisor-call-1", - createdAt: "2026-01-01T00:00:01.000Z", - status: "running", - }, - ], - }, - ], - transcript: [], - transcriptAvailable: true, - } satisfies ConversationDetailReport; - client.setQueryData(["conversation", "conversation-1"], detail); - - const html = renderConversationPage(dashboardData([session])); - - expect(html).not.toContain('aria-label="Execution activity"'); - expect(html).not.toContain("Execution Activity"); - expect(html).toContain("advisor"); - expect(html).not.toContain("advisor subagent"); - expect(html).toContain("running"); - expect(html.indexOf("advisor")).toBeGreaterThan( - html.indexOf('aria-label="Transcript view"'), - ); - }); - - it("uses the detail report for the View in Sentry conversation link", () => { - const summary = { - conversationId: "conversation-1", - cumulativeDurationMs: 0, - lastProgressAt: "2026-01-01T00:00:00.000Z", - lastSeenAt: "2026-01-01T00:00:00.000Z", - startedAt: "2026-01-01T00:00:00.000Z", - status: "completed", - surface: "slack", - displayTitle: "Conversation", - actorIdentity: { - email: "avery@example.com", - fullName: "Avery Example", - }, - } satisfies ConversationSummaryReport; - const detail = { - ...summary, - generatedAt: "2026-01-01T00:00:00.000Z", - sentryConversationUrl: - "https://sentry.example/explore/conversations/conversation-1/?project=1", - transcript: [], - transcriptAvailable: true, - } satisfies ConversationDetailReport; - client.setQueryData(["conversation", "conversation-1"], detail); - - const html = renderConversationPage(dashboardData([summary])); - - expect(html).toContain('href="/people/avery%40example.com"'); - expect(html).toContain("View in Sentry"); - expect(html).toContain( - "https://sentry.example/explore/conversations/conversation-1/?project=1", - ); - }); - - it("renders the selected personal conversation in the home workspace", () => { - const summary = { - conversationId: "conversation-1", - cumulativeDurationMs: 1_000, - displayTitle: "Personal conversation", - lastProgressAt: "2026-01-01T00:00:01.000Z", - lastSeenAt: "2026-01-01T00:00:02.000Z", - actorIdentity: { email: "morgan@example.com", fullName: "Morgan" }, - startedAt: "2026-01-01T00:00:00.000Z", - status: "completed", - surface: "slack", - } satisfies ConversationSummaryReport; - const data = dashboardData([summary]); - data.config.authRequired = true; - data.me = { user: { email: "morgan@example.com" } }; - client.setQueryData( - ["dashboard", "conversations", "morgan@example.com"], - data.conversations, - ); - client.setQueryData(["conversation", "conversation-1"], { - ...summary, - generatedAt: "2026-01-01T00:00:02.000Z", - transcript: [], - transcriptAvailable: true, - } satisfies ConversationDetailReport); - - const html = renderToStaticMarkup( - - - - } - path="/conversations/:conversationId" - /> - - - , - ); - - expect(html).toContain("Your conversations"); - expect(html).toContain('aria-label="Search your conversations"'); - expect(html).toContain('href="/conversations/conversation-1"'); - expect(html).toContain('aria-current="page"'); - expect(html).toContain("Personal conversation"); - }); - - it("uses React Router's decoded conversation id without decoding it again", () => { - const summary = { - conversationId: "conversation%one", - cumulativeDurationMs: 1_000, - displayTitle: "Percent conversation", - lastProgressAt: "2026-01-01T00:00:01.000Z", - lastSeenAt: "2026-01-01T00:00:01.000Z", - startedAt: "2026-01-01T00:00:00.000Z", - status: "completed", - surface: "slack", - } satisfies ConversationSummaryReport; - const data = dashboardData([summary]); - data.config.authRequired = true; - client.setQueryData( - ["dashboard", "conversations", "viewer@example.com"], - data.conversations, - ); - client.setQueryData(["conversation", "conversation%one"], { - ...summary, - generatedAt: "2026-01-01T00:00:01.000Z", - transcript: [], - transcriptAvailable: true, - } satisfies ConversationDetailReport); - - const html = renderToStaticMarkup( - - - - } - path="/conversations/:conversationId" - /> - - - , - ); - - expect(html).toContain("Percent conversation"); - }); - - it("shows the global feed when dashboard auth is disabled", () => { - const summary = { - conversationId: "conversation-1", - cumulativeDurationMs: 1_000, - displayTitle: "Local conversation", - lastProgressAt: "2026-01-01T00:00:01.000Z", - lastSeenAt: "2026-01-01T00:00:01.000Z", - actorIdentity: { email: "actor@example.com" }, - startedAt: "2026-01-01T00:00:00.000Z", - status: "completed", - surface: "slack", - } satisfies ConversationSummaryReport; - const data = dashboardData([summary]); - client.setQueryData( - ["dashboard", "conversations", "all"], - data.conversations, - ); - - const html = renderToStaticMarkup( - - - - - , - ); - - expect(html).toContain("Local conversation"); - }); - - it("renders system conversation metrics and plugin reports", () => { - vi.useFakeTimers(); - vi.setSystemTime(new Date("2026-01-05T00:00:00.000Z")); - - const conversationSummaries: ConversationSummaryReport[] = [ - { - channel: "C1", - channelName: "proj-alpha", - conversationId: "slack:C1:100", - cumulativeDurationMs: 1_000, - lastProgressAt: "2026-01-01T00:00:01.000Z", - lastSeenAt: "2026-01-01T00:00:02.000Z", - actorIdentity: { fullName: "Avery" }, - startedAt: "2026-01-01T00:00:00.000Z", - status: "completed", - surface: "slack", - displayTitle: "Conversation", - }, - { - channel: "D1", - conversationId: "slack:D1:200", - cumulativeDurationMs: 2_000, - lastProgressAt: "2026-01-01T00:02:01.000Z", - lastSeenAt: "2026-01-01T00:02:02.000Z", - actorIdentity: { fullName: "Avery" }, - startedAt: "2026-01-01T00:02:00.000Z", - status: "failed", - surface: "slack", - displayTitle: "Conversation", - }, - { - channel: "C2", - channelName: "old-project", - conversationId: "slack:C2:300", - cumulativeDurationMs: 5_000, - lastProgressAt: "2025-12-20T00:00:01.000Z", - lastSeenAt: "2025-12-20T00:00:02.000Z", - actorIdentity: { fullName: "Casey" }, - startedAt: "2025-12-20T00:00:00.000Z", - status: "completed", - surface: "slack", - displayTitle: "Old thread", - }, - ]; - const data = dashboardData(conversationSummaries); - data.plugins = [{ name: "github" }]; - data.pluginReports!.reports = [ - { - pluginName: "scheduler", - title: "Scheduler", - metrics: [{ label: "active", value: "2" }], - recordSets: [ - { - title: "Upcoming", - fields: [{ key: "task", label: "Task" }], - records: [{ id: "sched_1", values: { task: "sched_1" } }], - }, - ], - }, - ]; - data.skills = [{ name: "triage", pluginProvider: "github" }]; - - const systemHtml = renderToStaticMarkup( - - - , - ); - - expect(systemHtml).toContain(">System<"); - expect(systemHtml).toContain("Token usage"); - expect(systemHtml).toContain("Model spend"); - expect(systemHtml).toContain("Runtime"); - expect(systemHtml).toContain(">12k<"); - expect(systemHtml).toContain(">$4.56<"); - expect(systemHtml).toContain('role="tablist"'); - expect(systemHtml).toContain('aria-selected="true"'); - expect(systemHtml).toContain(">Plugins<"); - expect(systemHtml).toContain(">Skills<"); - expect(systemHtml).toContain(">Scheduler<"); - expect(systemHtml).toContain("github"); - expect(systemHtml).not.toContain(">triage<"); - expect(systemHtml).toContain("scheduler"); - expect(systemHtml).toContain("sched_1"); - - const skillsHtml = renderToStaticMarkup( - , - ); - expect(skillsHtml).toContain(">Skills<"); - expect(skillsHtml).toContain(">github<"); - expect(skillsHtml).toContain(">triage<"); - }); - - it("renders public locations as primary rows and collapses private activity", () => { - const data: LocationDirectoryReport = { - activityDays: [ - { - date: "2026-01-05", - privateConversations: 3, - publicConversations: 6, - }, - ], - generatedAt: "2026-01-05T00:00:00.000Z", - locations: [ - { - active: 1, - conversations: 4, - durationMs: 12_000, - failed: 1, - firstSeenAt: "2026-01-01T00:00:00.000Z", - id: "destination-1", - kind: "channel", - label: "#proj-alpha", - lastSeenAt: "2026-01-05T00:00:00.000Z", - provider: "slack", - providerDestinationId: "C1", - tokens: 12_500, - visibility: "public", - }, - { - active: 0, - conversations: 2, - durationMs: 4_000, - failed: 0, - firstSeenAt: "2026-01-02T00:00:00.000Z", - id: "destination-2", - kind: "channel", - label: "#other", - lastSeenAt: "2026-01-04T00:00:00.000Z", - provider: "slack", - providerDestinationId: "C2", - visibility: "public", - }, - ], - privateActivity: { - active: 0, - conversations: 3, - durationMs: 2_000, - failed: 0, - label: "Private activity", - }, - source: "conversation_index", - windowEnd: "2026-01-05T00:00:00.000Z", - windowStart: "2025-10-08T00:00:00.000Z", - }; - const html = renderToStaticMarkup( - - - , - ); - expect(html).toContain("1 of 2 public locations"); - expect(html).toContain("#proj-alpha"); - expect(html).not.toContain("#other"); - expect(html).not.toContain("C1"); - expect(html).toContain(">13k<"); - expect(html).not.toContain(">Errors<"); - expect(html).not.toContain(">Active<"); - expect(html).toContain("Private activity"); - expect(html).toContain("DMs, private channels, and unknown visibility"); - expect(html).toContain("Public and private conversations per day"); - }); - - it("keeps cached location rows visible after a refresh failure", () => { - const data: LocationDirectoryReport = { - activityDays: [], - generatedAt: "2026-01-05T00:00:00.000Z", - locations: [ - { - active: 0, - conversations: 1, - durationMs: 1_000, - failed: 0, - firstSeenAt: "2026-01-01T00:00:00.000Z", - id: "destination-1", - kind: "channel", - label: "#proj-alpha", - lastSeenAt: "2026-01-05T00:00:00.000Z", - provider: "slack", - providerDestinationId: "C1", - visibility: "public", - }, - ], - privateActivity: { - active: 0, - conversations: 0, - durationMs: 0, - failed: 0, - label: "Private activity", - }, - source: "conversation_index", - windowEnd: "2026-01-05T00:00:00.000Z", - windowStart: "2025-10-08T00:00:00.000Z", - }; - - const html = renderToStaticMarkup( - - - , - ); - - expect(html).toContain("Location telemetry refresh failed"); - expect(html).toContain("#proj-alpha"); - }); - - it("renders public location detail with people and recent conversations", () => { - const detail: LocationDetailReport = { - active: 0, - activityDays: [], - actors: [ - { - active: 0, - actor: { email: "avery@example.com", fullName: "Avery" }, - conversations: 1, - durationMs: 1_000, - failed: 0, - label: "avery@example.com", - }, - ], - conversations: 1, - durationMs: 1_000, - failed: 0, - firstSeenAt: "2026-01-01T00:00:00.000Z", - generatedAt: "2026-01-05T00:00:00.000Z", - id: "destination-1", - kind: "channel", - label: "#proj-alpha", - lastSeenAt: "2026-01-05T00:00:00.000Z", - provider: "slack", - providerDestinationId: "C1", - recentConversations: [ - { - channel: "C1", - channelName: "proj-alpha", - conversationId: "slack:C1:100", - cumulativeDurationMs: 1_000, - displayTitle: "Investigate checkout", - lastProgressAt: "2026-01-05T00:00:00.000Z", - lastSeenAt: "2026-01-05T00:00:00.000Z", - locationId: "destination-1", - startedAt: "2026-01-05T00:00:00.000Z", - status: "completed", - surface: "slack", - }, - ], - source: "conversation_index", - visibility: "public", - windowEnd: "2026-01-05T00:00:00.000Z", - windowStart: "2025-12-07T00:00:00.000Z", - }; - client.setQueryData(["dashboard", "locations", "destination-1"], detail); - const html = renderToStaticMarkup( - - - - } - path="/locations/:locationId" - /> - - - , - ); - expect(html).toContain("#proj-alpha"); - expect(html).toContain("Investigate checkout"); - expect(html).toContain('href="/people/avery%40example.com"'); - expect(html).toContain("Recent conversations"); - - const staleHtml = renderToStaticMarkup( - - - , - ); - expect(staleHtml).toContain("Location telemetry refresh failed"); - expect(staleHtml).toContain("#proj-alpha"); - }); - - it("renders a clear fallback for plugin records without fields", () => { + it("renders plugin records without declared fields safely", () => { const html = renderToStaticMarkup( { }, ], }, - ]} - />, - ); - - expect(html).toContain( - "Report records are unavailable because no fields were declared.", - ); - }); - - it("keeps plugin inventory available when conversation metrics fail", () => { - const data = dashboardData([]); - data.conversationStats = undefined; - data.conversationStatsError = true; - data.plugins = [{ name: "github" }]; - - const html = renderToStaticMarkup( - - - , - ); - - expect(html).toContain("Conversation metrics failed to load."); - expect(html).toContain(">Plugins<"); - expect(html).toContain(">github<"); - }); - - it("keeps cached conversation metrics visible after a refresh failure", () => { - const data = dashboardData([]); - data.conversationStatsError = true; - - const html = renderToStaticMarkup( - - - , + ]} + />, + ); + expect(html).toContain( + "Report records are unavailable because no fields were declared.", ); - - expect(html).toContain("Metrics refresh failed. Showing cached data."); - expect(html).toContain("Usage over time"); }); - it("renders system page when plugin reports are absent", () => { + it("keeps plugin inventory available when conversation metrics fail", () => { const data = dashboardData([]); + data.conversationStats = undefined; + data.conversationStatsError = true; data.plugins = [{ name: "github" }]; - delete data.pluginReports; const html = renderToStaticMarkup( @@ -1484,16 +582,14 @@ describe("dashboard telemetry components", () => { , ); + expect(html).toContain("Conversation metrics failed to load."); expect(html).toContain(">Plugins<"); - expect(html).toContain("github"); - expect(html).toContain("No plugins have been reported yet."); + expect(html).toContain(">github<"); }); - it("shows plugin reports as loading before the report query returns", () => { + it("keeps cached conversation metrics visible after a refresh failure", () => { const data = dashboardData([]); - data.pluginReportsLoading = true; - data.plugins = [{ name: "github" }]; - data.skills = [{ name: "triage", pluginProvider: "github" }]; + data.conversationStatsError = true; const html = renderToStaticMarkup( @@ -1501,36 +597,19 @@ describe("dashboard telemetry components", () => { , ); - expect(html).toContain("Loading plugin stats."); - expect(html).toContain(">…<"); - expect(html).not.toContain(">none<"); - expect(html).not.toContain("No plugins have been reported yet."); - }); - - it("shows plugin report failures without looking empty", () => { - const data = dashboardData([]); - data.pluginReportsError = true; - - const html = renderToStaticMarkup( - - - , + expect(html).toContain( + "Conversation metrics refresh failed. Showing cached data.", ); - - expect(html).toContain("Plugin stats failed to load."); - expect(html).not.toContain("No plugins have been reported yet."); + expect(html).toContain("90-day pulse"); }); - it("shows plugin report failures while keeping stale reports visible", () => { + it("does not report a completion rate before any conversation finishes", () => { const data = dashboardData([]); - data.pluginReportsError = true; - data.pluginReports!.reports = [ - { - metrics: [{ label: "active", value: "1" }], - pluginName: "scheduler", - title: "Scheduler", - }, - ]; + data.conversationStats = { + ...data.conversationStats!, + active: 2, + conversations: 2, + }; const html = renderToStaticMarkup( @@ -1538,684 +617,275 @@ describe("dashboard telemetry components", () => { , ); - expect(html).toContain("Plugin stats failed to load."); - expect(html).toContain(">Scheduler<"); - }); - - it("preserves unknown runtime in shared activity tooltips", () => { - const html = renderToStaticMarkup( - , - ); - - expect(html).toContain('aria-label="2026-01-01: 1 conversations, unknown"'); - }); - it("renders transcript copy as an icon-only control", () => { - const session = { - conversationId: "conversation-1", - cumulativeDurationMs: 0, - lastProgressAt: "2026-01-01T00:00:00.000Z", - lastSeenAt: "2026-01-01T00:00:00.000Z", - startedAt: "2026-01-01T00:00:00.000Z", - status: "completed", - surface: "slack", - displayTitle: "Readable transcript", - } satisfies ConversationSummaryReport; - const detail = { - ...session, - generatedAt: "2026-01-01T00:00:00.000Z", - transcript: [ - { - parts: [{ text: "hello", type: "text" }], - role: "user", - }, - ], - transcriptAvailable: true, - } satisfies ConversationDetailReport; - client.setQueryData(["conversation", "conversation-1"], detail); - - const html = renderConversationPage(dashboardData([session])); - const controls = html.slice( - html.indexOf('aria-label="Transcript view"'), - html.indexOf("hello"), - ); - const pageHeader = html.slice( - 0, - html.indexOf('aria-label="Transcript view"'), - ); - - expect(pageHeader).not.toContain('aria-label="Copy as Markdown"'); - expect(controls).toContain('aria-label="Copy as Markdown"'); - expect(controls).toContain("size-9"); - expect(controls).not.toContain(">Copy as Markdown<"); - }); - - it("keeps zero timestamps in tool metadata", () => { - const html = renderToStaticMarkup( - - - , - ); - - expect(html.match(/·/g) ?? []).toHaveLength(5); - expect(html).toContain("5ms · 2b ·"); - expect(html).toContain("hidden text-[#777] max-md:inline"); - expect(html).toContain( - 'hidden min-w-0 break-words text-[#888] max-md:inline">5ms', - ); - expect(html).toContain("max-md:block"); - }); - - it("highlights expandable tool summaries on hover", () => { - const html = renderToStaticMarkup( - - - , - ); - - expect(html).toContain("hover:text-white"); - expect(html).toContain("hover:[&_*]:text-white"); - expect(html).toContain( - 'hidden min-w-0 break-words text-[#888] max-md:inline">missing result', - ); - expect(html).toContain(" { - const html = renderToStaticMarkup( - - - , - ); - - expect(html).toContain(">arguments<"); - expect(html).toContain(">result<"); - expect(html).toContain("checkout latency"); - expect(html).toContain("environment"); - expect(html).toContain("production"); - expect(html).toContain(" { - const html = renderToStaticMarkup( - , - ); - - expect(html).toContain("command"); - expect(html).toContain("pnpm test"); - expect(html).toContain(" { - const html = renderToStaticMarkup( - , - ); - - expect(html).toContain("data"); - expect(html).toContain("count"); - expect(html).toContain("success"); - expect(html).toContain("ok"); - expect(html).not.toContain("{"data""); - }); - - it("leaves JSON-looking prose as a string", () => { - const html = renderToStaticMarkup( - , - ); - - expect(html).toContain("{"status": nope}"); - }); - - it("does not highlight static tool summaries as expandable", () => { - const html = renderToStaticMarkup( - - - , - ); - - expect(html).not.toContain("hover:text-white"); - expect(html).not.toContain(" { - const code = - '{ "message": "junior command failed: CACHE_URL is required" }'; - client.setQueryData( - ["highlight", "json", code], - '
junior command failed: CACHE_URL is required
', - ); - - const html = renderToStaticMarkup( - - - , - ); - - expect(html).toContain("overflow-hidden"); - expect(html).toContain("overflow-wrap:anywhere"); - expect(html).toContain("[&_.line]:block"); - expect(html).toContain("[&_.line]:whitespace-pre-wrap"); - expect(html).toContain("[&_code]:whitespace-normal"); - expect(html).toContain("[&_pre]:whitespace-normal"); - expect(html).not.toContain("[&_code]:whitespace-pre-wrap"); - }); - - it("uses compact highlighted code spacing for raw XML transcripts", () => { - const rawText = "\n Checking MCP output\n"; - client.setQueryData( - ["highlight", "xml", rawText], - '
<root>\n  <message>Checking MCP output</message>\n</root>
', - ); - - const turn = { - conversationId: "conversation-1", - cumulativeDurationMs: 0, - lastProgressAt: "2026-01-01T00:00:00.000Z", - lastSeenAt: "2026-01-01T00:00:00.000Z", - startedAt: "2026-01-01T00:00:00.000Z", - status: "completed", - surface: "internal", - displayTitle: "Raw XML", - transcript: [ - { - parts: [{ text: rawText, type: "text" }], - role: "assistant", - }, - ], - transcriptAvailable: true, - } as ConversationTranscript; - - const html = renderToStaticMarkup( - - - , - ); - - expect(html).toContain("[&_code]:whitespace-normal"); - expect(html).toContain("[&_pre]:whitespace-normal"); - expect(html).toContain("Checking MCP output"); - }); - - it("renders four consecutive tool calls behind a reveal disclosure", () => { - const html = renderToStaticMarkup( - - - , - ); - - expect(html).toContain('
{ - const html = renderToStaticMarkup( - - - , - ); - - expect(html).not.toContain("show"); - expect(html).toContain("tool-0"); - expect(html).toContain("tool-1"); - expect(html).toContain("tool-2"); + expect(html).toContain("No terminal outcomes"); + expect(html).not.toContain("100% healthy completion"); + expect(html).not.toContain("undefined%"); }); - it("renders thinking rows as collapsed disclosures", () => { - const turn = { - conversationId: "conversation-1", - lastProgressAt: "2026-01-01T00:00:10.000Z", - lastSeenAt: "2026-01-01T00:00:10.000Z", - startedAt: "2026-01-01T00:00:00.000Z", - status: "completed", - surface: "slack", - displayTitle: "Conversation", - transcript: [ - { - role: "assistant", - timestamp: Date.parse("2026-01-01T00:00:10.000Z"), - parts: [ - { - type: "thinking", - output: "checking the rollout\nlisting deploy windows", - }, - ], - }, - ], - transcriptAvailable: true, - } as ConversationTranscript; + it("renders system page when plugin reports are absent", () => { + const data = dashboardData([]); + data.plugins = [{ name: "github" }]; + delete data.pluginReports; const html = renderToStaticMarkup( - - - , + + + , ); - expect(html).toContain('aria-label="Thinking"'); - expect(html).toContain(""), - ); - // Summary shows truncated thinking text preview (not just a static label). - expect(summary).toContain("checking the rollout"); - expect(summary).toContain("listing deploy windows"); + expect(html).toContain(">Plugins<"); + expect(html).toContain("github"); + expect(html).toContain("No plugins have been reported yet."); }); - it("collapses short thinking rows by default", () => { - const turn = { - conversationId: "conversation-1", - lastProgressAt: "2026-01-01T00:00:10.000Z", - lastSeenAt: "2026-01-01T00:00:10.000Z", - startedAt: "2026-01-01T00:00:00.000Z", - status: "completed", - surface: "slack", - displayTitle: "Conversation", - transcript: [ - { - role: "assistant", - timestamp: Date.parse("2026-01-01T00:00:10.000Z"), - parts: [{ type: "thinking", output: "checking the rollout" }], - }, - ], - transcriptAvailable: true, - } as ConversationTranscript; + it("shows plugin reports as loading before the report query returns", () => { + const data = dashboardData([]); + data.pluginReportsLoading = true; + data.plugins = [{ name: "github" }]; + data.skills = [{ name: "triage", pluginProvider: "github" }]; const html = renderToStaticMarkup( - - - , + + + , ); - expect(html).toContain(""), - ); - // Summary shows the truncated thinking text so users can scan before expanding. - expect(summary).toContain("checking the rollout"); + expect(html).toContain("Loading plugin stats."); + expect(html).toContain(">…<"); + expect(html).not.toContain(">none<"); + expect(html).not.toContain("No plugins have been reported yet."); }); - it("does not render empty thinking as JSON", () => { - const turn = { - conversationId: "conversation-1", - lastProgressAt: "2026-01-01T00:00:10.000Z", - lastSeenAt: "2026-01-01T00:00:10.000Z", - startedAt: "2026-01-01T00:00:00.000Z", - status: "completed", - surface: "slack", - displayTitle: "Conversation", - transcript: [ - { - role: "assistant", - parts: [{ type: "thinking" }], - }, - ], - transcriptAvailable: true, - } as ConversationTranscript; + it("shows plugin report failures without looking empty", () => { + const data = dashboardData([]); + data.pluginReportsError = true; const html = renderToStaticMarkup( - - - , + + + , ); - expect(html).not.toContain("{}"); - expect(html).toContain("thinking"); + expect(html).toContain("Plugin stats failed to load."); + expect(html).not.toContain("No plugins have been reported yet."); }); - it("expands thinking rows during transcript search", () => { - const turn = { - conversationId: "conversation-1", - lastProgressAt: "2026-01-01T00:00:10.000Z", - lastSeenAt: "2026-01-01T00:00:10.000Z", - startedAt: "2026-01-01T00:00:00.000Z", - status: "completed", - surface: "slack", - displayTitle: "Conversation", - transcript: [ - { - role: "assistant", - timestamp: Date.parse("2026-01-01T00:00:10.000Z"), - parts: [ - { - type: "thinking", - output: "checking the rollout\nlisting deploy windows", - }, - ], - }, - ], - transcriptAvailable: true, - } as ConversationTranscript; + it("shows plugin report failures while keeping stale reports visible", () => { + const data = dashboardData([]); + data.pluginReportsError = true; + data.pluginReports!.reports = [ + { + metrics: [{ label: "active", value: "1" }], + pluginName: "scheduler", + title: "Scheduler", + }, + ]; const html = renderToStaticMarkup( - - - - - , + + + , ); - expect(html).toContain("checking the rollout"); - expect(html).toContain("listing deploy<"); + expect(html).toContain("Plugin stats failed to load."); + expect(html).toContain(">Scheduler<"); }); - it("keeps execution settings visible when transcript search has no matches", () => { - const turn = { - conversationId: "conversation-1", - lastProgressAt: "2026-01-01T00:00:10.000Z", - lastSeenAt: "2026-01-01T00:00:10.000Z", - modelId: "openai/gpt-5.6-sol", - reasoningLevel: "high", - startedAt: "2026-01-01T00:00:00.000Z", - status: "completed", - surface: "internal", - displayTitle: "Conversation", - transcript: [ - { - role: "assistant", - parts: [{ type: "text", text: "A visible response" }], - }, - ], - transcriptAvailable: true, - } as ConversationTranscript; - + it("preserves unknown runtime in shared activity tooltips", () => { const html = renderToStaticMarkup( - - - - - , + , ); - expect(html).toContain("gpt-5.6-sol"); - expect(html).toContain("(high)"); - expect(html).toContain("No events match your search."); - expect(html).not.toContain("A visible response"); + expect(html).toContain('aria-label="2026-01-01: 1 conversations, unknown"'); }); - - it("consolidates mixed tool-and-thinking run at threshold behind a reveal", () => { - // 2 tool calls + 2 thinking entries = 4 total = at threshold - const turn = { + it("renders transcript copy as an icon-only control", () => { + const session = { conversationId: "conversation-1", - lastProgressAt: "2026-01-01T00:00:10.000Z", - lastSeenAt: "2026-01-01T00:00:10.000Z", + cumulativeDurationMs: 0, + lastProgressAt: "2026-01-01T00:00:00.000Z", + lastSeenAt: "2026-01-01T00:00:00.000Z", startedAt: "2026-01-01T00:00:00.000Z", status: "completed", surface: "slack", - displayTitle: "Conversation", + displayTitle: "Readable transcript", + } satisfies ConversationSummaryReport; + const detail = { + ...session, + generatedAt: "2026-01-01T00:00:00.000Z", transcript: [ { - role: "assistant", - timestamp: Date.parse("2026-01-01T00:00:10.000Z"), - parts: [ - { id: "call-0", name: "tool-0", type: "tool_call" }, - { type: "thinking", output: "first thought" }, - { id: "call-1", name: "tool-1", type: "tool_call" }, - { type: "thinking", output: "second thought" }, - ], - }, - { - role: "assistant", - timestamp: Date.parse("2026-01-01T00:00:11.000Z"), - parts: [{ type: "text", text: "done" }], + parts: [{ text: "hello", type: "text" }], + role: "user", }, ], transcriptAvailable: true, - } as ConversationTranscript; + } satisfies ConversationDetailReport; + client.setQueryData(["conversation", "conversation-1"], detail); + + const html = renderConversationPage(dashboardData([session])); + const controls = html.slice( + html.indexOf('aria-label="Transcript view"'), + html.indexOf("hello"), + ); + const pageHeader = html.slice( + 0, + html.indexOf('aria-label="Transcript view"'), + ); + + expect(pageHeader).not.toContain('aria-label="Copy as Markdown"'); + expect(controls).toContain('aria-label="Copy as Markdown"'); + expect(controls).toContain("size-9"); + expect(controls).not.toContain(">Copy as Markdown<"); + }); + it("keeps zero timestamps in tool metadata", () => { const html = renderToStaticMarkup( - + , ); - // All four entries collapse into one reveal group. - expect(html).toContain('
5ms', + ); + expect(html).toContain("max-md:block"); }); - it("keeps mixed tool-and-thinking run below threshold expanded flat", () => { - // 1 tool + 2 thinking = 3 total, below threshold - const turn = { - conversationId: "conversation-1", - lastProgressAt: "2026-01-01T00:00:10.000Z", - lastSeenAt: "2026-01-01T00:00:10.000Z", - startedAt: "2026-01-01T00:00:00.000Z", - status: "completed", - surface: "slack", - displayTitle: "Conversation", - transcript: [ - { - role: "assistant", - timestamp: Date.parse("2026-01-01T00:00:10.000Z"), - parts: [ - { id: "call-0", name: "tool-0", type: "tool_call" }, - { type: "thinking", output: "first thought" }, - { type: "thinking", output: "second thought" }, - ], - }, - ], - transcriptAvailable: true, - } as ConversationTranscript; - + it("highlights expandable tool summaries on hover", () => { const html = renderToStaticMarkup( - + , ); - expect(html).not.toContain("show"); - expect(html).toContain("tool-0"); - expect(html).toContain("first thought"); - expect(html).toContain("second thought"); + expect(html).toContain("hover:text-white"); + expect(html).toContain("hover:[&_*]:text-white"); + expect(html).toContain( + 'hidden min-w-0 break-words text-[#888] max-md:inline">missing result', + ); + expect(html).toContain(" { - const turn = { - conversationId: "conversation-1", - lastProgressAt: "2026-01-01T00:00:10.000Z", - lastSeenAt: "2026-01-01T00:00:10.000Z", - startedAt: "2026-01-01T00:00:00.000Z", - status: "failed", - surface: "slack", - displayTitle: "Conversation", - transcript: [ - { - role: "assistant", - timestamp: Date.parse("2026-01-01T00:00:10.000Z"), - parts: [ - { id: "call-0", name: "tool-0", type: "tool_call" }, - { type: "thinking", output: "first thought" }, - { id: "call-1", name: "tool-1", type: "tool_call" }, - { type: "thinking", output: "second thought" }, - ], - }, - ], - transcriptAvailable: true, - } as ConversationTranscript; - - const html = renderToStaticMarkup( + it("renders structured tool inspector values without dumping one JSON blob", () => { + const toolHtml = renderToStaticMarkup( - + , ); + expect(toolHtml).toContain("arguments"); + expect(toolHtml).toContain("result"); + expect(toolHtml).toContain(", + ); + expect(valueHtml).toContain("pnpm test"); + expect(valueHtml).toContain("src/a.ts"); + expect(valueHtml).not.toContain("{"command""); }); - it("shows correct counts in mixed-run reveal label", () => { - // 5 tool calls + 2 thinking entries - const toolParts = Array.from({ length: 5 }, (_, i) => ({ - id: `call-${i}`, - name: `tool-${i}`, - type: "tool_call", - })); - const turn = { - conversationId: "conversation-1", - lastProgressAt: "2026-01-01T00:00:10.000Z", - lastSeenAt: "2026-01-01T00:00:10.000Z", - startedAt: "2026-01-01T00:00:00.000Z", - status: "completed", - surface: "slack", - displayTitle: "Conversation", - transcript: [ - { - role: "assistant", - timestamp: Date.parse("2026-01-01T00:00:10.000Z"), - parts: [ - ...toolParts, - { type: "thinking", output: "thought a" }, - { type: "thinking", output: "thought b" }, - ], - }, - { - role: "assistant", - timestamp: Date.parse("2026-01-01T00:00:11.000Z"), - parts: [{ type: "text", text: "done" }], - }, - ], - transcriptAvailable: true, - } as ConversationTranscript; - + it("contains highlighted code so long mobile lines cannot widen transcripts", () => { + const code = '{ "message": "CACHE_URL is required" }'; + client.setQueryData( + ["highlight", "json", code], + '
CACHE_URL is required
', + ); const html = renderToStaticMarkup( - + , ); - - expect(html).toContain("show 5 tool calls and 2 thinking entries"); + expect(html).toContain("overflow-hidden"); + expect(html).toContain("overflow-wrap:anywhere"); + expect(html).toContain("[&_.line]:whitespace-pre-wrap"); }); - it("collapses four consecutive pure-thinking entries behind a reveal", () => { - // 4 consecutive thinking entries collapse the same as tool runs - const turn = { - conversationId: "conversation-1", - lastProgressAt: "2026-01-01T00:00:10.000Z", - lastSeenAt: "2026-01-01T00:00:10.000Z", + it("renders cached canonical conversation detail through decoded routing", () => { + const summary: ConversationSummaryReport = { + conversationId: "slack:C1:123", + cumulativeDurationMs: 1_000, + displayTitle: "Cached conversation", + lastProgressAt: "2026-01-01T00:00:01.000Z", + lastSeenAt: "2026-01-01T00:00:01.000Z", startedAt: "2026-01-01T00:00:00.000Z", status: "completed", surface: "slack", - displayTitle: "Conversation", - transcript: [ - { - role: "assistant", - timestamp: Date.parse("2026-01-01T00:00:10.000Z"), - parts: [ - { type: "thinking", output: "thought 1" }, - { type: "thinking", output: "thought 2" }, - { type: "thinking", output: "thought 3" }, - { type: "thinking", output: "thought 4" }, - ], - }, - { + }; + const detail = conversation( + [ + event(0, { + type: "visible_message", + messageId: "cached-answer", role: "assistant", - timestamp: Date.parse("2026-01-01T00:00:11.000Z"), - parts: [{ type: "text", text: "done" }], - }, + text: "cached canonical answer", + }), ], - transcriptAvailable: true, - } as ConversationTranscript; - + { + conversationId: summary.conversationId, + displayTitle: summary.displayTitle, + }, + ); + client.setQueryData(["conversation", summary.conversationId], detail); const html = renderToStaticMarkup( - + + + + } + /> + + , ); - - expect(html).toContain('
{ @@ -55,11 +62,17 @@ describe("transcript bottom pinning", () => { const before = transcriptBottomVersion(activeTurn()); const after = transcriptBottomVersion( activeTurn({ - transcript: [ + events: [ { - role: "assistant", - timestamp: 1_000, - parts: [{ type: "text", text: "checking the deployment" }], + seq: 0, + contextEpoch: 0, + createdAt: "2026-01-01T00:00:01.000Z", + data: { + type: "visible_message", + messageId: "assistant-1", + role: "assistant", + text: "checking the deployment", + }, }, ], }), @@ -92,17 +105,21 @@ describe("transcript bottom pinning", () => { }); it("changes the tail version when an empty response gains a terminal outcome", () => { - const emptyResponse = { - role: "assistant" as const, - timestamp: 1_000, - parts: [], - }; - const before = transcriptBottomVersion( - activeTurn({ transcript: [emptyResponse] }), - ); + const before = transcriptBottomVersion(activeTurn({ events: [] })); const after = transcriptBottomVersion( activeTurn({ - transcript: [{ ...emptyResponse, outcome: "error" }], + events: [ + { + seq: 0, + contextEpoch: 0, + createdAt: "2026-01-01T00:00:01.000Z", + data: { + type: "turn_lifecycle", + turnId: "turn-1", + state: "failed", + }, + }, + ], }), ); diff --git a/packages/junior-dashboard/tests/transcriptContextEvent.test.tsx b/packages/junior-dashboard/tests/transcriptContextEvent.test.tsx index c032983c5..f41ef56d1 100644 --- a/packages/junior-dashboard/tests/transcriptContextEvent.test.tsx +++ b/packages/junior-dashboard/tests/transcriptContextEvent.test.tsx @@ -15,7 +15,7 @@ function withQueryClient(children: ReactNode) { } describe("transcript context events", () => { - it("renders compaction as a compact expandable event", () => { + it("renders compaction without the private summary or model", () => { const html = renderToStaticMarkup( withQueryClient( { event: { type: "context_compacted", createdAt: "2026-01-01T00:00:02.000Z", - modelId: "openai/gpt-5.4", - summary: "Earlier release checks passed.", - transcriptIndex: 0, }, }} timestamp={Date.parse("2026-01-01T00:00:02.000Z")} @@ -35,12 +32,12 @@ describe("transcript context events", () => { ); expect(html).toContain("Context compacted"); - expect(html).toContain("gpt-5.4"); - expect(html).toContain("View summary"); - expect(html).toContain("Earlier release checks passed."); + expect(html).toContain("Earlier context was summarized"); + expect(html).not.toContain("gpt-5.4"); + expect(html).not.toContain("Earlier release checks passed."); }); - it("renders a model handoff as an expandable raw user message", () => { + it("renders a structural model handoff without raw context", () => { const html = renderToStaticMarkup( withQueryClient( { event: { type: "model_handoff", createdAt: "2026-01-01T00:00:04.000Z", - fromModelId: "openai/gpt-5.4", - toModelId: "openai/gpt-5.6-sol", - message: - "Model handoff checkpoint. Continue the outstanding request now using this summary as the complete prior context:\n**Next:** Continue with the migration fix.", - transcriptIndex: 0, }, }} />, @@ -61,15 +53,13 @@ describe("transcript context events", () => { ); expect(html).toContain("Model handoff"); - expect(html).toContain("gpt-5.4"); - expect(html).toContain("gpt-5.6-sol"); - expect(html).toContain("Next:"); - expect(html).toContain("Continue with the migration fix."); + expect(html).toContain("Execution continued with a different model."); + expect(html).not.toContain("gpt-5.4"); + expect(html).not.toContain("gpt-5.6-sol"); + expect(html).not.toContain("Continue with the migration fix."); }); - it("reveals a transition summary while transcript search is active", () => { + it("does not reveal transition payload while search is active", () => { const html = renderToStaticMarkup( withQueryClient( @@ -79,8 +69,6 @@ describe("transcript context events", () => { event: { type: "context_compacted", createdAt: "2026-01-01T00:00:02.000Z", - summary: "Earlier release checks passed.", - transcriptIndex: 0, }, }} /> @@ -88,8 +76,8 @@ describe("transcript context events", () => { ), ); - expect(html).toContain(", +function conversation( + events: ConversationReportEvent[], ): ConversationTranscript { return { conversationId: "conversation-1", cumulativeDurationMs: 0, displayTitle: "Conversation", + eventHistory: { status: "available" }, + events, + generatedAt: "2026-01-01T00:01:00.000Z", lastProgressAt: "2026-01-01T00:00:00.000Z", lastSeenAt: "2026-01-01T00:00:00.000Z", startedAt: "2026-01-01T00:00:00.000Z", status: "completed", surface: "internal", - transcript: [], - transcriptAvailable: true, - ...overrides, }; } -describe("transcript render model", () => { - it("promotes terminal assistant outcomes to standalone failure entries", () => { - const messages = [ - { - role: "assistant", - outcome: "error", - timestamp: 1_000, - parts: [], - }, - { - role: "assistant", - outcome: "aborted", - timestamp: 2_000, - parts: [{ type: "text", text: "Partial response" }], - }, - ] as TranscriptMessage[]; - - const entries = groupTranscriptMessages(messages); - - expect(entries).toEqual([ - { kind: "failure", outcome: "error", timestamp: 1_000 }, - { - kind: "message", - message: { - role: "assistant", - outcome: "aborted", - timestamp: 2_000, - parts: [{ type: "text", text: "Partial response" }], - }, - }, - { kind: "failure", outcome: "aborted", timestamp: 2_000 }, - ]); - expect(entryMatchesSearch(entries[0]!, "failed")).toBe(true); - expect(entryMatchesSearch(entries[2]!, "aborted")).toBe(true); - }); - - it("promotes thinking parts to standalone transcript events", () => { - const messages = [ - { - role: "assistant", - timestamp: 1_000, - parts: [ - { type: "text", text: "first" }, - { type: "thinking", output: "inspect the inputs" }, - { type: "text", text: "second" }, - ], - }, - ] as TranscriptMessage[]; - - expect(groupTranscriptMessages(messages)).toEqual([ - { - kind: "message", - message: { - role: "assistant", - timestamp: 1_000, - parts: [{ type: "text", text: "first" }], - }, - }, - { - kind: "thinking", - part: { type: "thinking", output: "inspect the inputs" }, - timestamp: 1_000, - }, - { - kind: "message", - message: { +describe("canonical event transcript reduction", () => { + it("uses API sequence order even when timestamps are inverted", () => { + const messages = conversationTranscriptMessages( + conversation([ + event(3, "2026-01-01T00:00:03.000Z", { + type: "visible_message", + messageId: "first", + role: "user", + text: "first by sequence", + }), + event(4, "2026-01-01T00:00:01.000Z", { + type: "visible_message", + messageId: "second", role: "assistant", - timestamp: 1_000, - parts: [{ type: "text", text: "second" }], - }, - }, - ]); - }); - - it("matches tool results by id before falling back to tool name", () => { - const messages = [ - { - role: "assistant", - timestamp: 1_000, - parts: [{ type: "tool_call", id: "call-1", name: "search" }], - }, - { - role: "assistant", - timestamp: 1_100, - parts: [{ type: "tool_call", id: "call-2", name: "search" }], - }, - { - role: "toolResult", - timestamp: 2_000, - parts: [{ type: "tool_result", id: "call-2", name: "search" }], - }, - ] as TranscriptMessage[]; - - expect(groupTranscriptMessages(messages)).toEqual([ - { - call: { type: "tool_call", id: "call-1", name: "search" }, - kind: "tool", - timestamp: 1_000, - }, - { - call: { type: "tool_call", id: "call-2", name: "search" }, - kind: "tool", - result: { type: "tool_result", id: "call-2", name: "search" }, - resultTimestamp: 2_000, - timestamp: 1_100, - }, - ]); - }); - - it("does not group inline same-name tool parts with mismatched ids", () => { - expect( - groupTranscriptParts([ - { type: "tool_call", id: "call-1", name: "search" }, - { type: "tool_result", id: "call-2", name: "search" }, + text: "second by sequence", + }), ]), - ).toEqual([ - { - call: { type: "tool_call", id: "call-1", name: "search" }, - kind: "tool", - }, - { - kind: "tool", - result: { type: "tool_result", id: "call-2", name: "search" }, - }, - ]); - }); - - it("backfills activity tool calls so result-only transcript entries are paired", () => { - const turn = conversationTurn({ - activity: [ - { - type: "tool_execution", - id: "call-1", - toolCallId: "call-1", - toolName: "search", - createdAt: "2026-01-01T00:00:01.000Z", - status: "completed", - args: { query: "activity" }, - subagents: [], - }, - ], - transcript: [ - { - role: "toolResult", - timestamp: Date.parse("2026-01-01T00:00:02.000Z"), - parts: [{ type: "tool_result", id: "call-1", name: "search" }], - }, - ], - transcriptAvailable: true, - }); + ); - expect( - groupTranscriptMessages(conversationTranscriptMessages(turn)), - ).toEqual([ - { - call: { - type: "tool_call", - id: "call-1", - name: "search", - status: "completed", - input: { query: "activity" }, - }, - kind: "tool", - result: { type: "tool_result", id: "call-1", name: "search" }, - resultTimestamp: Date.parse("2026-01-01T00:00:02.000Z"), - timestamp: Date.parse("2026-01-01T00:00:01.000Z"), - }, + expect(messages.map((message) => message.parts[0]?.text)).toEqual([ + "first by sequence", + "second by sequence", ]); }); - it("preserves transcript order when activity rows have inverted tool timestamps", () => { - const turn = conversationTurn({ - activity: [ - { - type: "tool_execution", - id: "call-1", - toolCallId: "call-1", - toolName: "search", - createdAt: "2026-01-01T00:00:01.000Z", - status: "completed", - subagents: [], - }, - ], - transcript: [ - { + it("projects visible and redacted messages without duplicate model text", () => { + const messages = conversationTranscriptMessages( + conversation([ + event(0, "2026-01-01T00:00:00.000Z", { + type: "visible_message", + messageId: "visible", role: "assistant", - timestamp: Date.parse("2026-01-01T00:00:02.000Z"), - parts: [{ type: "tool_call", id: "call-1", name: "search" }], - }, - { - role: "toolResult", - timestamp: Date.parse("2026-01-01T00:00:01.000Z"), - parts: [{ type: "tool_result", id: "call-1", name: "search" }], - }, - ], - transcriptAvailable: true, - }); + text: "safe answer", + }), + event(1, "2026-01-01T00:00:01.000Z", { + type: "model_activity", + activities: ["thinking", "tool_result"], + }), + event(2, "2026-01-01T00:00:02.000Z", { + type: "visible_message", + messageId: "private", + role: "user", + redacted: true, + }), + ]), + ); - expect( - groupTranscriptMessages(conversationTranscriptMessages(turn)), - ).toEqual([ - { - call: { - type: "tool_call", - id: "call-1", - name: "search", - status: "completed", - }, - kind: "tool", - result: { type: "tool_result", id: "call-1", name: "search" }, - resultTimestamp: Date.parse("2026-01-01T00:00:01.000Z"), - timestamp: Date.parse("2026-01-01T00:00:02.000Z"), - }, - ]); + expect(messages).toHaveLength(2); + expect(messages[0]?.parts).toEqual([{ type: "text", text: "safe answer" }]); + expect(messages[1]?.parts).toEqual([{ type: "text", redacted: true }]); }); - it("adds subagent activity as transcript entries", () => { - const turn = conversationTurn({ - activity: [ - { - type: "tool_execution", - id: "advisor-call", - toolCallId: "advisor-call", - toolName: "advisor", - createdAt: "2026-01-01T00:00:01.000Z", - status: "running", - subagents: [ - { - type: "subagent", - id: "advisor-call", - subagentKind: "advisor", - parentToolCallId: "advisor-call", - createdAt: "2026-01-01T00:00:02.000Z", - status: "running", - }, - { - type: "subagent", - id: "advisor-call-2", - subagentKind: "advisor", - parentToolCallId: "advisor-call", - createdAt: "2026-01-01T00:00:03.000Z", - status: "completed", - outcome: "success", - }, - ], - }, - ], - transcript: [], - transcriptAvailable: true, - }); + it("renders tool starts as neutral structural events", () => { + const [message] = conversationTranscriptMessages( + conversation([ + event(0, "2026-01-01T00:00:00.000Z", { + type: "tool_started", + name: "search", + }), + ]), + ); - expect( - groupTranscriptMessages(conversationTranscriptMessages(turn)), - ).toEqual([ - { - call: { - type: "tool_call", - id: "advisor-call", - name: "advisor", - status: "running", - }, - kind: "tool", - timestamp: Date.parse("2026-01-01T00:00:01.000Z"), - }, - { - kind: "subagent", - part: { - type: "subagent", - id: "advisor-call", - subagentKind: "advisor", - parentToolCallId: "advisor-call", - status: "running", - }, - timestamp: Date.parse("2026-01-01T00:00:02.000Z"), - }, - { - kind: "subagent", - part: { - type: "subagent", - id: "advisor-call-2", - subagentKind: "advisor", - parentToolCallId: "advisor-call", - status: "completed", - outcome: "success", - }, - timestamp: Date.parse("2026-01-01T00:00:03.000Z"), - }, - ]); + expect(message?.parts).toEqual([{ type: "tool_call", name: "search" }]); + expect(message?.parts[0]).not.toHaveProperty("status"); }); - it("inserts context changes into the transcript in timestamp order", () => { - const turn = conversationTurn({ - contextEvents: [ - { + it("projects failures, context changes, and correlated child conversations", () => { + const messages = conversationTranscriptMessages( + conversation([ + event(0, "2026-01-01T00:00:00.000Z", { type: "context_compacted", - createdAt: "2026-01-01T00:00:02.000Z", - modelId: "openai/gpt-5.4", - summary: "Earlier investigation was summarized.", - transcriptIndex: 1, - }, - { + }), + event(1, "2026-01-01T00:00:01.000Z", { type: "model_handoff", - createdAt: "2026-01-01T00:00:04.000Z", - fromModelId: "openai/gpt-5.4", - toModelId: "openai/gpt-5.6-sol", - message: "Continue with the coding fix.", - transcriptIndex: 2, - }, - ], - transcript: [ - { - role: "user", - parts: [{ type: "text", text: "Investigate the release" }], - }, - { - role: "assistant", - parts: [{ type: "text", text: "The migration is suspect." }], - }, - { - role: "assistant", - parts: [{ type: "text", text: "I prepared the fix." }], - }, - ], - }); - - const entries = groupTranscriptMessages( - conversationTranscriptMessages(turn), + }), + event(2, "2026-01-01T00:00:02.000Z", { + type: "subagent_started", + childConversationId: "child-1", + subagentKind: "advisor", + historyMode: "shared", + }), + event(3, "2026-01-01T00:00:03.000Z", { + type: "subagent_ended", + childConversationId: "child-1", + subagentKind: "advisor", + historyMode: "shared", + outcome: "success", + }), + event(4, "2026-01-01T00:00:04.000Z", { + type: "turn_lifecycle", + turnId: "turn-1", + state: "failed", + }), + ]), ); + const entries = groupTranscriptMessages(messages); expect(entries.map((entry) => entry.kind)).toEqual([ - "message", "context", - "message", "context", - "message", + "subagent", + "failure", ]); - expect( - entries.some((entry) => entryMatchesSearch(entry, "gpt-5.6-sol")), - ).toBe(true); - expect( - entries.some((entry) => - entryMatchesSearch(entry, "earlier investigation"), - ), - ).toBe(true); - }); - - it("does not duplicate subagents from repeated activity snapshots", () => { - const turn = conversationTurn({ - activity: [ - { - type: "tool_execution", - id: "advisor-call", - toolCallId: "advisor-call", - toolName: "advisor", - createdAt: "2026-01-01T00:00:01.000Z", - status: "running", - subagents: [ - { - type: "subagent", - id: "advisor-subagent", - subagentKind: "advisor", - parentToolCallId: "advisor-call", - createdAt: "2026-01-01T00:00:02.000Z", - status: "running", - }, - ], - }, - { - type: "tool_execution", - id: "advisor-call", - toolCallId: "advisor-call", - toolName: "advisor", - createdAt: "2026-01-01T00:00:01.000Z", - status: "running", - subagents: [ - { - type: "subagent", - id: "advisor-subagent", - subagentKind: "advisor", - parentToolCallId: "advisor-call", - createdAt: "2026-01-01T00:00:02.000Z", - status: "running", - }, - ], - }, - ], - transcript: [], - transcriptAvailable: true, - }); - - expect( - groupTranscriptMessages(conversationTranscriptMessages(turn)), - ).toEqual([ - { - call: { - type: "tool_call", - id: "advisor-call", - name: "advisor", - status: "running", - }, - kind: "tool", - timestamp: Date.parse("2026-01-01T00:00:01.000Z"), - }, - { - kind: "subagent", - part: { - type: "subagent", - id: "advisor-subagent", - subagentKind: "advisor", - parentToolCallId: "advisor-call", - status: "running", - }, - timestamp: Date.parse("2026-01-01T00:00:02.000Z"), + expect(entries[2]).toMatchObject({ + part: { + childConversationId: "child-1", + outcome: "success", + status: "completed", }, - ]); - }); - - it("keeps subagent activity between an existing tool call and result", () => { - const turn = conversationTurn({ - activity: [ - { - type: "tool_execution", - id: "advisor-call", - toolCallId: "advisor-call", - toolName: "advisor", - createdAt: "2026-01-01T00:00:01.000Z", - status: "completed", - subagents: [ - { - type: "subagent", - id: "advisor-subagent", - subagentKind: "advisor", - parentToolCallId: "advisor-call", - createdAt: "2026-01-01T00:00:02.000Z", - status: "completed", - outcome: "success", - }, - ], - }, - ], - transcript: [ - { - role: "assistant", - timestamp: Date.parse("2026-01-01T00:00:01.000Z"), - parts: [{ type: "tool_call", id: "advisor-call", name: "advisor" }], - }, - { - role: "toolResult", - timestamp: Date.parse("2026-01-01T00:00:03.000Z"), - parts: [{ type: "tool_result", id: "advisor-call", name: "advisor" }], - }, - ], - transcriptAvailable: true, }); - - expect( - groupTranscriptMessages(conversationTranscriptMessages(turn)), - ).toEqual([ - { - call: { - type: "tool_call", - id: "advisor-call", - name: "advisor", - status: "completed", - }, - kind: "tool", - result: { - type: "tool_result", - id: "advisor-call", - name: "advisor", - }, - resultTimestamp: Date.parse("2026-01-01T00:00:03.000Z"), - timestamp: Date.parse("2026-01-01T00:00:01.000Z"), - }, - { - kind: "subagent", - part: { - type: "subagent", - id: "advisor-subagent", - subagentKind: "advisor", - parentToolCallId: "advisor-call", - status: "completed", - outcome: "success", - }, - timestamp: Date.parse("2026-01-01T00:00:02.000Z"), - }, - ]); - }); - - it("leaves ambiguous name-only tool results unpaired", () => { - const messages = [ - { - role: "assistant", - timestamp: 1_000, - parts: [{ type: "tool_call", name: "search" }], - }, - { - role: "assistant", - timestamp: 1_100, - parts: [{ type: "tool_call", name: "search" }], - }, - { - role: "toolResult", - timestamp: 2_000, - parts: [{ type: "tool_result", name: "search" }], - }, - ] as TranscriptMessage[]; - - expect(groupTranscriptMessages(messages)).toEqual([ - { - call: { type: "tool_call", name: "search" }, - kind: "tool", - timestamp: 1_000, - }, - { - call: { type: "tool_call", name: "search" }, - kind: "tool", - timestamp: 1_100, - }, - { - kind: "tool", - result: { type: "tool_result", name: "search" }, - resultTimestamp: 2_000, - }, - ]); }); - it("does not pair name-only results across message boundaries", () => { - const messages = [ - { - role: "assistant", - timestamp: 1_000, - parts: [{ type: "tool_call", name: "search" }], - }, - { - role: "assistant", - timestamp: 1_500, - parts: [{ type: "text", text: "Continuing after the search." }], - }, - { - role: "toolResult", - timestamp: 2_000, - parts: [{ type: "tool_result", name: "search" }], - }, - ] as TranscriptMessage[]; - - expect(groupTranscriptMessages(messages)).toEqual([ - { - call: { type: "tool_call", name: "search" }, - kind: "tool", - timestamp: 1_000, - }, - { - kind: "message", - message: { - role: "assistant", - timestamp: 1_500, - parts: [{ type: "text", text: "Continuing after the search." }], - }, - }, - { - kind: "tool", - result: { type: "tool_result", name: "search" }, - resultTimestamp: 2_000, - }, - ]); - }); - - it("pairs one name-only call across thinking activity", () => { - const messages = [ - { - role: "assistant", - timestamp: 1_000, - parts: [{ type: "tool_call", name: "search" }], - }, - { - role: "assistant", - timestamp: 1_500, - parts: [{ type: "thinking", output: "Waiting for search." }], - }, - { - role: "toolResult", - timestamp: 2_000, - parts: [{ type: "tool_result", name: "search" }], - }, - ] as TranscriptMessage[]; + it("projects only failed deliveries as neutral failure history", () => { + const entries = groupTranscriptMessages( + conversationTranscriptMessages( + conversation([ + event(0, "2026-01-01T00:00:00.000Z", { + type: "delivery", + deliveryId: "delivery-1", + state: "intended", + }), + event(1, "2026-01-01T00:00:01.000Z", { + type: "delivery", + deliveryId: "delivery-1", + state: "accepted", + }), + event(2, "2026-01-01T00:00:02.000Z", { + type: "delivery", + deliveryId: "delivery-2", + state: "failed", + }), + ]), + ), + ); - expect(groupTranscriptMessages(messages)).toEqual([ - { - call: { type: "tool_call", name: "search" }, - kind: "tool", - result: { type: "tool_result", name: "search" }, - resultTimestamp: 2_000, - timestamp: 1_000, - }, + expect(entries).toEqual([ { - kind: "thinking", - part: { type: "thinking", output: "Waiting for search." }, - timestamp: 1_500, + kind: "failure", + outcome: "delivery_failed", + timestamp: Date.parse("2026-01-01T00:00:02.000Z"), }, ]); + expect(entryMatchesSearch(entries[0]!, "delivery failed")).toBe(true); + expect(entryMatchesSearch(entries[0]!, "agent response failed")).toBe( + false, + ); }); - it("treats mixed ID metadata as ambiguous for name-only results", () => { - const messages = [ - { - role: "assistant", - timestamp: 1_000, - parts: [{ type: "tool_call", name: "search" }], - }, - { - role: "assistant", - timestamp: 1_100, - parts: [{ type: "tool_call", id: "call-2", name: "search" }], - }, - { - role: "toolResult", - timestamp: 2_000, - parts: [{ type: "tool_result", name: "search" }], - }, - ] as TranscriptMessage[]; - - expect(groupTranscriptMessages(messages).at(-1)).toEqual({ - kind: "tool", - result: { type: "tool_result", name: "search" }, - resultTimestamp: 2_000, - }); - }); - - it("matches derived activity rows in transcript search", () => { - const turn = conversationTurn({ - activity: [ - { - type: "tool_execution", - id: "advisor-call", - toolCallId: "advisor-call", - toolName: "advisor", - createdAt: "2026-01-01T00:00:01.000Z", - status: "running", - subagents: [ - { - type: "subagent", - id: "advisor-call", - subagentKind: "advisor", - parentToolCallId: "advisor-call", - createdAt: "2026-01-01T00:00:02.000Z", - status: "running", - }, - ], - }, - ], - transcript: [], - transcriptAvailable: true, - }); - + it("searches canonical tool, failure, context, and subagent rows", () => { const entries = groupTranscriptMessages( - conversationTranscriptMessages(turn), + conversationTranscriptMessages( + conversation([ + event(0, "2026-01-01T00:00:00.000Z", { + type: "tool_started", + name: "sentry.search", + }), + event(1, "2026-01-01T00:00:01.000Z", { + type: "subagent_started", + childConversationId: "child-1", + subagentKind: "advisor", + historyMode: "isolated", + }), + event(2, "2026-01-01T00:00:02.000Z", { + type: "context_compacted", + }), + event(3, "2026-01-01T00:00:03.000Z", { + type: "turn_lifecycle", + turnId: "turn-1", + state: "failed", + }), + ]), + ), ); - expect(entries.some((entry) => entryMatchesSearch(entry, "advisor"))).toBe( - true, - ); - expect(entries.some((entry) => entryMatchesSearch(entry, "running"))).toBe( - true, - ); - expect( - entries.some((entry) => entryMatchesSearch(entry, "not-present")), - ).toBe(false); + for (const query of ["sentry.search", "advisor", "compacted", "failed"]) { + expect(entries.some((entry) => entryMatchesSearch(entry, query))).toBe( + true, + ); + } }); +}); - it("matches tool activity status in transcript search", () => { - const turn = conversationTurn({ - activity: [ - { - type: "tool_execution", - id: "call-running", - toolCallId: "call-running", - toolName: "search", - createdAt: "2026-01-01T00:00:01.000Z", - status: "running", - subagents: [], - }, - ], - transcript: [], - transcriptAvailable: true, - }); - - const entries = groupTranscriptMessages( - conversationTranscriptMessages(turn), - ); - - expect(entries.some((entry) => entryMatchesSearch(entry, "running"))).toBe( - true, - ); +describe("transcript render grouping", () => { + it("keeps terminal failure outcomes as standalone entries", () => { + const messages: TranscriptViewMessage[] = [ + { role: "assistant", outcome: "error", timestamp: 1_000, parts: [] }, + ]; + expect(groupTranscriptMessages(messages)).toEqual([ + { kind: "failure", outcome: "error", timestamp: 1_000 }, + ]); }); }); diff --git a/packages/junior/src/api/conversations/activity.ts b/packages/junior/src/api/conversations/activity.ts deleted file mode 100644 index c07b54b41..000000000 --- a/packages/junior/src/api/conversations/activity.ts +++ /dev/null @@ -1,170 +0,0 @@ -import type { - ConversationEvent, - ConversationEventData, -} from "@/chat/conversations/history"; -import { redactedPayloadFields } from "./transcript"; -import type { - ConversationActivityReport, - ConversationActivityStatus, - ConversationSubagentActivityReport, -} from "./schema"; - -interface ActivityPayloadMetadata { - inputKeys?: string[]; - inputSizeBytes?: number; - inputSizeChars?: number; - inputType?: string; -} -function toolResultStatuses( - events: ConversationEvent[], -): Map { - const statuses = new Map(); - for (const event of events) { - if (event.data.type !== "message") continue; - const record = event.data.message as Record; - if (record.role !== "toolResult" || typeof record.toolCallId !== "string") { - continue; - } - statuses.set(record.toolCallId, record.isError ? "error" : "completed"); - } - return statuses; -} - -function activityPayloadFields( - args: unknown, - canExposePayload: boolean, -): ActivityPayloadMetadata & { args?: unknown; redacted?: boolean } { - if (args === undefined) { - return {}; - } - return canExposePayload - ? { args } - : { redacted: true, ...redactedPayloadFields("input", args) }; -} - -/** - * Build the current-run activity timeline from durable conversation events. - * - * Tool executions, subagent starts/ends, and their nesting are derived from the - * conversation's durable events instead of the legacy Redis session log; tool - * statuses come from durable model-message events. Redaction stays - * byte-compatible with the prior session-log path. - */ -export function buildConversationActivityFromEvents(args: { - canExposePayload: boolean; - events: ConversationEvent[]; -}): ConversationActivityReport[] { - const toolStatuses = toolResultStatuses(args.events); - const subagentEnds = new Map(); - const subagentsByToolCallId = new Map< - string, - ConversationSubagentActivityReport[] - >(); - const orphanSubagents: ConversationSubagentActivityReport[] = []; - - for (const event of args.events) { - if (event.data.type === "subagent_ended") { - subagentEnds.set( - event.data.subagentInvocationId, - event as SubagentEndedEvent, - ); - } - } - - for (const event of args.events) { - if (event.data.type !== "subagent_started") { - continue; - } - const start = event as SubagentStartedEvent; - const parentStatus = start.data.parentToolCallId - ? toolStatuses.get(start.data.parentToolCallId) - : undefined; - const activity = subagentActivityFromEvents( - start, - subagentEnds.get(start.data.subagentInvocationId), - { canExposeTranscript: args.canExposePayload, parentStatus }, - ); - if (start.data.parentToolCallId) { - subagentsByToolCallId.set(start.data.parentToolCallId, [ - ...(subagentsByToolCallId.get(start.data.parentToolCallId) ?? []), - activity, - ]); - continue; - } - orphanSubagents.push(activity); - } - - const rows: ConversationActivityReport[] = []; - for (const event of args.events) { - if (event.data.type !== "tool_execution_started") { - continue; - } - rows.push({ - type: "tool_execution", - id: event.data.toolCallId, - toolCallId: event.data.toolCallId, - toolName: event.data.toolName, - createdAt: new Date(event.createdAtMs).toISOString(), - status: toolStatuses.get(event.data.toolCallId) ?? "running", - subagents: subagentsByToolCallId.get(event.data.toolCallId) ?? [], - ...activityPayloadFields(event.data.args, args.canExposePayload), - }); - } - - return [...rows, ...orphanSubagents].sort( - (left, right) => - Date.parse(left.createdAt) - Date.parse(right.createdAt) || - left.id.localeCompare(right.id), - ); -} - -export type SubagentStartedEvent = ConversationEvent & { - data: Extract; -}; -export type SubagentEndedEvent = ConversationEvent & { - data: Extract; -}; - -/** Pair durable subagent start and end events into one activity report. */ -export function subagentActivityFromEvents( - start: SubagentStartedEvent, - end: SubagentEndedEvent | undefined, - options: { - canExposeTranscript?: boolean; - parentStatus?: ConversationActivityStatus; - } = {}, -): ConversationSubagentActivityReport { - return { - type: "subagent", - id: start.data.subagentInvocationId, - subagentKind: start.data.subagentKind, - ...(start.data.modelId ? { modelId: start.data.modelId } : {}), - ...(start.data.parentToolCallId - ? { parentToolCallId: start.data.parentToolCallId } - : {}), - ...(start.data.reasoningLevel - ? { reasoningLevel: start.data.reasoningLevel } - : {}), - createdAt: new Date(start.createdAtMs).toISOString(), - ...(end - ? { - endedAt: new Date(end.createdAtMs).toISOString(), - outcome: end.data.outcome, - status: end.data.outcome, - // Every subagent is a child conversation whose transcript loads on - // demand; expose the affordance only when the parent is public. - ...(options.canExposeTranscript ? { transcriptAvailable: true } : {}), - } - : { status: options.parentStatus ?? "running" }), - }; -} - -/** - * Read one child-agent transcript through its parent conversation. - * - * The parent records `subagent_started`/`subagent_ended` as durable events that - * name the child by `childConversationId`; the transcript is the child - * conversation's own projected Pi messages. `runId` is retained for the route - * signature but no longer scopes the lookup — subagent events live on the parent - * conversation regardless of the run that produced them. - */ diff --git a/packages/junior/src/api/conversations/detail-projection.ts b/packages/junior/src/api/conversations/detail-projection.ts index 0d71b584b..ddf4b9e5b 100644 --- a/packages/junior/src/api/conversations/detail-projection.ts +++ b/packages/junior/src/api/conversations/detail-projection.ts @@ -1,534 +1,61 @@ -import { isDeepStrictEqual } from "node:util"; import { canExposeConversationPayload } from "@/chat/conversation-privacy"; -import type { - ConversationEventStore, - ConversationEvent, - ConversationModelMessage, -} from "@/chat/conversations/history"; +import type { ConversationEvent } from "@/chat/conversations/history"; import type { Conversation } from "@/chat/conversations/store"; -import { projectConversationEventHistory } from "@/chat/conversations/event-projection"; -import { loadCurrentConversationEvents } from "@/chat/conversations/history-loader"; -import { stripRuntimeTurnContextMessages } from "@/chat/conversations/model-messages"; -import { getConversationEventStore } from "@/chat/db"; -import { projectVisibleConversationMessages } from "@/chat/conversations/visible-message-projection"; -import type { ConversationMessage } from "@/chat/state/conversation"; -import { - buildSentryConversationUrl, - buildSentryTraceUrl, -} from "@/chat/sentry-links"; -import { - buildConversationActivityFromEvents, - subagentActivityFromEvents, - type SubagentEndedEvent, - type SubagentStartedEvent, -} from "./activity"; +import { buildSentryConversationUrl } from "@/chat/sentry-links"; +import { projectConversationReportEvents } from "./events"; import { conversationSummaryFromStoredConversation } from "./projection"; -import { - countConversationMessages, - normalizeSubagentTranscriptMessage, - normalizeTranscriptMessage, - redactTranscriptMessage, - subagentTranscriptReport, - traceIdFromTranscript, -} from "./transcript"; -import type { - ConversationActivityReport, - ConversationContextEvent, - ConversationDetailReport, - ConversationSubagentTranscriptReport, - TranscriptMessage, -} from "./schema"; +import type { ConversationDetailReport } from "./schema"; -const COMPACTION_SUMMARY_PREFIXES = [ - "Context compaction summary for future Junior turns:", - "Context handoff summary for future Junior turns:", -] as const; -const MODEL_HANDOFF_SUMMARY_PREFIX = - "Model handoff checkpoint. Continue the outstanding request now using this summary as the complete prior context:"; - -type EpochStartedEvent = ConversationEvent & { - data: Extract; -}; - -function messageText(message: ConversationModelMessage): string { - return normalizeTranscriptMessage(message) - .parts.filter((part) => part.type === "text") - .map((part) => part.text ?? "") - .join("\n") - .trim(); -} - -function summaryAfterPrefix( - message: ConversationModelMessage, - prefixes: readonly string[], -): string | undefined { - const text = messageText(message); - const prefix = prefixes.find((candidate) => text.startsWith(candidate)); - if (!prefix) return undefined; - return text.slice(prefix.length).trim(); -} - -function summaryIndex( - messages: ConversationModelMessage[], - provenance: Array<{ authority: "context" | "instruction" }>, - prefixes: readonly string[], -): number { - for (let index = messages.length - 1; index >= 0; index -= 1) { - if ( - provenance[index]?.authority === "context" && - summaryAfterPrefix(messages[index]!, prefixes) !== undefined - ) { - return index; - } - } - return -1; -} - -function matchingPrefix( - left: ConversationModelMessage[], - right: ConversationModelMessage[], -): number { - const limit = Math.min(left.length, right.length); - for (let index = 0; index < limit; index += 1) { - if (!isDeepStrictEqual(left[index], right[index])) return index; - } - return limit; -} - -/** - * Rebuild the chronological execution once across context replacements. - * Synthetic summaries become context events, while copied replacement - * messages are omitted without collapsing later execution messages. - */ -function historyContent(args: { - canExposePayload: boolean; - events: ConversationEvent[]; -}): { - contextEvents: ConversationContextEvent[]; - messages: ConversationModelMessage[]; -} { - const contextEvents: ConversationContextEvent[] = []; - const messages: ConversationModelMessage[] = []; - const epochs = new Map(); - for (const event of args.events) { - const epoch = epochs.get(event.contextEpoch); - if (epoch) epoch.push(event); - else epochs.set(event.contextEpoch, [event]); - } - - let previousModelId: string | undefined; - let previousProjection: ConversationModelMessage[] = []; - for (const events of epochs.values()) { - const marker = events.find( - (event): event is EpochStartedEvent => - event.data.type === "context_epoch_started", - ); - const projection = projectConversationEventHistory(events); - const projected: ConversationModelMessage[] = []; - const projectedProvenance: typeof projection.provenance = []; - projection.messages.forEach((message, index) => { - for (const retained of stripRuntimeTurnContextMessages([message])) { - projected.push(retained); - projectedProvenance.push(projection.provenance[index]!); - } - }); - const replacementSummaryIndex = - marker?.data.reason === "compaction" - ? summaryIndex( - projected, - projectedProvenance, - COMPACTION_SUMMARY_PREFIXES, - ) - : marker?.data.reason === "handoff" - ? summaryIndex(projected, projectedProvenance, [ - MODEL_HANDOFF_SUMMARY_PREFIX, - ]) - : -1; - const summary = - marker?.data.reason === "compaction" && replacementSummaryIndex >= 0 - ? summaryAfterPrefix( - projected[replacementSummaryIndex]!, - COMPACTION_SUMMARY_PREFIXES, - ) - : undefined; - const handoffMessage = - marker?.data.reason === "handoff" && replacementSummaryIndex >= 0 - ? messageText(projected[replacementSummaryIndex]!) || undefined - : undefined; - - if (marker?.data.reason === "compaction") { - contextEvents.push({ - type: "context_compacted", - createdAt: new Date(marker.createdAtMs).toISOString(), - ...(marker.data.modelId ? { modelId: marker.data.modelId } : {}), - ...(args.canExposePayload && summary ? { summary } : {}), - transcriptIndex: messages.length, - }); - } else if (marker?.data.reason === "handoff") { - contextEvents.push({ - type: "model_handoff", - createdAt: new Date(marker.createdAtMs).toISOString(), - ...(previousModelId ? { fromModelId: previousModelId } : {}), - toModelId: marker.data.modelId, - ...(args.canExposePayload && handoffMessage - ? { message: handoffMessage } - : {}), - transcriptIndex: messages.length, - }); - } - - if (marker?.data.reason === "rollback") { - messages.push( - ...projected.slice(matchingPrefix(previousProjection, projected)), - ); - } else { - const copiedMessageIndexes = new Set(); - projected.forEach((message, index) => { - if (index === replacementSummaryIndex) return; - let copiedCompactionMessage = false; - if ( - marker?.data.reason === "compaction" && - replacementSummaryIndex >= 0 && - index < replacementSummaryIndex - ) { - const copiedIndex = messages.findIndex( - (candidate, candidateIndex) => - !copiedMessageIndexes.has(candidateIndex) && - isDeepStrictEqual(candidate, message), - ); - copiedCompactionMessage = copiedIndex >= 0; - if (copiedCompactionMessage) copiedMessageIndexes.add(copiedIndex); - } - if (!copiedCompactionMessage) messages.push(message); - }); - } - previousModelId = marker?.data.modelId ?? previousModelId; - previousProjection = projected; - } - - return { contextEvents, messages }; -} - -function completedTurnIntervals(events: ConversationEvent[]): Array<{ - startSeq: number; - terminalSeq: number; -}> { - const starts = new Map(); - const intervals: Array<{ startSeq: number; terminalSeq: number }> = []; - for (const event of events) { - if (event.data.type === "turn_started") { - starts.set(event.data.turnId, event.seq); - continue; - } - if ( - event.data.type !== "turn_completed" && - event.data.type !== "turn_failed" - ) { - continue; - } - const startSeq = starts.get(event.data.turnId); - if (startSeq !== undefined && startSeq <= event.seq) { - intervals.push({ startSeq, terminalSeq: event.seq }); - } - } - return intervals; -} - -function terminalOutcomeTranscript(args: { - events: ConversationEvent[]; - messages: ConversationModelMessage[]; -}): TranscriptMessage[] { - const intervals = completedTurnIntervals(args.events); - const lifecycleCoveredMessages = new Set( - args.events.flatMap((event) => - event.data.type === "message" && - intervals.some( - (interval) => - event.seq >= interval.startSeq && event.seq <= interval.terminalSeq, - ) - ? [event.data.message] - : [], - ), - ); - const legacyOutcomes = args.messages.flatMap((message) => { - if (lifecycleCoveredMessages.has(message)) return []; - const normalized = normalizeTranscriptMessage(message); - if ( - normalized.role !== "assistant" || - (normalized.outcome !== "error" && normalized.outcome !== "aborted") - ) { - return []; - } - return [ - { - role: "assistant" as const, - outcome: normalized.outcome, - parts: [], - ...(normalized.timestamp === undefined - ? {} - : { timestamp: normalized.timestamp }), - }, - ]; - }); - const lifecycleFailures = args.events.flatMap((event) => - event.data.type === "turn_failed" - ? [ - { - role: "assistant" as const, - outcome: "error" as const, - parts: [], - timestamp: event.createdAtMs, - }, - ] - : [], - ); - return [...legacyOutcomes, ...lifecycleFailures]; -} - -function mergeTranscriptChronologically( - messages: TranscriptMessage[], -): TranscriptMessage[] { - return messages - .map((message, index) => ({ message, index })) - .sort( - (left, right) => - (left.message.timestamp ?? Number.POSITIVE_INFINITY) - - (right.message.timestamp ?? Number.POSITIVE_INFINITY) || - left.index - right.index, - ) - .map(({ message }) => message); -} - -async function conversationContent(args: { - conversationId: string; - eventStore: ConversationEventStore; - canExposePayload: boolean; -}): Promise<{ - activity: ConversationActivityReport[]; - contextEvents: ConversationContextEvent[]; - traceId?: string; - transcript: TranscriptMessage[]; -}> { - const events = await args.eventStore.loadHistory(args.conversationId); - const history = historyContent({ - canExposePayload: args.canExposePayload, - events, - }); - const visibleMessages = projectVisibleConversationMessages(events); - const visibleTranscript = visibleMessages.map(visibleMessageTranscript); - const modelTranscript = history.messages.map(normalizeTranscriptMessage); - const transcript = mergeTranscriptChronologically([ - ...visibleTranscript, - ...terminalOutcomeTranscript({ events, messages: history.messages }), - ]); - const contextEvents = history.contextEvents.map((event) => ({ - ...event, - transcriptIndex: transcript.findIndex( - (message) => - (message.timestamp ?? Number.POSITIVE_INFINITY) > - new Date(event.createdAt).getTime(), - ), - })); - for (const event of contextEvents) { - if (event.transcriptIndex < 0) event.transcriptIndex = transcript.length; - } - return { - activity: buildConversationActivityFromEvents({ - canExposePayload: args.canExposePayload, - events, - }), - contextEvents, - ...(args.canExposePayload - ? { - traceId: traceIdFromTranscript([ - ...modelTranscript, - ...visibleTranscript, - ]), - } - : {}), - transcript, - }; -} - -function visibleMessageTranscript( - message: ConversationMessage, -): TranscriptMessage { - return { - role: message.role, - timestamp: message.createdAtMs, - parts: [{ type: "text", text: message.text }], - }; -} - -/** Build one conversation REST detail from durable SQL records. */ +/** Build one conversation REST detail from its canonical event history. */ export async function buildConversationDetail(args: { conversation: Conversation; durationMs: number; + effectiveVisibility?: Conversation["visibility"]; + events: ConversationEvent[]; + privacyConversationId?: string; locationId?: string; usage: ConversationDetailReport["cumulativeUsage"]; }): Promise { const { conversation } = args; const conversationId = conversation.conversationId; const nowMs = Date.now(); - const eventStore = getConversationEventStore(); const transcriptPurgedAtMs = conversation.transcriptPurgedAtMs; - const transcriptExpiredAt = - transcriptPurgedAtMs !== undefined - ? new Date(transcriptPurgedAtMs).toISOString() - : undefined; - - // Reporting reads the complete durable execution history. Context rebuilds - // become explicit events while copied replacement messages are de-duplicated. - // Purged conversations have no events to read. - const canExposeSqlContent = canExposeConversationPayload({ - conversationId, - visibility: conversation.visibility, + const { visibility: _storedVisibility, ...conversationWithoutVisibility } = + conversation; + const authorizedConversation: Conversation = { + ...conversationWithoutVisibility, + ...(args.effectiveVisibility + ? { visibility: args.effectiveVisibility } + : {}), + }; + const canExposePayload = canExposeConversationPayload({ + conversationId: args.privacyConversationId ?? conversationId, + visibility: args.effectiveVisibility, }); - const currentContent: Awaited> = - transcriptPurgedAtMs === undefined - ? await conversationContent({ - conversationId, - eventStore, - canExposePayload: canExposeSqlContent, - }) - : { activity: [], contextEvents: [], transcript: [] }; - - const currentTranscript = currentContent.transcript; - const traceId = canExposeSqlContent ? currentContent.traceId : undefined; - const sentryTraceUrl = traceId ? buildSentryTraceUrl(traceId) : undefined; + const events = transcriptPurgedAtMs === undefined ? args.events : []; const sentryConversationUrl = buildSentryConversationUrl(conversationId); return { ...conversationSummaryFromStoredConversation({ - conversation, + conversation: authorizedConversation, durationMs: args.durationMs, ...(args.locationId ? { locationId: args.locationId } : {}), usage: args.usage, }), - ...(traceId ? { traceId } : {}), - ...(sentryTraceUrl ? { sentryTraceUrl } : {}), - activity: currentContent.activity, - contextEvents: currentContent.contextEvents, - transcriptAvailable: - transcriptExpiredAt === undefined && - canExposeSqlContent && - currentTranscript.length > 0, - ...(currentTranscript.length > 0 - ? { - transcriptMessageCount: countConversationMessages(currentTranscript), - } - : {}), - ...(!canExposeSqlContent && transcriptExpiredAt === undefined - ? { - transcriptMetadata: currentTranscript.map(redactTranscriptMessage), - transcriptRedacted: true, - transcriptRedactionReason: "non_public_conversation" as const, - } - : {}), - ...(transcriptExpiredAt !== undefined - ? { - transcriptExpired: true, - transcriptExpiredAt, - transcriptMetadata: [], - } - : {}), - transcript: - transcriptExpiredAt === undefined && canExposeSqlContent - ? currentTranscript - : [], + events: projectConversationReportEvents({ canExposePayload, events }), + eventHistory: + transcriptPurgedAtMs !== undefined + ? { + status: "expired", + expiredAt: new Date(transcriptPurgedAtMs).toISOString(), + } + : canExposePayload + ? { status: "available" } + : { + status: "redacted", + reason: "non_public_conversation", + }, generatedAt: new Date(nowMs).toISOString(), ...(sentryConversationUrl ? { sentryConversationUrl } : {}), }; } - -/** Build one child-agent REST detail from durable SQL history. */ -export async function buildConversationSubagent( - conversation: Conversation, - subagentId: string, -): Promise { - const conversationId = conversation.conversationId; - const eventStore = getConversationEventStore(); - const parentEvents = await eventStore.loadHistory(conversationId); - - // Retention purge deletes the parent tree's events wholesale; present the - // subagent as expired rather than "not found" (data-redaction.md). - if (conversation?.transcriptPurgedAtMs !== undefined) { - return { - type: "subagent", - createdAt: new Date(0).toISOString(), - id: subagentId, - status: "completed", - subagentKind: "unknown", - transcript: [], - transcriptAvailable: false, - transcriptExpired: true, - transcriptExpiredAt: new Date( - conversation.transcriptPurgedAtMs, - ).toISOString(), - }; - } - - const start = parentEvents.find( - (event): event is SubagentStartedEvent => - event.data.type === "subagent_started" && - event.data.subagentInvocationId === subagentId, - ); - if (!start) { - return { - type: "subagent", - createdAt: new Date(0).toISOString(), - id: subagentId, - status: "error", - subagentKind: "unknown", - transcript: [], - transcriptAvailable: false, - unavailableReason: "not_found", - }; - } - const end = parentEvents.find( - (event): event is SubagentEndedEvent => - event.data.type === "subagent_ended" && - event.data.subagentInvocationId === subagentId, - ); - - const childConversationId = start.data.childConversationId; - const activity = subagentActivityFromEvents(start, end); - const subagentSentryConversationUrl = - buildSentryConversationUrl(childConversationId); - const conversationFields = { - subagentConversationId: childConversationId, - ...(subagentSentryConversationUrl ? { subagentSentryConversationUrl } : {}), - }; - - const canExposeTranscript = canExposeConversationPayload({ - conversationId, - visibility: conversation?.visibility, - }); - if (!canExposeTranscript) { - return subagentTranscriptReport(activity, { - ...conversationFields, - transcriptRedacted: true, - transcriptRedactionReason: "non_public_conversation", - }); - } - - const childEvents = await loadCurrentConversationEvents({ - conversationId: childConversationId, - }); - const childMessages = projectConversationEventHistory(childEvents).messages; - if (childMessages.length === 0) { - return subagentTranscriptReport(activity, { - ...conversationFields, - unavailableReason: "missing_transcript_ref", - }); - } - - const transcript = childMessages.map((message) => - normalizeSubagentTranscriptMessage(message, activity.subagentKind), - ); - return subagentTranscriptReport(activity, { - ...conversationFields, - transcript, - transcriptMessageCount: countConversationMessages(transcript), - }); -} diff --git a/packages/junior/src/api/conversations/detail.query.ts b/packages/junior/src/api/conversations/detail.query.ts index 6ae20ad0c..537534467 100644 --- a/packages/junior/src/api/conversations/detail.query.ts +++ b/packages/junior/src/api/conversations/detail.query.ts @@ -1,46 +1,42 @@ -import { logException } from "@/chat/logging"; -import { - listBoundedAgentTurnSessionSummariesForConversation, - type AgentTurnSessionSummary, -} from "@/chat/state/turn-session"; import { buildConversationDetail } from "./detail-projection"; -import { readConversationRecordFromSql } from "./list"; +import { getSqlExecutor } from "@/chat/db"; +import { createSqlConversationEventStore } from "@/chat/conversations/sql/history"; +import { withConversationEventLock } from "@/chat/conversations/sql/event-lock"; +import { resolveRootVisibility } from "@/chat/conversations/sql/privacy"; +import { readConversationRecordFromSql } from "./list.query"; import type { ConversationDetailReport } from "./schema"; -async function readLatestRun( - conversationId: string, -): Promise { - try { - return ( - await listBoundedAgentTurnSessionSummariesForConversation(conversationId) - ).find((summary) => summary.modelId || summary.reasoningLevel); - } catch (error) { - logException(error, "conversation_execution_settings_read_failed", { - conversationId, - }); - return undefined; - } -} - -/** Read one SQL conversation with its latest operational run settings. */ +/** Read one SQL conversation and its canonical, root-authorized event history. */ export async function readConversationDetailFromSql( conversationId: string, ): Promise { - const record = await readConversationRecordFromSql(conversationId); - if (!record) return undefined; - - const [report, latestRun] = await Promise.all([ - buildConversationDetail({ - ...record, - usage: record.usage ?? undefined, + const executor = getSqlExecutor(); + return executor.transaction(async () => + withConversationEventLock(executor, conversationId, async () => { + const { rootConversationId, visibility } = await resolveRootVisibility( + executor, + conversationId, + ); + const record = await readConversationRecordFromSql( + conversationId, + executor.db(), + ); + if (!record) return undefined; + const events = + await createSqlConversationEventStore(executor).loadHistory( + conversationId, + ); + const effectiveVisibility = + visibility === "public" || visibility === "private" + ? visibility + : undefined; + return buildConversationDetail({ + ...record, + effectiveVisibility, + events, + privacyConversationId: rootConversationId, + usage: record.usage ?? undefined, + }); }), - readLatestRun(conversationId), - ]); - return { - ...report, - ...(latestRun?.modelId ? { modelId: latestRun.modelId } : {}), - ...(latestRun?.reasoningLevel - ? { reasoningLevel: latestRun.reasoningLevel } - : {}), - }; + ); } diff --git a/packages/junior/src/api/conversations/list.query.ts b/packages/junior/src/api/conversations/list.query.ts new file mode 100644 index 000000000..fccb5faa6 --- /dev/null +++ b/packages/junior/src/api/conversations/list.query.ts @@ -0,0 +1,200 @@ +import { and, asc, desc, eq, isNull } from "drizzle-orm"; +import { getDb } from "@/chat/db"; +import type { Conversation } from "@/chat/conversations/store"; +import type { JuniorDatabase } from "@/db/db"; +import { + juniorConversations, + juniorDestinations, + juniorIdentities, +} from "@/db/schema"; +import { conversationSummaryFromStoredConversation } from "./projection"; +import type { ConversationFeed } from "./schema"; + +const CONVERSATION_FEED_LIMIT = 50; + +async function conversationRows( + db: JuniorDatabase, + limit: number, + actorEmail?: string, +) { + return db + .select({ + conversation: juniorConversations, + destinationId: juniorDestinations.id, + destinationVisibility: juniorDestinations.visibility, + identityDisplayName: juniorIdentities.displayName, + identityEmail: juniorIdentities.email, + identityHandle: juniorIdentities.handle, + identityProvider: juniorIdentities.provider, + identitySubjectId: juniorIdentities.providerSubjectId, + identityTenantId: juniorIdentities.providerTenantId, + }) + .from(juniorConversations) + .leftJoin( + juniorDestinations, + eq(juniorDestinations.id, juniorConversations.destinationId), + ) + .leftJoin( + juniorIdentities, + eq(juniorIdentities.id, juniorConversations.actorIdentityId), + ) + .where( + and( + isNull(juniorConversations.parentConversationId), + actorEmail + ? and( + eq(juniorIdentities.emailNormalized, actorEmail), + eq(juniorIdentities.emailVerified, true), + ) + : undefined, + ), + ) + .orderBy( + desc(juniorConversations.lastActivityAt), + asc(juniorConversations.conversationId), + ) + .limit(limit); +} + +type ConversationRow = Awaited>[number]; + +function conversationFromRow(row: ConversationRow): Conversation { + const value = row.conversation; + const actor = + row.identityProvider === "slack" + ? { + platform: "slack" as const, + ...(row.identityEmail ? { email: row.identityEmail } : {}), + ...(row.identityDisplayName + ? { fullName: row.identityDisplayName } + : {}), + ...(row.identitySubjectId + ? { slackUserId: row.identitySubjectId } + : {}), + ...(row.identityHandle ? { slackUserName: row.identityHandle } : {}), + ...(row.identityTenantId ? { teamId: row.identityTenantId } : {}), + } + : undefined; + return { + schemaVersion: 1, + conversationId: value.conversationId, + createdAtMs: value.createdAt.getTime(), + lastActivityAtMs: value.lastActivityAt.getTime(), + updatedAtMs: value.updatedAt.getTime(), + execution: { + status: value.executionStatus, + ...(value.runId ? { runId: value.runId } : {}), + ...(value.executionUpdatedAt + ? { updatedAtMs: value.executionUpdatedAt.getTime() } + : {}), + }, + ...(actor ? { actor } : {}), + ...(value.channelName ? { channelName: value.channelName } : {}), + ...(value.source ? { source: value.source } : {}), + ...(value.title ? { title: value.title } : {}), + ...(value.transcriptPurgedAt + ? { transcriptPurgedAtMs: value.transcriptPurgedAt.getTime() } + : {}), + ...(value.parentConversationId + ? { + lineage: { + parentConversationId: value.parentConversationId, + ...(value.rootConversationId + ? { rootConversationId: value.rootConversationId } + : {}), + ...(value.parentTurnId ? { parentTurnId: value.parentTurnId } : {}), + ...(value.parentEventSeq !== null + ? { parentEventSeq: value.parentEventSeq } + : {}), + ...(value.contextForkSeq !== null + ? { contextForkSeq: value.contextForkSeq } + : {}), + }, + } + : {}), + ...(row.destinationVisibility + ? { + visibility: + row.destinationVisibility === "public" ? "public" : "private", + } + : {}), + }; +} + +/** Read one normalized conversation record directly from its SQL row. */ +export async function readConversationRecordFromSql( + conversationId: string, + db: JuniorDatabase = getDb(), +): Promise< + | { + conversation: Conversation; + durationMs: number; + locationId?: string; + usage: ConversationRow["conversation"]["usage"]; + } + | undefined +> { + const rows = await db + .select({ + conversation: juniorConversations, + destinationId: juniorDestinations.id, + destinationVisibility: juniorDestinations.visibility, + identityDisplayName: juniorIdentities.displayName, + identityEmail: juniorIdentities.email, + identityHandle: juniorIdentities.handle, + identityProvider: juniorIdentities.provider, + identitySubjectId: juniorIdentities.providerSubjectId, + identityTenantId: juniorIdentities.providerTenantId, + }) + .from(juniorConversations) + .leftJoin( + juniorDestinations, + eq(juniorDestinations.id, juniorConversations.destinationId), + ) + .leftJoin( + juniorIdentities, + eq(juniorIdentities.id, juniorConversations.actorIdentityId), + ) + .where(eq(juniorConversations.conversationId, conversationId)) + .limit(1); + const row = rows[0]; + return row + ? { + conversation: conversationFromRow(row), + durationMs: row.conversation.durationMs, + ...(row.destinationVisibility === "public" && row.destinationId + ? { locationId: row.destinationId } + : {}), + usage: row.conversation.usage, + } + : undefined; +} + +/** + * Build a bounded dashboard feed, applying a normalized actor-email filter + * before the limit when one is provided. + */ +export async function readConversationFeedFromSql( + options: { actorEmail?: string; limit?: number } = {}, +): Promise { + const nowMs = Date.now(); + const rows = await conversationRows( + getDb(), + options.limit ?? CONVERSATION_FEED_LIMIT, + options.actorEmail, + ); + return { + conversations: rows.map((row) => + conversationSummaryFromStoredConversation({ + conversation: conversationFromRow(row), + durationMs: row.conversation.durationMs, + ...(row.destinationVisibility === "public" && row.destinationId + ? { locationId: row.destinationId } + : {}), + usage: row.conversation.usage ?? undefined, + }), + ), + generatedAt: new Date(nowMs).toISOString(), + source: "conversation_index", + }; +} diff --git a/packages/junior/src/api/conversations/schema.ts b/packages/junior/src/api/conversations/schema.ts index efed9281b..978ff973f 100644 --- a/packages/junior/src/api/conversations/schema.ts +++ b/packages/junior/src/api/conversations/schema.ts @@ -49,102 +49,6 @@ export const conversationSummaryReportSchema = z }) .strict(); -export const transcriptPartTypeSchema = z.enum([ - "text", - "thinking", - "tool_call", - "tool_result", - "unknown", -]); - -export const transcriptPartSchema = z - .object({ - bytes: z.number().optional(), - chars: z.number().optional(), - id: z.string().optional(), - input: z.unknown().optional(), - inputKeys: z.array(z.string()).optional(), - inputSizeBytes: z.number().optional(), - inputSizeChars: z.number().optional(), - inputType: z.string().optional(), - name: z.string().optional(), - output: z.unknown().optional(), - outputKeys: z.array(z.string()).optional(), - outputSizeBytes: z.number().optional(), - outputSizeChars: z.number().optional(), - outputType: z.string().optional(), - redacted: z.boolean().optional(), - sourceType: z.string().optional(), - text: z.string().optional(), - type: transcriptPartTypeSchema, - }) - .strict(); - -export const transcriptRoleSchema = z.enum([ - "assistant", - "system", - "tool", - "toolResult", - "unknown", - "user", -]); - -export const transcriptMessageSchema = z - .object({ - outcome: z.enum(["error", "aborted"]).optional(), - parts: z.array(transcriptPartSchema), - role: transcriptRoleSchema, - timestamp: z.number().optional(), - }) - .strict(); - -export const conversationActivityStatusSchema = z.enum([ - "aborted", - "completed", - "error", - "running", - "success", -]); - -export const conversationSubagentActivityReportSchema = z - .object({ - type: z.literal("subagent"), - createdAt: z.string(), - endedAt: z.string().optional(), - id: z.string(), - modelId: z.string().optional(), - outcome: z.enum(["success", "error", "aborted"]).optional(), - parentToolCallId: z.string().optional(), - reasoningLevel: z.string().optional(), - status: conversationActivityStatusSchema, - subagentKind: z.string(), - transcriptAvailable: z.boolean().optional(), - }) - .strict(); - -export const conversationToolActivityReportSchema = z - .object({ - type: z.literal("tool_execution"), - args: z.unknown().optional(), - createdAt: z.string(), - id: z.string(), - inputKeys: z.array(z.string()).optional(), - inputSizeBytes: z.number().optional(), - inputSizeChars: z.number().optional(), - inputType: z.string().optional(), - redacted: z.boolean().optional(), - status: conversationActivityStatusSchema, - subagents: z.array(conversationSubagentActivityReportSchema), - toolCallId: z.string(), - toolName: z.string(), - }) - .strict(); - -export const conversationActivityReportSchema = z.discriminatedUnion("type", [ - conversationToolActivityReportSchema, - conversationSubagentActivityReportSchema, -]); - const conversationReportVisibleMessageEventDataSchema = z .object({ type: z.literal("visible_message"), @@ -261,73 +165,72 @@ export const conversationReportEventSchema = z }) .strict(); -export const conversationContextEventSchema = z.discriminatedUnion("type", [ +/** Availability of the canonical event history attached to a detail report. */ +export const conversationEventHistorySchema = z.discriminatedUnion("status", [ + z.object({ status: z.literal("available") }).strict(), z .object({ - type: z.literal("context_compacted"), - createdAt: z.string(), - modelId: z.string().optional(), - summary: z.string().optional(), - transcriptIndex: z.number().int().nonnegative(), + status: z.literal("redacted"), + reason: z.literal("non_public_conversation"), }) .strict(), z .object({ - type: z.literal("model_handoff"), - createdAt: z.string(), - fromModelId: z.string().optional(), - message: z.string().optional(), - toModelId: z.string(), - transcriptIndex: z.number().int().nonnegative(), + status: z.literal("expired"), + expiredAt: z.string().datetime(), }) .strict(), ]); export const conversationDetailReportSchema = conversationSummaryReportSchema .extend({ - activity: z.array(conversationActivityReportSchema).optional(), - modelId: z.string().optional(), - reasoningLevel: z.string().optional(), - contextEvents: z.array(conversationContextEventSchema).optional(), - transcriptAvailable: z.boolean(), - transcriptMetadata: z.array(transcriptMessageSchema).optional(), - transcriptMessageCount: z.number().optional(), - transcriptRedacted: z.boolean().optional(), - transcriptRedactionReason: z.literal("non_public_conversation").optional(), - transcriptExpired: z.boolean().optional(), - transcriptExpiredAt: z.string().optional(), - transcript: z.array(transcriptMessageSchema), + events: z.array(conversationReportEventSchema), + eventHistory: conversationEventHistorySchema, generatedAt: z.string(), sentryConversationUrl: z.string().optional(), }) - .strict(); - -export const conversationSubagentTranscriptReportSchema = z - .object({ - type: z.literal("subagent"), - createdAt: z.string(), - endedAt: z.string().optional(), - id: z.string(), - modelId: z.string().optional(), - outcome: z.enum(["success", "error", "aborted"]).optional(), - parentToolCallId: z.string().optional(), - reasoningLevel: z.string().optional(), - status: conversationActivityStatusSchema, - subagentConversationId: z.string().optional(), - subagentKind: z.string(), - subagentSentryConversationUrl: z.string().optional(), - transcript: z.array(transcriptMessageSchema), - transcriptAvailable: z.boolean(), - transcriptMessageCount: z.number().optional(), - transcriptRedacted: z.boolean().optional(), - transcriptRedactionReason: z.literal("non_public_conversation").optional(), - transcriptExpired: z.boolean().optional(), - transcriptExpiredAt: z.string().optional(), - unavailableReason: z - .enum(["missing_transcript_range", "missing_transcript_ref", "not_found"]) - .optional(), - }) - .strict(); + .strict() + .superRefine((report, context) => { + if (report.eventHistory.status === "expired" && report.events.length > 0) { + context.addIssue({ + code: "custom", + path: ["events"], + message: "expired event history must not contain events", + }); + } + for (let index = 1; index < report.events.length; index += 1) { + if (report.events[index]!.seq <= report.events[index - 1]!.seq) { + context.addIssue({ + code: "custom", + path: ["events", index, "seq"], + message: "report event sequences must be strictly increasing", + }); + } + } + for (const [index, event] of report.events.entries()) { + if (event.data.type !== "visible_message") continue; + if ( + report.eventHistory.status === "redacted" && + event.data.redacted !== true + ) { + context.addIssue({ + code: "custom", + path: ["events", index, "data"], + message: "redacted event history must redact visible messages", + }); + } + if ( + report.eventHistory.status === "available" && + event.data.redacted === true + ) { + context.addIssue({ + code: "custom", + path: ["events", index, "data"], + message: "available event history must expose visible messages", + }); + } + } + }); export const conversationFeedSchema = z .object({ @@ -386,37 +289,18 @@ export type ActorIdentity = z.infer; export type ConversationSummaryReport = z.infer< typeof conversationSummaryReportSchema >; -export type TranscriptPartType = z.infer; -export type TranscriptPart = z.infer; -export type TranscriptRole = z.infer; -export type TranscriptMessage = z.infer; -export type ConversationContextEvent = z.infer< - typeof conversationContextEventSchema ->; -export type ConversationActivityStatus = z.infer< - typeof conversationActivityStatusSchema ->; -export type ConversationSubagentActivityReport = z.infer< - typeof conversationSubagentActivityReportSchema ->; -export type ConversationToolActivityReport = z.infer< - typeof conversationToolActivityReportSchema ->; -export type ConversationActivityReport = z.infer< - typeof conversationActivityReportSchema ->; export type ConversationReportEventData = z.infer< typeof conversationReportEventDataSchema >; export type ConversationReportEvent = z.infer< typeof conversationReportEventSchema >; +export type ConversationEventHistory = z.infer< + typeof conversationEventHistorySchema +>; export type ConversationDetailReport = z.infer< typeof conversationDetailReportSchema >; -export type ConversationSubagentTranscriptReport = z.infer< - typeof conversationSubagentTranscriptReportSchema ->; export type ConversationFeed = z.infer; export type ConversationStatsItem = z.infer; export type ConversationMetricDay = z.infer; diff --git a/packages/junior/src/api/conversations/transcript.ts b/packages/junior/src/api/conversations/transcript.ts deleted file mode 100644 index def1a0d59..000000000 --- a/packages/junior/src/api/conversations/transcript.ts +++ /dev/null @@ -1,431 +0,0 @@ -import { isRecord } from "@/chat/coerce"; -import type { ConversationModelMessage } from "@/chat/conversations/history"; -import { unwrapCurrentInstruction } from "@/chat/current-instruction"; -import { unescapeXml } from "@/chat/xml"; -import type { - ConversationSubagentActivityReport, - ConversationSubagentTranscriptReport, - TranscriptMessage, - TranscriptPart, - TranscriptRole, -} from "./schema"; - -const SAFE_METADATA_KEY_LIMIT = 20; -const LEGACY_ADVISOR_TASK_OPEN = "\n"; -const LEGACY_ADVISOR_TASK_CLOSE = "\n"; -const LEGACY_EXECUTOR_CONTEXT_OPEN = "\n"; -const LEGACY_EXECUTOR_CONTEXT_CLOSE = "\n"; - -/** Decode the advisor request wire format retained by historical SQL child rows. */ -function unwrapLegacyAdvisorTask(text: string): string | undefined { - // TODO(v0.97.0): Remove after raw advisor-task child messages are upgraded in SQL. - if ( - !text.startsWith(LEGACY_ADVISOR_TASK_OPEN) || - !text.endsWith(LEGACY_EXECUTOR_CONTEXT_CLOSE) - ) { - return undefined; - } - - const taskEnd = text.indexOf( - LEGACY_ADVISOR_TASK_CLOSE, - LEGACY_ADVISOR_TASK_OPEN.length, - ); - if (taskEnd < 0) return undefined; - - const contextStart = taskEnd + LEGACY_ADVISOR_TASK_CLOSE.length + 2; - if (!text.startsWith(LEGACY_EXECUTOR_CONTEXT_OPEN, contextStart)) { - return undefined; - } - - const task = text.slice(LEGACY_ADVISOR_TASK_OPEN.length, taskEnd); - const context = text.slice( - contextStart + LEGACY_EXECUTOR_CONTEXT_OPEN.length, - -LEGACY_EXECUTOR_CONTEXT_CLOSE.length, - ); - return `${unescapeXml(task)}\n\nExecutor context:\n${unescapeXml(context)}`; -} - -function textPart(text: string): TranscriptPart { - return { type: "text", text }; -} - -function recordField(value: Record, names: string[]): unknown { - for (const name of names) { - if (value[name] !== undefined) { - return value[name]; - } - } - return undefined; -} - -function thinkingOutput(part: Record): unknown { - const output = recordField(part, ["thinking", "text", "content", "output"]); - return isRecord(output) - ? recordField(output, ["thinking", "text", "content", "output"]) - : output; -} - -function isDisplayableTranscriptPart(part: TranscriptPart): boolean { - if (part.type !== "thinking") return true; - return typeof part.output === "string" && part.output.trim().length > 0; -} - -/** Normalize opaque model content parts for user-facing transcript output. */ -function normalizeTranscriptPart( - part: unknown, - options: { - unwrapCurrentTask?: boolean; - unwrapLegacyAdvisorTask?: boolean; - } = {}, -): TranscriptPart { - const displayText = (text: string) => { - if (options.unwrapCurrentTask) { - const instruction = unwrapCurrentInstruction(text); - if (instruction !== undefined) return instruction; - } - if (options.unwrapLegacyAdvisorTask) { - return unwrapLegacyAdvisorTask(text) ?? text; - } - return text; - }; - - if (typeof part === "string") { - return textPart(displayText(part)); - } - if (!isRecord(part)) { - return { type: "unknown", output: part }; - } - - const rawType = typeof part.type === "string" ? part.type : "unknown"; - if (rawType === "text") { - const text = recordField(part, ["text", "content"]); - return textPart( - typeof text === "string" - ? displayText(text) - : (JSON.stringify(text) ?? ""), - ); - } - if (rawType === "toolCall") { - return { - type: "tool_call", - ...(typeof part.id === "string" ? { id: part.id } : {}), - ...(typeof part.name === "string" ? { name: part.name } : {}), - input: recordField(part, ["arguments", "input", "args"]), - }; - } - if (rawType === "toolResult") { - return { - type: "tool_result", - ...(typeof part.id === "string" ? { id: part.id } : {}), - ...(typeof part.name === "string" ? { name: part.name } : {}), - output: recordField(part, ["result", "output", "content"]), - }; - } - if (rawType === "thinking") { - return { - type: "thinking", - output: thinkingOutput(part), - }; - } - - return { - type: "unknown", - ...(rawType !== "unknown" ? { sourceType: rawType } : {}), - output: part, - }; -} - -function normalizeToolResultMessage( - record: Record, -): TranscriptPart { - const content = record.content; - let output = content; - if (Array.isArray(content) && content.length === 1 && isRecord(content[0])) { - const extracted = recordField(content[0], [ - "text", - "content", - "output", - "result", - ]); - output = extracted !== undefined ? extracted : content; - } - return { - type: "tool_result", - ...(typeof record.toolCallId === "string" ? { id: record.toolCallId } : {}), - ...(typeof record.name === "string" - ? { name: record.name } - : typeof record.toolName === "string" - ? { name: record.toolName } - : {}), - output, - }; -} - -/** Normalize one durable model message into the reporting contract. */ -export function normalizeTranscriptMessage( - message: ConversationModelMessage, -): TranscriptMessage { - return normalizeMessage(message); -} - -/** Normalize a stored subagent message, including bounded legacy formats. */ -export function normalizeSubagentTranscriptMessage( - message: ConversationModelMessage, - subagentKind: string, -): TranscriptMessage { - return normalizeMessage(message, { - unwrapLegacyAdvisorTask: subagentKind === "advisor", - }); -} - -function normalizeMessage( - message: ConversationModelMessage, - options: { unwrapLegacyAdvisorTask?: boolean } = {}, -): TranscriptMessage { - const record = message as unknown as Record; - const content = record.content; - const role = transcriptRole(record.role); - return { - role, - ...(role === "assistant" && - (record.stopReason === "error" || record.stopReason === "aborted") - ? { outcome: record.stopReason } - : {}), - ...(typeof record.timestamp === "number" - ? { timestamp: record.timestamp } - : {}), - parts: - role === "toolResult" - ? [normalizeToolResultMessage(record)] - : Array.isArray(content) - ? content - .map((part) => - normalizeTranscriptPart(part, { - unwrapCurrentTask: role === "user", - unwrapLegacyAdvisorTask: - options.unwrapLegacyAdvisorTask && role === "user", - }), - ) - .filter(isDisplayableTranscriptPart) - : [ - normalizeTranscriptPart(content, { - unwrapCurrentTask: role === "user", - unwrapLegacyAdvisorTask: - options.unwrapLegacyAdvisorTask && role === "user", - }), - ], - }; -} - -function transcriptRole(role: unknown): TranscriptRole { - return role === "assistant" || - role === "system" || - role === "tool" || - role === "toolResult" || - role === "user" - ? role - : "unknown"; -} - -function serializedChars(value: unknown): number { - if (typeof value === "string") return value.length; - return JSON.stringify(value)?.length ?? 0; -} - -function serializedBytes(value: unknown): number { - const serialized = typeof value === "string" ? value : JSON.stringify(value); - return new TextEncoder().encode(serialized ?? "").byteLength; -} - -function payloadType(value: unknown): string { - return Array.isArray(value) ? "array" : typeof value; -} - -function payloadKeys(value: unknown): string[] | undefined { - if (!value || typeof value !== "object" || Array.isArray(value)) { - return undefined; - } - const keys = Object.keys(value as Record).slice( - 0, - SAFE_METADATA_KEY_LIMIT, - ); - return keys.length > 0 ? keys : undefined; -} - -/** Describe a redacted payload without exposing its contents. */ -export function redactedPayloadFields( - prefix: "input" | "output", - value: unknown, -) { - const keys = payloadKeys(value); - return { - [`${prefix}Type`]: payloadType(value), - [`${prefix}SizeBytes`]: serializedBytes(value), - [`${prefix}SizeChars`]: serializedChars(value), - ...(keys ? { [`${prefix}Keys`]: keys } : {}), - }; -} - -function redactTranscriptPart(part: TranscriptPart): TranscriptPart { - if (part.type === "text") { - return { - type: "text", - redacted: true, - bytes: serializedBytes(part.text ?? ""), - chars: serializedChars(part.text ?? ""), - }; - } - if (part.type === "thinking") { - return { - type: "thinking", - redacted: true, - ...redactedPayloadFields("output", part.output), - }; - } - if (part.type === "tool_call") { - return { - type: "tool_call", - redacted: true, - ...(part.id ? { id: part.id } : {}), - ...(part.name ? { name: part.name } : {}), - ...redactedPayloadFields("input", part.input), - }; - } - if (part.type === "tool_result") { - return { - type: "tool_result", - redacted: true, - ...(part.id ? { id: part.id } : {}), - ...(part.name ? { name: part.name } : {}), - ...redactedPayloadFields("output", part.output), - }; - } - return { - type: "unknown", - redacted: true, - ...(part.sourceType ? { sourceType: part.sourceType } : {}), - ...redactedPayloadFields("output", part.output ?? part.input ?? part.text), - }; -} - -/** Redact transcript payloads while retaining safe structural metadata. */ -export function redactTranscriptMessage( - message: TranscriptMessage, -): TranscriptMessage { - return { - role: message.role, - ...(message.outcome ? { outcome: message.outcome } : {}), - ...(typeof message.timestamp === "number" - ? { timestamp: message.timestamp } - : {}), - parts: message.parts.map(redactTranscriptPart), - }; -} - -function isConversationMessageRole(role: TranscriptRole): boolean { - return role === "user" || role === "assistant"; -} - -function hasTextPart(message: TranscriptMessage): boolean { - return message.parts.some((part) => { - if (part.type !== "text") return false; - if (part.redacted) return true; - return typeof part.text === "string" && part.text.trim().length > 0; - }); -} - -function isConversationMessage(message: TranscriptMessage): boolean { - if (!isConversationMessageRole(message.role)) return false; - if (message.role === "assistant") return hasTextPart(message); - return message.parts.length > 0; -} - -/** Count user-visible conversation messages in a normalized transcript. */ -export function countConversationMessages( - transcript: TranscriptMessage[], -): number { - return transcript.filter(isConversationMessage).length; -} - -/** Read the latest trace identifier carried by a transcript message. */ -export function traceIdFromTranscript( - transcript: TranscriptMessage[], -): string | undefined { - for (const message of transcript) { - for (const part of message.parts) { - const text = - part.text ?? - (typeof part.output === "string" - ? part.output - : typeof part.input === "string" - ? part.input - : undefined); - const match = text?.match( - /\btrace[_-]?id["']?\s*[:=]\s*["']?([a-f0-9]{16,32})\b/i, - ); - if (match?.[1]) { - return match[1]; - } - } - } - return undefined; -} - -/** Build a subagent transcript response from its durable activity metadata. */ -export function subagentTranscriptReport( - activity: ConversationSubagentActivityReport, - options: { - subagentConversationId?: string; - subagentSentryConversationUrl?: string; - transcript?: TranscriptMessage[]; - transcriptMessageCount?: number; - transcriptRedacted?: boolean; - transcriptRedactionReason?: "non_public_conversation"; - transcriptExpired?: boolean; - transcriptExpiredAt?: string; - unavailableReason?: ConversationSubagentTranscriptReport["unavailableReason"]; - } = {}, -): ConversationSubagentTranscriptReport { - return { - type: "subagent", - ...(options.subagentConversationId - ? { subagentConversationId: options.subagentConversationId } - : {}), - createdAt: activity.createdAt, - id: activity.id, - ...(activity.modelId ? { modelId: activity.modelId } : {}), - status: activity.status, - ...(options.subagentSentryConversationUrl - ? { subagentSentryConversationUrl: options.subagentSentryConversationUrl } - : {}), - subagentKind: activity.subagentKind, - transcript: options.transcript ?? [], - transcriptAvailable: Boolean(options.transcript?.length), - ...(activity.endedAt ? { endedAt: activity.endedAt } : {}), - ...(activity.outcome ? { outcome: activity.outcome } : {}), - ...(activity.parentToolCallId - ? { parentToolCallId: activity.parentToolCallId } - : {}), - ...(activity.reasoningLevel - ? { reasoningLevel: activity.reasoningLevel } - : {}), - ...(options.transcriptMessageCount !== undefined - ? { transcriptMessageCount: options.transcriptMessageCount } - : {}), - ...(options.transcriptRedacted - ? { transcriptRedacted: options.transcriptRedacted } - : {}), - ...(options.transcriptRedactionReason - ? { transcriptRedactionReason: options.transcriptRedactionReason } - : {}), - ...(options.transcriptExpired - ? { transcriptExpired: options.transcriptExpired } - : {}), - ...(options.transcriptExpiredAt - ? { transcriptExpiredAt: options.transcriptExpiredAt } - : {}), - ...(options.unavailableReason - ? { unavailableReason: options.unavailableReason } - : {}), - }; -} diff --git a/packages/junior/src/api/schema.ts b/packages/junior/src/api/schema.ts index 0b0f645d3..17f054968 100644 --- a/packages/junior/src/api/schema.ts +++ b/packages/junior/src/api/schema.ts @@ -2,19 +2,17 @@ export { dailyConversationActivitySchema } from "./activity"; export type { DailyConversationActivity } from "./activity"; export { conversationDetailReportSchema, + conversationEventHistorySchema, conversationFeedSchema, conversationReportEventDataSchema, conversationReportEventSchema, conversationStatsReportSchema, - conversationSubagentTranscriptReportSchema, } from "./conversations/schema"; export type { ActorIdentity, - ConversationActivityReport, - ConversationActivityStatus, - ConversationContextEvent, ConversationCost, ConversationDetailReport, + ConversationEventHistory, ConversationFeed, ConversationReportEvent, ConversationReportEventData, @@ -22,16 +20,9 @@ export type { ConversationMetricDay, ConversationStatsItem, ConversationStatsReport, - ConversationSubagentActivityReport, - ConversationSubagentTranscriptReport, ConversationSummaryReport, ConversationSurface, - ConversationToolActivityReport, ConversationUsage, - TranscriptMessage, - TranscriptPart, - TranscriptPartType, - TranscriptRole, } from "./conversations/schema"; export { actorDirectoryReportSchema, @@ -96,9 +87,6 @@ export const conversationFeedQuerySchema = z .optional(), }) .strict(); -export const subagentParamsSchema = conversationParamsSchema - .extend({ subagentId: z.string().min(1) }) - .strict(); export const personParamsSchema = z .object({ email: z.string().trim().min(1) }) .strict(); @@ -107,6 +95,5 @@ export const locationParamsSchema = z .strict(); export type ConversationParams = z.infer; -export type SubagentParams = z.infer; export type PersonParams = z.infer; export type LocationParams = z.infer; diff --git a/packages/junior/src/chat/conversations/README.md b/packages/junior/src/chat/conversations/README.md index 952a994a7..8b0c72b82 100644 --- a/packages/junior/src/chat/conversations/README.md +++ b/packages/junior/src/chat/conversations/README.md @@ -43,11 +43,12 @@ contract. Raw `ConversationEventData` is an internal persistence boundary and must not become a dashboard or external API payload. The reporting API owns a strict ordered event projection with only safe product -fields. It preserves canonical sequence order, sources display text only from -visible-message events, reduces model messages to content-free activity, and -omits provider receipts, delivery commands, failure codes, authorization -identifiers, arbitrary metadata, and persistence-envelope fields. The current -detail response and dashboard have not yet cut over to this projection. +fields. Conversation detail exposes that projection as its sole history, and +the dashboard derives its view directly from canonical sequence order. Display +text comes only from visible-message events; model messages become content-free +activity, while provider receipts, delivery commands, failure codes, +authorization identifiers, arbitrary metadata, and persistence-envelope fields +remain internal. ## Write Rules diff --git a/packages/junior/src/chat/conversations/retention.ts b/packages/junior/src/chat/conversations/retention.ts index 2165c45e8..6415657dd 100644 --- a/packages/junior/src/chat/conversations/retention.ts +++ b/packages/junior/src/chat/conversations/retention.ts @@ -9,11 +9,7 @@ */ import { logException, logInfo } from "@/chat/logging"; import type { JuniorSqlDatabase } from "@/db/db"; -import { - purgeConversationTree, - resolveRootVisibility, - selectExpiredRoots, -} from "./sql/purge"; +import { purgeConversationTree, selectExpiredRoots } from "./sql/purge"; const DAY_MS = 24 * 60 * 60 * 1000; @@ -110,10 +106,9 @@ export async function purgeConversation( conversationId: string, opts: { nowMs?: number } = {}, ): Promise { - const { visibility } = await resolveRootVisibility(executor, conversationId); await purgeConversationTree(executor, { rootConversationId: conversationId, - scrubMetadata: visibility !== "public", + scrubMetadataFromRootVisibility: true, nowMs: opts.nowMs ?? Date.now(), }); } diff --git a/packages/junior/src/chat/conversations/sql/privacy.ts b/packages/junior/src/chat/conversations/sql/privacy.ts new file mode 100644 index 000000000..1ba19eb72 --- /dev/null +++ b/packages/junior/src/chat/conversations/sql/privacy.ts @@ -0,0 +1,160 @@ +import { eq } from "drizzle-orm"; +import type { JuniorSqlDatabase } from "@/db/db"; +import { juniorConversations, juniorDestinations } from "@/db/schema"; +import type { JuniorDestinationVisibility } from "@/db/schema/destinations"; + +const MAX_LINEAGE_DEPTH = 32; + +interface LineageRow { + destinationId: string | null; + parentId: string | null; + rootId: string | null; +} + +interface LineageCandidate { + path: string[]; + rootConversationId: string; +} + +async function readLineageRow( + executor: JuniorSqlDatabase, + conversationId: string, + lock: boolean, +): Promise { + const query = executor + .db() + .select({ + parentId: juniorConversations.parentConversationId, + rootId: juniorConversations.rootConversationId, + destinationId: juniorConversations.destinationId, + }) + .from(juniorConversations) + .where(eq(juniorConversations.conversationId, conversationId)); + const rows = lock ? await query.for("share") : await query; + return rows[0]; +} + +async function traceLineage( + executor: JuniorSqlDatabase, + conversationId: string, +): Promise { + let currentId = conversationId; + let declaredRootId: string | null | undefined; + const path: string[] = []; + const seen = new Set(); + + while (!seen.has(currentId) && seen.size < MAX_LINEAGE_DEPTH) { + seen.add(currentId); + path.push(currentId); + const row = await readLineageRow(executor, currentId, false); + if (!row) return undefined; + + if (declaredRootId === undefined) declaredRootId = row.rootId; + if (row.parentId) { + if (!declaredRootId || row.rootId !== declaredRootId) return undefined; + currentId = row.parentId; + continue; + } + + if ( + row.rootId !== null || + (declaredRootId !== null && declaredRootId !== currentId) || + !row.destinationId + ) { + return undefined; + } + return { path, rootConversationId: currentId }; + } + + return undefined; +} + +function lockedLineageIsConsistent( + candidate: LineageCandidate, + rows: Map, +): boolean { + const declaredRootId = rows.get(candidate.path[0]!)?.rootId; + for (const [index, conversationId] of candidate.path.entries()) { + const row = rows.get(conversationId); + if (!row) return false; + const expectedParentId = candidate.path[index + 1] ?? null; + if (row.parentId !== expectedParentId) return false; + if (expectedParentId) { + if (!declaredRootId || row.rootId !== declaredRootId) return false; + continue; + } + if ( + row.rootId !== null || + (declaredRootId !== null && + declaredRootId !== candidate.rootConversationId) || + !row.destinationId + ) { + return false; + } + } + return true; +} + +/** + * Resolve a conversation's privacy authority from its persisted root. + * + * The lineage is discovered without locks, then locked and revalidated from + * root to requested conversation. Root-first ordering matches tree purges; + * callers that keep the transaction open receive a stable privacy decision. + * Missing, cyclic, over-depth, historically uncorrelated, or internally + * inconsistent lineage fails closed. + */ +export async function resolveRootVisibility( + executor: JuniorSqlDatabase, + conversationId: string, +): Promise<{ + rootConversationId: string; + visibility: JuniorDestinationVisibility | null; +}> { + const candidate = await traceLineage(executor, conversationId); + if (!candidate) { + return { rootConversationId: conversationId, visibility: null }; + } + + const rootRow = await readLineageRow( + executor, + candidate.rootConversationId, + true, + ); + if (!rootRow?.destinationId) { + return { + rootConversationId: candidate.rootConversationId, + visibility: null, + }; + } + const destinations = await executor + .db() + .select({ visibility: juniorDestinations.visibility }) + .from(juniorDestinations) + .where(eq(juniorDestinations.id, rootRow.destinationId)) + .for("share"); + + const lockedRows = new Map([ + [candidate.rootConversationId, rootRow], + ]); + for (const id of [...candidate.path].reverse().slice(1)) { + const row = await readLineageRow(executor, id, true); + if (!row) { + return { + rootConversationId: candidate.rootConversationId, + visibility: null, + }; + } + lockedRows.set(id, row); + } + if (!lockedLineageIsConsistent(candidate, lockedRows)) { + return { + rootConversationId: candidate.rootConversationId, + visibility: null, + }; + } + return { + rootConversationId: candidate.rootConversationId, + visibility: destinations[0]?.visibility ?? null, + }; +} diff --git a/packages/junior/src/chat/conversations/sql/purge.ts b/packages/junior/src/chat/conversations/sql/purge.ts index a58b52ce3..5f924cc12 100644 --- a/packages/junior/src/chat/conversations/sql/purge.ts +++ b/packages/junior/src/chat/conversations/sql/purge.ts @@ -1,12 +1,14 @@ import { and, asc, eq, inArray, isNull, sql } from "drizzle-orm"; import type { JuniorSqlDatabase } from "@/db/db"; +import type { JuniorDestinationVisibility } from "@/db/schema/destinations"; import { juniorConversationEvents, juniorConversationMessages, juniorConversations, juniorDestinations, } from "@/db/schema"; -import type { JuniorDestinationVisibility } from "@/db/schema/destinations"; +import { withConversationEventLock } from "./event-lock"; +import { resolveRootVisibility } from "./privacy"; /** An expired root conversation selected for purge, with its resolved visibility. */ export interface ExpiredRoot { @@ -22,28 +24,112 @@ export interface PurgeTreeResult { conversations: number; } -/** Collect a conversation and every descendant via `parent_conversation_id`. */ -async function conversationTreeIds( +interface ConversationTreeRow { + conversationId: string; + depth: number; + lastActivityAt: Date; + parentConversationId: string | null; +} + +const MAX_TREE_LOCK_PASSES = 32; + +/** Discover a root and its current descendants via `parent_conversation_id`. */ +async function discoverConversationTree( executor: JuniorSqlDatabase, - rootConversationId: string, -): Promise { - const all = new Set([rootConversationId]); - let frontier = [rootConversationId]; + root: Omit, +): Promise { + const all = new Map([ + [root.conversationId, { ...root, depth: 0 }], + ]); + let frontier = [root.conversationId]; + let depth = 1; while (frontier.length > 0) { const children = await executor .db() - .select({ id: juniorConversations.conversationId }) + .select({ + conversationId: juniorConversations.conversationId, + lastActivityAt: juniorConversations.lastActivityAt, + parentConversationId: juniorConversations.parentConversationId, + }) .from(juniorConversations) - .where(inArray(juniorConversations.parentConversationId, frontier)); + .where(inArray(juniorConversations.parentConversationId, frontier)) + .orderBy(asc(juniorConversations.conversationId)); frontier = []; for (const child of children) { - if (!all.has(child.id)) { - all.add(child.id); - frontier.push(child.id); + if (!all.has(child.conversationId)) { + all.set(child.conversationId, { ...child, depth }); + frontier.push(child.conversationId); + } + } + depth += 1; + } + return [...all.values()]; +} + +/** + * Lock a complete tree root-first and repeat discovery until every row in the + * final scan is locked. Locking known parents blocks normal FK-backed child + * creation; the repeat catches children attached to a not-yet-locked parent + * during an earlier scan. + */ +async function lockStableConversationTree( + executor: JuniorSqlDatabase, + root: Omit, +): Promise { + const locked = new Map([ + [root.conversationId, { ...root, depth: 0 }], + ]); + + for (let pass = 0; pass < MAX_TREE_LOCK_PASSES; pass += 1) { + const discovered = await discoverConversationTree(executor, root); + for (const candidate of discovered) { + const existing = locked.get(candidate.conversationId); + if (existing) { + if (existing.parentConversationId !== candidate.parentConversationId) { + return undefined; + } + continue; + } + await withConversationEventLock( + executor, + candidate.conversationId, + async () => undefined, + ); + const rows = await executor + .db() + .select({ + conversationId: juniorConversations.conversationId, + lastActivityAt: juniorConversations.lastActivityAt, + parentConversationId: juniorConversations.parentConversationId, + }) + .from(juniorConversations) + .where(eq(juniorConversations.conversationId, candidate.conversationId)) + .for("update"); + const row = rows[0]; + if (!row || row.parentConversationId !== candidate.parentConversationId) { + return undefined; } + locked.set(row.conversationId, { ...row, depth: candidate.depth }); + } + + const finalScan = await discoverConversationTree(executor, root); + if ( + finalScan.length === locked.size && + finalScan.every( + (row) => + locked.get(row.conversationId)?.parentConversationId === + row.parentConversationId, + ) + ) { + return [...locked.values()].sort( + (left, right) => + left.depth - right.depth || + left.conversationId.localeCompare(right.conversationId), + ); } } - return [...all]; + + return undefined; } /** @@ -68,6 +154,16 @@ export async function selectExpiredRoots( args.nowMs - args.privateWindowMs, ).toISOString(); const cutoff = sql`case when ${juniorDestinations.visibility} = 'public' then ${publicCutoff}::timestamptz else ${privateCutoff}::timestamptz end`; + const effectiveLastActivityAt = sql`( + with recursive conversation_tree(conversation_id, last_activity_at) as ( + select ${juniorConversations.conversationId}, ${juniorConversations.lastActivityAt} + union all + select child.conversation_id, child.last_activity_at + from junior_conversations child + join conversation_tree parent on child.parent_conversation_id = parent.conversation_id + ) + select max(tree.last_activity_at) from conversation_tree tree + )`; const hasTreeWork = sql`exists ( with recursive conversation_tree(conversation_id) as ( select ${juniorConversations.conversationId} @@ -113,12 +209,12 @@ export async function selectExpiredRoots( .where( and( isNull(juniorConversations.parentConversationId), - sql`${juniorConversations.lastActivityAt} < ${cutoff}`, + sql`${effectiveLastActivityAt} < ${cutoff}`, hasTreeWork, ), ) .orderBy( - asc(juniorConversations.lastActivityAt), + asc(effectiveLastActivityAt), asc(juniorConversations.conversationId), ) .limit(Math.max(0, args.limit)); @@ -140,14 +236,46 @@ export async function purgeConversationTree( args: { rootConversationId: string; scrubMetadata?: boolean; + scrubMetadataFromRootVisibility?: boolean; nowMs: number; retention?: { privateWindowMs: number; publicWindowMs: number }; }, ): Promise { return await executor.transaction(async () => { + const initialRoots = await executor + .db() + .select({ + conversationId: juniorConversations.conversationId, + destinationId: juniorConversations.destinationId, + lastActivityAt: juniorConversations.lastActivityAt, + parentConversationId: juniorConversations.parentConversationId, + }) + .from(juniorConversations) + .where(eq(juniorConversations.conversationId, args.rootConversationId)); + const initialRoot = initialRoots[0]; + if ( + !initialRoot || + (args.retention && initialRoot.parentConversationId !== null) + ) { + return { purged: false, conversations: 0 }; + } + const initialTree = await discoverConversationTree(executor, initialRoot); + for (const conversation of initialTree) { + await withConversationEventLock( + executor, + conversation.conversationId, + async () => undefined, + ); + } + const resolvedScrubMetadata = args.scrubMetadataFromRootVisibility + ? (await resolveRootVisibility(executor, args.rootConversationId)) + .visibility !== "public" + : args.scrubMetadata; + const roots = await executor .db() .select({ + conversationId: juniorConversations.conversationId, destinationId: juniorConversations.destinationId, lastActivityAt: juniorConversations.lastActivityAt, parentConversationId: juniorConversations.parentConversationId, @@ -165,17 +293,29 @@ export async function purgeConversationTree( .select({ visibility: juniorDestinations.visibility }) .from(juniorDestinations) .where(eq(juniorDestinations.id, root.destinationId)) + .for("share") : []; const isPublic = destinations[0]?.visibility === "public"; + const tree = await lockStableConversationTree(executor, { + conversationId: root.conversationId, + lastActivityAt: root.lastActivityAt, + parentConversationId: root.parentConversationId, + }); + if (!tree) { + return { purged: false, conversations: 0 }; + } if (args.retention) { const windowMs = isPublic ? args.retention.publicWindowMs : args.retention.privateWindowMs; - if (root.lastActivityAt.getTime() >= args.nowMs - windowMs) { + const effectiveLastActivityAt = Math.max( + ...tree.map((conversation) => conversation.lastActivityAt.getTime()), + ); + if (effectiveLastActivityAt >= args.nowMs - windowMs) { return { purged: false, conversations: 0 }; } } - const ids = await conversationTreeIds(executor, args.rootConversationId); + const ids = tree.map((conversation) => conversation.conversationId); await executor .db() .delete(juniorConversationEvents) @@ -189,7 +329,7 @@ export async function purgeConversationTree( .update(juniorConversations) .set({ transcriptPurgedAt: new Date(args.nowMs), - ...((args.retention ? !isPublic : args.scrubMetadata) + ...((args.retention ? !isPublic : resolvedScrubMetadata) ? { title: null, channelName: null, actor: null } : {}), }) @@ -197,43 +337,3 @@ export async function purgeConversationTree( return { purged: true, conversations: ids.length }; }); } - -/** - * Walk `parent_conversation_id` to the root and return the root's destination - * visibility. Retention resolves visibility at purge time rather than storing an - * expiry, so a public↔private flip takes effect on the next pass. - */ -export async function resolveRootVisibility( - executor: JuniorSqlDatabase, - conversationId: string, -): Promise<{ - rootConversationId: string; - visibility: JuniorDestinationVisibility | null; -}> { - let currentId = conversationId; - const seen = new Set(); - while (!seen.has(currentId)) { - seen.add(currentId); - const rows = await executor - .db() - .select({ - parentId: juniorConversations.parentConversationId, - visibility: juniorDestinations.visibility, - }) - .from(juniorConversations) - .leftJoin( - juniorDestinations, - eq(juniorDestinations.id, juniorConversations.destinationId), - ) - .where(eq(juniorConversations.conversationId, currentId)); - const row = rows[0]; - if (!row) { - return { rootConversationId: currentId, visibility: null }; - } - if (!row.parentId) { - return { rootConversationId: currentId, visibility: row.visibility }; - } - currentId = row.parentId; - } - return { rootConversationId: currentId, visibility: null }; -} diff --git a/packages/junior/tests/component/conversation-transcripts-sql.test.ts b/packages/junior/tests/component/conversation-transcripts-sql.test.ts index a3592cb8e..e5cc68b32 100644 --- a/packages/junior/tests/component/conversation-transcripts-sql.test.ts +++ b/packages/junior/tests/component/conversation-transcripts-sql.test.ts @@ -1,4 +1,6 @@ +import { fileURLToPath } from "node:url"; import { asc, eq } from "drizzle-orm"; +import { readMigrationFiles } from "drizzle-orm/migrator"; import { describe, expect, it } from "vitest"; import { createSqlConversationEventStore } from "@/chat/conversations/sql/history"; import { @@ -34,6 +36,9 @@ import { ConversationTurnLifecycleService } from "@/chat/conversations/turn-life const CONVERSATION_ID = "slack:C123:1718123456.000000"; const CHILD_CONVERSATION_ID = "advisor:child-1"; +const coreMigrationCount = readMigrationFiles({ + migrationsFolder: fileURLToPath(new URL("../../migrations", import.meta.url)), +}).length; async function listMessageRows( fixture: LocalJuniorSqlFixture, @@ -360,7 +365,7 @@ describe("conversation transcript SQL stores", () => { const [applied] = await fixture.sql.query<{ count: number }>( "SELECT count(*)::integer AS count FROM drizzle.__drizzle_junior_core", ); - expect(applied?.count).toBe(6); + expect(applied?.count).toBe(coreMigrationCount); } finally { await fixture.close(); } diff --git a/packages/junior/tests/component/conversations/retention.test.ts b/packages/junior/tests/component/conversations/retention.test.ts index 0b5a47335..23c3b5192 100644 --- a/packages/junior/tests/component/conversations/retention.test.ts +++ b/packages/junior/tests/component/conversations/retention.test.ts @@ -280,6 +280,42 @@ describe("retention purge job", () => { expect(await eventCount(fixture.sql, "child")).toBe(0); }); + it("uses the freshest activity in the tree for selection and destructive recheck", async () => { + const destinationId = await seedDestination(fixture.sql, "public"); + const nowMs = BASE_MS + 100 * DAY_MS; + await seedConversation(fixture.sql, { + conversationId: "old-root", + destinationId, + lastActivityAtMs: BASE_MS, + }); + await seedConversation(fixture.sql, { + conversationId: "fresh-child", + parentConversationId: "old-root", + lastActivityAtMs: nowMs - DAY_MS, + }); + + await expect( + selectExpiredRoots(fixture.sql, { + nowMs, + publicWindowMs: 90 * DAY_MS, + privateWindowMs: 14 * DAY_MS, + limit: 10, + }), + ).resolves.toEqual([]); + await expect( + purgeConversationTree(fixture.sql, { + rootConversationId: "old-root", + nowMs, + retention: { + publicWindowMs: 90 * DAY_MS, + privateWindowMs: 14 * DAY_MS, + }, + }), + ).resolves.toEqual({ purged: false, conversations: 0 }); + expect(await eventCount(fixture.sql, "old-root")).toBe(1); + expect(await eventCount(fixture.sql, "fresh-child")).toBe(1); + }); + it("purges an expired root whose remaining content exists only on a child", async () => { const dest = await seedDestination(fixture.sql, "public"); await seedConversation(fixture.sql, { diff --git a/packages/junior/tests/component/conversations/shared-lineage-projection.test.ts b/packages/junior/tests/component/conversations/shared-lineage-projection.test.ts index e594a6f42..0d8dd4eea 100644 --- a/packages/junior/tests/component/conversations/shared-lineage-projection.test.ts +++ b/packages/junior/tests/component/conversations/shared-lineage-projection.test.ts @@ -18,7 +18,7 @@ import { } from "@/chat/db"; import type { PiMessage } from "@/chat/pi/messages"; import { juniorConversations } from "@/db/schema"; -import { resolveRootVisibility } from "@/chat/conversations/sql/purge"; +import { resolveRootVisibility } from "@/chat/conversations/sql/privacy"; import { purgeConversation } from "@/chat/conversations/retention"; import { getAgentTurnSessionRecord, diff --git a/packages/junior/tests/component/services/subagent-lineage.test.ts b/packages/junior/tests/component/services/subagent-lineage.test.ts index 050496387..1a42bae21 100644 --- a/packages/junior/tests/component/services/subagent-lineage.test.ts +++ b/packages/junior/tests/component/services/subagent-lineage.test.ts @@ -4,7 +4,7 @@ import { eq } from "drizzle-orm"; import { migrateSchema } from "@/chat/conversations/sql/migrations"; import { createSqlConversationEventStore } from "@/chat/conversations/sql/history"; import { createSqlStore } from "@/chat/conversations/sql/store"; -import { resolveRootVisibility } from "@/chat/conversations/sql/purge"; +import { resolveRootVisibility } from "@/chat/conversations/sql/privacy"; import { SubagentLineageConflictError, SubagentLineageService, diff --git a/packages/junior/tests/integration/api/rest.test.ts b/packages/junior/tests/integration/api/rest.test.ts index 3c0d43da6..50e0fa70e 100644 --- a/packages/junior/tests/integration/api/rest.test.ts +++ b/packages/junior/tests/integration/api/rest.test.ts @@ -157,9 +157,11 @@ describe("Junior REST API", () => { "http://localhost/api/conversations", ); expect( - ((await defaultFeedAfterArchive.json()) as { - conversations: Array<{ conversationId: string }>; - }).conversations, + ( + (await defaultFeedAfterArchive.json()) as { + conversations: Array<{ conversationId: string }>; + } + ).conversations, ).not.toEqual( expect.arrayContaining([expect.objectContaining({ conversationId })]), ); @@ -187,6 +189,11 @@ describe("Junior REST API", () => { conversationId, }); + const removedSubagentDetail = await app.request( + `http://localhost/api/conversations/${encodeURIComponent(conversationId)}/subagents/legacy-subagent`, + ); + expect(removedSubagentDetail.status).toBe(404); + const people = await app.request("http://localhost/api/people"); expect(people.status).toBe(200); await expect(people.json()).resolves.toMatchObject({ diff --git a/packages/junior/tests/integration/dashboard-reporting.test.ts b/packages/junior/tests/integration/dashboard-reporting.test.ts index d5fc0939d..54fae548c 100644 --- a/packages/junior/tests/integration/dashboard-reporting.test.ts +++ b/packages/junior/tests/integration/dashboard-reporting.test.ts @@ -1,16 +1,17 @@ +import { eq } from "drizzle-orm"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import type { PiMessage } from "@/chat/pi/messages"; import { readConversationDetail } from "@/api/conversations/detail"; -import { readConversationSubagent as readConversationSubagentTranscriptReport } from "@/api/conversations/subagent"; -import { buildTurnFailureResponse } from "@/chat/logging"; -import { SubagentLineageService } from "@/chat/services/subagent-lineage"; - -vi.mock("@/chat/prompt", () => ({ - buildSystemPrompt: vi.fn(() => "[system prompt]"), - buildTurnContextPrompt: vi.fn(() => null), - JUNIOR_PERSONALITY: "", - JUNIOR_WORLD: null, -})); +import { createSqlConversationEventStore } from "@/chat/conversations/sql/history"; +import { createSqlConversationMessageStore } from "@/chat/conversations/sql/messages"; +import { purgeConversation } from "@/chat/conversations/retention"; +import type { PiMessage } from "@/chat/pi/messages"; +import { createPostgresJuniorSqlExecutor } from "@/db/postgres"; +import { + juniorConversationEvents, + juniorConversationMessages, + juniorConversations, + juniorDestinations, +} from "@/db/schema"; const ORIGINAL_ENV = { ...process.env }; const TEST_DATABASE_URL = ORIGINAL_ENV.DATABASE_URL; @@ -21,53 +22,215 @@ if (!TEST_DATABASE_URL) { ); } -async function readConversationDetailReport(conversationId: string) { - const report = await readConversationDetail(conversationId); - if (!report) throw new Error(`Missing SQL conversation ${conversationId}`); - return report; -} - -/** - * Record a source-confirmed public destination so reads may expose raw - * content, mirroring a live event whose channel_type was "channel". - */ -async function confirmPublicSlackConversation( +async function recordRoot( conversationId: string, - channelId = "C1", -) { + visibility: "private" | "public", +): Promise { const { getConversationStore } = await import("@/chat/db"); await getConversationStore().recordActivity({ conversationId, - destination: { platform: "slack", teamId: "T1", channelId }, - visibility: "public", + destination: { + platform: "slack", + teamId: "T-reporting", + channelId: `C-${conversationId}`, + }, + nowMs: 1, + source: "slack", + title: "Canonical event report", + visibility, }); } -async function recordVisibleTranscript( +async function appendVisibleHistory( conversationId: string, - messages: Array<{ - role: "assistant" | "system" | "user"; - text: string; - timestamp: number; - }>, -) { + text = "Visible answer", +): Promise { const { getConversationEventStore } = await import("@/chat/db"); - await getConversationEventStore().append( - conversationId, - messages.map((message, index) => ({ - idempotencyKey: `test-visible:${message.timestamp}:${index}:${message.role}`, + await getConversationEventStore().append(conversationId, [ + { data: { - type: "visible_message_recorded" as const, - messageId: `visible:${message.timestamp}:${index}:${message.role}`, - role: message.role, - text: message.text, + type: "visible_message_recorded", + messageId: `${conversationId}:visible`, + role: "assistant", + text, }, - createdAtMs: message.timestamp, - })), - ); + createdAtMs: 10, + }, + { + data: { + type: "message", + message: { + role: "assistant", + content: [{ type: "text", text: "private model-only duplicate" }], + timestamp: 11, + } as PiMessage, + }, + createdAtMs: 11, + }, + { + data: { + type: "tool_execution_started", + toolCallId: `${conversationId}:tool-call`, + toolName: "search", + args: { secret: "must not leave persistence" }, + }, + createdAtMs: 12, + }, + { + data: { + type: "turn_started", + turnId: `${conversationId}:turn`, + inputMessageIds: [`${conversationId}:visible`], + surface: "internal", + }, + createdAtMs: 13, + }, + { + data: { + type: "turn_completed", + turnId: `${conversationId}:turn`, + outcome: "success", + }, + createdAtMs: 14, + }, + { + data: { + type: "delivery_intended", + deliveryId: `${conversationId.replaceAll(":", "_")}_delivery`, + correlation: { kind: "turn", turnId: `${conversationId}:turn` }, + messageId: `${conversationId}:visible`, + deliveryKind: "assistant_reply", + provider: "slack", + partCount: 1, + }, + createdAtMs: 15, + }, + { + data: { + type: "delivery_accepted", + deliveryId: `${conversationId.replaceAll(":", "_")}_delivery`, + providerMessageIds: ["123.456"], + }, + createdAtMs: 16, + }, + { + data: { + type: "subagent_started", + subagentInvocationId: `${conversationId}:subagent-call`, + subagentKind: "review", + childConversationId: `${conversationId}:child`, + historyMode: "isolated", + }, + createdAtMs: 17, + }, + { + data: { + type: "subagent_ended", + subagentInvocationId: `${conversationId}:subagent-call`, + outcome: "success", + }, + createdAtMs: 18, + }, + ]); + await getConversationEventStore().startEpoch(conversationId, { + reason: "compaction", + modelProfile: "standard", + modelId: "private-model-id", + messages: [], + }); + await getConversationEventStore().startEpoch(conversationId, { + reason: "handoff", + modelProfile: "fast", + modelId: "private-handoff-model-id", + messages: [], + }); +} + +async function createChild(args: { + childConversationId: string; + parentConversationId: string; +}): Promise { + const { getConversationEventStore, getSubagentLineageService } = + await import("@/chat/db"); + await getConversationEventStore().append(args.parentConversationId, [ + { + data: { + type: "turn_started", + turnId: `${args.parentConversationId}:turn`, + inputMessageIds: [`${args.parentConversationId}:input`], + surface: "internal", + }, + createdAtMs: 2, + }, + ]); + await getSubagentLineageService().start({ + childConversationId: args.childConversationId, + historyMode: "isolated", + parentConversationId: args.parentConversationId, + parentTurnId: `${args.parentConversationId}:turn`, + subagentInvocationId: `${args.childConversationId}:call`, + subagentKind: "task", + nowMs: 3, + }); + await appendVisibleHistory(args.childConversationId, "Child answer"); +} + +async function requireDetail(conversationId: string) { + const detail = await readConversationDetail(conversationId); + if (!detail) throw new Error(`Missing detail for ${conversationId}`); + return detail; +} + +function deferred(): { promise: Promise; resolve: () => void } { + let resolve!: () => void; + const promise = new Promise((done) => { + resolve = done; + }); + return { promise, resolve }; +} + +async function waitUntilWaitingOnEventTable( + observer: ReturnType, +): Promise { + for (let attempt = 0; attempt < 200; attempt += 1) { + const [row] = await observer.query<{ count: number }>(` + select count(*)::integer as count + from pg_stat_activity + where datname = current_database() + and pid <> pg_backend_pid() + and wait_event_type = 'Lock' + and query ilike '%junior_conversation_events%' + `); + if ((row?.count ?? 0) > 0) return; + await new Promise((resolve) => setTimeout(resolve, 10)); + } + throw new Error("Detail read did not block while loading event history"); +} + +async function waitUntilApplicationWaitsOnLock( + observer: ReturnType, + applicationName: string, + queryFragment: string, +): Promise { + for (let attempt = 0; attempt < 200; attempt += 1) { + const [row] = await observer.query<{ count: number }>( + ` + select count(*)::integer as count + from pg_stat_activity + where datname = current_database() + and pid <> pg_backend_pid() + and wait_event_type = 'Lock' + and query ilike $1 + `, + [`%${queryFragment}%`], + ); + if ((row?.count ?? 0) > 0) return; + await new Promise((resolve) => setTimeout(resolve, 10)); + } + throw new Error(`${applicationName} did not reach the expected lock wait`); } -describe("dashboard reporting", () => { +describe("dashboard canonical event reporting", () => { beforeEach(async () => { process.env = { ...ORIGINAL_ENV, @@ -75,8 +238,6 @@ describe("dashboard reporting", () => { JUNIOR_STATE_ADAPTER: "memory", }; vi.resetModules(); - const { disconnectStateAdapter } = await import("@/chat/state/adapter"); - await disconnectStateAdapter(); }); afterEach(async () => { @@ -84,2179 +245,523 @@ describe("dashboard reporting", () => { const { closeDb } = await import("@/chat/db"); await closeDb(); await disconnectStateAdapter(); - vi.useRealTimers(); vi.resetModules(); process.env = { ...ORIGINAL_ENV }; }); - it("indexes recent turn session summaries", async () => { - const { listAgentTurnSessionSummaries, upsertAgentTurnSessionRecord } = - await import("@/chat/state/turn-session"); - - await upsertAgentTurnSessionRecord({ - modelId: "test/model", - conversationId: "slack:C1:111", - sessionId: "turn-1", - sliceId: 1, - state: "running", - piMessages: [], - }); - await upsertAgentTurnSessionRecord({ - modelId: "test/model", - conversationId: "slack:C1:111", - sessionId: "turn-1", - sliceId: 2, - state: "completed", - piMessages: [], - cumulativeDurationMs: 1_200, - errorMessage: "provider failed with sensitive details", - loadedSkillNames: ["triage"], - }); - await upsertAgentTurnSessionRecord({ - modelId: "test/model", - conversationId: "slack:C2:222", - sessionId: "turn-2", - sliceId: 1, - state: "awaiting_resume", - piMessages: [], - resumeReason: "timeout", - }); - - const summaries = await listAgentTurnSessionSummaries(); - const turn1 = summaries.find((summary) => summary.sessionId === "turn-1"); - const turn2 = summaries.find((summary) => summary.sessionId === "turn-2"); - - expect( - summaries.filter((summary) => summary.sessionId === "turn-1"), - ).toHaveLength(1); - expect(turn1).toMatchObject({ - conversationId: "slack:C1:111", - sessionId: "turn-1", - sliceId: 2, - state: "completed", - cumulativeDurationMs: 1_200, - loadedSkillNames: ["triage"], - }); - expect(turn1?.startedAtMs).toBeLessThanOrEqual(turn1?.updatedAtMs ?? 0); - expect(turn1).not.toHaveProperty("errorMessage"); - expect(turn2).toMatchObject({ - conversationId: "slack:C2:222", - cumulativeDurationMs: 0, - sessionId: "turn-2", - state: "awaiting_resume", - resumeReason: "timeout", - }); - }); - - it("mirrors local turn sessions as local conversation summaries", async () => { - const { recordAgentTurnSessionSummary } = - await import("@/chat/state/turn-session"); - const { getConversationStore } = await import("@/chat/db"); - const conversationId = "local:workspace:run-123"; - - await recordAgentTurnSessionSummary({ - conversationId, - destination: { - platform: "local", - conversationId, - }, - sessionId: "local-turn-1", - sliceId: 1, - state: "completed", - surface: "internal", - ttlMs: 60_000, - }); - - await expect( - getConversationStore().get({ - conversationId, - }), - ).resolves.toMatchObject({ - conversationId, - source: "local", - }); - }); - - it("redacts private conversation summaries", async () => { - const { getConversationStore } = await import("@/chat/db"); - const { listRecentConversationSummaries } = - await import("@/reporting/plugin-conversations"); - const conversationStore = getConversationStore(); - - await conversationStore.recordActivity({ - conversationId: "slack:G1:222", - channelName: "private-incident-room", - nowMs: 1_000, - source: "slack", - title: "Sensitive escalation", - }); - - const summaries = await listRecentConversationSummaries(); - - expect(JSON.stringify(summaries)).not.toContain("private-incident-room"); - expect(JSON.stringify(summaries)).not.toContain("Sensitive escalation"); - expect(summaries[0]).toMatchObject({ - conversationId: "slack:G1:222", - status: "completed", - }); - }); - - it("redacts C-prefixed conversations Slack reports as private", async () => { - const { getConversationStore } = await import("@/chat/db"); - const { listRecentConversationSummaries } = - await import("@/reporting/plugin-conversations"); - const conversationStore = getConversationStore(); - - // Modern Slack private channels use C-prefixed ids; the event said - // channel_type: group, so the destination is confirmed private. - await conversationStore.recordActivity({ - conversationId: "slack:C9:333", - channelName: "stealth-project", - destination: { platform: "slack", teamId: "T1", channelId: "C9" }, - nowMs: 1_000, - source: "slack", - title: "Stealth planning", - visibility: "private", - }); - - const summaries = await listRecentConversationSummaries(); - - expect(JSON.stringify(summaries)).not.toContain("stealth-project"); - expect(JSON.stringify(summaries)).not.toContain("Stealth planning"); - expect(summaries[0]).toMatchObject({ - conversationId: "slack:C9:333", - }); - }); - - it("redacts C-prefixed conversations without public visibility", async () => { - const { getConversationStore } = await import("@/chat/db"); - const { listRecentConversationSummaries } = - await import("@/reporting/plugin-conversations"); - const conversationStore = getConversationStore(); - - // Legacy-style row: no live signal ever marked this channel public. - await conversationStore.recordActivity({ - conversationId: "slack:C9:444", - channelName: "maybe-private-room", - destination: { platform: "slack", teamId: "T1", channelId: "C9" }, - nowMs: 1_000, - source: "slack", - title: "Private by default", - }); - - const summaries = await listRecentConversationSummaries(); - - expect(JSON.stringify(summaries)).not.toContain("maybe-private-room"); - expect(JSON.stringify(summaries)).not.toContain("Private by default"); - expect(summaries[0]).toMatchObject({ - channelName: "Private Conversation", - channelNameRedacted: true, - displayTitle: "Private Conversation", - }); - }); - - it("uses SQL title and visible events when model history is absent", async () => { - const { getConversationEventStore, getConversationStore } = - await import("@/chat/db"); - - await confirmPublicSlackConversation("slack:C1:details-only"); - await getConversationStore().recordActivity({ - conversationId: "slack:C1:details-only", - channelName: "proj-alpha", - source: "slack", - title: "SQL Title", - }); - await getConversationEventStore().append("slack:C1:details-only", [ - { - data: { - type: "visible_message_recorded", - messageId: "visible-only", - role: "user", - text: "Visible SQL message", - }, - createdAtMs: 1_000, - }, - ]); - - const report = await readConversationDetailReport("slack:C1:details-only"); - - expect(report).toMatchObject({ - conversationId: "slack:C1:details-only", - displayTitle: "SQL Title", - }); - expect(report).toMatchObject({ - transcriptAvailable: true, - transcriptMessageCount: 1, - transcript: [ - { - role: "user", - timestamp: 1_000, - parts: [{ type: "text", text: "Visible SQL message" }], - }, - ], - }); - }); - - it("reports conversation-index detail when conversation records are absent", async () => { - vi.useFakeTimers(); - vi.setSystemTime(new Date("2026-06-04T12:00:00.000Z")); - const { requestConversationWork } = - await import("@/chat/task-execution/store"); - - await requestConversationWork({ - conversationId: "slack:C1:index-only", - destination: { - platform: "slack", - teamId: "T1", - channelId: "C1", - }, - nowMs: Date.now(), - }); - - const report = await readConversationDetailReport("slack:C1:index-only"); - - expect(report).toMatchObject({ - conversationId: "slack:C1:index-only", - status: "active", - transcriptAvailable: false, - transcript: [], - }); - }); - - it("reports the complete SQL conversation transcript", async () => { - const { recordAgentTurnSessionSummary, upsertAgentTurnSessionRecord } = - await import("@/chat/state/turn-session"); - const { readConversationDetailFromSql } = - await import("@/api/conversations/detail.query"); + it("returns one ordered safe event log without legacy detail projections", async () => { + const conversationId = "slack:C-reporting:canonical-detail"; + await recordRoot(conversationId, "public"); + await appendVisibleHistory(conversationId); - await confirmPublicSlackConversation("slack:C1:222"); - await upsertAgentTurnSessionRecord({ - conversationId: "slack:C1:222", - sessionId: "turn-current", - sliceId: 1, - state: "completed", - cumulativeDurationMs: 1_200, - cumulativeUsage: { inputTokens: 100, outputTokens: 20 }, - modelId: "openai/gpt-5.5", - reasoningLevel: "high", - piMessages: [ - { - role: "user", - content: [{ type: "text", text: "previous question" }], - timestamp: 1, - }, - { - role: "assistant", - content: [{ type: "text", text: "previous answer" }], - timestamp: 2, - }, - { - role: "user", - content: [ - { - type: "text", - text: [ - "", - "prior context", - "", - "", - "", - "current question", - "", - ].join("\n"), - }, - ], - timestamp: 3, - }, - { - role: "assistant", - content: [ - { type: "thinking", thinking: "I should use a tool" }, - { type: "thinking", thinking: "" }, - { - type: "toolCall", - name: "search", - arguments: { query: "current question" }, - }, - ], - timestamp: 4, - }, - { - role: "toolResult", - toolCallId: "search-1", - name: "search", - content: [{ type: "text", text: "tool result" }], - timestamp: 5, - }, - { - role: "assistant", - content: [{ type: "text", text: "current answer" }], - timestamp: 6, - }, - ] as PiMessage[], - }); - await recordAgentTurnSessionSummary({ - conversationId: "slack:C1:222", - sessionId: "turn-running", - sliceId: 1, - state: "running", - }); - await recordVisibleTranscript("slack:C1:222", [ - { role: "user", text: "previous question", timestamp: 1 }, - { role: "assistant", text: "previous answer", timestamp: 2 }, - { role: "user", text: "current question", timestamp: 3 }, - { role: "assistant", text: "current answer", timestamp: 6 }, - ]); + const detail = await requireDetail(conversationId); - const report = await readConversationDetailFromSql("slack:C1:222"); - expect(report).toMatchObject({ - cumulativeDurationMs: 1_200, - cumulativeUsage: { totalTokens: 120 }, - modelId: "openai/gpt-5.5", - reasoningLevel: "high", - transcriptMessageCount: 4, - }); - expect(report?.transcript).toEqual([ + expect(detail.eventHistory).toEqual({ status: "available" }); + expect(detail.events.map((event) => event.data)).toEqual([ { - role: "user", - timestamp: 1, - parts: [{ type: "text", text: "previous question" }], + type: "visible_message", + messageId: `${conversationId}:visible`, + role: "assistant", + text: "Visible answer", }, + { type: "tool_started", name: "search" }, { - role: "assistant", - timestamp: 2, - parts: [{ type: "text", text: "previous answer" }], + type: "turn_lifecycle", + turnId: `${conversationId}:turn`, + state: "started", }, { - role: "user", - timestamp: 3, - parts: [{ type: "text", text: "current question" }], + type: "turn_lifecycle", + turnId: `${conversationId}:turn`, + state: "succeeded", }, { - role: "assistant", - timestamp: 6, - parts: [{ type: "text", text: "current answer" }], + type: "delivery", + deliveryId: `${conversationId.replaceAll(":", "_")}_delivery`, + state: "intended", }, - ]); - }); - - it("reports private execution activity as safe metadata", async () => { - const { upsertAgentTurnSessionRecord } = - await import("@/chat/state/turn-session"); - const { getConversationEventStore } = await import("@/chat/db"); - - await upsertAgentTurnSessionRecord({ - modelId: "test/model", - conversationId: "slack:G1:activity", - sessionId: "turn-activity", - sliceId: 1, - state: "completed", - turnStartMessageIndex: 0, - piMessages: [ - { - role: "user", - content: [{ type: "text", text: "current question" }], - timestamp: 1, - }, - { - role: "toolResult", - toolCallId: "advisor-call-1", - name: "advisor", - content: [{ type: "text", text: "advisor result" }], - timestamp: 4, - }, - ] as PiMessage[], - }); - // Activity now derives from durable conversation events, not the Redis session log. - await getConversationEventStore().append("slack:G1:activity", [ { - data: { - type: "tool_execution_started", - toolCallId: "advisor-call-1", - toolName: "advisor", - args: { question: "private question", context: "private context" }, - }, - createdAtMs: 2, + type: "delivery", + deliveryId: `${conversationId.replaceAll(":", "_")}_delivery`, + state: "accepted", }, { - data: { - type: "subagent_started", - subagentInvocationId: "advisor-call-1", - subagentKind: "advisor", - parentToolCallId: "advisor-call-1", - childConversationId: "advisor:slack:G1:activity", - historyMode: "shared", - modelId: "openai/gpt-5.6-sol", - reasoningLevel: "high", - }, - createdAtMs: 3, + type: "subagent_started", + childConversationId: `${conversationId}:child`, + subagentKind: "review", + historyMode: "isolated", }, { - data: { - type: "subagent_ended", - subagentInvocationId: "advisor-call-1", - outcome: "success", - }, - createdAtMs: 5, + type: "subagent_ended", + childConversationId: `${conversationId}:child`, + subagentKind: "review", + historyMode: "isolated", + outcome: "success", }, + { type: "context_compacted" }, + { type: "model_handoff" }, ]); - await getConversationEventStore().startEpoch("slack:G1:activity", { - reason: "compaction", - modelProfile: "standard", - modelId: "test/model", - messages: [ - { - message: { - role: "user", - content: [ - { - type: "text", - text: "Context compaction summary for future Junior turns:\nprivate incident details", - }, - ], - timestamp: 6, - } as PiMessage, - createdAtMs: 6, - }, - ], - }); - - const report = await readConversationDetailReport("slack:G1:activity"); - - expect(report.activity).toEqual([ - expect.objectContaining({ - type: "tool_execution", - toolCallId: "advisor-call-1", - toolName: "advisor", - status: "completed", - redacted: true, - // jsonb round-trips object keys in length-then-byte order. - inputKeys: ["context", "question"], - subagents: [ - expect.objectContaining({ - type: "subagent", - id: "advisor-call-1", - outcome: "success", - parentToolCallId: "advisor-call-1", - modelId: "openai/gpt-5.6-sol", - reasoningLevel: "high", - status: "success", - subagentKind: "advisor", - }), - ], - }), - ]); - expect(report.contextEvents).toEqual([ - expect.objectContaining({ - type: "context_compacted", - modelId: "test/model", - }), - ]); - expect(report.contextEvents?.[0]).not.toHaveProperty("summary"); - expect(JSON.stringify(report.activity)).not.toContain("private question"); - expect(JSON.stringify(report)).not.toContain("private incident details"); + const eventSeqs = detail.events.map((event) => event.seq); + expect(eventSeqs).toEqual(eventSeqs.slice().sort((a, b) => a - b)); + expect(JSON.stringify(detail)).not.toContain( + "private model-only duplicate", + ); + expect(JSON.stringify(detail)).not.toContain("must not leave persistence"); + expect(JSON.stringify(detail)).not.toContain("private-model-id"); + for (const removed of [ + "activity", + "contextEvents", + "modelId", + "reasoningLevel", + "transcript", + "transcriptAvailable", + "transcriptMetadata", + "transcriptMessageCount", + ]) { + expect(detail).not.toHaveProperty(removed); + } }); - it("loads subagent transcript history from the child conversation", async () => { - const { getConversationEventStore, getSqlExecutor } = - await import("@/chat/db"); - - const conversationId = "slack:C1:subagent-slices"; - await confirmPublicSlackConversation(conversationId); - const childConversationId = `task:${conversationId}`; - const eventStore = getConversationEventStore(); - const lineage = new SubagentLineageService(getSqlExecutor()); - - await eventStore.append(conversationId, [ - { - data: { - type: "turn_started", - turnId: "turn-subagent-slices", - inputMessageIds: ["subagent-slices-input"], - surface: "slack", - }, - createdAtMs: 1, - }, - ]); + it("redacts visible content for a private root while retaining structure", async () => { + const conversationId = "slack:C-reporting:private-detail"; + await recordRoot(conversationId, "private"); + await appendVisibleHistory(conversationId); - await lineage.start({ - childConversationId, - historyMode: "shared", - modelId: "openai/gpt-5.6-sol", - parentConversationId: conversationId, - parentToolCallId: "task-plan", - parentTurnId: "turn-subagent-slices", - reasoningLevel: "high", - subagentInvocationId: "task-plan", - subagentKind: "task", - nowMs: 3, - }); - await eventStore.append(childConversationId, [ - { - data: { - type: "message", - message: { - role: "user", - content: [ - { - type: "text", - text: "first subagent question\n\nExecutor context:\nfirst packet", - }, - ], - timestamp: 10, - } as PiMessage, - provenance: { authority: "instruction" }, - }, - createdAtMs: 10, - }, - { - data: { - type: "message", - message: { - role: "assistant", - content: [{ type: "text", text: "first subagent answer" }], - timestamp: 20, - } as PiMessage, - }, - createdAtMs: 20, - }, - { - data: { - type: "message", - message: { - role: "user", - content: [{ type: "text", text: "second subagent question" }], - timestamp: 30, - } as PiMessage, - }, - createdAtMs: 30, - }, - { - data: { - type: "message", - message: { - role: "assistant", - content: [{ type: "text", text: "second subagent answer" }], - timestamp: 40, - } as PiMessage, - }, - createdAtMs: 40, - }, - { - data: { - type: "message", - message: { - role: "user", - content: [ - { - type: "text", - text: - "\nReview <change>.\n\n\n" + - "\nUse A & B.\n", - }, - ], - timestamp: 50, - } as PiMessage, - }, - createdAtMs: 50, - }, - ]); + const detail = await requireDetail(conversationId); - await lineage.finish({ - parentConversationId: conversationId, - parentTurnId: "turn-subagent-slices", - subagentInvocationId: "task-plan", - outcome: "success", - nowMs: 25, + expect(detail.eventHistory).toEqual({ + status: "redacted", + reason: "non_public_conversation", }); - // Preserve coverage for historical references that predate immutable - // per-invocation lineage and reused one advisor child transcript. - await eventStore.append(conversationId, [ - { - data: { - type: "subagent_started", - subagentInvocationId: "task-review", - subagentKind: "task", - parentToolCallId: "task-review", - childConversationId, - historyMode: "shared", - modelId: "openai/gpt-5.6-sol", - reasoningLevel: "high", - }, - createdAtMs: 31, - }, - { - data: { - type: "subagent_ended", - subagentInvocationId: "task-review", - outcome: "success", - }, - createdAtMs: 45, - }, - { - data: { - type: "subagent_started", - subagentInvocationId: "legacy-advisor", - subagentKind: "advisor", - childConversationId, - historyMode: "shared", - }, - createdAtMs: 50, - }, - { - data: { - type: "subagent_ended", - subagentInvocationId: "legacy-advisor", - outcome: "success", - }, - createdAtMs: 55, - }, - ]); - - const first = await readConversationSubagentTranscriptReport( - conversationId, - "task-plan", - ); - const second = await readConversationSubagentTranscriptReport( - conversationId, - "task-review", - ); - const legacyAdvisor = await readConversationSubagentTranscriptReport( - conversationId, - "legacy-advisor", - ); - - expect(first.subagentConversationId).toBe(childConversationId); - expect(first.modelId).toBe("openai/gpt-5.6-sol"); - expect(first.reasoningLevel).toBe("high"); - expect(first.transcriptAvailable).toBe(true); - expect(JSON.stringify(first.transcript)).toContain( - "first subagent question", - ); - expect(JSON.stringify(first.transcript)).toContain( - "first packet", - ); - expect(JSON.stringify(first.transcript)).toContain( - "second subagent answer", - ); - expect(second.subagentConversationId).toBe(childConversationId); - expect(JSON.stringify(second.transcript)).toContain( - "first subagent answer", - ); - expect(first.transcript.at(-1)?.parts[0]).toEqual({ - type: "text", - text: - "\nReview <change>.\n\n\n" + - "\nUse A & B.\n", + expect(detail.events[0]?.data).toEqual({ + type: "visible_message", + messageId: `${conversationId}:visible`, + role: "assistant", + redacted: true, }); - expect(legacyAdvisor.transcript.at(-1)?.parts[0]).toEqual({ - type: "text", - text: "Review .\n\nExecutor context:\nUse A & B.", + expect(detail.events[1]?.data).toEqual({ + type: "tool_started", + name: "search", }); + expect(JSON.stringify(detail)).not.toContain("Visible answer"); }); - it("redacts advisor subagent transcript history for private conversations", async () => { - const { getConversationEventStore, getSqlExecutor } = - await import("@/chat/db"); - - const conversationId = "slack:D1:advisor-private"; - const toolCallId = "advisor-private"; - const privateAdvisorText = "private advisor question"; - const childConversationId = `advisor:${conversationId}`; - const eventStore = getConversationEventStore(); - - await eventStore.append(conversationId, [ - { - data: { - type: "turn_started", - turnId: "turn-advisor-private", - inputMessageIds: ["advisor-private-input"], - surface: "slack", - }, - createdAtMs: 1, - }, - ]); + it("authorizes children from their persisted root and rejects forged or malformed lineage", async () => { + const publicRoot = "slack:C-reporting:public-root"; + const publicChild = "child:reporting-public"; + await recordRoot(publicRoot, "public"); + await createChild({ + childConversationId: publicChild, + parentConversationId: publicRoot, + }); + expect((await requireDetail(publicChild)).eventHistory).toEqual({ + status: "available", + }); + const { getConversationStore, getDb } = await import("@/chat/db"); + const [rootRow] = await getDb() + .select({ destinationId: juniorConversations.destinationId }) + .from(juniorConversations) + .where(eq(juniorConversations.conversationId, publicRoot)); + if (!rootRow?.destinationId) throw new Error("Missing root destination"); + await getDb() + .update(juniorDestinations) + .set({ visibility: "private" }) + .where(eq(juniorDestinations.id, rootRow.destinationId)); + expect((await requireDetail(publicChild)).eventHistory.status).toBe( + "redacted", + ); + await getDb() + .update(juniorDestinations) + .set({ visibility: "public" }) + .where(eq(juniorDestinations.id, rootRow.destinationId)); + expect((await requireDetail(publicChild)).eventHistory.status).toBe( + "available", + ); - await new SubagentLineageService(getSqlExecutor()).start({ - childConversationId, - historyMode: "shared", - parentConversationId: conversationId, - parentToolCallId: toolCallId, - parentTurnId: "turn-advisor-private", - subagentInvocationId: toolCallId, - subagentKind: "advisor", - nowMs: 3, + const privateRoot = "slack:C-reporting:private-root"; + const privateChild = "child:reporting-private"; + await recordRoot(privateRoot, "private"); + await createChild({ + childConversationId: privateChild, + parentConversationId: privateRoot, }); - await eventStore.append(childConversationId, [ - { - data: { - type: "message", - message: { - role: "user", - content: [{ type: "text", text: privateAdvisorText }], - timestamp: 10, - } as PiMessage, - }, - createdAtMs: 10, + await getConversationStore().recordActivity({ + conversationId: privateChild, + channelName: "forged-public-child-channel", + destination: { + platform: "slack", + teamId: "T-reporting", + channelId: "C-forged-child", }, - ]); - await new SubagentLineageService(getSqlExecutor()).finish({ - parentConversationId: conversationId, - parentTurnId: "turn-advisor-private", - subagentInvocationId: toolCallId, - outcome: "success", - nowMs: 10, + title: "Forged public child title", + visibility: "public", }); - - const transcript = await readConversationSubagentTranscriptReport( - conversationId, - toolCallId, + const privateChildDetail = await requireDetail(privateChild); + expect(privateChildDetail.eventHistory.status).toBe("redacted"); + expect(JSON.stringify(privateChildDetail)).not.toContain( + "Forged public child title", ); - - expect(transcript.subagentConversationId).toBe(childConversationId); - expect(transcript.transcriptAvailable).toBe(false); - expect(transcript.transcriptRedacted).toBe(true); - expect(transcript.transcript).toEqual([]); - expect(JSON.stringify(transcript)).not.toContain(privateAdvisorText); - }); - - it("derives unfinished subagent status from completed parent tool results", async () => { - const { upsertAgentTurnSessionRecord } = - await import("@/chat/state/turn-session"); - const { getConversationEventStore } = await import("@/chat/db"); - - await upsertAgentTurnSessionRecord({ - modelId: "test/model", - conversationId: "slack:C1:activity-parent-result", - sessionId: "turn-parent-result", - sliceId: 1, - state: "completed", - turnStartMessageIndex: 0, - piMessages: [ - { - role: "user", - content: [{ type: "text", text: "current question" }], - timestamp: 1, - }, - { - role: "toolResult", - toolCallId: "advisor-call-parent", - name: "advisor", - content: [{ type: "text", text: "advisor result" }], - timestamp: 4, - }, - ] as PiMessage[], - }); - // The subagent has no end event; its status derives from the parent tool's - // completed result in the current epoch projection. - await getConversationEventStore().append( - "slack:C1:activity-parent-result", - [ - { - data: { - type: "tool_execution_started", - toolCallId: "advisor-call-parent", - toolName: "advisor", - args: { question: "public question" }, - }, - createdAtMs: 2, - }, - { - data: { - type: "subagent_started", - subagentInvocationId: "advisor-call-parent", - subagentKind: "advisor", - parentToolCallId: "advisor-call-parent", - childConversationId: "advisor:slack:C1:activity-parent-result", - historyMode: "shared", - }, - createdAtMs: 3, - }, - ], + expect(JSON.stringify(privateChildDetail)).not.toContain( + "forged-public-child-channel", ); - const report = await readConversationDetailReport( - "slack:C1:activity-parent-result", + const forgedRoot = "slack:C-reporting:forged-root"; + await recordRoot(forgedRoot, "public"); + await getDb() + .update(juniorConversations) + .set({ rootConversationId: forgedRoot }) + .where(eq(juniorConversations.conversationId, publicChild)); + expect((await requireDetail(publicChild)).eventHistory.status).toBe( + "redacted", ); - - expect(report.activity).toEqual([ - expect.objectContaining({ - type: "tool_execution", - status: "completed", - subagents: [ - expect.objectContaining({ - type: "subagent", - id: "advisor-call-parent", - parentToolCallId: "advisor-call-parent", - status: "completed", - }), - ], - }), - ]); }); - it("keeps the complete visible transcript when steering adds a message", async () => { - const { upsertAgentTurnSessionRecord } = - await import("@/chat/state/turn-session"); - - await confirmPublicSlackConversation("slack:C1:steering-transcript"); - await upsertAgentTurnSessionRecord({ - modelId: "test/model", - conversationId: "slack:C1:steering-transcript", - sessionId: "turn-steering", - sliceId: 1, - state: "completed", - turnStartMessageIndex: 2, - piMessages: [ - { - role: "user", - content: [{ type: "text", text: "previous question" }], - timestamp: 1, - }, - { - role: "assistant", - content: [{ type: "text", text: "previous answer" }], - timestamp: 2, - }, - { - role: "user", - content: [{ type: "text", text: "hello" }], - timestamp: 3, - }, - { - role: "assistant", - content: [{ type: "text", text: "working" }], - timestamp: 4, - }, - { - role: "user", - content: [{ type: "text", text: "steering message" }], - timestamp: 5, - }, - { - role: "assistant", - content: [{ type: "text", text: "done" }], - timestamp: 6, - }, - ] as PiMessage[], + it("lets requested-row expiry win and stamps both root and child purges", async () => { + const rootConversationId = "slack:C-reporting:purged-root"; + const childConversationId = "child:reporting-purged"; + await recordRoot(rootConversationId, "public"); + await createChild({ + childConversationId, + parentConversationId: rootConversationId, }); - await recordVisibleTranscript("slack:C1:steering-transcript", [ - { role: "user", text: "previous question", timestamp: 1 }, - { role: "assistant", text: "previous answer", timestamp: 2 }, - { role: "user", text: "hello", timestamp: 3 }, - { role: "assistant", text: "working", timestamp: 4 }, - { role: "user", text: "steering message", timestamp: 5 }, - { role: "assistant", text: "done", timestamp: 6 }, - ]); + const { getSqlExecutor } = await import("@/chat/db"); - const report = await readConversationDetailReport( - "slack:C1:steering-transcript", - ); - expect(report).toMatchObject({ - transcriptMessageCount: 6, + await purgeConversation(getSqlExecutor(), rootConversationId, { + nowMs: 50, }); - expect(report.transcript).toEqual([ - { - role: "user", - timestamp: 1, - parts: [{ type: "text", text: "previous question" }], - }, - { - role: "assistant", - timestamp: 2, - parts: [{ type: "text", text: "previous answer" }], - }, - { - role: "user", - timestamp: 3, - parts: [{ type: "text", text: "hello" }], - }, - { - role: "assistant", - timestamp: 4, - parts: [{ type: "text", text: "working" }], - }, - { - role: "user", - timestamp: 5, - parts: [{ type: "text", text: "steering message" }], - }, - { - role: "assistant", - timestamp: 6, - parts: [{ type: "text", text: "done" }], - }, - ]); + + for (const conversationId of [rootConversationId, childConversationId]) { + const detail = await requireDetail(conversationId); + expect(detail.eventHistory).toEqual({ + status: "expired", + expiredAt: new Date(50).toISOString(), + }); + expect(detail.events).toEqual([]); + } }); - it("reports a conversation directly from SQL without a secondary execution index", async () => { - const { getConversationEventStore, getConversationStore } = - await import("@/chat/db"); - await getConversationStore().recordActivity({ - conversationId: "slack:C1:999", - destination: { - platform: "slack", - teamId: "T1", - channelId: "C1", - }, - source: "slack", - visibility: "public", - }); - await getConversationEventStore().append("slack:C1:999", [ - { - data: { - type: "visible_message_recorded", - messageId: "target-question", - role: "user", - text: "target question", - }, - createdAtMs: 1, - }, - ]); - - const report = await readConversationDetailReport("slack:C1:999"); - expect(report).toMatchObject({ - conversationId: "slack:C1:999", - transcriptAvailable: true, - }); - expect(report.transcript).toEqual([ - { - role: "user", - timestamp: 1, - parts: [{ type: "text", text: "target question" }], - }, - ]); - }); - - it("extracts trace ids from authorized model history outside the visible transcript", async () => { - const { getConversationEventStore } = await import("@/chat/db"); - const conversationId = "slack:C1:model-trace"; - const traceId = "0123456789abcdef0123456789abcdef"; - await confirmPublicSlackConversation(conversationId); - await getConversationEventStore().append(conversationId, [ - { - data: { - type: "message", - message: { - role: "toolResult", - toolCallId: "trace-tool", - toolName: "lookup", - isError: false, - content: [{ type: "text", text: `trace_id=${traceId}` }], - timestamp: 2, - } as PiMessage, - }, - createdAtMs: 2, - }, - ]); - await recordVisibleTranscript(conversationId, [ - { role: "user", text: "look it up", timestamp: 1 }, - { role: "assistant", text: "done", timestamp: 3 }, - ]); - - const report = await readConversationDetailReport(conversationId); - - expect(report.traceId).toBe(traceId); - expect(JSON.stringify(report.transcript)).not.toContain(traceId); - expect(report.transcript).toEqual([ - { - role: "user", - timestamp: 1, - parts: [{ type: "text", text: "look it up" }], - }, - { - role: "assistant", - timestamp: 3, - parts: [{ type: "text", text: "done" }], - }, - ]); - }); - - it("reports terminal assistant outcomes without exposing provider errors", async () => { - const { getConversationEventStore, getConversationStore } = - await import("@/chat/db"); - const errorConversationId = "slack:C1:terminal-error"; - const abortedConversationId = "slack:D1:terminal-aborted"; - const sensitiveError = - "xAI 503 credential=secret-provider-token upstream payload"; - - await confirmPublicSlackConversation(errorConversationId); - await getConversationEventStore().append(errorConversationId, [ - { - data: { - type: "message", - message: { - role: "assistant", - api: "openai-responses", - content: [], - model: "grok-4.5", - provider: "xai", - stopReason: "error", - errorMessage: sensitiveError, - timestamp: 2, - usage: { - input: 0, - output: 0, - cacheRead: 0, - cacheWrite: 0, - totalTokens: 0, - cost: { - input: 0, - output: 0, - cacheRead: 0, - cacheWrite: 0, - total: 0, - }, - }, - } as PiMessage, - }, - createdAtMs: 2, - }, - ]); - - await getConversationStore().recordActivity({ - conversationId: abortedConversationId, - destination: { - platform: "slack", - teamId: "T1", - channelId: "D1", - }, - source: "slack", - visibility: "private", - }); - await getConversationEventStore().append(abortedConversationId, [ - { - data: { - type: "message", - message: { - role: "assistant", - api: "openai-responses", - content: [], - model: "grok-4.5", - provider: "xai", - stopReason: "aborted", - errorMessage: sensitiveError, - timestamp: 3, - usage: { - input: 0, - output: 0, - cacheRead: 0, - cacheWrite: 0, - totalTokens: 0, - cost: { - input: 0, - output: 0, - cacheRead: 0, - cacheWrite: 0, - total: 0, - }, - }, - } as PiMessage, - }, - createdAtMs: 3, - }, - ]); - - const errorReport = await readConversationDetailReport(errorConversationId); - const abortedReport = await readConversationDetailReport( - abortedConversationId, - ); - - expect(errorReport.transcript).toEqual([ - { - role: "assistant", - outcome: "error", - parts: [], - timestamp: 2, - }, - ]); - expect(errorReport.transcriptMessageCount).toBe(0); - expect(abortedReport).toMatchObject({ - transcript: [], - transcriptAvailable: false, - transcriptMetadata: [ - { - role: "assistant", - outcome: "aborted", - parts: [], - timestamp: 3, - }, - ], - transcriptRedacted: true, - }); - expect(JSON.stringify({ errorReport, abortedReport })).not.toContain( - sensitiveError, - ); - expect(JSON.stringify({ errorReport, abortedReport })).not.toContain( - '"provider":"xai"', - ); - }); - - it("reports one safe lifecycle failure marker for public and private details", async () => { - const { getConversationEventStore, getConversationStore } = - await import("@/chat/db"); - const publicConversationId = "slack:C1:lifecycle-failure-public"; - const privateConversationId = "slack:D1:lifecycle-failure-private"; - const sensitiveError = - "raw-error-sentinel https://provider.invalid/private?token=secret"; - const eventId = "0123456789abcdef0123456789abcdef"; - - await confirmPublicSlackConversation(publicConversationId); - await getConversationEventStore().append(publicConversationId, [ - { - data: { - type: "visible_message_recorded", - messageId: "public-user", - role: "user", - text: "please retry", - }, - createdAtMs: 1, - }, - { - data: { - type: "turn_started", - turnId: "turn-public", - inputMessageIds: ["public-user"], - surface: "slack", - }, - createdAtMs: 2, - }, - { - data: { - type: "message", - message: { - role: "assistant", - content: [], - stopReason: "error", - errorMessage: sensitiveError, - timestamp: 3, - } as unknown as PiMessage, - }, - createdAtMs: 3, - }, - { - data: { - type: "visible_message_recorded", - messageId: "public-fallback", - role: "assistant", - text: buildTurnFailureResponse(eventId), - }, - createdAtMs: 4, - }, - { - data: { - type: "turn_failed", - turnId: "turn-public", - failureCode: "model_execution_failed", - eventId, - }, - createdAtMs: 5, - }, - ]); - - await getConversationStore().recordActivity({ - conversationId: privateConversationId, - destination: { - platform: "slack", - teamId: "T1", - channelId: "D1", - }, - source: "slack", - visibility: "private", - }); - await getConversationEventStore().append(privateConversationId, [ - { - data: { - type: "turn_started", - turnId: "turn-private", - inputMessageIds: ["private-user"], - surface: "slack", - }, - createdAtMs: 10, - }, - { - data: { - type: "visible_message_recorded", - messageId: "private-fallback", - role: "assistant", - text: buildTurnFailureResponse(eventId), - }, - createdAtMs: 10_500, - }, - { - data: { - type: "turn_failed", - turnId: "turn-private", - failureCode: "delivery_failed", - eventId, - }, - createdAtMs: 11_000, - }, - ]); - - const publicReport = - await readConversationDetailReport(publicConversationId); - const publicOutcomeMarkers = publicReport.transcript.filter( - (message) => message.outcome === "error", - ); - expect(publicOutcomeMarkers).toEqual([ - { role: "assistant", outcome: "error", parts: [], timestamp: 5 }, - ]); - expect(publicReport.transcript).toContainEqual({ - role: "assistant", - parts: [{ type: "text", text: buildTurnFailureResponse(eventId) }], - timestamp: 4, - }); - - const privateReport = await readConversationDetailReport( - privateConversationId, - ); - expect(privateReport.transcript).toEqual([]); - expect(privateReport.transcriptMetadata).toContainEqual({ - role: "assistant", - parts: [expect.objectContaining({ type: "text", redacted: true })], - timestamp: 10_500, - }); - expect(privateReport.transcriptMetadata).toContainEqual({ - role: "assistant", - outcome: "error", - parts: [], - timestamp: 11_000, - }); - const publicSerialized = JSON.stringify(publicReport); - expect(publicSerialized).toContain(eventId); - expect(publicSerialized).not.toContain(sensitiveError); - expect(publicSerialized).not.toContain("model_execution_failed"); - - const privateSerialized = JSON.stringify(privateReport); - expect(privateSerialized).not.toContain(eventId); - expect(privateSerialized).not.toContain("delivery_failed"); - expect(privateSerialized).not.toContain(sensitiveError); - }); - - it("merges safe terminal outcomes chronologically with visible messages", async () => { - const { getConversationEventStore } = await import("@/chat/db"); - const conversationId = "slack:C1:terminal-ordered"; - const sensitiveError = "provider=xai credential=secret"; - await confirmPublicSlackConversation(conversationId); - await getConversationEventStore().append(conversationId, [ - { - data: { - type: "message", - message: { - role: "assistant", - api: "openai-responses", - content: [], - model: "grok-4.5", - provider: "xai", - stopReason: "error", - errorMessage: sensitiveError, - timestamp: 2, - usage: { - input: 0, - output: 0, - cacheRead: 0, - cacheWrite: 0, - totalTokens: 0, - cost: { - input: 0, - output: 0, - cacheRead: 0, - cacheWrite: 0, - total: 0, - }, - }, - } as PiMessage, - }, - createdAtMs: 2, - }, - ]); - await recordVisibleTranscript(conversationId, [ - { role: "user", text: "please retry", timestamp: 1 }, - { role: "assistant", text: "safe fallback", timestamp: 3 }, - ]); - - const report = await readConversationDetailReport(conversationId); - - expect(report.transcript).toEqual([ - { - role: "user", - timestamp: 1, - parts: [{ type: "text", text: "please retry" }], - }, - { role: "assistant", outcome: "error", parts: [], timestamp: 2 }, - { - role: "assistant", - timestamp: 3, - parts: [{ type: "text", text: "safe fallback" }], - }, - ]); - expect(report.transcriptMessageCount).toBe(2); - expect(JSON.stringify(report)).not.toContain(sensitiveError); - expect(JSON.stringify(report)).not.toContain('"provider":"xai"'); - }); - - it("keeps SQL detail available when optional execution settings fail", async () => { - const { getConversationEventStore, getConversationStore } = - await import("@/chat/db"); - const { getStateAdapter } = await import("@/chat/state/adapter"); - const conversationId = "slack:C1:settings-unavailable"; - await getConversationStore().recordActivity({ - conversationId, - destination: { - platform: "slack", - teamId: "T1", - channelId: "C1", - }, - source: "slack", - visibility: "public", - }); - await getConversationEventStore().append(conversationId, [ - { - data: { - type: "visible_message_recorded", - messageId: "available-transcript", - role: "user", - text: "available transcript", - }, - createdAtMs: 1, - }, - ]); - vi.spyOn(getStateAdapter(), "getList").mockRejectedValueOnce( - new Error("execution settings unavailable"), - ); - - const report = await readConversationDetailReport(conversationId); - - expect(report).toMatchObject({ - conversationId, - transcriptAvailable: true, - transcript: [ - { - role: "user", - timestamp: 1, - parts: [{ type: "text", text: "available transcript" }], - }, - ], - }); - expect(report).not.toHaveProperty("modelId"); - expect(report).not.toHaveProperty("reasoningLevel"); - }); - - it("reports multiple exchanges as one complete visible transcript", async () => { - const { upsertAgentTurnSessionRecord } = - await import("@/chat/state/turn-session"); - - await confirmPublicSlackConversation("slack:C1:333"); - await upsertAgentTurnSessionRecord({ - modelId: "test/model", - conversationId: "slack:C1:333", - destination: { - platform: "slack", - teamId: "T1", - channelId: "C1", - }, - source: { - platform: "slack", - type: "pub", - teamId: "T1", - channelId: "C1", - threadTs: "333", - }, - sessionId: "turn-one", - sliceId: 1, - state: "completed", - piMessages: [ - { - role: "user", - content: [{ type: "text", text: "first question" }], - timestamp: 1, - }, - { - role: "assistant", - content: [{ type: "text", text: "first answer" }], - timestamp: 2, - }, - ] as PiMessage[], - }); - await upsertAgentTurnSessionRecord({ - modelId: "test/model", - conversationId: "slack:C1:333", - destination: { - platform: "slack", - teamId: "T1", - channelId: "C1", - }, - source: { - platform: "slack", - type: "pub", - teamId: "T1", - channelId: "C1", - threadTs: "333", - }, - sessionId: "turn-two", - sliceId: 1, - state: "completed", - piMessages: [ - { - role: "user", - content: [{ type: "text", text: "first question" }], - timestamp: 1, - }, - { - role: "assistant", - content: [{ type: "text", text: "first answer" }], - timestamp: 2, - }, - { - role: "user", - content: [{ type: "text", text: "second question" }], - timestamp: 3, - }, - { - role: "assistant", - content: [{ type: "text", text: "second answer" }], - timestamp: 4, - }, - ] as PiMessage[], - }); - await recordVisibleTranscript("slack:C1:333", [ - { role: "user", text: "first question", timestamp: 1 }, - { role: "assistant", text: "first answer", timestamp: 2 }, - { role: "user", text: "second question", timestamp: 3 }, - { role: "assistant", text: "second answer", timestamp: 4 }, - ]); - - const report = await readConversationDetailReport("slack:C1:333"); - expect(report).toMatchObject({ conversationId: "slack:C1:333" }); - expect(report.transcript).toEqual([ - { - role: "user", - timestamp: 1, - parts: [{ type: "text", text: "first question" }], - }, - { - role: "assistant", - timestamp: 2, - parts: [{ type: "text", text: "first answer" }], - }, - { - role: "user", - timestamp: 3, - parts: [{ type: "text", text: "second question" }], - }, - { - role: "assistant", - timestamp: 4, - parts: [{ type: "text", text: "second answer" }], - }, - ]); - }); - - it("redacts dashboard transcripts for non-public conversations", async () => { - const { upsertAgentTurnSessionRecord } = - await import("@/chat/state/turn-session"); - const { persistThreadStateById } = - await import("@/chat/runtime/thread-state"); - const privateToolArgs = Object.fromEntries( - Array.from({ length: 25 }, (_, index) => [ - `privateKey${index}`, - `private value ${index}`, - ]), - ); - - // Store the generated title in thread state — the canonical location. - await persistThreadStateById("slack:D1:222", { - artifacts: { assistantTitle: "sensitive generated thread title" }, - }); - - await upsertAgentTurnSessionRecord({ - modelId: "test/model", - conversationId: "slack:D1:222", - sessionId: "turn-private", - sliceId: 1, - state: "completed", - channelName: "secret-dm-name", - actor: { - email: "david@sentry.io", - platform: "slack", - teamId: "T1", - userId: "U1", - }, - piMessages: [ - { - role: "user", - content: [{ type: "text", text: "private question" }], - timestamp: 1, - }, - { - role: "assistant", - content: [ - { type: "text", text: "private answer" }, - { - type: "toolCall", - name: "search", - arguments: privateToolArgs, + it("linearizes detail reads before concurrent appends, privacy flips, and purges", async () => { + const rootConversationId = "slack:C-reporting:concurrent-detail"; + const childConversationId = "child:reporting-concurrent-detail"; + await recordRoot(rootConversationId, "public"); + await createChild({ + childConversationId, + parentConversationId: rootConversationId, + }); + + const blocker = createPostgresJuniorSqlExecutor({ + applicationName: "junior-reporting-table-blocker", + connectionString: TEST_DATABASE_URL, + }); + const observer = createPostgresJuniorSqlExecutor({ + applicationName: "junior-reporting-lock-observer", + connectionString: TEST_DATABASE_URL, + }); + const mutator = createPostgresJuniorSqlExecutor({ + applicationName: "junior-reporting-mutator", + connectionString: TEST_DATABASE_URL, + }); + const appender = createPostgresJuniorSqlExecutor({ + applicationName: "junior-reporting-appender", + connectionString: TEST_DATABASE_URL, + }); + const purger = createPostgresJuniorSqlExecutor({ + applicationName: "junior-reporting-purger", + connectionString: TEST_DATABASE_URL, + }); + const tableLocked = deferred(); + const releaseTable = deferred(); + const blockerDone = blocker.transaction(async () => { + await blocker.execute( + "LOCK TABLE junior_conversation_events IN ACCESS EXCLUSIVE MODE", + ); + tableLocked.resolve(); + await releaseTable.promise; + }); + + try { + await tableLocked.promise; + const completionOrder: string[] = []; + const detailPromise = requireDetail(childConversationId).then( + (detail) => { + completionOrder.push("read"); + return detail; + }, + ); + await waitUntilWaitingOnEventTable(observer); + + const [rootRow] = await observer + .db() + .select({ destinationId: juniorConversations.destinationId }) + .from(juniorConversations) + .where(eq(juniorConversations.conversationId, rootConversationId)); + if (!rootRow?.destinationId) throw new Error("Missing destination"); + + const flipPromise = mutator + .db() + .update(juniorDestinations) + .set({ visibility: "private" }) + .where(eq(juniorDestinations.id, rootRow.destinationId)) + .then(() => { + completionOrder.push("privacy-flip"); + }); + const appendPromise = createSqlConversationEventStore(appender) + .append(childConversationId, [ + { + data: { + type: "visible_message_recorded", + messageId: `${childConversationId}:late-visible`, + role: "assistant", + text: "Late private payload", }, - ], - timestamp: 2, - }, - ] as PiMessage[], - traceId: "0123456789abcdef0123456789abcdef", - }); - await recordVisibleTranscript("slack:D1:222", [ - { role: "user", text: "private question", timestamp: 1 }, - { role: "assistant", text: "private answer", timestamp: 2 }, - ]); - - const report = await readConversationDetailReport("slack:D1:222"); - - expect(report).toMatchObject({ - displayTitle: "Direct Message", - channelName: "Direct Message", - channelNameRedacted: true, - conversationId: "slack:D1:222", - actorIdentity: { - email: "david@sentry.io", - slackUserId: "U1", - }, - transcriptAvailable: false, - transcriptMessageCount: 2, - transcriptRedacted: true, - transcriptRedactionReason: "non_public_conversation", - transcript: [], - }); - expect(report).not.toHaveProperty("actor"); - expect(JSON.stringify(report)).not.toContain("private question"); - expect(JSON.stringify(report)).not.toContain("private answer"); - expect(JSON.stringify(report)).not.toContain("private value"); - expect(JSON.stringify(report)).not.toContain( - "sensitive generated thread title", - ); - expect(JSON.stringify(report)).not.toContain("secret-dm-name"); - expect(report.transcriptMetadata).toEqual([ - { - role: "user", - parts: [{ type: "text", bytes: 16, chars: 16, redacted: true }], - timestamp: 1, - }, - { - role: "assistant", - parts: [{ type: "text", bytes: 14, chars: 14, redacted: true }], - timestamp: 2, - }, - ]); - }); - - it("marks expired private transcripts as privacy redacted", async () => { - const { recordAgentTurnSessionSummary } = - await import("@/chat/state/turn-session"); - - await recordAgentTurnSessionSummary({ - conversationId: "slack:D1:333", - sessionId: "turn-private-expired", - sliceId: 1, - state: "completed", - }); - - const report = await readConversationDetailReport("slack:D1:333"); - - expect(report).toMatchObject({ - displayTitle: "Direct Message", - channelName: "Direct Message", - channelNameRedacted: true, - conversationId: "slack:D1:333", - transcriptAvailable: false, - transcriptMetadata: [], - transcriptRedacted: true, - transcriptRedactionReason: "non_public_conversation", - transcript: [], - }); - }); - - it("presents purged conversation content as expired under retention", async () => { - const { upsertAgentTurnSessionRecord } = - await import("@/chat/state/turn-session"); - const { getSqlExecutor } = await import("@/chat/db"); - const { purgeConversation } = - await import("@/chat/conversations/retention"); - - const conversationId = "slack:C1:purged"; - await confirmPublicSlackConversation(conversationId); - await upsertAgentTurnSessionRecord({ - modelId: "test/model", - conversationId, - sessionId: "turn-purged", - sliceId: 1, - state: "completed", - piMessages: [ - { - role: "user", - content: [{ type: "text", text: "public question" }], - timestamp: 1, - }, - { - role: "assistant", - content: [{ type: "text", text: "public answer" }], - timestamp: 2, - }, - ] as PiMessage[], - }); - - // Retention deletes content wholesale and stamps transcript_purged_at. - await purgeConversation(getSqlExecutor(), conversationId, { - nowMs: Date.now(), - }); - - const report = await readConversationDetailReport(conversationId); - expect(report).toMatchObject({ - conversationId, - transcriptAvailable: false, - transcriptExpired: true, - transcriptMetadata: [], - transcript: [], - }); - // Expiry under retention is distinct from privacy redaction, even though - // this conversation is public. - expect(report).not.toHaveProperty("transcriptRedacted"); - expect(report.transcriptExpiredAt).toEqual(expect.any(String)); - expect(JSON.stringify(report)).not.toContain("public question"); - expect(JSON.stringify(report)).not.toContain("public answer"); - }); - - it("reports complete history around a compaction without copied messages", async () => { - const { getConversationEventStore } = await import("@/chat/db"); - - const conversationId = "slack:C1:compaction"; - await confirmPublicSlackConversation(conversationId); - const eventStore = getConversationEventStore(); - - // Epoch 0: execution that remains visible after a later context rebuild. - await eventStore.append(conversationId, [ - { - data: { - type: "message", - message: { - role: "user", - content: [ - { - type: "text", - text: "Context compaction summary for future Junior turns:\nThis is quoted documentation, not a generated summary.", - }, - ], - timestamp: 0, - } as PiMessage, - }, - createdAtMs: 0, - }, - { - data: { - type: "message", - message: { - role: "user", - content: [{ type: "text", text: "old question" }], - timestamp: 1, - } as PiMessage, - }, - createdAtMs: 1, - }, - { - data: { - type: "message", - message: { - role: "user", - content: [{ type: "text", text: "current question" }], - timestamp: 2, - } as PiMessage, - provenance: { authority: "instruction" }, - }, - createdAtMs: 2, - }, - { - data: { - type: "message", - message: { - role: "user", - content: [{ type: "text", text: "current question" }], - timestamp: 2, - } as PiMessage, - provenance: { authority: "instruction" }, - }, - createdAtMs: 2, - }, - { - data: { - type: "tool_execution_started", - toolCallId: "old-tool", - toolName: "search", - }, - createdAtMs: 3, - }, - ]); - // Compaction copies the latest user intent and adds a generated summary. - await eventStore.startEpoch(conversationId, { - modelId: "test/model", - reason: "compaction", - modelProfile: "standard", - messages: [ - { - message: { - role: "user", - content: [ - { - type: "text", - text: "Context compaction summary for future Junior turns:\nThis is quoted documentation, not a generated summary.", - }, - ], - timestamp: 0, - } as PiMessage, - provenance: { authority: "instruction" }, - createdAtMs: 0, - }, - { - message: { - role: "user", - content: [{ type: "text", text: "current question" }], - timestamp: 2, - } as PiMessage, - provenance: { authority: "instruction" }, - createdAtMs: 2, - }, - { - message: { - role: "user", - content: [{ type: "text", text: "current question" }], - timestamp: 2, - } as PiMessage, - provenance: { authority: "instruction" }, - createdAtMs: 2, - }, - { - message: { - role: "user", - content: [ - { - type: "text", - text: "Context compaction summary for future Junior turns:\nThe earlier search found the relevant deployment.", - }, - ], - timestamp: 4, - } as PiMessage, - provenance: { authority: "context" }, - createdAtMs: 4, - }, - ], - }); - // A current-epoch tool execution the report should surface. - await eventStore.append(conversationId, [ - { - data: { - type: "message", - message: { - role: "user", - content: [ - { - type: "text", - text: "Context compaction summary for future Junior turns:\nPlease explain this marker to the user.", - }, - ], - timestamp: 4.5, - } as PiMessage, - provenance: { authority: "instruction" }, - }, - createdAtMs: 4.5, - }, - { - data: { - type: "tool_execution_started", - toolCallId: "new-tool", - toolName: "search", - args: { q: "current question" }, - }, - createdAtMs: 5, - }, - ]); - await recordVisibleTranscript(conversationId, [ - { - role: "user", - text: "Context compaction summary for future Junior turns:\nThis is quoted documentation, not a generated summary.", - timestamp: 0, - }, - { role: "user", text: "old question", timestamp: 1 }, - { role: "user", text: "current question", timestamp: 2 }, - { - role: "user", - text: "Context compaction summary for future Junior turns:\nPlease explain this marker to the user.", - timestamp: 4.5, - }, - ]); - - const report = await readConversationDetailReport(conversationId); - const currentRun = report; - const toolIds = (currentRun?.activity ?? []) - .filter((row) => row.type === "tool_execution") - .map((row) => row.toolCallId); - - expect(toolIds).toEqual(["old-tool", "new-tool"]); - expect(currentRun.contextEvents).toEqual([ - expect.objectContaining({ - type: "context_compacted", - modelId: "test/model", - summary: "The earlier search found the relevant deployment.", - }), - ]); - expect(JSON.stringify(currentRun.transcript)).toContain("old question"); - expect(JSON.stringify(currentRun.transcript)).toContain("current question"); - expect(JSON.stringify(currentRun.transcript)).toContain( - "This is quoted documentation, not a generated summary.", - ); - expect(JSON.stringify(currentRun.transcript)).toContain( - "Please explain this marker to the user.", - ); - expect(JSON.stringify(currentRun.transcript)).not.toContain( - "The earlier search found the relevant deployment.", - ); - expect( - currentRun.transcript.filter((message) => - JSON.stringify(message).includes("current question"), - ), - ).toHaveLength(1); - }); - - it("reports the original execution and continuation around a model handoff", async () => { - const { getConversationEventStore } = await import("@/chat/db"); - const conversationId = "slack:C1:handoff-reporting"; - await confirmPublicSlackConversation(conversationId); - const eventStore = getConversationEventStore(); - - await eventStore.startEpoch(conversationId, { - reason: "initial", - modelProfile: "standard", - modelId: "openai/gpt-5.4", - messages: [ - { - message: { - role: "user", - content: [ - { - type: "text", - text: "Model handoff checkpoint. Continue the outstanding request now using this summary as the complete prior context:\nThis is quoted documentation, not a generated checkpoint.", - }, - ], - timestamp: 0, - } as PiMessage, - createdAtMs: 0, - }, - { - message: { - role: "user", - content: [{ type: "text", text: "Investigate the release" }], - timestamp: 1, - } as PiMessage, - createdAtMs: 1, - }, - { - message: { - role: "user", - content: [{ type: "text", text: "Investigate the release" }], - timestamp: 1, - } as PiMessage, - createdAtMs: 1, - }, - { - message: { - role: "assistant", - content: [ - { - type: "toolCall", - id: "handoff-call", - name: "handoff", - arguments: { profile: "handoff" }, - }, - ], - timestamp: 2, - } as unknown as PiMessage, - createdAtMs: 2, - }, - ], - }); - await eventStore.startEpoch(conversationId, { - reason: "handoff", - modelProfile: "handoff", - modelId: "openai/gpt-5.6-sol", - messages: [ - { - message: { - role: "user", - content: [ - { - type: "text", - text: "Model handoff checkpoint. Continue the outstanding request now using this summary as the complete prior context:\nThe release migration fails because its constraint is created too late.", - }, - ], - timestamp: 3, - } as PiMessage, - createdAtMs: 3, - }, - { - message: { - role: "user", - content: [ - { - type: "text", - text: "\nBootstrap metadata\n", - }, - ], - timestamp: 3, - } as PiMessage, - createdAtMs: 3, - }, - ], - }); - await eventStore.append(conversationId, [ - { - data: { - type: "message", - message: { - role: "assistant", - content: [{ type: "text", text: "I prepared the ordering fix." }], - timestamp: 4, - } as PiMessage, - }, - createdAtMs: 4, - }, - ]); - await recordVisibleTranscript(conversationId, [ - { - role: "user", - text: "Model handoff checkpoint. Continue the outstanding request now using this summary as the complete prior context:\nThis is quoted documentation, not a generated checkpoint.", - timestamp: 0, - }, - { role: "user", text: "Investigate the release", timestamp: 1 }, - { - role: "assistant", - text: "I prepared the ordering fix.", - timestamp: 4, - }, - ]); - - const report = await readConversationDetailReport(conversationId); - - expect(report.contextEvents).toEqual([ - expect.objectContaining({ - type: "model_handoff", - fromModelId: "openai/gpt-5.4", - toModelId: "openai/gpt-5.6-sol", - message: - "Model handoff checkpoint. Continue the outstanding request now using this summary as the complete prior context:\nThe release migration fails because its constraint is created too late.", - }), - ]); - expect(JSON.stringify(report.transcript)).toContain( - "Investigate the release", - ); - expect(JSON.stringify(report.transcript)).toContain( - "This is quoted documentation, not a generated checkpoint.", - ); - expect( - report.transcript.filter((message) => - JSON.stringify(message).includes("Investigate the release"), - ), - ).toHaveLength(1); - expect(JSON.stringify(report.transcript)).toContain( - "I prepared the ordering fix.", - ); - expect(JSON.stringify(report.transcript)).not.toContain( - "The release migration fails because its constraint is created too late.", - ); - expect(JSON.stringify(report.transcript)).not.toContain( - "runtime-turn-context", - ); - }); - - it("reports divergent rollback history without repeating the shared prefix", async () => { - const { getConversationEventStore } = await import("@/chat/db"); - const conversationId = "slack:C1:rollback-reporting"; - await confirmPublicSlackConversation(conversationId); - const eventStore = getConversationEventStore(); - const shared = { - role: "user", - content: [{ type: "text", text: "Regenerate the answer" }], - timestamp: 1, - } as PiMessage; - - await eventStore.startEpoch(conversationId, { - reason: "initial", - modelProfile: "standard", - modelId: "openai/gpt-5.4", - messages: [ - { message: shared, createdAtMs: 1 }, - { - message: { - role: "assistant", - content: [{ type: "text", text: "Original answer" }], - timestamp: 2, - } as PiMessage, - createdAtMs: 2, - }, - ], - }); - await recordVisibleTranscript(conversationId, [ - { role: "user", text: "Regenerate the answer", timestamp: 1 }, - { role: "assistant", text: "Original answer", timestamp: 2 }, - { role: "assistant", text: "Regenerated answer", timestamp: 3 }, - ]); - await eventStore.startEpoch(conversationId, { - reason: "rollback", - modelProfile: "standard", - modelId: "openai/gpt-5.4", - messages: [ - { message: shared, createdAtMs: 1 }, - { - message: { - role: "assistant", - content: [{ type: "text", text: "Regenerated answer" }], - timestamp: 3, - } as PiMessage, - createdAtMs: 3, - }, - ], - }); - - const report = await readConversationDetailReport(conversationId); - const serialized = report.transcript.map((message) => - JSON.stringify(message), - ); - - expect(report.contextEvents).toEqual([]); - expect( - serialized.filter((message) => message.includes("Regenerate the answer")), - ).toHaveLength(1); - expect( - serialized.filter((message) => message.includes("Original answer")), - ).toHaveLength(1); - expect( - serialized.filter((message) => message.includes("Regenerated answer")), - ).toHaveLength(1); - }); - - it("reports ordered compaction and handoff events with a once-only transcript", async () => { - const { getConversationEventStore } = await import("@/chat/db"); - const conversationId = "slack:C1:compaction-handoff-reporting"; - await confirmPublicSlackConversation(conversationId); - const eventStore = getConversationEventStore(); - const original = { - role: "user", - content: [{ type: "text", text: "Finish the release work" }], - timestamp: 1, - } as PiMessage; - - await eventStore.startEpoch(conversationId, { - reason: "initial", - modelProfile: "standard", - modelId: "openai/gpt-5.4", - messages: [{ message: original, createdAtMs: 1 }], - }); - await eventStore.startEpoch(conversationId, { - reason: "compaction", - modelProfile: "standard", - modelId: "openai/gpt-5.4", - messages: [ - { message: original, createdAtMs: 1 }, - { - message: { - role: "user", - content: [ - { - type: "text", - text: "Context compaction summary for future Junior turns:\nThe release plan is ready for implementation.", - }, - ], - timestamp: 2, - } as PiMessage, - createdAtMs: 2, - }, - ], - }); - await eventStore.append(conversationId, [ - { - data: { - type: "message", - message: { + createdAtMs: 100, + }, + ]) + .then(() => { + completionOrder.push("append"); + }); + const purgePromise = purgeConversation(purger, rootConversationId, { + nowMs: 200, + }).then(() => { + completionOrder.push("purge"); + }); + + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(completionOrder).toEqual([]); + + releaseTable.resolve(); + await blockerDone; + const [detail] = await Promise.all([ + detailPromise, + flipPromise, + appendPromise, + purgePromise, + ]); + + expect(completionOrder[0]).toBe("read"); + expect(detail.eventHistory).toEqual({ status: "available" }); + expect(JSON.stringify(detail)).toContain("Child answer"); + + const finalDetail = await requireDetail(childConversationId); + expect(["expired", "redacted"]).toContain( + finalDetail.eventHistory.status, + ); + expect(JSON.stringify(finalDetail)).not.toContain("Late private payload"); + } finally { + releaseTable.resolve(); + await blockerDone.catch(() => undefined); + await Promise.all([ + blocker.close(), + observer.close(), + mutator.close(), + appender.close(), + purger.close(), + ]); + } + }, 15_000); + + it("deletes physical child event and message rows when an append wins before tree purge", async () => { + const rootConversationId = "slack:C-reporting:append-purge-root"; + const childConversationId = "child:reporting-append-purge"; + await recordRoot(rootConversationId, "public"); + await createChild({ + childConversationId, + parentConversationId: rootConversationId, + }); + + const blocker = createPostgresJuniorSqlExecutor({ + applicationName: "junior-append-purge-table-blocker", + connectionString: TEST_DATABASE_URL, + }); + const observer = createPostgresJuniorSqlExecutor({ + applicationName: "junior-append-purge-observer", + connectionString: TEST_DATABASE_URL, + }); + const writer = createPostgresJuniorSqlExecutor({ + applicationName: "junior-append-purge-writer", + connectionString: TEST_DATABASE_URL, + }); + const purger = createPostgresJuniorSqlExecutor({ + applicationName: "junior-append-purge-purger", + connectionString: TEST_DATABASE_URL, + }); + const tablesLocked = deferred(); + const releaseTables = deferred(); + const blockerDone = blocker.transaction(async () => { + await blocker.execute( + "LOCK TABLE junior_conversation_events, junior_conversation_messages IN ACCESS EXCLUSIVE MODE", + ); + tablesLocked.resolve(); + await releaseTables.promise; + }); + + try { + await tablesLocked.promise; + const completionOrder: string[] = []; + const writerPromise = createSqlConversationMessageStore(writer) + .record(childConversationId, [ + { + messageId: "concurrent-child-message", role: "assistant", - content: [{ type: "text", text: "Compacted continuation" }], - timestamp: 3, - } as PiMessage, - }, - createdAtMs: 3, - }, - ]); - await eventStore.startEpoch(conversationId, { - reason: "handoff", - modelProfile: "handoff", - modelId: "openai/gpt-5.6-sol", - messages: [ - { - message: { - role: "user", - content: [ - { - type: "text", - text: "Model handoff checkpoint. Continue the outstanding request now using this summary as the complete prior context:\nImplement the prepared release plan.", - }, - ], - timestamp: 4, - } as PiMessage, - createdAtMs: 4, - }, - ], - }); - await eventStore.append(conversationId, [ - { - data: { - type: "message", - message: { + text: "concurrent physical child payload", + createdAtMs: Date.now(), + }, + ]) + .then(() => completionOrder.push("writer")); + await waitUntilApplicationWaitsOnLock( + observer, + "junior-append-purge-writer", + "junior_conversation_messages", + ); + + const purgePromise = purgeConversation(purger, rootConversationId, { + nowMs: Date.now(), + }).then(() => completionOrder.push("purge")); + await waitUntilApplicationWaitsOnLock( + observer, + "junior-append-purge-purger", + "pg_advisory_xact_lock", + ); + expect(completionOrder).toEqual([]); + + releaseTables.resolve(); + await blockerDone; + await Promise.all([writerPromise, purgePromise]); + expect(completionOrder).toEqual(["writer", "purge"]); + + for (const conversationId of [rootConversationId, childConversationId]) { + const events = await observer + .db() + .select() + .from(juniorConversationEvents) + .where(eq(juniorConversationEvents.conversationId, conversationId)); + const messages = await observer + .db() + .select() + .from(juniorConversationMessages) + .where(eq(juniorConversationMessages.conversationId, conversationId)); + expect(events).toEqual([]); + expect(messages).toEqual([]); + } + } finally { + releaseTables.resolve(); + await blockerDone.catch(() => undefined); + await Promise.all([ + blocker.close(), + observer.close(), + writer.close(), + purger.close(), + ]); + } + }, 15_000); + + it("rediscovers a child attached while known descendants are being locked", async () => { + const rootConversationId = "slack:C-reporting:membership-race-root"; + const firstChildId = "child:a-reporting-membership-race"; + const secondChildId = "child:z-reporting-membership-race"; + const lateGrandchildId = "child:late-reporting-membership-race"; + await recordRoot(rootConversationId, "public"); + await createChild({ + childConversationId: firstChildId, + parentConversationId: rootConversationId, + }); + await createChild({ + childConversationId: secondChildId, + parentConversationId: rootConversationId, + }); + + const blocker = createPostgresJuniorSqlExecutor({ + applicationName: "junior-membership-race-blocker", + connectionString: TEST_DATABASE_URL, + }); + const observer = createPostgresJuniorSqlExecutor({ + applicationName: "junior-membership-race-observer", + connectionString: TEST_DATABASE_URL, + }); + const creator = createPostgresJuniorSqlExecutor({ + applicationName: "junior-membership-race-creator", + connectionString: TEST_DATABASE_URL, + }); + const purger = createPostgresJuniorSqlExecutor({ + applicationName: "junior-membership-race-purger", + connectionString: TEST_DATABASE_URL, + }); + const childLocked = deferred(); + const releaseChild = deferred(); + const blockerDone = blocker.transaction(async () => { + await blocker + .db() + .select() + .from(juniorConversations) + .where(eq(juniorConversations.conversationId, firstChildId)) + .for("update"); + childLocked.resolve(); + await releaseChild.promise; + }); + + try { + await childLocked.promise; + const purgePromise = purgeConversation(purger, rootConversationId, { + nowMs: Date.now(), + }); + await waitUntilApplicationWaitsOnLock( + observer, + "junior-membership-race-purger", + "junior_conversations", + ); + + const now = new Date(); + await creator.db().insert(juniorConversations).values({ + conversationId: lateGrandchildId, + parentConversationId: secondChildId, + createdAt: now, + lastActivityAt: now, + updatedAt: now, + executionStatus: "idle", + }); + await createSqlConversationMessageStore(creator).record( + lateGrandchildId, + [ + { + messageId: "late-grandchild-message", role: "assistant", - content: [{ type: "text", text: "Handoff continuation" }], - timestamp: 5, - } as PiMessage, - }, - createdAtMs: 5, - }, - ]); - await recordVisibleTranscript(conversationId, [ - { role: "user", text: "Finish the release work", timestamp: 1 }, - { role: "assistant", text: "Compacted continuation", timestamp: 3 }, - { role: "assistant", text: "Handoff continuation", timestamp: 5 }, - ]); - - const report = await readConversationDetailReport(conversationId); - const transcript = JSON.stringify(report.transcript); - - expect(report.contextEvents).toEqual([ - expect.objectContaining({ - type: "context_compacted", - modelId: "openai/gpt-5.4", - summary: "The release plan is ready for implementation.", - }), - expect.objectContaining({ - type: "model_handoff", - fromModelId: "openai/gpt-5.4", - toModelId: "openai/gpt-5.6-sol", - message: - "Model handoff checkpoint. Continue the outstanding request now using this summary as the complete prior context:\nImplement the prepared release plan.", - }), - ]); - expect(report.contextEvents?.[0]?.transcriptIndex).toBeLessThanOrEqual( - report.contextEvents?.[1]?.transcriptIndex ?? -1, - ); - expect( - report.transcript.filter((message) => - JSON.stringify(message).includes("Finish the release work"), - ), - ).toHaveLength(1); - expect(transcript).toContain("Compacted continuation"); - expect(transcript).toContain("Handoff continuation"); - expect(transcript).not.toContain("Context compaction summary"); - expect(transcript).not.toContain("Model handoff checkpoint"); - }); + text: "late grandchild payload", + createdAtMs: now.getTime(), + }, + ], + ); + + releaseChild.resolve(); + await blockerDone; + await purgePromise; + + const events = await observer + .db() + .select() + .from(juniorConversationEvents) + .where(eq(juniorConversationEvents.conversationId, lateGrandchildId)); + const messages = await observer + .db() + .select() + .from(juniorConversationMessages) + .where(eq(juniorConversationMessages.conversationId, lateGrandchildId)); + const [conversation] = await observer + .db() + .select({ transcriptPurgedAt: juniorConversations.transcriptPurgedAt }) + .from(juniorConversations) + .where(eq(juniorConversations.conversationId, lateGrandchildId)); + expect(events).toEqual([]); + expect(messages).toEqual([]); + expect(conversation?.transcriptPurgedAt).toBeInstanceOf(Date); + } finally { + releaseChild.resolve(); + await blockerDone.catch(() => undefined); + await Promise.all([ + blocker.close(), + observer.close(), + creator.close(), + purger.close(), + ]); + } + }, 15_000); }); diff --git a/packages/junior/tests/integration/reporting-support.test.ts b/packages/junior/tests/integration/reporting-support.test.ts new file mode 100644 index 000000000..008055505 --- /dev/null +++ b/packages/junior/tests/integration/reporting-support.test.ts @@ -0,0 +1,188 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const ORIGINAL_ENV = { ...process.env }; +const TEST_DATABASE_URL = ORIGINAL_ENV.DATABASE_URL; + +if (!TEST_DATABASE_URL) { + throw new Error("DATABASE_URL is required for reporting integration tests"); +} + +describe("reporting support", () => { + beforeEach(async () => { + process.env = { + ...ORIGINAL_ENV, + DATABASE_URL: TEST_DATABASE_URL, + JUNIOR_STATE_ADAPTER: "memory", + }; + vi.resetModules(); + const { disconnectStateAdapter } = await import("@/chat/state/adapter"); + await disconnectStateAdapter(); + }); + + afterEach(async () => { + const { disconnectStateAdapter } = await import("@/chat/state/adapter"); + const { closeDb } = await import("@/chat/db"); + await closeDb(); + await disconnectStateAdapter(); + vi.resetModules(); + process.env = { ...ORIGINAL_ENV }; + }); + + it("indexes only the latest safe turn-session summary", async () => { + const { listAgentTurnSessionSummaries, upsertAgentTurnSessionRecord } = + await import("@/chat/state/turn-session"); + const conversationId = "slack:C-reporting-support:summary-index"; + + await upsertAgentTurnSessionRecord({ + modelId: "test/model", + conversationId, + sessionId: "reporting-support-turn", + sliceId: 1, + state: "running", + piMessages: [], + }); + await upsertAgentTurnSessionRecord({ + modelId: "test/model", + conversationId, + sessionId: "reporting-support-turn", + sliceId: 2, + state: "completed", + piMessages: [], + cumulativeDurationMs: 1_200, + errorMessage: "provider failed with sensitive details", + loadedSkillNames: ["triage"], + }); + + const matching = (await listAgentTurnSessionSummaries()).filter( + (summary) => summary.sessionId === "reporting-support-turn", + ); + + expect(matching).toHaveLength(1); + expect(matching[0]).toMatchObject({ + conversationId, + sessionId: "reporting-support-turn", + sliceId: 2, + state: "completed", + cumulativeDurationMs: 1_200, + loadedSkillNames: ["triage"], + }); + expect(matching[0]).not.toHaveProperty("errorMessage"); + }); + + it("lists recent conversations for plugin operational reports", async () => { + const { getConversationStore } = await import("@/chat/db"); + const { listRecentConversationSummaries } = + await import("@/reporting/plugin-conversations"); + const conversationId = "slack:C-reporting-support:plugin-feed"; + + await getConversationStore().recordActivity({ + conversationId, + channelName: "reporting-support-incidents", + destination: { + platform: "slack", + teamId: "T-reporting-support", + channelId: "C-reporting-support", + }, + nowMs: Date.now(), + source: "slack", + title: "Reporting support incident", + visibility: "public", + }); + + const summary = (await listRecentConversationSummaries(100)).find( + (item) => item.conversationId === conversationId, + ); + expect(summary).toMatchObject({ + channelName: "reporting-support-incidents", + conversationId, + displayTitle: expect.any(String), + source: "slack", + status: "completed", + }); + }); + + it("mirrors local turn sessions into the SQL conversation store", async () => { + const { recordAgentTurnSessionSummary } = + await import("@/chat/state/turn-session"); + const { getConversationStore } = await import("@/chat/db"); + const conversationId = "local:reporting-support:run"; + + await recordAgentTurnSessionSummary({ + conversationId, + destination: { platform: "local", conversationId }, + sessionId: "reporting-support-local-turn", + sliceId: 1, + state: "completed", + surface: "internal", + ttlMs: 60_000, + }); + + await expect( + getConversationStore().get({ conversationId }), + ).resolves.toMatchObject({ conversationId, source: "local" }); + }); + + it.each([ + { + name: "a G-prefixed private conversation", + conversationId: "slack:G-reporting-support:private", + channelId: undefined, + channelName: "reporting-support-private-room", + title: "Reporting support sensitive escalation", + visibility: undefined, + }, + { + name: "a source-confirmed private C-prefixed conversation", + conversationId: "slack:C-reporting-support:confirmed-private", + channelId: "C-reporting-support-private", + channelName: "reporting-support-stealth-project", + title: "Reporting support stealth planning", + visibility: "private" as const, + }, + { + name: "a C-prefixed conversation without confirmed public visibility", + conversationId: "slack:C-reporting-support:unknown", + channelId: "C-reporting-support-unknown", + channelName: "reporting-support-maybe-private-room", + title: "Reporting support private by default", + visibility: undefined, + }, + ])("redacts $name", async (testCase) => { + const { getConversationStore } = await import("@/chat/db"); + const { listRecentConversationSummaries } = + await import("@/reporting/plugin-conversations"); + + await getConversationStore().recordActivity({ + conversationId: testCase.conversationId, + channelName: testCase.channelName, + ...(testCase.channelId + ? { + destination: { + platform: "slack" as const, + teamId: "T-reporting-support", + channelId: testCase.channelId, + }, + } + : {}), + nowMs: Date.now(), + source: "slack", + title: testCase.title, + ...(testCase.visibility ? { visibility: testCase.visibility } : {}), + }); + + const summaries = await listRecentConversationSummaries(100); + const serialized = JSON.stringify(summaries); + const summary = summaries.find( + (item) => item.conversationId === testCase.conversationId, + ); + + expect(serialized).not.toContain(testCase.channelName); + expect(serialized).not.toContain(testCase.title); + expect(summary).toMatchObject({ + conversationId: testCase.conversationId, + channelName: "Private Conversation", + channelNameRedacted: true, + displayTitle: "Private Conversation", + }); + }); +}); diff --git a/packages/junior/tests/integration/runtime/agent-run-provider-retry.test.ts b/packages/junior/tests/integration/runtime/agent-run-provider-retry.test.ts index c405d8cea..1c4c2c24e 100644 --- a/packages/junior/tests/integration/runtime/agent-run-provider-retry.test.ts +++ b/packages/junior/tests/integration/runtime/agent-run-provider-retry.test.ts @@ -602,23 +602,13 @@ describe("executeAgentRun provider retry", () => { expect(serializedMessages).toContain("actually do the other thing"); const report = await readConversationDetail("slack:C123:1712345.0001"); - const transcript = report?.transcript ?? []; - expect(JSON.stringify(transcript)).toContain("previous question"); - expect(transcript).toHaveLength(5); - expect(transcript[2]).toMatchObject({ - role: "user", - timestamp: expect.any(Number), - parts: expect.arrayContaining([{ type: "text", text: "help me" }]), - }); - expect(transcript[3]).toEqual({ - role: "user", - timestamp: 2_000, - parts: [{ type: "text", text: "actually do the other thing" }], - }); - expect(transcript[4]).toEqual({ - role: "assistant", - parts: [{ type: "text", text: "Steered." }], - }); + const visibleText = report?.events.flatMap((event) => + event.data.type === "visible_message" && event.data.text + ? [event.data.text] + : [], + ); + expect(visibleText).not.toContain("previous question"); + expect(visibleText).toEqual([]); }); it("parks the turn when the worker asks to yield at a Pi boundary", async () => { diff --git a/packages/junior/tests/unit/api/conversation-events.test.ts b/packages/junior/tests/unit/api/conversation-events.test.ts index d6d4adf12..207dcd55c 100644 --- a/packages/junior/tests/unit/api/conversation-events.test.ts +++ b/packages/junior/tests/unit/api/conversation-events.test.ts @@ -1,6 +1,9 @@ import { describe, expect, it } from "vitest"; import { projectConversationReportEvents } from "@/api/conversations/events"; -import { conversationReportEventSchema } from "@/api/conversations/schema"; +import { + conversationDetailReportSchema, + conversationReportEventSchema, +} from "@/api/conversations/schema"; import { conversationEventSchema, type ConversationEvent, @@ -427,4 +430,127 @@ describe("conversation report event projection", () => { }).success, ).toBe(false); }); + + it("rejects non-increasing event sequences at the detail boundary", () => { + const summary = { + displayTitle: "Report", + cumulativeDurationMs: 0, + conversationId: "conversation-1", + status: "completed" as const, + startedAt: "2026-07-15T12:00:00.000Z", + lastSeenAt: "2026-07-15T12:00:00.000Z", + lastProgressAt: "2026-07-15T12:00:00.000Z", + surface: "internal" as const, + generatedAt: "2026-07-15T12:00:00.000Z", + eventHistory: { status: "available" as const }, + }; + const reportEvent = (seq: number) => ({ + seq, + contextEpoch: 0, + createdAt: "2026-07-15T12:00:00.000Z", + data: { + type: "turn_lifecycle" as const, + turnId: `turn-${seq}`, + state: "started" as const, + }, + }); + + expect( + conversationDetailReportSchema.safeParse({ + ...summary, + events: [reportEvent(1), reportEvent(3)], + }).success, + ).toBe(true); + expect( + conversationDetailReportSchema.safeParse({ + ...summary, + events: [reportEvent(3), reportEvent(3)], + }).success, + ).toBe(false); + expect( + conversationDetailReportSchema.safeParse({ + ...summary, + events: [reportEvent(3), reportEvent(2)], + }).success, + ).toBe(false); + }); + + it("enforces event-history privacy invariants at the detail boundary", () => { + const summary = { + displayTitle: "Report", + cumulativeDurationMs: 0, + conversationId: "conversation-privacy", + status: "completed" as const, + startedAt: "2026-07-15T12:00:00.000Z", + lastSeenAt: "2026-07-15T12:00:00.000Z", + lastProgressAt: "2026-07-15T12:00:00.000Z", + surface: "internal" as const, + generatedAt: "2026-07-15T12:00:00.000Z", + }; + const visibleEvent = ( + data: + | { text: string; redacted?: never } + | { redacted: true; text?: never }, + ) => ({ + seq: 1, + contextEpoch: 0, + createdAt: "2026-07-15T12:00:00.000Z", + data: { + type: "visible_message" as const, + messageId: "message-1", + role: "assistant" as const, + ...data, + }, + }); + + expect( + conversationDetailReportSchema.safeParse({ + ...summary, + eventHistory: { status: "expired", expiredAt: summary.generatedAt }, + events: [visibleEvent({ redacted: true })], + }).success, + ).toBe(false); + expect( + conversationDetailReportSchema.safeParse({ + ...summary, + eventHistory: { + status: "redacted", + reason: "non_public_conversation", + }, + events: [visibleEvent({ text: "must not be exposed" })], + }).success, + ).toBe(false); + expect( + conversationDetailReportSchema.safeParse({ + ...summary, + eventHistory: { status: "available" }, + events: [visibleEvent({ redacted: true })], + }).success, + ).toBe(false); + + expect( + conversationDetailReportSchema.safeParse({ + ...summary, + eventHistory: { status: "expired", expiredAt: summary.generatedAt }, + events: [], + }).success, + ).toBe(true); + expect( + conversationDetailReportSchema.safeParse({ + ...summary, + eventHistory: { + status: "redacted", + reason: "non_public_conversation", + }, + events: [visibleEvent({ redacted: true })], + }).success, + ).toBe(true); + expect( + conversationDetailReportSchema.safeParse({ + ...summary, + eventHistory: { status: "available" }, + events: [visibleEvent({ text: "safe public text" })], + }).success, + ).toBe(true); + }); }); diff --git a/packages/junior/tests/unit/conversations/privacy.test.ts b/packages/junior/tests/unit/conversations/privacy.test.ts new file mode 100644 index 000000000..fa09e87c0 --- /dev/null +++ b/packages/junior/tests/unit/conversations/privacy.test.ts @@ -0,0 +1,167 @@ +import { describe, expect, it } from "vitest"; +import { resolveRootVisibility } from "@/chat/conversations/sql/privacy"; +import type { JuniorSqlDatabase } from "@/db/db"; +import { juniorConversations, juniorDestinations } from "@/db/schema"; + +interface ConversationRow { + destinationId: string | null; + parentId: string | null; + rootId: string | null; +} + +function scriptedExecutor(args: { + conversations: Array; + visibility?: "private" | "public"; +}): JuniorSqlDatabase { + const conversations = [...args.conversations]; + const db = { + select: () => { + let source: unknown; + const readRows = () => { + if (source === juniorConversations) { + const row = conversations.shift(); + return row ? [row] : []; + } + if (source === juniorDestinations) { + return args.visibility ? [{ visibility: args.visibility }] : []; + } + throw new Error("Unexpected privacy resolver table"); + }; + const query = { + from(table: unknown) { + source = table; + return query; + }, + where() { + const rows = Promise.resolve(readRows()); + return Object.assign(rows, { + for: async () => rows, + }); + }, + }; + return query; + }, + }; + return { + db: () => db, + transaction: async (callback: () => Promise): Promise => + callback(), + withLock: async ( + _name: string, + callback: () => Promise, + ): Promise => callback(), + } as unknown as JuniorSqlDatabase; +} + +describe("resolveRootVisibility", () => { + it("resolves visibility only from a consistent root destination", async () => { + const result = await resolveRootVisibility( + scriptedExecutor({ + conversations: [ + { destinationId: null, parentId: "root", rootId: "root" }, + { destinationId: "destination", parentId: null, rootId: null }, + { destinationId: "destination", parentId: null, rootId: null }, + { destinationId: null, parentId: "root", rootId: "root" }, + ], + visibility: "public", + }), + "child", + ); + + expect(result).toEqual({ + rootConversationId: "root", + visibility: "public", + }); + }); + + it.each([ + { + name: "a missing requested conversation", + rows: [], + }, + { + name: "a missing parent", + rows: [{ destinationId: null, parentId: "missing", rootId: "root" }], + }, + { + name: "a historical child without a declared root", + rows: [{ destinationId: null, parentId: "root", rootId: null }], + }, + { + name: "an inconsistent intermediate root", + rows: [ + { destinationId: null, parentId: "parent", rootId: "root" }, + { destinationId: null, parentId: "root", rootId: "other-root" }, + ], + }, + { + name: "a missing root destination", + rows: [{ destinationId: null, parentId: null, rootId: null }], + }, + ])("fails closed for $name", async ({ rows }) => { + await expect( + resolveRootVisibility( + scriptedExecutor({ conversations: rows }), + "requested", + ), + ).resolves.toMatchObject({ visibility: null }); + }); + + it("fails closed when lineage contains a cycle", async () => { + const result = await resolveRootVisibility( + scriptedExecutor({ + conversations: [ + { destinationId: null, parentId: "parent", rootId: "root" }, + { destinationId: null, parentId: "requested", rootId: "root" }, + ], + }), + "requested", + ); + + expect(result.visibility).toBeNull(); + }); + + it("fails closed when lineage changes between discovery and locking", async () => { + const result = await resolveRootVisibility( + scriptedExecutor({ + conversations: [ + { destinationId: null, parentId: "root", rootId: "root" }, + { + destinationId: "destination", + parentId: null, + rootId: null, + }, + { + destinationId: "destination", + parentId: null, + rootId: null, + }, + { + destinationId: null, + parentId: "different-root", + rootId: "different-root", + }, + ], + visibility: "public", + }), + "child", + ); + + expect(result.visibility).toBeNull(); + }); + + it("fails closed when lineage exceeds the traversal bound", async () => { + const conversations = Array.from({ length: 32 }, (_, index) => ({ + destinationId: null, + parentId: `node-${index + 1}`, + rootId: "root", + })); + + const result = await resolveRootVisibility( + scriptedExecutor({ conversations }), + "node-0", + ); + + expect(result.visibility).toBeNull(); + }); +}); From 766825db0154a604ae6d40dc1a366103414d62a2 Mon Sep 17 00:00:00 2001 From: David Cramer Date: Wed, 15 Jul 2026 10:09:57 -0700 Subject: [PATCH 11/58] ref(conversations): Remove lazy legacy history Make SQL events the only live history source for Pi projection and visible hydration. Keep the Redis decoder and backfill writer available only through the explicit operator upgrade path. Co-Authored-By: GPT-5 Codex --- .../changes/conversation-event-log/tasks.md | 10 +- .../junior/src/chat/conversations/README.md | 23 ++- .../src/chat/conversations/history-loader.ts | 31 ---- .../src/chat/conversations/projection.ts | 11 +- .../chat/conversations/visible-messages.ts | 21 +-- .../conversation-history/advisor-session.ts} | 2 - .../conversation-history/import.ts} | 71 +------ .../legacy-history-import.ts | 21 +-- .../conversation-history}/session-log.ts | 2 +- .../state-conversation-store.ts} | 2 +- .../migrations/conversations-history-sql.ts | 8 +- .../upgrade/migrations/conversations-sql.ts | 2 +- .../conversation-history-import.test.ts} | 174 ++---------------- .../conversation-history-conversion.test.ts} | 6 +- .../conversation-history-session-log.test.ts} | 4 +- 15 files changed, 82 insertions(+), 306 deletions(-) delete mode 100644 packages/junior/src/chat/conversations/history-loader.ts rename packages/junior/src/{chat/conversations/legacy-advisor-session.ts => cli/upgrade/migrations/conversation-history/advisor-session.ts} (89%) rename packages/junior/src/{chat/conversations/legacy-import.ts => cli/upgrade/migrations/conversation-history/import.ts} (74%) rename packages/junior/src/{chat/conversations/sql => cli/upgrade/migrations/conversation-history}/legacy-history-import.ts (97%) rename packages/junior/src/{chat/state => cli/upgrade/migrations/conversation-history}/session-log.ts (99%) rename packages/junior/src/{chat/conversations/state.ts => cli/upgrade/migrations/conversation-history/state-conversation-store.ts} (93%) rename packages/junior/tests/component/{conversations/legacy-import.test.ts => cli/conversation-history-import.test.ts} (82%) rename packages/junior/tests/unit/{conversations/legacy-history-import.test.ts => cli/conversation-history-conversion.test.ts} (97%) rename packages/junior/tests/unit/{state/session-log.test.ts => cli/conversation-history-session-log.test.ts} (96%) diff --git a/openspec/changes/conversation-event-log/tasks.md b/openspec/changes/conversation-event-log/tasks.md index e18a7948b..8a026ee84 100644 --- a/openspec/changes/conversation-event-log/tasks.md +++ b/openspec/changes/conversation-event-log/tasks.md @@ -10,8 +10,8 @@ authorization observations, and bounded sequence projection. - [x] Hard-cut storage writes and SQL rows to canonical events without changing API payloads or runtime behavior. -- [x] Restrict legacy `pi_message` to the bounded Redis/import and rolling-worker - compatibility seams; never expose raw persistence data through the API. +- [x] Restrict legacy `pi_message` to the explicit operator backfill; never + expose it through runtime conversation modules or reporting APIs. - [x] Add focused adapter parity tests and update owning module documentation. ## 2. Canonical Event Writes @@ -55,9 +55,11 @@ - [x] Make the visible-message table an explicit event-backed read model. - [x] Hydrate runtime and primary conversation-detail transcripts from visible events instead of the message-table projection or model messages. +- [x] Remove lazy Redis/advisor history import from every runtime read and move + the retained backfill decoder/writer under `cli/upgrade` only. - [ ] Make remaining aggregate stores explicit read models. -- [ ] Remove obsolete transcript reconstruction and legacy compatibility after - migration completion. +- [ ] Delete the explicit operator Redis backfill after every supported + deployment has crossed its migration horizon. - [ ] Verify retention, purge, redaction, idempotency, replay parity, and event tree behavior. - [ ] Move durable invariants into owning code and documentation, then delete diff --git a/packages/junior/src/chat/conversations/README.md b/packages/junior/src/chat/conversations/README.md index 8b0c72b82..0db2fec3e 100644 --- a/packages/junior/src/chat/conversations/README.md +++ b/packages/junior/src/chat/conversations/README.md @@ -1,7 +1,7 @@ # Conversation Storage This module owns the durable product record for conversations, visible messages, -conversation events, compaction boundaries, search, retention, and legacy import. +conversation events, compaction boundaries, search, and retention. ## Records @@ -23,7 +23,8 @@ conversation events, compaction boundaries, search, retention, and legacy import markers preserve that relationship; compaction, handoff, and isolated epochs are self-contained. Child sequence cursors count only child-local events, and commits reject any mutation of the inherited prefix. -- Provider payloads and old state-store mirrors are migration inputs, not +- Provider payloads and old state-store mirrors are inputs only to the explicit + operator migration under `cli/upgrade`; they are not runtime dependencies or canonical product records. The schemas and migrations under `sql/` are authoritative. @@ -34,9 +35,10 @@ timestamp. The `(conversation_id, seq)` key is both stable event identity and the lease-fencing tripwire. The Junior-owned `message` event retains an opaque model-continuity payload and -canonical provenance. The Pi adapter is the only module that interprets that -payload as a Pi message; the legacy Redis `pi_message` shape exists only as a -bounded import input. +canonical provenance. The Pi adapter is the only runtime module that +interprets that payload as a Pi message. The legacy Redis `pi_message` shape +exists only in the explicit operator backfill and is absent from the live +conversation module graph. Reporting APIs must project events into an authorized, redacted product contract. Raw `ConversationEventData` is an internal persistence boundary and @@ -61,7 +63,8 @@ remain internal. one first-writer-wins `turn_completed` or `turn_failed` terminal event. - Persist only allowlisted failure classifications and opaque Sentry event IDs; raw exceptions, provider payloads, and URLs are not lifecycle data. -- Restore state from durable events rather than a duplicate transcript cache. +- Restore state directly from durable SQL events rather than a duplicate + transcript cache or lazy legacy import. - Compaction replaces prior model context without rewriting visible history. - Imports and migrations are idempotent and preserve stable conversation IDs. - Establish child lineage and its parent `subagent_started` reference through @@ -90,8 +93,12 @@ Follow `../../../../../policies/data-redaction.md` and - Schema changes are expand-first and compatible with the currently deployed reader and writer during rollout. - Data rewrites use explicit migrations or resumable import code. -- Legacy fields remain readable only for the migration window and are removed - after the new authority is verified. +- Live workers never read Redis session logs, advisor blobs, or legacy + thread-state to restore history. Operators must run `junior upgrade` before + serving retained pre-cutover conversations. +- The Redis decoder and backfill writer remain under `cli/upgrade` only for the + bounded operator-migration horizon; they can be deleted independently once + every supported deployment has completed that upgrade. - Drain every 0.103.x worker before applying the visible-message schema cut, which drops the temporary `junior_agent_steps` compatibility view and its functions. diff --git a/packages/junior/src/chat/conversations/history-loader.ts b/packages/junior/src/chat/conversations/history-loader.ts deleted file mode 100644 index 8eb854a28..000000000 --- a/packages/junior/src/chat/conversations/history-loader.ts +++ /dev/null @@ -1,31 +0,0 @@ -/** Bounded legacy import followed by canonical conversation-event reads. */ -import { getConversationEventStore } from "@/chat/db"; -import { ensureLegacyConversationImport } from "./legacy-import"; -import type { ConversationEvent } from "./history"; - -interface ScopedConversation { - conversationId: string; -} - -/** Ensure any pre-cutover transcript is represented in the canonical event log. */ -export async function ensureConversationEventHistory( - args: ScopedConversation, -): Promise { - await ensureLegacyConversationImport({ conversationId: args.conversationId }); -} - -/** Load the current event epoch after the bounded lazy legacy import. */ -export async function loadCurrentConversationEvents( - args: ScopedConversation, -): Promise { - await ensureConversationEventHistory(args); - return getConversationEventStore().loadCurrentEpoch(args.conversationId); -} - -/** Load complete event history after the bounded lazy legacy import. */ -export async function loadConversationEventHistory( - args: ScopedConversation, -): Promise { - await ensureConversationEventHistory(args); - return getConversationEventStore().loadHistory(args.conversationId); -} diff --git a/packages/junior/src/chat/conversations/projection.ts b/packages/junior/src/chat/conversations/projection.ts index 2e322e5a6..e3dfd34c5 100644 --- a/packages/junior/src/chat/conversations/projection.ts +++ b/packages/junior/src/chat/conversations/projection.ts @@ -26,10 +26,6 @@ import type { JuniorSqlDatabase } from "@/db/db"; import { createSqlStore } from "@/chat/conversations/sql/store"; import { createSqlConversationEventStore } from "@/chat/conversations/sql/history"; import { withConversationEventLock } from "@/chat/conversations/sql/event-lock"; -import { - ensureConversationEventHistory, - loadCurrentConversationEvents, -} from "@/chat/conversations/history-loader"; import { projectConversationEvents, type PiConversationEventProjection, @@ -363,7 +359,6 @@ async function loadComposedProjection(args: { async function loadCurrentComposedProjection( args: ScopedConversation, ): Promise { - await ensureConversationEventHistory(args); const eventStore = getConversationEventStore(); return await loadComposedProjection({ conversationId: args.conversationId, @@ -391,7 +386,6 @@ export async function loadConversationProjection( export async function openConversationProjection( args: ScopedConversation & { modelId: string }, ): Promise { - await ensureConversationEventHistory(args); const eventStore = getConversationEventStore(); const events = await eventStore.loadCurrentEpoch(args.conversationId); const composed = await loadComposedProjection({ @@ -448,7 +442,6 @@ export async function loadTurnProjection(args: { }) | undefined > { - await ensureConversationEventHistory(args); const eventStore = getConversationEventStore(); // A record that committed no messages materializes the live projection, the // same way count-based records with a zero cursor did. @@ -495,7 +488,9 @@ export async function loadTurnProjection(args: { export async function loadConnectedMcpProviders( args: ScopedConversation, ): Promise { - const events = await loadCurrentConversationEvents(args); + const events = await getConversationEventStore().loadCurrentEpoch( + args.conversationId, + ); return connectedMcpProvidersFromEvents(events); } diff --git a/packages/junior/src/chat/conversations/visible-messages.ts b/packages/junior/src/chat/conversations/visible-messages.ts index 2c8e6825b..55c230628 100644 --- a/packages/junior/src/chat/conversations/visible-messages.ts +++ b/packages/junior/src/chat/conversations/visible-messages.ts @@ -6,8 +6,10 @@ * `ThreadConversationState.messages` is the current-turn working set. No * transcript data is persisted to `thread-state`. */ -import { getConversationMessageStore } from "@/chat/db"; -import { loadConversationEventHistory } from "./history-loader"; +import { + getConversationEventStore, + getConversationMessageStore, +} from "@/chat/db"; import { projectVisibleConversationCompactions } from "./visible-compactions"; import { projectVisibleConversationMessages } from "./visible-message-projection"; import { toStoredConversationMessage } from "./visible-message-serializer"; @@ -18,13 +20,8 @@ import type { ThreadConversationState } from "@/chat/state/conversation"; * Replace the in-memory working set with the durable event-log transcript, * excluding messages already folded into a visible compaction summary. * - * Hydrate is a first-read boundary, so it must trigger the once-only Redis→SQL - * lazy import before reading events: consumers that hydrate before any model - * projection read (turn-dedupe, delivered-message redelivery guards, - * channel-context assembly) would otherwise make correctness decisions on an - * empty transcript for promotion-window stragglers whose history is still only - * in legacy Redis. The import is idempotent (skips when SQL event rows exist) - * and no-ops cheaply when there is nothing legacy to read. + * SQL events are the only live history authority. The operator upgrade command + * owns any pre-cutover Redis import before new workers serve the conversation. */ export async function hydrateConversationMessages(args: { conversation: ThreadConversationState; @@ -34,9 +31,9 @@ export async function hydrateConversationMessages(args: { args.conversation.messages = []; return; } - const events = await loadConversationEventHistory({ - conversationId: args.conversationId, - }); + const events = await getConversationEventStore().loadHistory( + args.conversationId, + ); args.conversation.compactions = projectVisibleConversationCompactions(events); const coveredIds = new Set( args.conversation.compactions.flatMap( diff --git a/packages/junior/src/chat/conversations/legacy-advisor-session.ts b/packages/junior/src/cli/upgrade/migrations/conversation-history/advisor-session.ts similarity index 89% rename from packages/junior/src/chat/conversations/legacy-advisor-session.ts rename to packages/junior/src/cli/upgrade/migrations/conversation-history/advisor-session.ts index 87f1b7187..5f0b1bbdd 100644 --- a/packages/junior/src/chat/conversations/legacy-advisor-session.ts +++ b/packages/junior/src/cli/upgrade/migrations/conversation-history/advisor-session.ts @@ -2,8 +2,6 @@ import { z } from "zod"; import { piMessageSchema, type PiMessage } from "@/chat/pi/messages"; import { getStateAdapter } from "@/chat/state/adapter"; -// TODO(v0.104.0): Remove this reader with the bounded Redis-to-SQL import path -// after the legacy import horizon. const legacyAdvisorMessagesSchema = z.array(piMessageSchema); /** Read historical advisor messages during the bounded Redis-to-SQL import. */ diff --git a/packages/junior/src/chat/conversations/legacy-import.ts b/packages/junior/src/cli/upgrade/migrations/conversation-history/import.ts similarity index 74% rename from packages/junior/src/chat/conversations/legacy-import.ts rename to packages/junior/src/cli/upgrade/migrations/conversation-history/import.ts index 19a34ad35..87a0471fd 100644 --- a/packages/junior/src/chat/conversations/legacy-import.ts +++ b/packages/junior/src/cli/upgrade/migrations/conversation-history/import.ts @@ -1,20 +1,17 @@ /** - * One-time Redis→SQL conversation import (bulk + lazy), deletion-scoped. + * One-time Redis→SQL conversation import for the operator upgrade command. * - * A single per-conversation import unit shared by `junior upgrade` (bulk, - * bounded newest-first) and the lazy first-read straggler path. It converts the + * A single per-conversation import unit used by `junior upgrade` (bulk, + * bounded newest-first). It converts the * legacy session log into `junior_conversation_events`, imports the advisor * session blob as a child conversation, and converts `thread-state` visible * messages into canonical events plus their rebuildable SQL search projection. * Import is idempotent per conversation: canonical event rows seal completed * imports. It never fabricates import-time timestamps. * - * This module and its lazy hook are removed wholesale after the legacy Redis TTL - * horizon passes; keeping it separate keeps that deletion mechanical. + * This module is intentionally outside the runtime conversation graph. Remove + * it after the legacy Redis operator-migration horizon passes. */ -// TODO(v0.104.0): Remove this module and its advisor-session reader after the -// legacy Redis-to-SQL import horizon. -import { eq } from "drizzle-orm"; import { z } from "zod"; import { type ConversationCompaction, @@ -27,21 +24,19 @@ import { readSessionLogEntries, type SessionLogEntry, type SessionLogStore, -} from "@/chat/state/session-log"; +} from "./session-log"; import { createLegacyAdvisorSessionReader, type LegacyAdvisorSessionReader, -} from "@/chat/conversations/legacy-advisor-session"; +} from "./advisor-session"; import type { JuniorSqlDatabase } from "@/db/db"; -import { juniorConversations } from "@/db/schema"; -import { getConversationEventStore, getSqlExecutor } from "@/chat/db"; -import { createSqlConversationEventStore } from "./sql/history"; -import type { Conversation } from "./store"; +import { createSqlConversationEventStore } from "@/chat/conversations/sql/history"; +import type { Conversation } from "@/chat/conversations/store"; import { convertAdvisorMessages, convertLegacySessionLog, writeLegacyImport, -} from "./sql/legacy-history-import"; +} from "./legacy-history-import"; const legacyVisibleMessageSchema = z.object({ id: z.string(), @@ -244,49 +239,3 @@ export async function importConversationFromLegacy( return { imported }; } - -/** - * Lazy first-read import for a straggler the old deployment touched during - * promotion. Runs under the conversation lease the worker already holds before - * any turn/resume projection read; idempotent skip-if-rows-exist makes re-entry - * safe. Missing legacy keys produce empty reads for genuinely new conversations; - * actual Redis read failures surface through normal worker recovery. - */ -export async function ensureLegacyConversationImport(args: { - conversationId: string; -}): Promise { - const eventStore = getConversationEventStore(); - if ((await eventStore.loadCurrentEpoch(args.conversationId)).length > 0) { - return; - } - const entries = await readSessionLogEntries({ - conversationId: args.conversationId, - }); - const snapshot = await loadThreadStateSnapshot(args.conversationId); - const visible = snapshot.messages; - if ( - entries.length === 0 && - visible.length === 0 && - snapshot.compactions.length === 0 - ) { - return; - } - const executor = getSqlExecutor(); - const purged = await executor - .db() - .select({ transcriptPurgedAt: juniorConversations.transcriptPurgedAt }) - .from(juniorConversations) - .where(eq(juniorConversations.conversationId, args.conversationId)); - if (purged[0]?.transcriptPurgedAt) { - return; - } - await importConversationFromLegacy(args.conversationId, { - executor, - sessionLogStore: { read: async () => entries }, - loadVisibleMessages: async () => visible, - legacyCompactions: snapshot.compactions, - ...(snapshot.lastActivityAtMs === undefined - ? {} - : { legacyLastActivityAtMs: snapshot.lastActivityAtMs }), - }); -} diff --git a/packages/junior/src/chat/conversations/sql/legacy-history-import.ts b/packages/junior/src/cli/upgrade/migrations/conversation-history/legacy-history-import.ts similarity index 97% rename from packages/junior/src/chat/conversations/sql/legacy-history-import.ts rename to packages/junior/src/cli/upgrade/migrations/conversation-history/legacy-history-import.ts index 92ab9b8db..b45881f9f 100644 --- a/packages/junior/src/chat/conversations/sql/legacy-history-import.ts +++ b/packages/junior/src/cli/upgrade/migrations/conversation-history/legacy-history-import.ts @@ -1,39 +1,36 @@ /** - * One-time Redis→SQL legacy history importer (import-only, deletion-scoped). + * One-time Redis→SQL legacy history importer used only by `junior upgrade`. * * Translates the legacy `junior:agent-session-log:` list shape into * `junior_conversation_events` rows: `sessionId` markers become context epochs, * `projection_reset` entries explode into a `context_epoch_started` marker plus * per-message rows, advisor `transcriptRef` links become `childConversationId`, * and legacy v1 provenance normalizes exactly as the legacy reducer does. This - * whole module is removed with the lazy-import path after the Redis TTL horizon; - * its self-contained child-id formula reproduces historical advisor ids during - * that bounded import. + * whole module is removed after the operator-migration horizon; its + * self-contained child-id formula reproduces historical advisor ids during + * that bounded backfill. */ import { eq, sql } from "drizzle-orm"; import type { JuniorSqlDatabase } from "@/db/db"; import { sanitizePostgresJson } from "@/db/postgres-json"; import type { PiMessage } from "@/chat/pi/messages"; import { unescapeXml } from "@/chat/xml"; -import type { NewConversationMessage } from "../messages"; -import { - legacyActorProvenance, - type SessionLogEntry, -} from "@/chat/state/session-log"; +import type { NewConversationMessage } from "@/chat/conversations/messages"; +import { legacyActorProvenance, type SessionLogEntry } from "./session-log"; import { contextProvenance, type ConversationMessageProvenance, -} from "../provenance"; +} from "@/chat/conversations/provenance"; import { conversationEventDataSchema, type ConversationEventData, -} from "../history"; +} from "@/chat/conversations/history"; import { juniorConversationEvents, juniorConversationMessages, juniorConversations, } from "@/db/schema"; -import { withConversationEventLock } from "./event-lock"; +import { withConversationEventLock } from "@/chat/conversations/sql/event-lock"; const INITIAL_SESSION_ID = "session_0"; const ADVISOR_TASK_OPEN = "\n"; diff --git a/packages/junior/src/chat/state/session-log.ts b/packages/junior/src/cli/upgrade/migrations/conversation-history/session-log.ts similarity index 99% rename from packages/junior/src/chat/state/session-log.ts rename to packages/junior/src/cli/upgrade/migrations/conversation-history/session-log.ts index 49e3d0e6e..9613a894c 100644 --- a/packages/junior/src/chat/state/session-log.ts +++ b/packages/junior/src/cli/upgrade/migrations/conversation-history/session-log.ts @@ -1,5 +1,5 @@ /** - * Legacy Redis Pi session-log read port. + * Legacy Redis Pi session-log reader for the operator SQL backfill. * * SQL conversation events are the sole durable history authority. This module * exists only to decode and read pre-cutover Redis entries during the bounded diff --git a/packages/junior/src/chat/conversations/state.ts b/packages/junior/src/cli/upgrade/migrations/conversation-history/state-conversation-store.ts similarity index 93% rename from packages/junior/src/chat/conversations/state.ts rename to packages/junior/src/cli/upgrade/migrations/conversation-history/state-conversation-store.ts index 58b5455d2..5950bf1ad 100644 --- a/packages/junior/src/chat/conversations/state.ts +++ b/packages/junior/src/cli/upgrade/migrations/conversation-history/state-conversation-store.ts @@ -5,7 +5,7 @@ import { recordConversationExecution as recordTaskConversationExecution, } from "@/chat/task-execution/state"; import type { StateAdapter } from "chat"; -import type { ConversationStore } from "./store"; +import type { ConversationStore } from "@/chat/conversations/store"; /** Create the legacy-import conversation record store backed by task-execution state. */ export function createStateConversationStore( diff --git a/packages/junior/src/cli/upgrade/migrations/conversations-history-sql.ts b/packages/junior/src/cli/upgrade/migrations/conversations-history-sql.ts index ea544ac11..65a2587d3 100644 --- a/packages/junior/src/cli/upgrade/migrations/conversations-history-sql.ts +++ b/packages/junior/src/cli/upgrade/migrations/conversations-history-sql.ts @@ -1,9 +1,9 @@ import { getChatConfig } from "@/chat/config"; -import { importConversationFromLegacy } from "@/chat/conversations/legacy-import"; -import { createStateConversationStore } from "@/chat/conversations/state"; -import type { LegacyAdvisorSessionReader } from "@/chat/conversations/legacy-advisor-session"; +import { importConversationFromLegacy } from "./conversation-history/import"; +import { createStateConversationStore } from "./conversation-history/state-conversation-store"; +import type { LegacyAdvisorSessionReader } from "./conversation-history/advisor-session"; import type { ConversationMessage as ThreadConversationMessage } from "@/chat/state/conversation"; -import type { SessionLogStore } from "@/chat/state/session-log"; +import type { SessionLogStore } from "./conversation-history/session-log"; import { createJuniorSqlExecutor } from "@/db/executor"; import type { JuniorSqlExecutor } from "@/db/db"; import type { MigrationContext, MigrationResult } from "../types"; diff --git a/packages/junior/src/cli/upgrade/migrations/conversations-sql.ts b/packages/junior/src/cli/upgrade/migrations/conversations-sql.ts index bb24c4283..6375755f1 100644 --- a/packages/junior/src/cli/upgrade/migrations/conversations-sql.ts +++ b/packages/junior/src/cli/upgrade/migrations/conversations-sql.ts @@ -1,6 +1,6 @@ import { getChatConfig } from "@/chat/config"; import { createSqlStore, type SqlStore } from "@/chat/conversations/sql/store"; -import { createStateConversationStore } from "@/chat/conversations/state"; +import { createStateConversationStore } from "./conversation-history/state-conversation-store"; import { addAgentTurnUsage } from "@/chat/usage"; import { listAgentTurnSessionSummariesForConversations } from "@/chat/state/turn-session"; import { createJuniorSqlExecutor } from "@/db/executor"; diff --git a/packages/junior/tests/component/conversations/legacy-import.test.ts b/packages/junior/tests/component/cli/conversation-history-import.test.ts similarity index 82% rename from packages/junior/tests/component/conversations/legacy-import.test.ts rename to packages/junior/tests/component/cli/conversation-history-import.test.ts index 0af5702b1..7bcd2e9b3 100644 --- a/packages/junior/tests/component/conversations/legacy-import.test.ts +++ b/packages/junior/tests/component/cli/conversation-history-import.test.ts @@ -10,23 +10,20 @@ import { } from "@/chat/db"; import { disconnectStateAdapter, getStateAdapter } from "@/chat/state/adapter"; import { requestConversationWork } from "@/chat/task-execution/store"; -import { - ensureLegacyConversationImport, - importConversationFromLegacy, -} from "@/chat/conversations/legacy-import"; +import { importConversationFromLegacy } from "@/cli/upgrade/migrations/conversation-history/import"; import { createSqlConversationEventStore } from "@/chat/conversations/sql/history"; import { createSqlConversationMessageStore } from "@/chat/conversations/sql/messages"; import { migrateSchema } from "@/chat/conversations/sql/migrations"; import { hydrateConversationMessages } from "@/chat/conversations/visible-messages"; import { coerceThreadConversationState } from "@/chat/state/conversation"; import { migrateConversationHistoryToSql } from "@/cli/upgrade/migrations/conversations-history-sql"; -import type { LegacyAdvisorSessionReader } from "@/chat/conversations/legacy-advisor-session"; +import type { LegacyAdvisorSessionReader } from "@/cli/upgrade/migrations/conversation-history/advisor-session"; import type { Conversation } from "@/chat/conversations/store"; import type { PiMessage } from "@/chat/pi/messages"; import type { SessionLogEntry, SessionLogStore, -} from "@/chat/state/session-log"; +} from "@/cli/upgrade/migrations/conversation-history/session-log"; import type { ConversationMessage as ThreadConversationMessage } from "@/chat/state/conversation"; import { CONVERSATION_ID, @@ -116,7 +113,7 @@ function staticAdvisorStore(messages: PiMessage[]): LegacyAdvisorSessionReader { }; } -describe("legacy conversation import", () => { +describe("operator conversation history import", () => { beforeEach(async () => { process.env.JUNIOR_STATE_ADAPTER = "memory"; await disconnectStateAdapter(); @@ -460,121 +457,6 @@ describe("legacy conversation import", () => { } }, 20_000); - it("lazily imports a straggler with a Redis log but no SQL rows, once", async () => { - const { executor, eventStore } = await processSqlStores(); - - const stateAdapter = getStateAdapter(); - await stateAdapter.connect(); - await stateAdapter.set(`junior:agent-session-log:${CONVERSATION_ID}`, [ - { - schemaVersion: 2, - type: "pi_message", - sessionId: "session_0", - message: userMessage("straggler", 70), - }, - ]); - await stateAdapter.set(`thread-state:${CONVERSATION_ID}`, { - conversation: { - messages: [], - stats: { updatedAtMs: 900 }, - }, - }); - - await ensureLegacyConversationImport({ conversationId: CONVERSATION_ID }); - const history = await eventStore.loadHistory(CONVERSATION_ID); - expect(history).toHaveLength(1); - expect(history[0]!.data.type).toBe("message"); - expect(history[0]!.createdAtMs).toBe(70); - const [conversation] = await executor - .db() - .select({ lastActivityAt: juniorConversations.lastActivityAt }) - .from(juniorConversations) - .where(eq(juniorConversations.conversationId, CONVERSATION_ID)); - expect(conversation?.lastActivityAt.getTime()).toBe(900); - - // Second read is idempotent: no duplicate rows. - await ensureLegacyConversationImport({ conversationId: CONVERSATION_ID }); - expect(await eventStore.loadHistory(CONVERSATION_ID)).toHaveLength(1); - }, 20_000); - - it("loadConnectedMcpProviders triggers the lazy import for a straggler", async () => { - await processSqlStores(); - - // A Redis-only straggler whose durable MCP connection fact has not been - // imported yet: the provider read must not miss it. - const stateAdapter = getStateAdapter(); - await stateAdapter.connect(); - await stateAdapter.set(`junior:agent-session-log:${CONVERSATION_ID}`, [ - { - schemaVersion: 2, - type: "pi_message", - sessionId: "session_0", - message: userMessage("straggler", 70), - }, - { - schemaVersion: 2, - type: "mcp_provider_connected", - sessionId: "session_0", - provider: "linear", - }, - ]); - - const { loadConnectedMcpProviders } = - await import("@/chat/conversations/projection"); - await expect( - loadConnectedMcpProviders({ conversationId: CONVERSATION_ID }), - ).resolves.toEqual(["linear"]); - }, 20_000); - - it("hydrate triggers the lazy import for a Redis-only straggler and preserves replied + author", async () => { - await processSqlStores(); - - // Seed ONLY legacy Redis thread-state (a user message with a delivery mark - // and an author) and no SQL rows: the promotion-window straggler shape. - const stateAdapter = getStateAdapter(); - await stateAdapter.connect(); - await stateAdapter.set(`thread-state:${CONVERSATION_ID}`, { - conversation: { - schemaVersion: 1, - compactions: [ - { - id: "legacy-compaction", - summary: "Older imported context", - coveredMessageIds: ["older-message"], - createdAtMs: 90, - }, - ], - messages: [ - { - id: "m1", - role: "user", - text: "legacy hello", - createdAtMs: 100, - author: { userId: "U123", userName: "alice", fullName: "Alice" }, - meta: { replied: true }, - }, - ], - }, - }); - - const conversation = coerceThreadConversationState({}); - await hydrateConversationMessages({ - conversation, - conversationId: CONVERSATION_ID, - }); - - const hydratedUser = conversation.messages.find( - (message) => message.id === "m1", - ); - expect(hydratedUser?.text).toBe("legacy hello"); - expect(hydratedUser?.author?.userId).toBe("U123"); - expect(hydratedUser?.author?.userName).toBe("alice"); - expect(hydratedUser?.meta?.replied).toBe(true); - expect(conversation.compactions).toEqual([ - expect.objectContaining({ id: "legacy-compaction" }), - ]); - }, 20_000); - it("hydrates visible messages from events without message-table rows", async () => { const { eventStore } = await processSqlStores(); await eventStore.append(CONVERSATION_ID, [ @@ -605,36 +487,29 @@ describe("legacy conversation import", () => { ]); }, 20_000); - it("does not resurrect purged SQL history from legacy Redis", async () => { - const { executor, eventStore } = await processSqlStores(); - - await executor - .db() - .insert(juniorConversations) - .values({ - conversationId: CONVERSATION_ID, - createdAt: new Date(100), - lastActivityAt: new Date(100), - updatedAt: new Date(100), - executionStatus: "idle", - transcriptPurgedAt: new Date(200), - }); + it("does not consult legacy Redis during live hydration", async () => { + const { eventStore } = await processSqlStores(); const stateAdapter = getStateAdapter(); await stateAdapter.connect(); - await stateAdapter.set(`junior:agent-session-log:${CONVERSATION_ID}`, [ - { - schemaVersion: 2, - type: "pi_message", - sessionId: "session_0", - message: userMessage("must stay purged", 50), + await stateAdapter.set(`thread-state:${CONVERSATION_ID}`, { + conversation: { + messages: [ + { + id: "redis-only", + role: "user", + text: "legacy text must stay outside the runtime", + createdAtMs: 100, + }, + ], }, - ]); + }); const conversation = coerceThreadConversationState({}); await hydrateConversationMessages({ conversation, conversationId: CONVERSATION_ID, }); + expect(conversation.messages).toEqual([]); expect(await eventStore.loadHistory(CONVERSATION_ID)).toEqual([]); }, 20_000); @@ -685,19 +560,6 @@ describe("legacy conversation import", () => { } }, 20_000); - it("surfaces legacy Redis read failures during lazy import", async () => { - await processSqlStores(); - const stateAdapter = getStateAdapter(); - await stateAdapter.connect(); - vi.spyOn(stateAdapter, "getList").mockRejectedValueOnce( - new Error("legacy Redis unavailable"), - ); - - await expect( - ensureLegacyConversationImport({ conversationId: CONVERSATION_ID }), - ).rejects.toThrow("legacy Redis unavailable"); - }, 20_000); - it("bulk-imports legacy Redis history through the upgrade migration", async () => { const fixture = await createLocalJuniorSqlFixture(); const stateAdapter = getStateAdapter(); diff --git a/packages/junior/tests/unit/conversations/legacy-history-import.test.ts b/packages/junior/tests/unit/cli/conversation-history-conversion.test.ts similarity index 97% rename from packages/junior/tests/unit/conversations/legacy-history-import.test.ts rename to packages/junior/tests/unit/cli/conversation-history-conversion.test.ts index be10146c8..d76672e50 100644 --- a/packages/junior/tests/unit/conversations/legacy-history-import.test.ts +++ b/packages/junior/tests/unit/cli/conversation-history-conversion.test.ts @@ -2,9 +2,9 @@ import { describe, expect, it } from "vitest"; import { convertAdvisorMessages, convertLegacySessionLog, -} from "@/chat/conversations/sql/legacy-history-import"; +} from "@/cli/upgrade/migrations/conversation-history/legacy-history-import"; import type { PiMessage } from "@/chat/pi/messages"; -import type { SessionLogEntry } from "@/chat/state/session-log"; +import type { SessionLogEntry } from "@/cli/upgrade/migrations/conversation-history/session-log"; const CONVERSATION_ID = "slack:C1:1710000.0001"; const FALLBACK_MS = 1_000; @@ -39,7 +39,7 @@ function piEntry( } as SessionLogEntry; } -describe("convertLegacySessionLog", () => { +describe("operator legacy conversation history conversion", () => { it("keeps a single session in epoch 0 with sequential seq and message timestamps", () => { const { events, advisorChildConversationId } = convertLegacySessionLog({ conversationId: CONVERSATION_ID, diff --git a/packages/junior/tests/unit/state/session-log.test.ts b/packages/junior/tests/unit/cli/conversation-history-session-log.test.ts similarity index 96% rename from packages/junior/tests/unit/state/session-log.test.ts rename to packages/junior/tests/unit/cli/conversation-history-session-log.test.ts index a0b6c8273..e7438665c 100644 --- a/packages/junior/tests/unit/state/session-log.test.ts +++ b/packages/junior/tests/unit/cli/conversation-history-session-log.test.ts @@ -5,7 +5,7 @@ import { readSessionLogEntries, type SessionLogEntry, type SessionLogStore, -} from "@/chat/state/session-log"; +} from "@/cli/upgrade/migrations/conversation-history/session-log"; function userMessage(text: string): PiMessage { return { @@ -21,7 +21,7 @@ function memoryStore(values: unknown[]): SessionLogStore { }; } -describe("legacy session log decode", () => { +describe("operator legacy session log decode", () => { it("preserves ordered legacy entry discriminants for the SQL importer", async () => { const message = userMessage("legacy question"); const values = [ From 3ceed61f5fcfe294d9021b43e52689e6758eb2c8 Mon Sep 17 00:00:00 2001 From: David Cramer Date: Wed, 15 Jul 2026 10:13:40 -0700 Subject: [PATCH 12/58] docs(conversations): Clarify event log authority Document canonical metadata, non-history aggregates, completed event reporting work, and the bounded migration and delivery follow-ups that remain intentionally open. Co-Authored-By: GPT-5 Codex --- .../changes/conversation-event-log/tasks.md | 18 ++++++++++-------- .../junior/src/chat/conversations/README.md | 10 ++++++++-- .../junior/src/chat/conversations/store.ts | 5 ++++- 3 files changed, 22 insertions(+), 11 deletions(-) diff --git a/openspec/changes/conversation-event-log/tasks.md b/openspec/changes/conversation-event-log/tasks.md index 8a026ee84..a792186a5 100644 --- a/openspec/changes/conversation-event-log/tasks.md +++ b/openspec/changes/conversation-event-log/tasks.md @@ -25,9 +25,10 @@ - [x] Keep Pi messages derived rather than separately persisted. - [x] Add strict turn-start and mutually exclusive terminal outcome variants, then cut the local runtime over at its input/delivery persistence bounds. -- [ ] Add durable delivery intent/receipt reconciliation and delivery-attempt - events, then cut Slack/dispatch lifecycle writers over without a - destination-acceptance/process-crash gap. +- [x] Add durable delivery intent/receipt reconciliation and delivery-attempt + events for ordinary Slack finalized replies. +- [ ] Cut dispatch, resumed, private-authorization, and nonstandard notice + delivery paths over without a destination-acceptance/process-crash gap. - [x] Persist visible-message and reply facts as canonical events while updating the hydration/search table as an atomic read model. - [x] Backfill existing visible-message rows with a bounded, verified migration @@ -36,11 +37,11 @@ ## 3. Event API And Dashboard -- [ ] Return one ordered, authorized, privacy-safe event array from conversation +- [x] Return one ordered, authorized, privacy-safe event array from conversation detail reporting. -- [ ] Move turn grouping, tool activity, failures, compactions, and delivery +- [x] Move turn grouping, tool activity, failures, compactions, and delivery presentation into the dashboard client. -- [ ] Remove the parallel transcript, activity, and context-event API views once +- [x] Remove the parallel transcript, activity, and context-event API views once all consumers use events. ## 4. Child Conversations @@ -57,10 +58,11 @@ events instead of the message-table projection or model messages. - [x] Remove lazy Redis/advisor history import from every runtime read and move the retained backfill decoder/writer under `cli/upgrade` only. -- [ ] Make remaining aggregate stores explicit read models. +- [x] Document conversation execution/usage aggregates and Redis turn-session + summaries as non-history reporting and control projections. - [ ] Delete the explicit operator Redis backfill after every supported deployment has crossed its migration horizon. -- [ ] Verify retention, purge, redaction, idempotency, replay parity, and event +- [x] Verify retention, purge, redaction, idempotency, replay parity, and event tree behavior. - [ ] Move durable invariants into owning code and documentation, then delete this completed plan. diff --git a/packages/junior/src/chat/conversations/README.md b/packages/junior/src/chat/conversations/README.md index 0db2fec3e..2efcacd34 100644 --- a/packages/junior/src/chat/conversations/README.md +++ b/packages/junior/src/chat/conversations/README.md @@ -5,14 +5,20 @@ conversation events, compaction boundaries, search, and retention. ## Records -- Conversation rows identify the source, destination, participants, visibility, - and lifecycle metadata. +- Conversation rows own canonical source, destination, participant, visibility, + lineage, and retention metadata. Their execution cursor, status, duration, + and usage columns are materialized reporting/control aggregates, not + transcript or event-history authority. - Visible-message events are the destination-facing user and assistant history. `junior_conversation_messages` is their rebuildable search read model, never a hydration source or second history authority. - Conversation events are the versioned, append-only execution history used to restore Pi state. Pi messages and context are projections of this log, not a second authority. +- Redis turn-session records retain bounded resumability metadata and event + sequence cursors. Their summary indexes and the SQL execution/usage columns + are operational projections; Pi state is materialized from conversation + events rather than stored in either aggregate. - Context epochs identify replacement boundaries created by compaction or model handoff. - Child rows carry immutable parent/root, parent-turn, and exact parent-event diff --git a/packages/junior/src/chat/conversations/store.ts b/packages/junior/src/chat/conversations/store.ts index 9f8c3798d..9180f0342 100644 --- a/packages/junior/src/chat/conversations/store.ts +++ b/packages/junior/src/chat/conversations/store.ts @@ -81,7 +81,10 @@ export interface ConversationStore { /** Source-confirmed visibility from the current event's signal only. */ visibility?: ConversationPrivacy; }): Promise; - /** Store task-execution metadata for long-term conversation history. */ + /** + * Materialize execution and usage aggregates beside canonical metadata. + * These fields serve reporting and runtime control, never history hydration. + */ recordExecution(args: { channelName?: string; conversationId: string; From 1374d234a1d4a86a9b92fa546a84e66423a777a2 Mon Sep 17 00:00:00 2001 From: David Cramer Date: Wed, 15 Jul 2026 10:26:34 -0700 Subject: [PATCH 13/58] test(migrations): Apply lineage schema in event fixture Co-Authored-By: GPT-5 Codex --- .../component/cli/conversation-event-migration.test.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/packages/junior/tests/component/cli/conversation-event-migration.test.ts b/packages/junior/tests/component/cli/conversation-event-migration.test.ts index f382ad5e5..bfa355ac6 100644 --- a/packages/junior/tests/component/cli/conversation-event-migration.test.ts +++ b/packages/junior/tests/component/cli/conversation-event-migration.test.ts @@ -186,6 +186,9 @@ describe("conversation event migration", () => { const visibleMessageEvents = migrationStatements( "0005_visible_message_events.sql", ); + const conversationLineage = migrationStatements( + "0007_conversation_lineage.sql", + ); const conversationId = "conversation-visible-events"; try { @@ -216,6 +219,10 @@ describe("conversation event migration", () => { (statement) => fixture.sql.execute(statement), visibleMessageEvents, ); + await executeStatements( + (statement) => fixture.sql.execute(statement), + conversationLineage, + ); const context = { io: { info: () => {} }, From e0fd8134c679c10ba2cde467a04a9e5e5c1d9074 Mon Sep 17 00:00:00 2001 From: David Cramer Date: Wed, 15 Jul 2026 10:50:07 -0700 Subject: [PATCH 14/58] docs(migrations): Remove completed event plan Co-Authored-By: GPT-5 Codex --- .../conversation-event-log/proposal.md | 87 -------------- .../changes/conversation-event-log/tasks.md | 68 ----------- packages/junior/migrations/README.md | 106 ++++++++++++------ 3 files changed, 72 insertions(+), 189 deletions(-) delete mode 100644 openspec/changes/conversation-event-log/proposal.md delete mode 100644 openspec/changes/conversation-event-log/tasks.md diff --git a/openspec/changes/conversation-event-log/proposal.md b/openspec/changes/conversation-event-log/proposal.md deleted file mode 100644 index a89304766..000000000 --- a/openspec/changes/conversation-event-log/proposal.md +++ /dev/null @@ -1,87 +0,0 @@ -# Canonical Conversation Event Log - -## Summary - -Make one ordered, append-only Junior conversation event log the canonical -history for model context, operator history, delivery facts, and child-agent -lineage. Pi and the dashboard consume separate projections of the same events. - -## Motivation - -Before this change, Junior reconstructed related views from agent steps, delivered -messages, turn-session records, activity records, and context events. Pi needs -an exact model context, while the dashboard needs chronological operational -history. Treating both as variants of one transcript makes failures difficult -to correlate and encourages duplicate message representations. - -The pre-cutover store persisted ordered agent steps and filtered host-only steps -out of Pi context. This change makes that event boundary explicit and extends it -rather than creating another transcript or lifecycle sidecar. - -## Design - -`ConversationEvent` is the canonical persisted history contract. Every event -has stable conversation ordering, a timestamp, and a versioned payload. The -initial cutover formalized the durable event kinds Junior already stored. A -later vertical slice adds correlated start and first-terminal-wins outcome -events to the local runtime; Slack, dispatch, delivery attempts, and lineage -follow at their own owning boundaries. - -The storage cutover rewrites existing `pi_message` rows into Junior-owned -`message` events. Their opaque continuity payload is interpreted as a Pi -message only by the Pi adapter; Pi no longer owns the persisted event contract. - -Two primary adapters consume the log: - -- the Pi adapter selects model-relevant events and deterministically produces - `PiMessage[]`; -- the conversation detail API projects persistence events into a separate - Junior-owned, authorized, redacted reporting-event contract, and the - dashboard builds its own timeline presentation from that safe contract. - -The raw persistence union is never an API or dashboard contract. In particular, -legacy Pi roles, provider fields, and permissive message payloads do not cross -the reporting boundary merely because they are readable during migration. - -Conversation list/search tables may remain materialized read models. Queue -wakeups, leases, resumability cursors, and credentials remain mutable control -state rather than conversation history. - -Subagents use independent conversation streams in the same event system. Child -conversation metadata records parent/root lineage and a shared-context fork -point when needed. Parent events reference child execution without copying the -child event stream. - -## Rollout - -1. Formalize the event contract, isolate Pi projection, rename the physical - table, persist schema versions, and cut every application reader and writer - over to canonical Junior events. -2. Rewrite legacy model-message rows, drain all 0.103.x workers, then apply the - hard schema cut that removes their compatibility view. Backfill - visible-message rows while workers remain stopped; start new workers only - after fail-closed zero-gap verification passes. -3. Add correlated local lifecycle events and project failures as safe detail - markers. Add Slack/dispatch writers only with durable delivery - intent/receipt reconciliation so process death cannot strand accepted work. -4. Expose privacy-safe events from the detail API and move timeline shaping to - the dashboard. -5. Add child-conversation lineage and context-fork projection. -6. Remove obsolete transcript reconstruction and demote remaining message - stores to explicit read models. - -## Non-Goals - -- Event-sourcing queue leases, credentials, or other mutable control state. -- Sending delivery, retry, failure, or provider diagnostics into Pi context. -- Copying child conversation events into parent conversations. -- Duplicating physical event rows or retaining parallel application stores. -- Persisting raw provider errors or private payloads as operational metadata. - -## Compatibility - -The in-place table rename preserves sequence identity and payload bytes. A -temporary SQL view supports the first migration phase, but the visible-message -cutover is intentionally not rolling: 0.103.x workers must be drained before -the view and its functions are dropped. New workers start only after the final -backfill verifies zero gaps. diff --git a/openspec/changes/conversation-event-log/tasks.md b/openspec/changes/conversation-event-log/tasks.md deleted file mode 100644 index a792186a5..000000000 --- a/openspec/changes/conversation-event-log/tasks.md +++ /dev/null @@ -1,68 +0,0 @@ -# Tasks - -## 1. Event Contract And Pi Adapter - -- [x] Define the runtime-schema-owned `ConversationEvent` envelope and event - data union over only the existing ordered history kinds. -- [x] Isolate deterministic `ConversationEvent[] -> PiMessage[]` projection in - the Pi-owned module. -- [x] Preserve message boundaries, provenance, context epochs, model binding, - authorization observations, and bounded sequence projection. -- [x] Hard-cut storage writes and SQL rows to canonical events without changing - API payloads or runtime behavior. -- [x] Restrict legacy `pi_message` to the explicit operator backfill; never - expose it through runtime conversation modules or reporting APIs. -- [x] Add focused adapter parity tests and update owning module documentation. - -## 2. Canonical Event Writes - -- [x] Use `(conversationId, seq)` as stable event identity and persist schema - versions on every physical event row. -- [x] Persist Junior-owned message, tool, authorization, subagent, and context - events through `ConversationEventStore`. -- [x] Rewrite legacy SQL message rows in bounded, retry-safe batches while a - database-only compatibility view supports rolling old workers. -- [x] Keep Pi messages derived rather than separately persisted. -- [x] Add strict turn-start and mutually exclusive terminal outcome variants, - then cut the local runtime over at its input/delivery persistence bounds. -- [x] Add durable delivery intent/receipt reconciliation and delivery-attempt - events for ordinary Slack finalized replies. -- [ ] Cut dispatch, resumed, private-authorization, and nonstandard notice - delivery paths over without a destination-acceptance/process-crash gap. -- [x] Persist visible-message and reply facts as canonical events while updating - the hydration/search table as an atomic read model. -- [x] Backfill existing visible-message rows with a bounded, verified migration - while workers are stopped, after removing the drained 0.103.x workers' - compatibility view. - -## 3. Event API And Dashboard - -- [x] Return one ordered, authorized, privacy-safe event array from conversation - detail reporting. -- [x] Move turn grouping, tool activity, failures, compactions, and delivery - presentation into the dashboard client. -- [x] Remove the parallel transcript, activity, and context-event API views once - all consumers use events. - -## 4. Child Conversations - -- [x] Add parent/root conversation lineage and parent turn/event correlation. -- [x] Record subagent start and finish references without copying child events. -- [x] Represent shared context with an immutable parent sequence fork point. -- [x] Verify isolated and shared child Pi projections and inherited privacy. - -## 5. Cleanup And Verification - -- [x] Make the visible-message table an explicit event-backed read model. -- [x] Hydrate runtime and primary conversation-detail transcripts from visible - events instead of the message-table projection or model messages. -- [x] Remove lazy Redis/advisor history import from every runtime read and move - the retained backfill decoder/writer under `cli/upgrade` only. -- [x] Document conversation execution/usage aggregates and Redis turn-session - summaries as non-history reporting and control projections. -- [ ] Delete the explicit operator Redis backfill after every supported - deployment has crossed its migration horizon. -- [x] Verify retention, purge, redaction, idempotency, replay parity, and event - tree behavior. -- [ ] Move durable invariants into owning code and documentation, then delete - this completed plan. diff --git a/packages/junior/migrations/README.md b/packages/junior/migrations/README.md index e27b300b4..17403d3e5 100644 --- a/packages/junior/migrations/README.md +++ b/packages/junior/migrations/README.md @@ -1,36 +1,74 @@ # SQL migrations for `@sentry/junior` -This is a standard Drizzle migration folder. `drizzle-kit generate` owns each -SQL file, snapshot, and journal entry; `junior upgrade` applies the folder with -Drizzle ORM's migrator before any data backfills run. - -1. Edit the schema under `src/db/schema/`. -2. Run `pnpm --filter @sentry/junior db:generate --name `. -3. Commit the generated SQL file and `meta/` changes together. - -The `0000_initial.sql` baseline represents the schema already deployed by the -pre-Drizzle Junior migration runner. During upgrade, existing installations -adopt that baseline once; new installations execute it normally. All later -migrations are applied by Drizzle in journal order. Baseline adoption requires -the legacy `junior_agent_steps` base table and no -`junior_conversation_events` table. A post-cutover schema without its Drizzle -journal fails closed for operator repair instead of inferring completed -migrations from mutable table shape. Every expected legacy migration record -must retain its exact historical checksum; an ID alone cannot prove which SQL -ran. Legacy metrics adoption likewise requires either none or all four metric -columns, with the legacy metrics record agreeing with that physical state. The -later search index and `metric_run_id` column must also be absent because only -the Drizzle journal may prove those immutable migrations ran. - -`0004_conversation_events.sql` temporarily creates `junior_agent_steps` as an -updatable 0.103.x compatibility view. It maps legacy `pi_message` reads and -writes to canonical `message` events while the first event rewrite runs. - -`0005_visible_message_events.sql` is the hard cut: drain every 0.103.x worker -before applying it because it drops the legacy view and its functions. Run the -final visible-message backfill next and require zero-gap verification before -starting new workers. - -`0007_conversation_lineage.sql` expands conversation metadata with immutable -child lineage and fork correlation. The subsequent bounded upgrade backfill -fills historical root IDs only; unknown historical fork points stay null. +This directory is the ordered Drizzle migration history for Junior's core SQL +schema. The TypeScript schema under `src/db/schema/` is the current contract; +the SQL files and `meta/` records describe how an existing database reaches +that contract. + +## Generate a migration + +1. Change the owning schema under `src/db/schema/`. +2. Run + `pnpm --filter @sentry/junior db:generate --name ` from the + repository root. +3. Review the generated SQL for locking, table rewrites, defaults, constraints, + indexes, and compatibility with the currently deployed workers. +4. Commit the SQL file, `meta/_journal.json`, and generated snapshot together. + +Drizzle compares the latest snapshot with the TypeScript schema. Snapshots are +generation inputs, not a substitute for the executable SQL history. Do not +rename, reorder, delete, or rewrite an applied migration, its snapshot, or its +journal entry; correct an already-shipped schema with a new migration. +Migration timestamps and journal order must remain append-only. + +## Apply migrations + +`junior upgrade` applies core schema migrations before running application data +backfills. `src/chat/conversations/sql/migrations.ts` resolves this packaged +directory in both source and built CLI layouts, serializes migration with the +core migration lock, and lets Drizzle record applied entries in +`drizzle.__drizzle_junior_core`. Re-running upgrade is expected and must not +reapply journaled SQL. + +Schema migration and data migration are separate concerns: + +- SQL files establish tables, columns, constraints, indexes, and temporary + database compatibility objects. +- Upgrade migrations under `src/cli/upgrade/migrations/` perform bounded, + rerunnable data adoption and backfills after the required schema exists. + +Keep destructive or non-rolling cutovers explicit. If old and new workers +cannot safely share the intermediate schema, drain the incompatible workers, +apply the schema migration, run and verify the required backfill, and only then +start the new workers. The owning module README must document any active +cutover gate; migration filenames are not durable operational documentation. + +## Legacy adoption + +The initial Drizzle baseline represents schema that predated the Drizzle +journal. Upgrade may adopt that baseline only when the legacy migration records +and physical schema match the exact state recognized by +`migrations.ts`. Adoption validates historical checksums and requires complete, +internally consistent schema markers. Ambiguous, partial, or post-cutover state +fails closed for operator repair instead of inferring success from mutable +table shape. + +This adoption path is the only exception to normal journal application. New +installations execute the baseline normally, and every later migration is +proved solely by the Drizzle journal. + +## Verification invariants + +- A migration must work on both a new database and every supported upgrade + state. +- Running schema migration twice must be safe because the journal prevents a + second application. +- Concurrent upgrade attempts must serialize on the migration lock. +- Generated SQL and metadata must stay synchronized with the schema change. +- Backfills must be bounded, rerunnable, and verify their completion criteria + before an incompatible cutover proceeds. + +The migration integration coverage in +`tests/integration/conversation-sql.test.ts` owns fresh-install, legacy-adoption, +ordering, locking, and failure-contract checks. Feature tests own the behavior +that becomes possible after a schema change. From 688fb9b4481b6c98421a90e55fcfc41a35e817ef Mon Sep 17 00:00:00 2001 From: David Cramer Date: Wed, 15 Jul 2026 10:56:26 -0700 Subject: [PATCH 15/58] fix(dashboard): Preserve error and location semantics Co-Authored-By: GPT-5 Codex --- .../components/SubagentTranscriptDrawer.tsx | 7 ++- .../src/client/components/transcriptStyles.ts | 10 +++- .../src/mock-conversations.ts | 37 ++++++++++++-- .../tests/dashboard-mock-routes.test.ts | 47 ++++++++++++++++- .../tests/telemetry-components.test.tsx | 51 ++++++++++++++++++- 5 files changed, 142 insertions(+), 10 deletions(-) diff --git a/packages/junior-dashboard/src/client/components/SubagentTranscriptDrawer.tsx b/packages/junior-dashboard/src/client/components/SubagentTranscriptDrawer.tsx index 67b32597e..2f19ef61b 100644 --- a/packages/junior-dashboard/src/client/components/SubagentTranscriptDrawer.tsx +++ b/packages/junior-dashboard/src/client/components/SubagentTranscriptDrawer.tsx @@ -125,8 +125,13 @@ function DrawerEmptyState(props: { children: string; tone?: "default" | "error"; }) { + const isError = props.tone === "error"; return ( -
+
{props.children}
); diff --git a/packages/junior-dashboard/src/client/components/transcriptStyles.ts b/packages/junior-dashboard/src/client/components/transcriptStyles.ts index 2ea2ce248..3e494d4bd 100644 --- a/packages/junior-dashboard/src/client/components/transcriptStyles.ts +++ b/packages/junior-dashboard/src/client/components/transcriptStyles.ts @@ -6,6 +6,12 @@ export function mutedTranscriptMetaClass(size = "text-[0.82rem]"): string { } /** Share the transcript empty/unavailable frame across top-level and segment views. */ -export function transcriptEmptyClass(): string { - return "rounded-lg border border-white/[0.07] bg-white/[0.025] p-5 text-[0.88rem] leading-relaxed text-white/45"; +export function transcriptEmptyClass( + tone: "default" | "error" = "default", +): string { + const colors = + tone === "error" + ? "border-rose-300/25 bg-rose-300/[0.07] text-rose-100" + : "border-white/[0.07] bg-white/[0.025] text-white/45"; + return `rounded-lg border p-5 text-[0.88rem] leading-relaxed ${colors}`; } diff --git a/packages/junior-dashboard/src/mock-conversations.ts b/packages/junior-dashboard/src/mock-conversations.ts index 68ea5e94a..e04c42dc9 100644 --- a/packages/junior-dashboard/src/mock-conversations.ts +++ b/packages/junior-dashboard/src/mock-conversations.ts @@ -902,19 +902,46 @@ function publicMockLocation( }; } +/** Find conversation time bounds without relying on feed order. */ +export function conversationTimeBounds( + summaries: readonly [ + ConversationSummaryReport, + ...ConversationSummaryReport[], + ], +): Pick { + let firstSeenAt = summaries[0].startedAt; + let lastSeenAt = summaries[0].lastSeenAt; + for (const summary of summaries) { + if (Date.parse(summary.startedAt) < Date.parse(firstSeenAt)) { + firstSeenAt = summary.startedAt; + } + if (Date.parse(summary.lastSeenAt) > Date.parse(lastSeenAt)) { + lastSeenAt = summary.lastSeenAt; + } + } + return { firstSeenAt, lastSeenAt }; +} + function publicLocation( summaries: ConversationSummaryReport[], channel: string, ): LocationSummaryReport { - const matching = summaries.filter((summary) => summary.channel === channel); - const item = statsItem(locationLabel(matching[0]!)); - matching.forEach((summary) => addSummary(item, summary)); + const [first, ...rest] = summaries.filter( + (summary) => summary.channel === channel, + ); + if (!first) throw new Error(`Missing mock summaries for ${channel}`); + const matching: [ConversationSummaryReport, ...ConversationSummaryReport[]] = + [first, ...rest]; + const item = statsItem(locationLabel(first)); + for (const summary of matching) { + addSummary(item, summary); + } + const bounds = conversationTimeBounds(matching); return { ...item, - firstSeenAt: matching.at(-1)!.startedAt, + ...bounds, id: `mock:${channel}`, kind: "channel", - lastSeenAt: matching[0]!.lastSeenAt, provider: "slack", providerDestinationId: channel, visibility: "public", diff --git a/packages/junior-dashboard/tests/dashboard-mock-routes.test.ts b/packages/junior-dashboard/tests/dashboard-mock-routes.test.ts index 467e3d915..9618b20d3 100644 --- a/packages/junior-dashboard/tests/dashboard-mock-routes.test.ts +++ b/packages/junior-dashboard/tests/dashboard-mock-routes.test.ts @@ -2,14 +2,59 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { actorDirectoryReportSchema, conversationDetailReportSchema, + type ConversationSummaryReport, } from "@sentry/junior/api/schema"; import { createDashboardApp } from "../src/app"; -import { DASHBOARD_QA_CONVERSATION_ID } from "../src/mock-conversations"; +import { + conversationTimeBounds, + DASHBOARD_QA_CONVERSATION_ID, +} from "../src/mock-conversations"; describe("dashboard canonical-event mock routes", () => { afterEach(() => vi.useRealTimers()); + it("derives public location bounds independently of summary order", () => { + const summary = ( + conversationId: string, + startedAt: string, + lastSeenAt: string, + ): ConversationSummaryReport => ({ + channel: "CQA123", + channelName: "proj-checkout", + conversationId, + cumulativeDurationMs: 1_000, + displayTitle: conversationId, + lastProgressAt: lastSeenAt, + lastSeenAt, + startedAt, + status: "completed", + surface: "slack", + }); + const summaries = [ + summary("middle", "2026-01-02T00:00:00.000Z", "2026-01-10T00:00:00.000Z"), + summary( + "earliest", + "2026-01-01T00:00:00.000Z", + "2026-01-05T00:00:00.000Z", + ), + summary("latest", "2026-01-03T00:00:00.000Z", "2026-01-20T00:00:00.000Z"), + ]; + + const ordered = conversationTimeBounds([ + summaries[0]!, + ...summaries.slice(1), + ]); + const reversedSummaries = [...summaries].reverse(); + const reversed = conversationTimeBounds([ + reversedSummaries[0]!, + ...reversedSummaries.slice(1), + ]); + expect(reversed).toEqual(ordered); + expect(ordered.firstSeenAt).toBe("2026-01-01T00:00:00.000Z"); + expect(ordered.lastSeenAt).toBe("2026-01-20T00:00:00.000Z"); + }); + it("serves canonical detail, directory, and aggregate reports", async () => { vi.useFakeTimers({ now: new Date("2026-05-30T00:00:00.000Z") }); const app = createDashboardApp({ diff --git a/packages/junior-dashboard/tests/telemetry-components.test.tsx b/packages/junior-dashboard/tests/telemetry-components.test.tsx index d994e9888..1ae0b8035 100644 --- a/packages/junior-dashboard/tests/telemetry-components.test.tsx +++ b/packages/junior-dashboard/tests/telemetry-components.test.tsx @@ -1,5 +1,5 @@ import { renderToStaticMarkup } from "react-dom/server"; -import { QueryClientProvider } from "@tanstack/react-query"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; import { MemoryRouter, Route, Routes } from "react-router"; import { afterEach, describe, expect, it } from "vitest"; import type { @@ -314,6 +314,55 @@ describe("dashboard canonical-event components", () => { expect(html).toContain("Open conversation"); }); + it("announces child conversation load failures with an error tone", () => { + const errorClient = new QueryClient({ + defaultOptions: { + queries: { refetchOnMount: false, retry: false, retryOnMount: false }, + }, + }); + const error = new Error("unavailable"); + errorClient.getQueryCache().build( + errorClient, + { queryKey: ["conversation", "child-error"] }, + { + data: undefined, + dataUpdateCount: 0, + dataUpdatedAt: 0, + error, + errorUpdateCount: 1, + errorUpdatedAt: Date.now(), + fetchFailureCount: 1, + fetchFailureReason: error, + fetchMeta: null, + fetchStatus: "idle", + isInvalidated: false, + status: "error", + }, + ); + const target: SubagentTranscriptTarget = { + conversationId: "child-error", + part: { + type: "subagent", + id: "child-error", + childConversationId: "child-error", + status: "completed", + outcome: "error", + subagentKind: "advisor", + }, + }; + + const html = renderToStaticMarkup( + + + {}} /> + + , + ); + expect(html).toContain("Conversation failed to load."); + expect(html).toContain('data-tone="error"'); + expect(html).toContain('role="alert"'); + }); + it("renders actor profiles with activity without recent conversations", () => { const profile: ActorProfileReport = { activityDays: [ From 721992c735f283174c444e8a7b4f61fd21de9a0e Mon Sep 17 00:00:00 2001 From: David Cramer Date: Wed, 15 Jul 2026 10:58:55 -0700 Subject: [PATCH 16/58] ref(migrations): Remove dead event compatibility phase Co-Authored-By: GPT-5 Codex --- .../migrations/0005_conversation_events.sql | 63 +--------- .../0005_visible_message_events.sql | 5 +- .../junior/src/chat/conversations/README.md | 9 +- .../cli/conversation-event-migration.test.ts | 115 ++++-------------- 4 files changed, 30 insertions(+), 162 deletions(-) diff --git a/packages/junior/migrations/0005_conversation_events.sql b/packages/junior/migrations/0005_conversation_events.sql index bbb9923ac..af9a1c706 100644 --- a/packages/junior/migrations/0005_conversation_events.sql +++ b/packages/junior/migrations/0005_conversation_events.sql @@ -2,65 +2,4 @@ ALTER TABLE "junior_agent_steps" RENAME TO "junior_conversation_events";--> stat ALTER TABLE "junior_conversation_events" RENAME CONSTRAINT "junior_agent_steps_conversation_id_seq_pk" TO "junior_conversation_events_conversation_id_seq_pk";--> statement-breakpoint ALTER TABLE "junior_conversation_events" RENAME CONSTRAINT "junior_agent_steps_conversation_id_junior_conversations_conversation_id_fk" TO "junior_conversation_events_conversation_id_junior_conversations_conversation_id_fk";--> statement-breakpoint ALTER INDEX "junior_agent_steps_epoch_idx" RENAME TO "junior_conversation_events_epoch_idx";--> statement-breakpoint -ALTER TABLE "junior_conversation_events" ADD COLUMN "schema_version" integer DEFAULT 1 NOT NULL;--> statement-breakpoint --- Temporary 0.103.x compatibility; 0005 removes the view and functions after --- the required worker drain. -CREATE VIEW "junior_agent_steps" AS -SELECT - "conversation_id", - "seq", - "context_epoch", - CASE WHEN "type" = 'message' THEN 'pi_message' ELSE "type" END AS "type", - "role", - "payload", - "created_at" -FROM "junior_conversation_events";--> statement-breakpoint -CREATE FUNCTION "junior_agent_steps_insert_compat"() -RETURNS trigger -LANGUAGE plpgsql -AS $$ -BEGIN - INSERT INTO "junior_conversation_events" ( - "conversation_id", - "seq", - "context_epoch", - "schema_version", - "type", - "role", - "payload", - "created_at" - ) VALUES ( - NEW."conversation_id", - NEW."seq", - NEW."context_epoch", - 1, - CASE WHEN NEW."type" = 'pi_message' THEN 'message' ELSE NEW."type" END, - NEW."role", - CASE - WHEN NEW."type" = 'pi_message' - AND jsonb_typeof(NEW."payload") = 'object' - THEN NEW."payload" - 'schemaVersion' - ELSE NEW."payload" - END, - NEW."created_at" - ); - RETURN NEW; -END; -$$;--> statement-breakpoint -CREATE TRIGGER "junior_agent_steps_insert_compat_trigger" -INSTEAD OF INSERT ON "junior_agent_steps" -FOR EACH ROW EXECUTE FUNCTION "junior_agent_steps_insert_compat"();--> statement-breakpoint -CREATE FUNCTION "junior_agent_steps_delete_compat"() -RETURNS trigger -LANGUAGE plpgsql -AS $$ -BEGIN - DELETE FROM "junior_conversation_events" - WHERE "conversation_id" = OLD."conversation_id" - AND "seq" = OLD."seq"; - RETURN OLD; -END; -$$;--> statement-breakpoint -CREATE TRIGGER "junior_agent_steps_delete_compat_trigger" -INSTEAD OF DELETE ON "junior_agent_steps" -FOR EACH ROW EXECUTE FUNCTION "junior_agent_steps_delete_compat"(); +ALTER TABLE "junior_conversation_events" ADD COLUMN "schema_version" integer DEFAULT 1 NOT NULL; diff --git a/packages/junior/migrations/0005_visible_message_events.sql b/packages/junior/migrations/0005_visible_message_events.sql index 02046117f..d0d0774ad 100644 --- a/packages/junior/migrations/0005_visible_message_events.sql +++ b/packages/junior/migrations/0005_visible_message_events.sql @@ -1,5 +1,2 @@ ALTER TABLE "junior_conversation_events" ADD COLUMN "idempotency_key" text;--> statement-breakpoint -CREATE UNIQUE INDEX "junior_conversation_events_idempotency_idx" ON "junior_conversation_events" USING btree ("conversation_id","idempotency_key");--> statement-breakpoint -DROP VIEW "junior_agent_steps";--> statement-breakpoint -DROP FUNCTION "junior_agent_steps_insert_compat"();--> statement-breakpoint -DROP FUNCTION "junior_agent_steps_delete_compat"(); +CREATE UNIQUE INDEX "junior_conversation_events_idempotency_idx" ON "junior_conversation_events" USING btree ("conversation_id","idempotency_key"); diff --git a/packages/junior/src/chat/conversations/README.md b/packages/junior/src/chat/conversations/README.md index 2efcacd34..8a11e2510 100644 --- a/packages/junior/src/chat/conversations/README.md +++ b/packages/junior/src/chat/conversations/README.md @@ -97,7 +97,8 @@ Follow `../../../../../policies/data-redaction.md` and ## Deployment Safety - Schema changes are expand-first and compatible with the currently deployed - reader and writer during rollout. + reader and writer during normal rollouts. Explicit hard cutovers document + their worker-stop gate here. - Data rewrites use explicit migrations or resumable import code. - Live workers never read Redis session logs, advisor blobs, or legacy thread-state to restore history. Operators must run `junior upgrade` before @@ -105,9 +106,9 @@ Follow `../../../../../policies/data-redaction.md` and - The Redis decoder and backfill writer remain under `cli/upgrade` only for the bounded operator-migration horizon; they can be deleted independently once every supported deployment has completed that upgrade. -- Drain every 0.103.x worker before applying the visible-message schema cut, - which drops the temporary `junior_agent_steps` compatibility view and its - functions. +- Stop every pre-event-log worker before running the event-table schema cut. + The migration renames `junior_agent_steps` directly and intentionally + provides no rolling compatibility view. - Run the final rerunnable visible-message backfill while workers remain stopped. Its fail-closed zero-gap verification is the cutover gate; start new workers only after it passes. diff --git a/packages/junior/tests/component/cli/conversation-event-migration.test.ts b/packages/junior/tests/component/cli/conversation-event-migration.test.ts index bfa355ac6..b3eebd47d 100644 --- a/packages/junior/tests/component/cli/conversation-event-migration.test.ts +++ b/packages/junior/tests/component/cli/conversation-event-migration.test.ts @@ -174,7 +174,7 @@ describe("conversation event migration", () => { expect(append).toHaveBeenCalledTimes(2); }); - it("backfills drained old-worker rows after removing their compatibility view", async () => { + it("backfills visible-message read-model rows after the hard cutover", async () => { const fixture = await createLocalJuniorSqlFixture(); const migrationsBeforeVisibleEvents = [ "0000_initial.sql", @@ -244,13 +244,13 @@ describe("conversation event migration", () => { await fixture.sql.execute( `INSERT INTO junior_conversation_messages ( conversation_id, message_id, role, text, created_at - ) VALUES ($1, 'old-worker', 'assistant', 'compatibility', $2)`, + ) VALUES ($1, 'late', 'assistant', 'late read model', $2)`, [conversationId, new Date("2026-07-14T10:00:03.000Z")], ); await fixture.sql.execute( `UPDATE junior_conversation_messages SET meta = $2::jsonb, replied_at = $3 - WHERE conversation_id = $1 AND message_id = 'old-worker'`, + WHERE conversation_id = $1 AND message_id = 'late'`, [ conversationId, JSON.stringify({ slackTs: "123.456" }), @@ -260,7 +260,7 @@ describe("conversation event migration", () => { await fixture.sql.execute( `INSERT INTO junior_conversation_messages ( conversation_id, message_id, role, text, replied_at, created_at - ) VALUES ($1, 'old-worker-replied', 'user', 'already replied', $2, $3)`, + ) VALUES ($1, 'late-replied', 'user', 'already replied', $2, $3)`, [ conversationId, new Date("2026-07-14T10:00:04.500Z"), @@ -303,48 +303,32 @@ describe("conversation event migration", () => { const events = await createSqlConversationEventStore( fixture.sql, ).loadHistory(conversationId); - const [compatibility] = await fixture.sql.query<{ - relation: string | null; - functions: number; - }>( - `SELECT - to_regclass('public.junior_agent_steps')::text AS relation, - ( - SELECT count(*)::integer - FROM pg_proc - WHERE proname IN ( - 'junior_agent_steps_insert_compat', - 'junior_agent_steps_delete_compat' - ) - ) AS functions`, - ); - expect(compatibility).toEqual({ relation: null, functions: 0 }); expect( events.filter( (event) => event.data.type === "visible_message_recorded" && - event.data.messageId === "old-worker", + event.data.messageId === "late", ), ).toHaveLength(1); expect( events.filter( (event) => event.data.type === "visible_message_replied" && - event.data.messageId === "old-worker-replied", + event.data.messageId === "late-replied", ), ).toHaveLength(1); expect( events.filter( (event) => event.data.type === "visible_message_metadata_updated" && - event.data.messageId === "old-worker", + event.data.messageId === "late", ), ).toHaveLength(0); expect( events.filter( (event) => event.data.type === "visible_message_replied" && - event.data.messageId === "old-worker", + event.data.messageId === "late", ), ).toHaveLength(1); expect( @@ -366,7 +350,7 @@ describe("conversation event migration", () => { } }, 20_000); - it("renames history, preserves rows, and provides rolling compatibility", async () => { + it("renames history, preserves rows, and rewrites legacy event payloads", async () => { const fixture = await createLocalJuniorSqlFixture(); const initial = migrationStatements("0000_initial.sql"); const conversationEvents = migrationStatements( @@ -418,6 +402,20 @@ describe("conversation event migration", () => { (statement) => fixture.sql.execute(statement), conversationEvents, ); + await expect( + fixture.sql.query<{ + deleteFunction: string | null; + insertFunction: string | null; + relation: string | null; + }>( + `SELECT + to_regclass('public.junior_agent_steps')::text AS relation, + to_regprocedure('public.junior_agent_steps_insert_compat()')::text AS "insertFunction", + to_regprocedure('public.junior_agent_steps_delete_compat()')::text AS "deleteFunction"`, + ), + ).resolves.toEqual([ + { deleteFunction: null, insertFunction: null, relation: null }, + ]); await expect( fixture.sql.query<{ @@ -513,72 +511,6 @@ ORDER BY seq WHERE type = 'pi_message'`, ), ).resolves.toEqual([{ count: 0 }]); - await expect( - fixture.sql.query<{ seq: number; type: string }>( - `SELECT seq, type FROM junior_agent_steps ORDER BY seq`, - ), - ).resolves.toEqual([ - { seq: 1, type: "pi_message" }, - { seq: 2, type: "authorization_completed" }, - ]); - - await fixture.sql.execute( - `INSERT INTO junior_agent_steps ( - conversation_id, - seq, - context_epoch, - type, - role, - payload, - created_at - ) VALUES ($1, 3, 3, 'pi_message', 'user', $2::jsonb, $3)`, - [ - "conversation-one", - JSON.stringify({ - schemaVersion: 99, - message: { role: "user", content: "hello" }, - }), - new Date("2026-07-14T10:02:00.000Z"), - ], - ); - await expect( - fixture.sql.query<{ - payload: Record; - schemaVersion: number; - type: string; - }>( - ` -SELECT - schema_version AS "schemaVersion", - type, - payload -FROM junior_conversation_events -WHERE conversation_id = $1 AND seq = 3 -`, - ["conversation-one"], - ), - ).resolves.toEqual([ - { - payload: { message: { role: "user", content: "hello" } }, - schemaVersion: 1, - type: "message", - }, - ]); - - await fixture.sql.execute( - `DELETE FROM junior_agent_steps - WHERE conversation_id = $1 AND seq = 3`, - ["conversation-one"], - ); - await expect( - fixture.sql.query<{ count: number }>( - `SELECT count(*)::integer AS count - FROM junior_conversation_events - WHERE conversation_id = $1 AND seq = 3`, - ["conversation-one"], - ), - ).resolves.toEqual([{ count: 0 }]); - await expect( migrateConversationEventData( { io: { info: () => {} }, stateAdapter: getStateAdapter() }, @@ -599,7 +531,6 @@ WHERE table_schema = 'public' ORDER BY table_name `), ).resolves.toEqual([ - { name: "junior_agent_steps", type: "VIEW" }, { name: "junior_conversation_events", type: "BASE TABLE" }, ]); await expect( From 2ff728556c2a887d8ede8f859a08ebb912325b6d Mon Sep 17 00:00:00 2001 From: David Cramer Date: Wed, 15 Jul 2026 11:14:15 -0700 Subject: [PATCH 17/58] ref(conversations): Remove speculative event machinery Co-Authored-By: GPT-5 Codex --- .../src/mock-conversations.ts | 4 - .../tests/markdownExport.test.ts | 1 - .../tests/telemetry-components.test.tsx | 2 - .../tests/transcriptRenderModel.test.ts | 3 - .../migrations/0007_conversation_lineage.sql | 13 - .../junior/migrations/meta/0007_snapshot.json | 1305 ----------------- packages/junior/migrations/meta/_journal.json | 7 - .../junior/src/api/conversations/events.ts | 2 - .../src/api/conversations/list.query.ts | 10 - .../junior/src/api/conversations/schema.ts | 2 - .../junior/src/chat/conversations/README.md | 40 +- .../chat/conversations/event-projection.ts | 96 -- .../junior/src/chat/conversations/history.ts | 8 - .../src/chat/conversations/model-messages.ts | 63 - .../src/chat/conversations/projection.ts | 364 +---- .../src/chat/conversations/sql/privacy.ts | 26 +- .../src/chat/conversations/sql/purge.ts | 85 +- .../src/chat/conversations/sql/store.ts | 14 - .../junior/src/chat/conversations/store.ts | 4 - packages/junior/src/chat/db.ts | 9 - .../junior/src/chat/pi/conversation-events.ts | 80 +- packages/junior/src/chat/pi/transcript.ts | 49 +- .../src/chat/services/subagent-lineage.ts | 418 ------ packages/junior/src/cli/upgrade.ts | 2 - .../legacy-history-import.ts | 69 +- .../migrations/conversation-lineage.ts | 130 -- .../junior/src/db/schema/conversations.ts | 36 - .../cli/conversation-event-migration.test.ts | 7 - .../cli/conversation-history-import.test.ts | 9 - .../conversation-transcripts-sql.test.ts | 1 - .../component/conversations/retention.test.ts | 9 +- .../shared-lineage-projection.test.ts | 581 -------- .../services/subagent-lineage.test.ts | 495 ------- .../api/conversations/list.test.ts | 32 +- .../api/conversations/stats.test.ts | 30 +- .../integration/dashboard-reporting.test.ts | 160 +- .../unit/api/conversation-events.test.ts | 11 +- .../unit/chat/pi/conversation-events.test.ts | 27 +- .../pi/transcript.test.ts} | 29 +- .../conversation-history-conversion.test.ts | 1 - .../tests/unit/conversations/privacy.test.ts | 34 +- 41 files changed, 289 insertions(+), 3979 deletions(-) delete mode 100644 packages/junior/migrations/0007_conversation_lineage.sql delete mode 100644 packages/junior/migrations/meta/0007_snapshot.json delete mode 100644 packages/junior/src/chat/conversations/event-projection.ts delete mode 100644 packages/junior/src/chat/conversations/model-messages.ts delete mode 100644 packages/junior/src/chat/services/subagent-lineage.ts delete mode 100644 packages/junior/src/cli/upgrade/migrations/conversation-lineage.ts delete mode 100644 packages/junior/tests/component/conversations/shared-lineage-projection.test.ts delete mode 100644 packages/junior/tests/component/services/subagent-lineage.test.ts rename packages/junior/tests/unit/{conversations/model-messages.test.ts => chat/pi/transcript.test.ts} (59%) diff --git a/packages/junior-dashboard/src/mock-conversations.ts b/packages/junior-dashboard/src/mock-conversations.ts index e04c42dc9..803b0d0df 100644 --- a/packages/junior-dashboard/src/mock-conversations.ts +++ b/packages/junior-dashboard/src/mock-conversations.ts @@ -157,26 +157,22 @@ function dashboardQaConversation(nowMs: number): ConversationDetailReport { type: "subagent_started", childConversationId: DASHBOARD_QA_PLAN_ID, subagentKind: "advisor", - historyMode: "shared", }), reportEvent(3, iso(Date.parse(startedAt), 20_000), { type: "subagent_ended", childConversationId: DASHBOARD_QA_PLAN_ID, subagentKind: "advisor", - historyMode: "shared", outcome: "success", }), reportEvent(4, iso(Date.parse(startedAt), 25_000), { type: "subagent_started", childConversationId: DASHBOARD_QA_REVIEW_ID, subagentKind: "advisor", - historyMode: "shared", }), reportEvent(5, iso(Date.parse(startedAt), 44_000), { type: "subagent_ended", childConversationId: DASHBOARD_QA_REVIEW_ID, subagentKind: "advisor", - historyMode: "shared", outcome: "success", }), reportEvent(6, iso(Date.parse(startedAt), 50_000), { diff --git a/packages/junior-dashboard/tests/markdownExport.test.ts b/packages/junior-dashboard/tests/markdownExport.test.ts index eccb62f6d..b4aa351cf 100644 --- a/packages/junior-dashboard/tests/markdownExport.test.ts +++ b/packages/junior-dashboard/tests/markdownExport.test.ts @@ -80,7 +80,6 @@ describe("dashboard canonical-event Markdown export", () => { type: "subagent_started", childConversationId: "child-1", subagentKind: "advisor", - historyMode: "shared", }), event(2, { type: "context_compacted" }), event(3, { diff --git a/packages/junior-dashboard/tests/telemetry-components.test.tsx b/packages/junior-dashboard/tests/telemetry-components.test.tsx index 1ae0b8035..15a2cf7f1 100644 --- a/packages/junior-dashboard/tests/telemetry-components.test.tsx +++ b/packages/junior-dashboard/tests/telemetry-components.test.tsx @@ -255,13 +255,11 @@ describe("dashboard canonical-event components", () => { type: "subagent_started", childConversationId: "child-1", subagentKind: "advisor", - historyMode: "shared", }), event(1, { type: "subagent_ended", childConversationId: "child-1", subagentKind: "advisor", - historyMode: "shared", outcome: "success", }), ])} diff --git a/packages/junior-dashboard/tests/transcriptRenderModel.test.ts b/packages/junior-dashboard/tests/transcriptRenderModel.test.ts index b1e50cf62..6e9c92e3e 100644 --- a/packages/junior-dashboard/tests/transcriptRenderModel.test.ts +++ b/packages/junior-dashboard/tests/transcriptRenderModel.test.ts @@ -117,13 +117,11 @@ describe("canonical event transcript reduction", () => { type: "subagent_started", childConversationId: "child-1", subagentKind: "advisor", - historyMode: "shared", }), event(3, "2026-01-01T00:00:03.000Z", { type: "subagent_ended", childConversationId: "child-1", subagentKind: "advisor", - historyMode: "shared", outcome: "success", }), event(4, "2026-01-01T00:00:04.000Z", { @@ -198,7 +196,6 @@ describe("canonical event transcript reduction", () => { type: "subagent_started", childConversationId: "child-1", subagentKind: "advisor", - historyMode: "isolated", }), event(2, "2026-01-01T00:00:02.000Z", { type: "context_compacted", diff --git a/packages/junior/migrations/0007_conversation_lineage.sql b/packages/junior/migrations/0007_conversation_lineage.sql deleted file mode 100644 index 396f49826..000000000 --- a/packages/junior/migrations/0007_conversation_lineage.sql +++ /dev/null @@ -1,13 +0,0 @@ -ALTER TABLE "junior_conversations" ADD COLUMN "root_conversation_id" text;--> statement-breakpoint -ALTER TABLE "junior_conversations" ADD COLUMN "parent_turn_id" text;--> statement-breakpoint -ALTER TABLE "junior_conversations" ADD COLUMN "parent_event_seq" integer;--> statement-breakpoint -ALTER TABLE "junior_conversations" ADD COLUMN "context_fork_seq" integer;--> statement-breakpoint -ALTER TABLE "junior_conversations" ADD CONSTRAINT "junior_conversations_root_conversation_id_junior_conversations_conversation_id_fk" FOREIGN KEY ("root_conversation_id") REFERENCES "public"."junior_conversations"("conversation_id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint -CREATE INDEX "junior_conversations_root_idx" ON "junior_conversations" USING btree ("root_conversation_id");--> statement-breakpoint -CREATE INDEX "junior_conversations_parent_event_idx" ON "junior_conversations" USING btree ("parent_conversation_id","parent_event_seq");--> statement-breakpoint -ALTER TABLE "junior_conversations" ADD CONSTRAINT "junior_conversations_not_own_parent_check" CHECK ("junior_conversations"."parent_conversation_id" is null or "junior_conversations"."parent_conversation_id" <> "junior_conversations"."conversation_id");--> statement-breakpoint -ALTER TABLE "junior_conversations" ADD CONSTRAINT "junior_conversations_not_own_root_check" CHECK ("junior_conversations"."root_conversation_id" is null or "junior_conversations"."root_conversation_id" <> "junior_conversations"."conversation_id");--> statement-breakpoint -ALTER TABLE "junior_conversations" ADD CONSTRAINT "junior_conversations_context_fork_check" CHECK ("junior_conversations"."context_fork_seq" is null or ("junior_conversations"."parent_event_seq" is not null and "junior_conversations"."context_fork_seq" <= "junior_conversations"."parent_event_seq"));--> statement-breakpoint -ALTER TABLE "junior_conversations" ADD CONSTRAINT "junior_conversations_top_level_lineage_check" CHECK ("junior_conversations"."parent_conversation_id" is not null or ("junior_conversations"."root_conversation_id" is null and "junior_conversations"."parent_turn_id" is null and "junior_conversations"."parent_event_seq" is null and "junior_conversations"."context_fork_seq" is null));--> statement-breakpoint -ALTER TABLE "junior_conversations" ADD CONSTRAINT "junior_conversations_parent_correlation_check" CHECK (("junior_conversations"."parent_turn_id" is null) = ("junior_conversations"."parent_event_seq" is null));--> statement-breakpoint -ALTER TABLE "junior_conversations" ADD CONSTRAINT "junior_conversations_correlated_root_check" CHECK ("junior_conversations"."parent_turn_id" is null or "junior_conversations"."root_conversation_id" is not null); diff --git a/packages/junior/migrations/meta/0007_snapshot.json b/packages/junior/migrations/meta/0007_snapshot.json deleted file mode 100644 index 533b36fd0..000000000 --- a/packages/junior/migrations/meta/0007_snapshot.json +++ /dev/null @@ -1,1305 +0,0 @@ -{ - "id": "f0793000-d703-4d2c-bc57-362bfa1a50a1", - "prevId": "46db5bc7-e30a-41bc-8194-abe7b42f6699", - "version": "7", - "dialect": "postgresql", - "tables": { - "public.junior_conversation_events": { - "name": "junior_conversation_events", - "schema": "", - "columns": { - "conversation_id": { - "name": "conversation_id", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "seq": { - "name": "seq", - "type": "integer", - "primaryKey": false, - "notNull": true - }, - "context_epoch": { - "name": "context_epoch", - "type": "integer", - "primaryKey": false, - "notNull": true - }, - "schema_version": { - "name": "schema_version", - "type": "integer", - "primaryKey": false, - "notNull": true, - "default": 1 - }, - "idempotency_key": { - "name": "idempotency_key", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "type": { - "name": "type", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "role": { - "name": "role", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "payload": { - "name": "payload", - "type": "jsonb", - "primaryKey": false, - "notNull": true - }, - "created_at": { - "name": "created_at", - "type": "timestamp with time zone", - "primaryKey": false, - "notNull": true - } - }, - "indexes": { - "junior_conversation_events_epoch_idx": { - "name": "junior_conversation_events_epoch_idx", - "columns": [ - { - "expression": "conversation_id", - "isExpression": false, - "asc": true, - "nulls": "last" - }, - { - "expression": "context_epoch", - "isExpression": false, - "asc": true, - "nulls": "last" - }, - { - "expression": "seq", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - }, - "junior_conversation_events_idempotency_idx": { - "name": "junior_conversation_events_idempotency_idx", - "columns": [ - { - "expression": "conversation_id", - "isExpression": false, - "asc": true, - "nulls": "last" - }, - { - "expression": "idempotency_key", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": true, - "concurrently": false, - "method": "btree", - "with": {} - } - }, - "foreignKeys": { - "junior_conversation_events_conversation_id_junior_conversations_conversation_id_fk": { - "name": "junior_conversation_events_conversation_id_junior_conversations_conversation_id_fk", - "tableFrom": "junior_conversation_events", - "tableTo": "junior_conversations", - "columnsFrom": ["conversation_id"], - "columnsTo": ["conversation_id"], - "onDelete": "no action", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": { - "junior_conversation_events_conversation_id_seq_pk": { - "name": "junior_conversation_events_conversation_id_seq_pk", - "columns": ["conversation_id", "seq"] - } - }, - "uniqueConstraints": {}, - "policies": {}, - "checkConstraints": {}, - "isRLSEnabled": false - }, - "public.junior_conversation_messages": { - "name": "junior_conversation_messages", - "schema": "", - "columns": { - "conversation_id": { - "name": "conversation_id", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "message_id": { - "name": "message_id", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "role": { - "name": "role", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "author_identity_id": { - "name": "author_identity_id", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "text": { - "name": "text", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "meta": { - "name": "meta", - "type": "jsonb", - "primaryKey": false, - "notNull": false - }, - "replied_at": { - "name": "replied_at", - "type": "timestamp with time zone", - "primaryKey": false, - "notNull": false - }, - "created_at": { - "name": "created_at", - "type": "timestamp with time zone", - "primaryKey": false, - "notNull": true - } - }, - "indexes": { - "junior_conversation_messages_activity_idx": { - "name": "junior_conversation_messages_activity_idx", - "columns": [ - { - "expression": "conversation_id", - "isExpression": false, - "asc": true, - "nulls": "last" - }, - { - "expression": "created_at", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - }, - "junior_conversation_messages_search_idx": { - "name": "junior_conversation_messages_search_idx", - "columns": [ - { - "expression": "to_tsvector('english', \"text\")", - "asc": true, - "isExpression": true, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "gin", - "with": {} - } - }, - "foreignKeys": { - "junior_conversation_messages_conversation_id_junior_conversations_conversation_id_fk": { - "name": "junior_conversation_messages_conversation_id_junior_conversations_conversation_id_fk", - "tableFrom": "junior_conversation_messages", - "tableTo": "junior_conversations", - "columnsFrom": ["conversation_id"], - "columnsTo": ["conversation_id"], - "onDelete": "no action", - "onUpdate": "no action" - }, - "junior_conversation_messages_author_identity_id_junior_identities_id_fk": { - "name": "junior_conversation_messages_author_identity_id_junior_identities_id_fk", - "tableFrom": "junior_conversation_messages", - "tableTo": "junior_identities", - "columnsFrom": ["author_identity_id"], - "columnsTo": ["id"], - "onDelete": "no action", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": { - "junior_conversation_messages_conversation_id_message_id_pk": { - "name": "junior_conversation_messages_conversation_id_message_id_pk", - "columns": ["conversation_id", "message_id"] - } - }, - "uniqueConstraints": {}, - "policies": {}, - "checkConstraints": {}, - "isRLSEnabled": false - }, - "public.junior_conversations": { - "name": "junior_conversations", - "schema": "", - "columns": { - "conversation_id": { - "name": "conversation_id", - "type": "text", - "primaryKey": true, - "notNull": true - }, - "schema_version": { - "name": "schema_version", - "type": "integer", - "primaryKey": false, - "notNull": true, - "default": 1 - }, - "source": { - "name": "source", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "origin_type": { - "name": "origin_type", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "origin_id": { - "name": "origin_id", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "origin_run_id": { - "name": "origin_run_id", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "destination_id": { - "name": "destination_id", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "destination_json": { - "name": "destination_json", - "type": "jsonb", - "primaryKey": false, - "notNull": false - }, - "actor_identity_id": { - "name": "actor_identity_id", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "creator_identity_id": { - "name": "creator_identity_id", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "credential_subject_identity_id": { - "name": "credential_subject_identity_id", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "actor_json": { - "name": "actor_json", - "type": "jsonb", - "primaryKey": false, - "notNull": false - }, - "channel_name": { - "name": "channel_name", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "title": { - "name": "title", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "created_at": { - "name": "created_at", - "type": "timestamp with time zone", - "primaryKey": false, - "notNull": true - }, - "last_activity_at": { - "name": "last_activity_at", - "type": "timestamp with time zone", - "primaryKey": false, - "notNull": true - }, - "updated_at": { - "name": "updated_at", - "type": "timestamp with time zone", - "primaryKey": false, - "notNull": true - }, - "execution_updated_at": { - "name": "execution_updated_at", - "type": "timestamp with time zone", - "primaryKey": false, - "notNull": false - }, - "execution_status": { - "name": "execution_status", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "run_id": { - "name": "run_id", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "last_checkpoint_at": { - "name": "last_checkpoint_at", - "type": "timestamp with time zone", - "primaryKey": false, - "notNull": false - }, - "last_enqueued_at": { - "name": "last_enqueued_at", - "type": "timestamp with time zone", - "primaryKey": false, - "notNull": false - }, - "parent_conversation_id": { - "name": "parent_conversation_id", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "transcript_purged_at": { - "name": "transcript_purged_at", - "type": "timestamp with time zone", - "primaryKey": false, - "notNull": false - }, - "duration_ms": { - "name": "duration_ms", - "type": "integer", - "primaryKey": false, - "notNull": true, - "default": 0 - }, - "usage_json": { - "name": "usage_json", - "type": "jsonb", - "primaryKey": false, - "notNull": false - }, - "execution_duration_ms": { - "name": "execution_duration_ms", - "type": "integer", - "primaryKey": false, - "notNull": true, - "default": 0 - }, - "execution_usage_json": { - "name": "execution_usage_json", - "type": "jsonb", - "primaryKey": false, - "notNull": false - }, - "metric_run_id": { - "name": "metric_run_id", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "root_conversation_id": { - "name": "root_conversation_id", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "parent_turn_id": { - "name": "parent_turn_id", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "parent_event_seq": { - "name": "parent_event_seq", - "type": "integer", - "primaryKey": false, - "notNull": false - }, - "context_fork_seq": { - "name": "context_fork_seq", - "type": "integer", - "primaryKey": false, - "notNull": false - } - }, - "indexes": { - "junior_conversations_last_activity_idx": { - "name": "junior_conversations_last_activity_idx", - "columns": [ - { - "expression": "last_activity_at", - "isExpression": false, - "asc": false, - "nulls": "last" - }, - { - "expression": "conversation_id", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - }, - "junior_conversations_active_idx": { - "name": "junior_conversations_active_idx", - "columns": [ - { - "expression": "coalesce(\"execution_updated_at\", \"updated_at\")", - "asc": true, - "isExpression": true, - "nulls": "last" - }, - { - "expression": "conversation_id", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "where": "\"junior_conversations\".\"execution_status\" <> 'idle'", - "concurrently": false, - "method": "btree", - "with": {} - }, - "junior_conversations_destination_activity_idx": { - "name": "junior_conversations_destination_activity_idx", - "columns": [ - { - "expression": "destination_id", - "isExpression": false, - "asc": true, - "nulls": "last" - }, - { - "expression": "last_activity_at", - "isExpression": false, - "asc": false, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - }, - "junior_conversations_actor_activity_idx": { - "name": "junior_conversations_actor_activity_idx", - "columns": [ - { - "expression": "actor_identity_id", - "isExpression": false, - "asc": true, - "nulls": "last" - }, - { - "expression": "last_activity_at", - "isExpression": false, - "asc": false, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - }, - "junior_conversations_origin_idx": { - "name": "junior_conversations_origin_idx", - "columns": [ - { - "expression": "origin_type", - "isExpression": false, - "asc": true, - "nulls": "last" - }, - { - "expression": "origin_id", - "isExpression": false, - "asc": true, - "nulls": "last" - }, - { - "expression": "last_activity_at", - "isExpression": false, - "asc": false, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - }, - "junior_conversations_parent_idx": { - "name": "junior_conversations_parent_idx", - "columns": [ - { - "expression": "parent_conversation_id", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - }, - "junior_conversations_parent_event_idx": { - "name": "junior_conversations_parent_event_idx", - "columns": [ - { - "expression": "parent_conversation_id", - "isExpression": false, - "asc": true, - "nulls": "last" - }, - { - "expression": "parent_event_seq", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - }, - "junior_conversations_root_idx": { - "name": "junior_conversations_root_idx", - "columns": [ - { - "expression": "root_conversation_id", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - } - }, - "foreignKeys": { - "junior_conversations_destination_id_junior_destinations_id_fk": { - "name": "junior_conversations_destination_id_junior_destinations_id_fk", - "tableFrom": "junior_conversations", - "tableTo": "junior_destinations", - "columnsFrom": ["destination_id"], - "columnsTo": ["id"], - "onDelete": "no action", - "onUpdate": "no action" - }, - "junior_conversations_actor_identity_id_junior_identities_id_fk": { - "name": "junior_conversations_actor_identity_id_junior_identities_id_fk", - "tableFrom": "junior_conversations", - "tableTo": "junior_identities", - "columnsFrom": ["actor_identity_id"], - "columnsTo": ["id"], - "onDelete": "no action", - "onUpdate": "no action" - }, - "junior_conversations_creator_identity_id_junior_identities_id_fk": { - "name": "junior_conversations_creator_identity_id_junior_identities_id_fk", - "tableFrom": "junior_conversations", - "tableTo": "junior_identities", - "columnsFrom": ["creator_identity_id"], - "columnsTo": ["id"], - "onDelete": "no action", - "onUpdate": "no action" - }, - "junior_conversations_credential_subject_identity_id_junior_identities_id_fk": { - "name": "junior_conversations_credential_subject_identity_id_junior_identities_id_fk", - "tableFrom": "junior_conversations", - "tableTo": "junior_identities", - "columnsFrom": ["credential_subject_identity_id"], - "columnsTo": ["id"], - "onDelete": "no action", - "onUpdate": "no action" - }, - "junior_conversations_parent_conversation_id_junior_conversations_conversation_id_fk": { - "name": "junior_conversations_parent_conversation_id_junior_conversations_conversation_id_fk", - "tableFrom": "junior_conversations", - "tableTo": "junior_conversations", - "columnsFrom": ["parent_conversation_id"], - "columnsTo": ["conversation_id"], - "onDelete": "no action", - "onUpdate": "no action" - }, - "junior_conversations_root_conversation_id_junior_conversations_conversation_id_fk": { - "name": "junior_conversations_root_conversation_id_junior_conversations_conversation_id_fk", - "tableFrom": "junior_conversations", - "tableTo": "junior_conversations", - "columnsFrom": ["root_conversation_id"], - "columnsTo": ["conversation_id"], - "onDelete": "no action", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "policies": {}, - "checkConstraints": { - "junior_conversations_not_own_parent_check": { - "name": "junior_conversations_not_own_parent_check", - "value": "\"junior_conversations\".\"parent_conversation_id\" is null or \"junior_conversations\".\"parent_conversation_id\" <> \"junior_conversations\".\"conversation_id\"" - }, - "junior_conversations_not_own_root_check": { - "name": "junior_conversations_not_own_root_check", - "value": "\"junior_conversations\".\"root_conversation_id\" is null or \"junior_conversations\".\"root_conversation_id\" <> \"junior_conversations\".\"conversation_id\"" - }, - "junior_conversations_context_fork_check": { - "name": "junior_conversations_context_fork_check", - "value": "\"junior_conversations\".\"context_fork_seq\" is null or (\"junior_conversations\".\"parent_event_seq\" is not null and \"junior_conversations\".\"context_fork_seq\" <= \"junior_conversations\".\"parent_event_seq\")" - }, - "junior_conversations_top_level_lineage_check": { - "name": "junior_conversations_top_level_lineage_check", - "value": "\"junior_conversations\".\"parent_conversation_id\" is not null or (\"junior_conversations\".\"root_conversation_id\" is null and \"junior_conversations\".\"parent_turn_id\" is null and \"junior_conversations\".\"parent_event_seq\" is null and \"junior_conversations\".\"context_fork_seq\" is null)" - }, - "junior_conversations_parent_correlation_check": { - "name": "junior_conversations_parent_correlation_check", - "value": "(\"junior_conversations\".\"parent_turn_id\" is null) = (\"junior_conversations\".\"parent_event_seq\" is null)" - }, - "junior_conversations_correlated_root_check": { - "name": "junior_conversations_correlated_root_check", - "value": "\"junior_conversations\".\"parent_turn_id\" is null or \"junior_conversations\".\"root_conversation_id\" is not null" - } - }, - "isRLSEnabled": false - }, - "public.junior_destinations": { - "name": "junior_destinations", - "schema": "", - "columns": { - "id": { - "name": "id", - "type": "text", - "primaryKey": true, - "notNull": true - }, - "provider": { - "name": "provider", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "provider_tenant_id": { - "name": "provider_tenant_id", - "type": "text", - "primaryKey": false, - "notNull": true, - "default": "''" - }, - "provider_destination_id": { - "name": "provider_destination_id", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "kind": { - "name": "kind", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "parent_destination_id": { - "name": "parent_destination_id", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "display_name": { - "name": "display_name", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "visibility": { - "name": "visibility", - "type": "text", - "primaryKey": false, - "notNull": true, - "default": "'unknown'" - }, - "metadata_json": { - "name": "metadata_json", - "type": "jsonb", - "primaryKey": false, - "notNull": false - }, - "created_at": { - "name": "created_at", - "type": "timestamp with time zone", - "primaryKey": false, - "notNull": true - }, - "updated_at": { - "name": "updated_at", - "type": "timestamp with time zone", - "primaryKey": false, - "notNull": true - } - }, - "indexes": { - "junior_destinations_provider_destination_uidx": { - "name": "junior_destinations_provider_destination_uidx", - "columns": [ - { - "expression": "provider", - "isExpression": false, - "asc": true, - "nulls": "last" - }, - { - "expression": "provider_tenant_id", - "isExpression": false, - "asc": true, - "nulls": "last" - }, - { - "expression": "provider_destination_id", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": true, - "concurrently": false, - "method": "btree", - "with": {} - }, - "junior_destinations_provider_kind_idx": { - "name": "junior_destinations_provider_kind_idx", - "columns": [ - { - "expression": "provider", - "isExpression": false, - "asc": true, - "nulls": "last" - }, - { - "expression": "kind", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "policies": {}, - "checkConstraints": {}, - "isRLSEnabled": false - }, - "public.junior_identities": { - "name": "junior_identities", - "schema": "", - "columns": { - "id": { - "name": "id", - "type": "text", - "primaryKey": true, - "notNull": true - }, - "kind": { - "name": "kind", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "provider": { - "name": "provider", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "provider_tenant_id": { - "name": "provider_tenant_id", - "type": "text", - "primaryKey": false, - "notNull": true, - "default": "''" - }, - "provider_subject_id": { - "name": "provider_subject_id", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "display_name": { - "name": "display_name", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "handle": { - "name": "handle", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "email": { - "name": "email", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "avatar_url": { - "name": "avatar_url", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "metadata_json": { - "name": "metadata_json", - "type": "jsonb", - "primaryKey": false, - "notNull": false - }, - "created_at": { - "name": "created_at", - "type": "timestamp with time zone", - "primaryKey": false, - "notNull": true - }, - "updated_at": { - "name": "updated_at", - "type": "timestamp with time zone", - "primaryKey": false, - "notNull": true - }, - "user_id": { - "name": "user_id", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "email_normalized": { - "name": "email_normalized", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "email_verified": { - "name": "email_verified", - "type": "boolean", - "primaryKey": false, - "notNull": true, - "default": false - } - }, - "indexes": { - "junior_identities_provider_subject_uidx": { - "name": "junior_identities_provider_subject_uidx", - "columns": [ - { - "expression": "provider", - "isExpression": false, - "asc": true, - "nulls": "last" - }, - { - "expression": "provider_tenant_id", - "isExpression": false, - "asc": true, - "nulls": "last" - }, - { - "expression": "provider_subject_id", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": true, - "concurrently": false, - "method": "btree", - "with": {} - }, - "junior_identities_user_idx": { - "name": "junior_identities_user_idx", - "columns": [ - { - "expression": "user_id", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - }, - "junior_identities_verified_email_idx": { - "name": "junior_identities_verified_email_idx", - "columns": [ - { - "expression": "email_normalized", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "where": "\"junior_identities\".\"email_verified\" = true AND \"junior_identities\".\"email_normalized\" IS NOT NULL", - "concurrently": false, - "method": "btree", - "with": {} - }, - "junior_identities_kind_provider_idx": { - "name": "junior_identities_kind_provider_idx", - "columns": [ - { - "expression": "kind", - "isExpression": false, - "asc": true, - "nulls": "last" - }, - { - "expression": "provider", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - } - }, - "foreignKeys": { - "junior_identities_user_id_junior_users_id_fk": { - "name": "junior_identities_user_id_junior_users_id_fk", - "tableFrom": "junior_identities", - "tableTo": "junior_users", - "columnsFrom": ["user_id"], - "columnsTo": ["id"], - "onDelete": "no action", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "policies": {}, - "checkConstraints": {}, - "isRLSEnabled": false - }, - "public.junior_pending_deliveries": { - "name": "junior_pending_deliveries", - "schema": "", - "columns": { - "delivery_id": { - "name": "delivery_id", - "type": "text", - "primaryKey": true, - "notNull": true - }, - "conversation_id": { - "name": "conversation_id", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "turn_id": { - "name": "turn_id", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "message_id": { - "name": "message_id", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "provider": { - "name": "provider", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "delivery_kind": { - "name": "delivery_kind", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "command_json": { - "name": "command_json", - "type": "jsonb", - "primaryKey": false, - "notNull": true - }, - "part_states_json": { - "name": "part_states_json", - "type": "jsonb", - "primaryKey": false, - "notNull": true - }, - "next_part_index": { - "name": "next_part_index", - "type": "integer", - "primaryKey": false, - "notNull": true, - "default": 0 - }, - "attempt_count": { - "name": "attempt_count", - "type": "integer", - "primaryKey": false, - "notNull": true, - "default": 0 - }, - "next_attempt_at": { - "name": "next_attempt_at", - "type": "timestamp with time zone", - "primaryKey": false, - "notNull": true - }, - "lease_owner": { - "name": "lease_owner", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "lease_version": { - "name": "lease_version", - "type": "integer", - "primaryKey": false, - "notNull": true, - "default": 0 - }, - "lease_expires_at": { - "name": "lease_expires_at", - "type": "timestamp with time zone", - "primaryKey": false, - "notNull": false - }, - "last_attempt_at": { - "name": "last_attempt_at", - "type": "timestamp with time zone", - "primaryKey": false, - "notNull": false - }, - "created_at": { - "name": "created_at", - "type": "timestamp with time zone", - "primaryKey": false, - "notNull": true - }, - "updated_at": { - "name": "updated_at", - "type": "timestamp with time zone", - "primaryKey": false, - "notNull": true - } - }, - "indexes": { - "junior_pending_deliveries_conversation_turn_idx": { - "name": "junior_pending_deliveries_conversation_turn_idx", - "columns": [ - { - "expression": "conversation_id", - "isExpression": false, - "asc": true, - "nulls": "last" - }, - { - "expression": "turn_id", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": true, - "concurrently": false, - "method": "btree", - "with": {} - }, - "junior_pending_deliveries_retry_idx": { - "name": "junior_pending_deliveries_retry_idx", - "columns": [ - { - "expression": "next_attempt_at", - "isExpression": false, - "asc": true, - "nulls": "last" - }, - { - "expression": "delivery_id", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - } - }, - "foreignKeys": { - "junior_pending_deliveries_conversation_id_fk": { - "name": "junior_pending_deliveries_conversation_id_fk", - "tableFrom": "junior_pending_deliveries", - "tableTo": "junior_conversations", - "columnsFrom": ["conversation_id"], - "columnsTo": ["conversation_id"], - "onDelete": "cascade", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "policies": {}, - "checkConstraints": { - "junior_pending_deliveries_cursor_check": { - "name": "junior_pending_deliveries_cursor_check", - "value": "\"junior_pending_deliveries\".\"next_part_index\" >= 0" - }, - "junior_pending_deliveries_attempt_count_check": { - "name": "junior_pending_deliveries_attempt_count_check", - "value": "\"junior_pending_deliveries\".\"attempt_count\" >= 0" - }, - "junior_pending_deliveries_lease_version_check": { - "name": "junior_pending_deliveries_lease_version_check", - "value": "\"junior_pending_deliveries\".\"lease_version\" >= 0" - } - }, - "isRLSEnabled": false - }, - "public.junior_users": { - "name": "junior_users", - "schema": "", - "columns": { - "id": { - "name": "id", - "type": "text", - "primaryKey": true, - "notNull": true - }, - "primary_email": { - "name": "primary_email", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "primary_email_normalized": { - "name": "primary_email_normalized", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "display_name": { - "name": "display_name", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "created_at": { - "name": "created_at", - "type": "timestamp with time zone", - "primaryKey": false, - "notNull": true - }, - "updated_at": { - "name": "updated_at", - "type": "timestamp with time zone", - "primaryKey": false, - "notNull": true - } - }, - "indexes": { - "junior_users_primary_email_normalized_uidx": { - "name": "junior_users_primary_email_normalized_uidx", - "columns": [ - { - "expression": "primary_email_normalized", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": true, - "concurrently": false, - "method": "btree", - "with": {} - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "policies": {}, - "checkConstraints": {}, - "isRLSEnabled": false - } - }, - "enums": {}, - "schemas": {}, - "sequences": {}, - "roles": {}, - "policies": {}, - "views": {}, - "_meta": { - "columns": {}, - "schemas": {}, - "tables": {} - } -} diff --git a/packages/junior/migrations/meta/_journal.json b/packages/junior/migrations/meta/_journal.json index ff5f4f19a..d5e259957 100644 --- a/packages/junior/migrations/meta/_journal.json +++ b/packages/junior/migrations/meta/_journal.json @@ -50,13 +50,6 @@ "when": 1784089383071, "tag": "0005_visible_message_events", "breakpoints": true - }, - { - "idx": 7, - "version": "7", - "when": 1784127545103, - "tag": "0007_conversation_lineage", - "breakpoints": true } ] } diff --git a/packages/junior/src/api/conversations/events.ts b/packages/junior/src/api/conversations/events.ts index ca456c73c..cd34366e0 100644 --- a/packages/junior/src/api/conversations/events.ts +++ b/packages/junior/src/api/conversations/events.ts @@ -10,7 +10,6 @@ import { interface SubagentReference { childConversationId: string; - historyMode: "isolated" | "shared"; subagentKind: string; } @@ -127,7 +126,6 @@ export function projectConversationReportEvents(args: { if (event.data.type === "subagent_started") { const reference: SubagentReference = { childConversationId: event.data.childConversationId, - historyMode: event.data.historyMode, subagentKind: event.data.subagentKind, }; subagents.set(event.data.subagentInvocationId, reference); diff --git a/packages/junior/src/api/conversations/list.query.ts b/packages/junior/src/api/conversations/list.query.ts index fccb5faa6..52f75a23e 100644 --- a/packages/junior/src/api/conversations/list.query.ts +++ b/packages/junior/src/api/conversations/list.query.ts @@ -99,16 +99,6 @@ function conversationFromRow(row: ConversationRow): Conversation { ? { lineage: { parentConversationId: value.parentConversationId, - ...(value.rootConversationId - ? { rootConversationId: value.rootConversationId } - : {}), - ...(value.parentTurnId ? { parentTurnId: value.parentTurnId } : {}), - ...(value.parentEventSeq !== null - ? { parentEventSeq: value.parentEventSeq } - : {}), - ...(value.contextForkSeq !== null - ? { contextForkSeq: value.contextForkSeq } - : {}), }, } : {}), diff --git a/packages/junior/src/api/conversations/schema.ts b/packages/junior/src/api/conversations/schema.ts index 978ff973f..38b2f0294 100644 --- a/packages/junior/src/api/conversations/schema.ts +++ b/packages/junior/src/api/conversations/schema.ts @@ -127,7 +127,6 @@ const conversationReportSubagentStartedEventDataSchema = z type: z.literal("subagent_started"), childConversationId: z.string().min(1), subagentKind: z.string().min(1), - historyMode: z.enum(["isolated", "shared"]), }) .strict(); @@ -136,7 +135,6 @@ const conversationReportSubagentEndedEventDataSchema = z type: z.literal("subagent_ended"), childConversationId: z.string().min(1), subagentKind: z.string().min(1), - historyMode: z.enum(["isolated", "shared"]), outcome: z.enum(["success", "error", "aborted"]), }) .strict(); diff --git a/packages/junior/src/chat/conversations/README.md b/packages/junior/src/chat/conversations/README.md index 8a11e2510..e78f59c94 100644 --- a/packages/junior/src/chat/conversations/README.md +++ b/packages/junior/src/chat/conversations/README.md @@ -6,7 +6,7 @@ conversation events, compaction boundaries, search, and retention. ## Records - Conversation rows own canonical source, destination, participant, visibility, - lineage, and retention metadata. Their execution cursor, status, duration, + retention metadata, and an optional parent conversation link. Their execution cursor, status, duration, and usage columns are materialized reporting/control aggregates, not transcript or event-history authority. - Visible-message events are the destination-facing user and assistant history. @@ -21,14 +21,9 @@ conversation events, compaction boundaries, search, and retention. events rather than stored in either aggregate. - Context epochs identify replacement boundaries created by compaction or model handoff. -- Child rows carry immutable parent/root, parent-turn, and exact parent-event - correlation. Shared children additionally retain that parent sequence as - their context fork; isolated children retain no fork. -- The Pi adapter recursively composes a shared child's pinned ancestor prefix - with only the child's current local epoch. Initial and inheriting rollback - markers preserve that relationship; compaction, handoff, and isolated epochs - are self-contained. Child sequence cursors count only child-local events, and - commits reject any mutation of the inherited prefix. +- Historical advisor imports may create an isolated child conversation linked + to its parent. Each conversation still owns a complete local event stream; + parent history is never composed into a child's Pi projection. - Provider payloads and old state-store mirrors are inputs only to the explicit operator migration under `cli/upgrade`; they are not runtime dependencies or canonical product records. @@ -73,16 +68,11 @@ remain internal. transcript cache or lazy legacy import. - Compaction replaces prior model context without rewriting visible history. - Imports and migrations are idempotent and preserve stable conversation IDs. -- Establish child lineage and its parent `subagent_started` reference through - the SQL-backed lineage service in one transaction. Retries must match the - original parent, root, turn, event, and history mode exactly. -- Never copy inherited Pi messages into the child log. Resolve every restore - path through the lineage-aware Pi projection and bound each ancestor at its - immutable fork before reading the next descendant. -- Shared projection has an explicit maximum lineage depth. Parent event reads - currently load the parent's full retained history before selecting the fork; - a bounded store read remains a future optimization rather than a second - projection contract. +- Historical advisor import writes the child parent link and the parent's + `subagent_started` and `subagent_ended` references. There is currently no live + runtime producer for child conversations. +- Project Pi state from only the conversation's local current epoch. Do not + duplicate parent messages into child streams or recursively compose history. ## Visibility And Retention @@ -113,8 +103,6 @@ Follow `../../../../../policies/data-redaction.md` and stopped. Its fail-closed zero-gap verification is the cutover gate; start new workers only after it passes. - Purge and migration jobs operate in bounded batches and are safe to retry. -- The lineage backfill fills historical roots only. Missing historical turn, - event, and fork correlation remains null and therefore isolated. Representative coverage lives in `packages/junior/tests/integration/conversation-sql.test.ts` and the @@ -136,8 +124,8 @@ visible/session/terminal writes can still leave a started turn without a terminal event. The next delivery slice must add durable intent/receipt reconciliation for Slack before claiming crash-safe terminality. -Subagent executions own separate child event streams. The parent records only -idempotent start/end references, and child events are never copied into the -parent. New child creation requires an existing parent turn and complete -lineage; only a metadata-bare row created by an earlier child event append may -be upgraded. Reparenting an existing conversation is rejected. +Imported historical advisor executions own separate child event streams. The +parent records start/end references and the child stores only its local events. +Dashboard detail authorizes a child through its parent chain and excludes child +rows from top-level conversation aggregates. Purging a root purges the complete +descendant subtree. diff --git a/packages/junior/src/chat/conversations/event-projection.ts b/packages/junior/src/chat/conversations/event-projection.ts deleted file mode 100644 index 77fc9d9bd..000000000 --- a/packages/junior/src/chat/conversations/event-projection.ts +++ /dev/null @@ -1,96 +0,0 @@ -/** - * Provider-neutral projection of Junior conversation events. - * - * This reducer preserves opaque model messages while aligning their durable - * provenance and event sequence. Provider adapters validate the opaque payload - * only when they turn this projection into provider-specific state. - */ -import type { ModelProfile } from "@/chat/model-profile"; -import { - contextProvenance, - type ConversationMessageProvenance, -} from "./provenance"; -import type { - ConversationEvent, - ConversationEventData, - ConversationModelMessage, -} from "./history"; - -type MessageEventData = Extract; -type AuthorizationCompletedEventData = Extract< - ConversationEventData, - { type: "authorization_completed" } ->; - -/** Opaque model context projected from ordered Junior conversation events. */ -export interface ConversationEventProjection { - messages: ConversationModelMessage[]; - provenance: ConversationMessageProvenance[]; - seqs: number[]; - modelProfile: ModelProfile; - modelId: string | undefined; -} - -function authorizationObservationMessage( - data: AuthorizationCompletedEventData, - createdAtMs: number, -): ConversationModelMessage { - const label = data.kind === "mcp" ? "MCP authorization" : "Authorization"; - return { - role: "user", - content: [ - { - type: "text", - text: `${label} completed for provider "${data.provider}". Continue the blocked request and retry the provider operation if needed.`, - }, - ], - timestamp: createdAtMs, - } as ConversationModelMessage; -} - -function messageEventProvenance( - data: MessageEventData, -): ConversationMessageProvenance { - return data.provenance ?? contextProvenance; -} - -/** - * Project ordered Junior events into provider-neutral model context. - * - * Host-only events are filtered, completed authorization becomes a synthetic - * observation, and `maxSeq` reproduces an exact committed boundary. - */ -export function projectConversationEventHistory( - events: ConversationEvent[], - options?: { maxSeq?: number }, -): ConversationEventProjection { - const messages: ConversationModelMessage[] = []; - const provenance: ConversationMessageProvenance[] = []; - const seqs: number[] = []; - let modelProfile: ModelProfile = "standard"; - let modelId: string | undefined; - - for (const event of events) { - if (options?.maxSeq !== undefined && event.seq > options.maxSeq) break; - if (event.data.type === "context_epoch_started") { - modelProfile = event.data.modelProfile ?? "standard"; - modelId = event.data.modelId; - continue; - } - if (event.data.type === "message") { - messages.push(event.data.message); - provenance.push(messageEventProvenance(event.data)); - seqs.push(event.seq); - continue; - } - if (event.data.type === "authorization_completed") { - messages.push( - authorizationObservationMessage(event.data, event.createdAtMs), - ); - provenance.push(contextProvenance); - seqs.push(event.seq); - } - } - - return { messages, provenance, seqs, modelProfile, modelId }; -} diff --git a/packages/junior/src/chat/conversations/history.ts b/packages/junior/src/chat/conversations/history.ts index 70bc56a97..d612c2df9 100644 --- a/packages/junior/src/chat/conversations/history.ts +++ b/packages/junior/src/chat/conversations/history.ts @@ -45,7 +45,6 @@ const contextEpochStartedEventDataSchema = z.union([ reason: z.literal("initial"), modelProfile: z.literal("standard"), modelId: z.string().min(1), - inheritsLineageContext: z.literal(true).optional(), }) .strict(), z @@ -80,7 +79,6 @@ const contextEpochStartedEventDataSchema = z.union([ reason: z.literal("rollback"), modelProfile: z.undefined().optional(), modelId: z.undefined().optional(), - inheritsLineageContext: z.literal(true).optional(), }) .strict(), z @@ -89,7 +87,6 @@ const contextEpochStartedEventDataSchema = z.union([ reason: z.literal("rollback"), modelProfile: modelProfileSchema, modelId: z.string().min(1), - inheritsLineageContext: z.literal(true).optional(), }) .strict(), ]); @@ -110,7 +107,6 @@ export const contextEpochStartSchema = z.discriminatedUnion("reason", [ modelProfile: z.literal("standard"), modelId: z.string().min(1), messages: z.array(contextEpochMessageSchema), - inheritsLineageContext: z.literal(true).optional(), }) .strict(), z @@ -135,7 +131,6 @@ export const contextEpochStartSchema = z.discriminatedUnion("reason", [ modelProfile: modelProfileSchema, modelId: z.string().min(1), messages: z.array(contextEpochMessageSchema), - inheritsLineageContext: z.literal(true).optional(), }) .strict(), ]); @@ -305,8 +300,6 @@ const subagentStartedEventDataSchema = z parentToolCallId: z.string().min(1).optional(), reasoningLevel: z.string().min(1).optional(), childConversationId: z.string().min(1), - parentTurnId: z.string().min(1).optional(), - historyMode: z.union([z.literal("isolated"), z.literal("shared")]), }) .strict(); @@ -314,7 +307,6 @@ const subagentEndedEventDataSchema = z .object({ type: z.literal("subagent_ended"), subagentInvocationId: z.string().min(1), - parentTurnId: z.string().min(1).optional(), outcome: z.union([ z.literal("success"), z.literal("error"), diff --git a/packages/junior/src/chat/conversations/model-messages.ts b/packages/junior/src/chat/conversations/model-messages.ts deleted file mode 100644 index 3c05ea9e4..000000000 --- a/packages/junior/src/chat/conversations/model-messages.ts +++ /dev/null @@ -1,63 +0,0 @@ -/** Shape-only utilities for opaque model messages stored in conversation events. */ -import { TURN_CONTEXT_TAG } from "@/chat/turn-context-tag"; -import type { ConversationModelMessage } from "./history"; - -const RUNTIME_TURN_CONTEXT_START = `<${TURN_CONTEXT_TAG}>`; - -function userMessageContent( - message: ConversationModelMessage, -): unknown[] | undefined { - const record = message as { role?: unknown; content?: unknown }; - return record.role === "user" && Array.isArray(record.content) - ? record.content - : undefined; -} - -function isRuntimeTurnContextPart(part: unknown): boolean { - return ( - part !== null && - typeof part === "object" && - (part as { type?: unknown }).type === "text" && - typeof (part as { text?: unknown }).text === "string" && - (part as { text: string }).text.startsWith(RUNTIME_TURN_CONTEXT_START) - ); -} - -/** Return whether opaque model history carries volatile runtime bootstrap context. */ -export function hasRuntimeTurnContextMessages( - messages: ConversationModelMessage[], -): boolean { - return messages.some((message) => - userMessageContent(message)?.some(isRuntimeTurnContextPart), - ); -} - -/** Keep only volatile runtime bootstrap messages needed during context replacement. */ -export function retainRuntimeTurnContextMessages< - T extends ConversationModelMessage, ->(messages: T[]): T[] { - return messages.flatMap((message) => { - const runtimeContent = - userMessageContent(message)?.filter(isRuntimeTurnContextPart) ?? []; - return runtimeContent.length > 0 - ? ([{ ...message, content: runtimeContent }] as T[]) - : []; - }); -} - -/** Remove volatile runtime bootstrap parts without interpreting provider fields. */ -export function stripRuntimeTurnContextMessages< - T extends ConversationModelMessage, ->(messages: T[]): T[] { - return messages.flatMap((message) => { - const content = userMessageContent(message); - if (!content) return [message]; - - const nextContent = content.filter( - (part) => !isRuntimeTurnContextPart(part), - ); - if (nextContent.length === content.length) return [message]; - if (nextContent.length === 0) return []; - return [{ ...message, content: nextContent } as T]; - }); -} diff --git a/packages/junior/src/chat/conversations/projection.ts b/packages/junior/src/chat/conversations/projection.ts index e3dfd34c5..99d8eb739 100644 --- a/packages/junior/src/chat/conversations/projection.ts +++ b/packages/junior/src/chat/conversations/projection.ts @@ -3,7 +3,7 @@ * * Materializes model-visible Pi context and derived run facts such as connected * providers. Storage and commit lifecycle stay in the conversations domain; - * provider-neutral event reduction stays here while Pi validates at its adapter. + * the Pi adapter owns event reduction and opaque-message validation. */ import { isDeepStrictEqual } from "node:util"; import type { PiMessage } from "@/chat/pi/messages"; @@ -14,21 +14,13 @@ import { import type { AuthorizationKind, ConversationEvent, - ConversationEventStore, } from "@/chat/conversations/history"; -import type { ConversationStore } from "@/chat/conversations/store"; -import { - getConversationEventStore, - getConversationStore, - getSqlExecutor, -} from "@/chat/db"; +import { getConversationEventStore, getSqlExecutor } from "@/chat/db"; import type { JuniorSqlDatabase } from "@/db/db"; -import { createSqlStore } from "@/chat/conversations/sql/store"; import { createSqlConversationEventStore } from "@/chat/conversations/sql/history"; import { withConversationEventLock } from "@/chat/conversations/sql/event-lock"; import { projectConversationEvents, - type PiConversationEventProjection, type PiConversationProjection, } from "@/chat/pi/conversation-events"; @@ -123,263 +115,24 @@ interface ScopedConversation { conversationId: string; } -interface ComposedConversationProjection { - inheritsLineageContext: boolean; - inherited: PiConversationEventProjection; - local: PiConversationEventProjection; - projection: PiConversationProjection; -} - -/** Maximum number of shared parent edges composed into one Pi projection. */ -export const MAX_LINEAGE_PROJECTION_DEPTH = 32; - -/** Raised when durable child lineage cannot safely identify one pinned context. */ -export class ConversationLineageProjectionError extends Error { - constructor(message: string) { - super(message); - this.name = "ConversationLineageProjectionError"; - } -} - -function emptyEventProjection(): PiConversationEventProjection { - return { - messages: [], - provenance: [], - seqs: [], - modelProfile: "standard", - modelId: undefined, - }; -} - -function epochInheritsLineage(events: ConversationEvent[]): boolean { - const markers = events.filter( - (event) => event.data.type === "context_epoch_started", - ); - if (markers.length > 1) { - throw new ConversationLineageProjectionError( - "conversation epoch has conflicting context markers", - ); - } - if (markers.length === 0) { - return true; - } - const marker = markers[0]!; - return ( - marker.data.type === "context_epoch_started" && - (marker.data.reason === "initial" || marker.data.reason === "rollback") && - marker.data.inheritsLineageContext === true - ); -} - -function epochAtFork(args: { - childConversationId: string; - forkSeq: number; - history: ConversationEvent[]; -}): ConversationEvent[] { - const fork = args.history.find((event) => event.seq === args.forkSeq); - if (!fork) { - throw new ConversationLineageProjectionError( - `shared context fork is missing for ${args.childConversationId}`, - ); - } - return args.history.filter( - (event) => - event.seq <= args.forkSeq && event.contextEpoch === fork.contextEpoch, - ); -} - -async function loadComposedProjection(args: { - conversationId: string; - depth?: number; - localEvents: ConversationEvent[]; - eventStore: ConversationEventStore; - conversationStore: ConversationStore; - visited?: ReadonlySet; -}): Promise { - const local = projectConversationEvents(args.localEvents); - const conversation = await args.conversationStore.get({ - conversationId: args.conversationId, - }); - if (!conversation) { - const inherited = emptyEventProjection(); - return { - inheritsLineageContext: false, - inherited, - local, - projection: local, - }; - } - const lineage = conversation.lineage; - if (!lineage) { - const inherited = emptyEventProjection(); - return { - inheritsLineageContext: false, - inherited, - local, - projection: local, - }; - } - const { parentConversationId } = lineage; - const hasParentEvent = lineage.parentEventSeq !== undefined; - const hasParentTurn = lineage.parentTurnId !== undefined; - if ( - hasParentEvent !== hasParentTurn || - lineage.rootConversationId === undefined || - (lineage.contextForkSeq !== undefined && !hasParentEvent) - ) { - throw new ConversationLineageProjectionError( - `conversation lineage is incomplete for ${args.conversationId}`, - ); - } - const visited = new Set(args.visited ?? []); - if (visited.has(args.conversationId)) { - throw new ConversationLineageProjectionError( - `conversation lineage contains a cycle at ${args.conversationId}`, - ); - } - visited.add(args.conversationId); - if (visited.has(parentConversationId)) { - throw new ConversationLineageProjectionError( - `conversation lineage contains a cycle at ${parentConversationId}`, - ); - } - - const parent = await args.conversationStore.get({ - conversationId: parentConversationId, - }); - if (!parent) { - throw new ConversationLineageProjectionError( - `shared lineage parent is missing for ${args.conversationId}`, - ); - } - const expectedRoot = - parent.lineage === undefined - ? parent.conversationId - : parent.lineage.rootConversationId; - if (expectedRoot === undefined) { - throw new ConversationLineageProjectionError( - `conversation lineage parent is incomplete for ${args.conversationId}`, - ); - } - if (lineage.rootConversationId !== expectedRoot) { - throw new ConversationLineageProjectionError( - `shared lineage root conflicts for ${args.conversationId}`, - ); - } - if (!hasParentEvent || !hasParentTurn) { - const inherited = emptyEventProjection(); - return { - inheritsLineageContext: false, - inherited, - local, - projection: local, - }; - } - - // A correlated child can only be created from a canonical parent start - // event, so this read never needs the legacy importer. - const parentHistory = await args.eventStore.loadHistory(parentConversationId); - const parentEventSeq = lineage.parentEventSeq!; - const parentTurnId = lineage.parentTurnId!; - const fork = parentHistory.find((event) => event.seq === parentEventSeq); - if ( - !fork || - fork.data.type !== "subagent_started" || - fork.data.childConversationId !== args.conversationId || - fork.data.parentTurnId !== parentTurnId - ) { - throw new ConversationLineageProjectionError( - `conversation lineage reference conflicts for ${args.conversationId}`, - ); - } - const contextForkSeq = lineage.contextForkSeq; - if (contextForkSeq === undefined) { - if (fork.data.historyMode !== "isolated") { - throw new ConversationLineageProjectionError( - `isolated lineage reference conflicts for ${args.conversationId}`, - ); - } - const inherited = emptyEventProjection(); - return { - inheritsLineageContext: false, - inherited, - local, - projection: local, - }; - } - if (fork.data.historyMode !== "shared" || parentEventSeq !== contextForkSeq) { - throw new ConversationLineageProjectionError( - `shared lineage fork conflicts for ${args.conversationId}`, - ); - } - if (!epochInheritsLineage(args.localEvents)) { - const inherited = emptyEventProjection(); - return { - inheritsLineageContext: false, - inherited, - local, - projection: local, - }; - } - const depth = args.depth ?? 0; - if (depth >= MAX_LINEAGE_PROJECTION_DEPTH) { - throw new ConversationLineageProjectionError( - `conversation lineage exceeds maximum projection depth for ${args.conversationId}`, - ); - } - const parentProjection = await loadComposedProjection({ - conversationId: parentConversationId, - depth: depth + 1, - localEvents: epochAtFork({ - childConversationId: args.conversationId, - forkSeq: contextForkSeq, - history: parentHistory, - }), - eventStore: args.eventStore, - conversationStore: args.conversationStore, - visited, - }); - const inherited: PiConversationEventProjection = { - ...parentProjection.projection, - seqs: [...parentProjection.inherited.seqs, ...parentProjection.local.seqs], - }; - return { - inheritsLineageContext: true, - inherited, - local, - projection: { - messages: [...inherited.messages, ...local.messages], - provenance: [...inherited.provenance, ...local.provenance], - modelProfile: local.modelProfile, - modelId: local.modelId, - }, - }; -} - -async function loadCurrentComposedProjection( - args: ScopedConversation, -): Promise { - const eventStore = getConversationEventStore(); - return await loadComposedProjection({ - conversationId: args.conversationId, - localEvents: await eventStore.loadCurrentEpoch(args.conversationId), - eventStore, - conversationStore: getConversationStore(), - }); -} - /** Load the current-epoch Pi projection for a conversation. */ export async function loadProjection( args: ScopedConversation, ): Promise { - return (await loadCurrentComposedProjection(args)).projection.messages; + const events = await getConversationEventStore().loadCurrentEpoch( + args.conversationId, + ); + return projectConversationEvents(events).messages; } /** Load the current-epoch Pi projection with aligned per-message provenance. */ export async function loadConversationProjection( args: ScopedConversation, ): Promise { - return (await loadCurrentComposedProjection(args)).projection; + const events = await getConversationEventStore().loadCurrentEpoch( + args.conversationId, + ); + return projectConversationEvents(events); } /** Open a standard initial epoch before a conversation's first model request. */ @@ -388,13 +141,7 @@ export async function openConversationProjection( ): Promise { const eventStore = getConversationEventStore(); const events = await eventStore.loadCurrentEpoch(args.conversationId); - const composed = await loadComposedProjection({ - conversationId: args.conversationId, - localEvents: events, - eventStore, - conversationStore: getConversationStore(), - }); - const projection = composed.projection; + const projection = projectConversationEvents(events); if (events.some((event) => event.data.type === "context_epoch_started")) { return projection; } @@ -405,9 +152,6 @@ export async function openConversationProjection( modelProfile: "standard", modelId: args.modelId, messages: [], - ...(composed.inheritsLineageContext - ? { inheritsLineageContext: true as const } - : {}), }); return { messages: projection.messages, @@ -446,16 +190,13 @@ export async function loadTurnProjection(args: { // A record that committed no messages materializes the live projection, the // same way count-based records with a zero cursor did. if (args.committedSeq < 0) { - const composed = await loadComposedProjection({ - conversationId: args.conversationId, - localEvents: await eventStore.loadCurrentEpoch(args.conversationId), - eventStore, - conversationStore: getConversationStore(), - }); + const projection = projectConversationEvents( + await eventStore.loadCurrentEpoch(args.conversationId), + ); return { - ...composed.projection, - localMessageStartIndex: composed.inherited.messages.length, - seqs: composed.local.seqs, + ...projection, + localMessageStartIndex: 0, + seqs: projection.seqs, }; } const history = await eventStore.loadHistory(args.conversationId); @@ -471,16 +212,11 @@ export async function loadTurnProjection(args: { const localEvents = args.includeTail ? epochEvents : epochEvents.filter((event) => event.seq <= args.committedSeq); - const composed = await loadComposedProjection({ - conversationId: args.conversationId, - localEvents, - eventStore, - conversationStore: getConversationStore(), - }); + const projection = projectConversationEvents(localEvents); return { - ...composed.projection, - localMessageStartIndex: composed.inherited.messages.length, - seqs: composed.local.seqs, + ...projection, + localMessageStartIndex: 0, + seqs: projection.seqs, }; } @@ -546,48 +282,17 @@ async function commitMessagesLocked( ): ReturnType { const eventStore = createSqlConversationEventStore(executor); const currentEvents = await eventStore.loadCurrentEpoch(args.conversationId); - const composed = await loadComposedProjection({ - conversationId: args.conversationId, - localEvents: currentEvents, - eventStore, - conversationStore: createSqlStore(executor), - }); - const inheritedMessageCount = composed.inherited.messages.length; - if ( - countMatchingPrefix( - composed.inherited.messages, - args.messages.slice(0, inheritedMessageCount), - ) !== inheritedMessageCount - ) { - throw new ConversationLineageProjectionError( - `commit mutated inherited context for ${args.conversationId}`, - ); - } - if ( - args.provenance && - !isDeepStrictEqual( - args.provenance.slice(0, inheritedMessageCount), - composed.inherited.provenance, - ) - ) { - throw new ConversationLineageProjectionError( - `commit mutated inherited provenance for ${args.conversationId}`, - ); - } - const nextLocalMessages = args.messages.slice(inheritedMessageCount); + const current = projectConversationEvents(currentEvents); + const nextLocalMessages = args.messages; const matchingPrefix = countMatchingPrefix( - composed.local.messages, + current.messages, nextLocalMessages, ); const nextLocalProvenance = resolveCommitProvenance({ - existing: composed.local, + existing: current, nextMessages: nextLocalMessages, matchingPrefix, - ...(args.provenance - ? { - explicitProvenance: args.provenance.slice(inheritedMessageCount), - } - : {}), + ...(args.provenance ? { explicitProvenance: args.provenance } : {}), ...(args.trailingMessageProvenance ? { trailingMessageProvenance: args.trailingMessageProvenance } : {}), @@ -600,8 +305,7 @@ async function commitMessagesLocked( ); if ( currentEvents.length === 0 || - (!hasContextEpochMarker && - matchingPrefix === composed.local.messages.length) + (!hasContextEpochMarker && matchingPrefix === current.messages.length) ) { const initialMessages = nextLocalMessages.slice(matchingPrefix); await eventStore.startEpoch(args.conversationId, { @@ -613,11 +317,8 @@ async function commitMessagesLocked( createdAtMs: messageTimestamp(message), provenance: nextLocalProvenance[matchingPrefix + index]!, })), - ...(composed.inheritsLineageContext - ? { inheritsLineageContext: true as const } - : {}), }); - } else if (matchingPrefix === composed.local.messages.length) { + } else if (matchingPrefix === current.messages.length) { const newMessages = nextLocalMessages.slice(matchingPrefix); await eventStore.append( args.conversationId, @@ -633,16 +334,13 @@ async function commitMessagesLocked( } else { await eventStore.startEpoch(args.conversationId, { reason: "rollback", - modelProfile: composed.local.modelProfile, + modelProfile: current.modelProfile, modelId: args.modelId, messages: nextLocalMessages.map((message, index) => ({ message, createdAtMs: messageTimestamp(message), provenance: nextLocalProvenance[index]!, })), - ...(composed.inheritsLineageContext - ? { inheritsLineageContext: true as const } - : {}), }); } const committed = projectConversationEvents( @@ -650,9 +348,9 @@ async function commitMessagesLocked( ); return { committedSeq: committed.seqs.at(-1) ?? -1, - localMessageStartIndex: inheritedMessageCount, + localMessageStartIndex: 0, messageSeqs: committed.seqs, - provenance: [...composed.inherited.provenance, ...nextLocalProvenance], + provenance: nextLocalProvenance, }; } diff --git a/packages/junior/src/chat/conversations/sql/privacy.ts b/packages/junior/src/chat/conversations/sql/privacy.ts index 1ba19eb72..73d56c101 100644 --- a/packages/junior/src/chat/conversations/sql/privacy.ts +++ b/packages/junior/src/chat/conversations/sql/privacy.ts @@ -8,7 +8,6 @@ const MAX_LINEAGE_DEPTH = 32; interface LineageRow { destinationId: string | null; parentId: string | null; - rootId: string | null; } interface LineageCandidate { @@ -25,7 +24,6 @@ async function readLineageRow( .db() .select({ parentId: juniorConversations.parentConversationId, - rootId: juniorConversations.rootConversationId, destinationId: juniorConversations.destinationId, }) .from(juniorConversations) @@ -39,7 +37,6 @@ async function traceLineage( conversationId: string, ): Promise { let currentId = conversationId; - let declaredRootId: string | null | undefined; const path: string[] = []; const seen = new Set(); @@ -49,18 +46,12 @@ async function traceLineage( const row = await readLineageRow(executor, currentId, false); if (!row) return undefined; - if (declaredRootId === undefined) declaredRootId = row.rootId; if (row.parentId) { - if (!declaredRootId || row.rootId !== declaredRootId) return undefined; currentId = row.parentId; continue; } - if ( - row.rootId !== null || - (declaredRootId !== null && declaredRootId !== currentId) || - !row.destinationId - ) { + if (!row.destinationId) { return undefined; } return { path, rootConversationId: currentId }; @@ -73,22 +64,12 @@ function lockedLineageIsConsistent( candidate: LineageCandidate, rows: Map, ): boolean { - const declaredRootId = rows.get(candidate.path[0]!)?.rootId; for (const [index, conversationId] of candidate.path.entries()) { const row = rows.get(conversationId); if (!row) return false; const expectedParentId = candidate.path[index + 1] ?? null; if (row.parentId !== expectedParentId) return false; - if (expectedParentId) { - if (!declaredRootId || row.rootId !== declaredRootId) return false; - continue; - } - if ( - row.rootId !== null || - (declaredRootId !== null && - declaredRootId !== candidate.rootConversationId) || - !row.destinationId - ) { + if (!expectedParentId && !row.destinationId) { return false; } } @@ -101,8 +82,7 @@ function lockedLineageIsConsistent( * The lineage is discovered without locks, then locked and revalidated from * root to requested conversation. Root-first ordering matches tree purges; * callers that keep the transaction open receive a stable privacy decision. - * Missing, cyclic, over-depth, historically uncorrelated, or internally - * inconsistent lineage fails closed. + * Missing, cyclic, over-depth, or concurrently changed lineage fails closed. */ export async function resolveRootVisibility( executor: JuniorSqlDatabase, diff --git a/packages/junior/src/chat/conversations/sql/purge.ts b/packages/junior/src/chat/conversations/sql/purge.ts index 5f924cc12..3ff6e3e7d 100644 --- a/packages/junior/src/chat/conversations/sql/purge.ts +++ b/packages/junior/src/chat/conversations/sql/purge.ts @@ -31,8 +31,6 @@ interface ConversationTreeRow { parentConversationId: string | null; } -const MAX_TREE_LOCK_PASSES = 32; - /** Discover a root and its current descendants via `parent_conversation_id`. */ async function discoverConversationTree( executor: JuniorSqlDatabase, @@ -66,72 +64,6 @@ async function discoverConversationTree( return [...all.values()]; } -/** - * Lock a complete tree root-first and repeat discovery until every row in the - * final scan is locked. Locking known parents blocks normal FK-backed child - * creation; the repeat catches children attached to a not-yet-locked parent - * during an earlier scan. - */ -async function lockStableConversationTree( - executor: JuniorSqlDatabase, - root: Omit, -): Promise { - const locked = new Map([ - [root.conversationId, { ...root, depth: 0 }], - ]); - - for (let pass = 0; pass < MAX_TREE_LOCK_PASSES; pass += 1) { - const discovered = await discoverConversationTree(executor, root); - for (const candidate of discovered) { - const existing = locked.get(candidate.conversationId); - if (existing) { - if (existing.parentConversationId !== candidate.parentConversationId) { - return undefined; - } - continue; - } - await withConversationEventLock( - executor, - candidate.conversationId, - async () => undefined, - ); - const rows = await executor - .db() - .select({ - conversationId: juniorConversations.conversationId, - lastActivityAt: juniorConversations.lastActivityAt, - parentConversationId: juniorConversations.parentConversationId, - }) - .from(juniorConversations) - .where(eq(juniorConversations.conversationId, candidate.conversationId)) - .for("update"); - const row = rows[0]; - if (!row || row.parentConversationId !== candidate.parentConversationId) { - return undefined; - } - locked.set(row.conversationId, { ...row, depth: candidate.depth }); - } - - const finalScan = await discoverConversationTree(executor, root); - if ( - finalScan.length === locked.size && - finalScan.every( - (row) => - locked.get(row.conversationId)?.parentConversationId === - row.parentConversationId, - ) - ) { - return [...locked.values()].sort( - (left, right) => - left.depth - right.depth || - left.conversationId.localeCompare(right.conversationId), - ); - } - } - - return undefined; -} - /** * Select expired root conversations for purge, oldest activity first. * @@ -296,12 +228,25 @@ export async function purgeConversationTree( .for("share") : []; const isPublic = destinations[0]?.visibility === "public"; - const tree = await lockStableConversationTree(executor, { + const tree = await discoverConversationTree(executor, { conversationId: root.conversationId, lastActivityAt: root.lastActivityAt, parentConversationId: root.parentConversationId, }); - if (!tree) { + const initiallyDiscovered = new Map( + initialTree.map((conversation) => [ + conversation.conversationId, + conversation.parentConversationId, + ]), + ); + if ( + tree.length !== initiallyDiscovered.size || + tree.some( + (conversation) => + initiallyDiscovered.get(conversation.conversationId) !== + conversation.parentConversationId, + ) + ) { return { purged: false, conversations: 0 }; } if (args.retention) { diff --git a/packages/junior/src/chat/conversations/sql/store.ts b/packages/junior/src/chat/conversations/sql/store.ts index b355efa4f..cdabe8a98 100644 --- a/packages/junior/src/chat/conversations/sql/store.ts +++ b/packages/junior/src/chat/conversations/sql/store.ts @@ -314,16 +314,6 @@ function conversationFromRow(readRow: ConversationReadRow): Conversation { ? { lineage: { parentConversationId: row.parentConversationId, - ...(row.rootConversationId - ? { rootConversationId: row.rootConversationId } - : {}), - ...(row.parentTurnId ? { parentTurnId: row.parentTurnId } : {}), - ...(row.parentEventSeq !== null - ? { parentEventSeq: row.parentEventSeq } - : {}), - ...(row.contextForkSeq !== null - ? { contextForkSeq: row.contextForkSeq } - : {}), }, } : {}), @@ -875,10 +865,6 @@ export class SqlStore implements ConversationStore { : dateFromMs(conversation.execution.lastEnqueuedAtMs), parentConversationId: conversation.lineage?.parentConversationId ?? null, - rootConversationId: conversation.lineage?.rootConversationId ?? null, - parentTurnId: conversation.lineage?.parentTurnId ?? null, - parentEventSeq: conversation.lineage?.parentEventSeq ?? null, - contextForkSeq: conversation.lineage?.contextForkSeq ?? null, }) .onConflictDoUpdate({ target: juniorConversations.conversationId, diff --git a/packages/junior/src/chat/conversations/store.ts b/packages/junior/src/chat/conversations/store.ts index 9180f0342..c1b5d7078 100644 --- a/packages/junior/src/chat/conversations/store.ts +++ b/packages/junior/src/chat/conversations/store.ts @@ -29,11 +29,7 @@ export interface ConversationExecution { /** Immutable parent correlation for a child conversation. */ export interface ConversationLineage { - contextForkSeq?: number; parentConversationId: string; - parentEventSeq?: number; - parentTurnId?: string; - rootConversationId?: string; } export interface Conversation { diff --git a/packages/junior/src/chat/db.ts b/packages/junior/src/chat/db.ts index e649bde81..2d11c9618 100644 --- a/packages/junior/src/chat/db.ts +++ b/packages/junior/src/chat/db.ts @@ -9,7 +9,6 @@ import { createSqlConversationSearchStore } from "@/chat/conversations/sql/searc import type { ConversationSearchStore } from "@/chat/conversations/search"; import type { JuniorDatabase, JuniorSqlExecutor } from "@/db/db"; import { createJuniorSqlExecutor } from "@/db/executor"; -import { SubagentLineageService } from "@/chat/services/subagent-lineage"; let current: | { @@ -20,7 +19,6 @@ let current: eventStore: ConversationEventStore; messageStore: ConversationMessageStore; searchStore: ConversationSearchStore; - subagentLineage: SubagentLineageService; } | undefined; @@ -61,7 +59,6 @@ export function getSqlExecutor(): JuniorSqlExecutor { eventStore: createSqlConversationEventStore(db), messageStore: createSqlConversationMessageStore(db), searchStore: createSqlConversationSearchStore(db), - subagentLineage: new SubagentLineageService(db), }; } return current.db; @@ -96,12 +93,6 @@ export function getConversationSearchStore(): ConversationSearchStore { return current!.searchStore; } -/** Return the SQL-backed immutable child-conversation lineage service. */ -export function getSubagentLineageService(): SubagentLineageService { - getSqlExecutor(); - return current!.subagentLineage; -} - /** Close the process SQL database when it has been opened. */ export async function closeDb(): Promise { const previous = current; diff --git a/packages/junior/src/chat/pi/conversation-events.ts b/packages/junior/src/chat/pi/conversation-events.ts index 2c12c8ea8..005937095 100644 --- a/packages/junior/src/chat/pi/conversation-events.ts +++ b/packages/junior/src/chat/pi/conversation-events.ts @@ -1,14 +1,25 @@ /** * Pi adapter for Junior conversation events. * - * Conversation storage owns the canonical ordered event log and its generic - * projection. This module validates the opaque projected messages as Pi state. + * Conversation storage owns the canonical ordered event log. This module is + * the sole boundary that interprets its opaque messages as Pi state. */ import type { ModelProfile } from "@/chat/model-profile"; -import type { ConversationEvent } from "@/chat/conversations/history"; -import { projectConversationEventHistory } from "@/chat/conversations/event-projection"; +import type { + ConversationEvent, + ConversationEventData, +} from "@/chat/conversations/history"; import { piMessageSchema, type PiMessage } from "@/chat/pi/messages"; -import type { ConversationMessageProvenance } from "@/chat/conversations/provenance"; +import { + contextProvenance, + type ConversationMessageProvenance, +} from "@/chat/conversations/provenance"; + +type MessageEventData = Extract; +type AuthorizationCompletedEventData = Extract< + ConversationEventData, + { type: "authorization_completed" } +>; /** Pi context projected from one Junior conversation epoch. */ export interface PiConversationProjection { @@ -23,6 +34,29 @@ export interface PiConversationEventProjection extends PiConversationProjection seqs: number[]; } +function authorizationObservationMessage( + data: AuthorizationCompletedEventData, + createdAtMs: number, +): PiMessage { + const label = data.kind === "mcp" ? "MCP authorization" : "Authorization"; + return piMessageSchema.parse({ + role: "user", + content: [ + { + type: "text", + text: `${label} completed for provider "${data.provider}". Continue the blocked request and retry the provider operation if needed.`, + }, + ], + timestamp: createdAtMs, + }); +} + +function messageEventProvenance( + data: MessageEventData, +): ConversationMessageProvenance { + return data.provenance ?? contextProvenance; +} + /** * Project ordered Junior events into Pi context. * @@ -33,11 +67,33 @@ export function projectConversationEvents( events: ConversationEvent[], options?: { maxSeq?: number }, ): PiConversationEventProjection { - const projection = projectConversationEventHistory(events, options); - return { - ...projection, - messages: projection.messages.map((message) => - piMessageSchema.parse(message), - ), - }; + const messages: PiMessage[] = []; + const provenance: ConversationMessageProvenance[] = []; + const seqs: number[] = []; + let modelProfile: ModelProfile = "standard"; + let modelId: string | undefined; + + for (const event of events) { + if (options?.maxSeq !== undefined && event.seq > options.maxSeq) break; + if (event.data.type === "context_epoch_started") { + modelProfile = event.data.modelProfile ?? "standard"; + modelId = event.data.modelId; + continue; + } + if (event.data.type === "message") { + messages.push(piMessageSchema.parse(event.data.message)); + provenance.push(messageEventProvenance(event.data)); + seqs.push(event.seq); + continue; + } + if (event.data.type === "authorization_completed") { + messages.push( + authorizationObservationMessage(event.data, event.createdAtMs), + ); + provenance.push(contextProvenance); + seqs.push(event.seq); + } + } + + return { messages, provenance, seqs, modelProfile, modelId }; } diff --git a/packages/junior/src/chat/pi/transcript.ts b/packages/junior/src/chat/pi/transcript.ts index 39704c863..ecba641b0 100644 --- a/packages/junior/src/chat/pi/transcript.ts +++ b/packages/junior/src/chat/pi/transcript.ts @@ -12,12 +12,27 @@ import type { ToolResultMessage, } from "@earendil-works/pi-ai"; import { unwrapCurrentInstruction } from "@/chat/current-instruction"; -import { - hasRuntimeTurnContextMessages, - retainRuntimeTurnContextMessages, - stripRuntimeTurnContextMessages, -} from "@/chat/conversations/model-messages"; import type { PiMessage } from "@/chat/pi/messages"; +import { TURN_CONTEXT_TAG } from "@/chat/turn-context-tag"; + +const RUNTIME_TURN_CONTEXT_START = `<${TURN_CONTEXT_TAG}>`; + +function userMessageContent(message: PiMessage): unknown[] | undefined { + const record = message as { role?: unknown; content?: unknown }; + return record.role === "user" && Array.isArray(record.content) + ? record.content + : undefined; +} + +function isRuntimeTurnContextPart(part: unknown): boolean { + return ( + part !== null && + typeof part === "object" && + (part as { type?: unknown }).type === "text" && + typeof (part as { text?: unknown }).text === "string" && + (part as { text: string }).text.startsWith(RUNTIME_TURN_CONTEXT_START) + ); +} // Prior-thread context blocks the runtime embeds inside the same user-turn text // that carries the block (see buildUserTurnText and @@ -127,12 +142,20 @@ export function trimTrailingAssistantMessages( /** Return whether Pi history already carries session bootstrap context. */ export function hasRuntimeTurnContext(messages: PiMessage[]): boolean { - return hasRuntimeTurnContextMessages(messages); + return messages.some((message) => + userMessageContent(message)?.some(isRuntimeTurnContextPart), + ); } /** Keep only volatile bootstrap messages needed by an in-turn context replacement. */ export function retainRuntimeTurnContext(messages: PiMessage[]): PiMessage[] { - return retainRuntimeTurnContextMessages(messages); + return messages.flatMap((message) => { + const runtimeContent = + userMessageContent(message)?.filter(isRuntimeTurnContextPart) ?? []; + return runtimeContent.length > 0 + ? [{ ...message, content: runtimeContent } as PiMessage] + : []; + }); } /** @@ -158,5 +181,15 @@ export function instructionTextForProjection(text: string): string { /** Remove volatile runtime context before reusing messages as history. */ export function stripRuntimeTurnContext(messages: PiMessage[]): PiMessage[] { - return stripRuntimeTurnContextMessages(messages); + return messages.flatMap((message) => { + const content = userMessageContent(message); + if (!content) return [message]; + + const nextContent = content.filter( + (part) => !isRuntimeTurnContextPart(part), + ); + if (nextContent.length === content.length) return [message]; + if (nextContent.length === 0) return []; + return [{ ...message, content: nextContent } as PiMessage]; + }); } diff --git a/packages/junior/src/chat/services/subagent-lineage.ts b/packages/junior/src/chat/services/subagent-lineage.ts deleted file mode 100644 index a5143fb51..000000000 --- a/packages/junior/src/chat/services/subagent-lineage.ts +++ /dev/null @@ -1,418 +0,0 @@ -import { isDeepStrictEqual } from "node:util"; -import { eq } from "drizzle-orm"; -import type { JuniorSqlDatabase } from "@/db/db"; -import { juniorConversations } from "@/db/schema"; -import type { - ConversationEvent, - ConversationEventData, -} from "@/chat/conversations/history"; -import { createSqlConversationEventStore } from "@/chat/conversations/sql/history"; -import { withConversationEventLock } from "@/chat/conversations/sql/event-lock"; - -export type SubagentHistoryMode = "isolated" | "shared"; - -export interface StartSubagentReferenceInput { - childConversationId: string; - historyMode: SubagentHistoryMode; - modelId?: string; - nowMs?: number; - parentConversationId: string; - parentToolCallId?: string; - parentTurnId: string; - reasoningLevel?: string; - subagentInvocationId: string; - subagentKind: string; -} - -export interface SubagentLineage { - childConversationId: string; - contextForkSeq: number | null; - parentConversationId: string; - parentEventSeq: number; - parentTurnId: string; - rootConversationId: string; -} - -export interface FinishSubagentReferenceInput { - nowMs?: number; - outcome: "aborted" | "error" | "success"; - parentConversationId: string; - parentTurnId: string; - subagentInvocationId: string; -} - -/** Raised when a retry attempts to change an immutable subagent reference. */ -export class SubagentLineageConflictError extends Error { - constructor(message: string) { - super(message); - this.name = "SubagentLineageConflictError"; - } -} - -function startedKey(subagentInvocationId: string): string { - return `subagent:${subagentInvocationId}:started`; -} - -function endedKey(subagentInvocationId: string): string { - return `subagent:${subagentInvocationId}:ended`; -} - -function eventWithKey( - history: ConversationEvent[], - idempotencyKey: string, -): ConversationEvent | undefined { - return history.find((event) => event.idempotencyKey === idempotencyKey); -} - -function expectedStartData( - input: StartSubagentReferenceInput, -): Extract { - return { - type: "subagent_started", - subagentInvocationId: input.subagentInvocationId, - subagentKind: input.subagentKind, - ...(input.modelId ? { modelId: input.modelId } : {}), - ...(input.parentToolCallId - ? { parentToolCallId: input.parentToolCallId } - : {}), - ...(input.reasoningLevel ? { reasoningLevel: input.reasoningLevel } : {}), - childConversationId: input.childConversationId, - parentTurnId: input.parentTurnId, - historyMode: input.historyMode, - }; -} - -function expectedEndData( - input: FinishSubagentReferenceInput, -): Extract { - return { - type: "subagent_ended", - subagentInvocationId: input.subagentInvocationId, - parentTurnId: input.parentTurnId, - outcome: input.outcome, - }; -} - -function assertExactEvent( - event: ConversationEvent | undefined, - expected: ConversationEventData, - label: string, -): asserts event is ConversationEvent { - if (!event || !isDeepStrictEqual(event.data, expected)) { - throw new SubagentLineageConflictError( - `${label} conflicts with the persisted subagent reference`, - ); - } -} - -/** - * Own immutable SQL lineage and parent-stream references for subagent runs. - * Child execution events remain in the child conversation's event stream. - */ -export class SubagentLineageService { - constructor(private readonly executor: JuniorSqlDatabase) {} - - /** Append the parent start reference and establish the correlated child row atomically. */ - async start(input: StartSubagentReferenceInput): Promise { - if (input.childConversationId === input.parentConversationId) { - throw new SubagentLineageConflictError( - "A subagent conversation cannot be its own parent", - ); - } - const eventStore = createSqlConversationEventStore(this.executor); - const expected = expectedStartData(input); - - return await withConversationEventLock( - this.executor, - input.parentConversationId, - async () => - await this.executor.transaction(async () => { - const rootConversationId = await this.resolveRootConversationId( - input.parentConversationId, - ); - let history = await eventStore.loadHistory( - input.parentConversationId, - ); - if ( - !history.some( - (event) => - event.data.type === "turn_started" && - event.data.turnId === input.parentTurnId, - ) - ) { - throw new SubagentLineageConflictError( - "Subagent start does not match a parent turn", - ); - } - const existingInvocation = history.find( - (event) => - event.data.type === "subagent_started" && - event.data.subagentInvocationId === input.subagentInvocationId, - ); - const key = startedKey(input.subagentInvocationId); - let start = eventWithKey(history, key); - if (!start && existingInvocation) { - throw new SubagentLineageConflictError( - "Subagent invocation already has a different start reference", - ); - } - if (!start) { - await eventStore.append(input.parentConversationId, [ - { - data: expected, - idempotencyKey: key, - createdAtMs: input.nowMs ?? Date.now(), - }, - ]); - history = await eventStore.loadHistory(input.parentConversationId); - start = eventWithKey(history, key); - } - assertExactEvent(start, expected, "Subagent start retry"); - - const contextForkSeq = - input.historyMode === "shared" ? start.seq : null; - await this.establishChild({ - childConversationId: input.childConversationId, - contextForkSeq, - nowMs: input.nowMs ?? Date.now(), - parentConversationId: input.parentConversationId, - parentEventSeq: start.seq, - parentTurnId: input.parentTurnId, - rootConversationId, - }); - return { - childConversationId: input.childConversationId, - contextForkSeq, - parentConversationId: input.parentConversationId, - parentEventSeq: start.seq, - parentTurnId: input.parentTurnId, - rootConversationId, - }; - }), - ); - } - - /** Append the idempotent terminal reference after validating its exact start correlation. */ - async finish(input: FinishSubagentReferenceInput): Promise { - const eventStore = createSqlConversationEventStore(this.executor); - const expected = expectedEndData(input); - await withConversationEventLock( - this.executor, - input.parentConversationId, - async () => - await this.executor.transaction(async () => { - let history = await eventStore.loadHistory( - input.parentConversationId, - ); - const start = history.find( - (event) => - event.data.type === "subagent_started" && - event.data.subagentInvocationId === input.subagentInvocationId, - ); - if ( - !start || - start.data.type !== "subagent_started" || - start.data.parentTurnId !== input.parentTurnId - ) { - throw new SubagentLineageConflictError( - "Subagent finish does not match an exact parent start reference", - ); - } - await this.assertChildMatchesStart( - input.parentConversationId, - await this.resolveRootConversationId(input.parentConversationId), - start, - ); - - const key = endedKey(input.subagentInvocationId); - let end = eventWithKey(history, key); - if (!end) { - await eventStore.append(input.parentConversationId, [ - { - data: expected, - idempotencyKey: key, - createdAtMs: input.nowMs ?? Date.now(), - }, - ]); - history = await eventStore.loadHistory(input.parentConversationId); - end = eventWithKey(history, key); - } - assertExactEvent(end, expected, "Subagent finish retry"); - }), - ); - } - - private async establishChild(lineage: SubagentLineage & { nowMs: number }) { - const at = new Date(lineage.nowMs); - await this.executor - .db() - .insert(juniorConversations) - .values({ - conversationId: lineage.childConversationId, - parentConversationId: lineage.parentConversationId, - rootConversationId: lineage.rootConversationId, - parentTurnId: lineage.parentTurnId, - parentEventSeq: lineage.parentEventSeq, - contextForkSeq: lineage.contextForkSeq, - createdAt: at, - lastActivityAt: at, - updatedAt: at, - executionStatus: "idle", - }) - .onConflictDoNothing({ target: juniorConversations.conversationId }); - - const rows = await this.executor - .db() - .select() - .from(juniorConversations) - .where( - eq(juniorConversations.conversationId, lineage.childConversationId), - ) - .for("update"); - const child = rows[0]; - if (!child) { - throw new Error("Subagent child conversation was not created"); - } - const isBare = - child.parentConversationId === null && - child.rootConversationId === null && - child.parentTurnId === null && - child.parentEventSeq === null && - child.contextForkSeq === null && - child.transcriptPurgedAt === null && - child.source === null && - child.originType === null && - child.originId === null && - child.originRunId === null && - child.destinationId === null && - child.destination === null && - child.actorIdentityId === null && - child.creatorIdentityId === null && - child.credentialSubjectIdentityId === null && - child.actor === null && - child.channelName === null && - child.title === null && - child.executionUpdatedAt === null && - child.executionStatus === "idle" && - child.runId === null && - child.lastCheckpointAt === null && - child.lastEnqueuedAt === null && - child.durationMs === 0 && - child.usage === null && - child.executionDurationMs === 0 && - child.executionUsage === null && - child.metricRunId === null; - if (isBare) { - await this.executor - .db() - .update(juniorConversations) - .set({ - parentConversationId: lineage.parentConversationId, - rootConversationId: lineage.rootConversationId, - parentTurnId: lineage.parentTurnId, - parentEventSeq: lineage.parentEventSeq, - contextForkSeq: lineage.contextForkSeq, - }) - .where( - eq(juniorConversations.conversationId, lineage.childConversationId), - ); - return; - } - if ( - child.parentConversationId !== lineage.parentConversationId || - child.rootConversationId !== lineage.rootConversationId || - child.parentTurnId !== lineage.parentTurnId || - child.parentEventSeq !== lineage.parentEventSeq || - child.contextForkSeq !== lineage.contextForkSeq - ) { - throw new SubagentLineageConflictError( - "Subagent child conversation already has different lineage", - ); - } - } - - private async assertChildMatchesStart( - parentConversationId: string, - rootConversationId: string, - start: ConversationEvent, - ): Promise { - if (start.data.type !== "subagent_started") { - throw new SubagentLineageConflictError( - "Subagent start reference has an invalid event type", - ); - } - const rows = await this.executor - .db() - .select() - .from(juniorConversations) - .where( - eq(juniorConversations.conversationId, start.data.childConversationId), - ) - .for("update"); - const child = rows[0]; - const expectedFork = start.data.historyMode === "shared" ? start.seq : null; - if ( - !child || - child.parentConversationId !== parentConversationId || - child.rootConversationId !== rootConversationId || - child.parentTurnId !== start.data.parentTurnId || - child.parentEventSeq !== start.seq || - child.contextForkSeq !== expectedFork - ) { - throw new SubagentLineageConflictError( - "Subagent child lineage no longer matches its parent start reference", - ); - } - } - - private async resolveRootConversationId( - parentConversationId: string, - ): Promise { - let currentId = parentConversationId; - const seen = new Set(); - let declaredRootConversationId: string | undefined; - while (!seen.has(currentId)) { - seen.add(currentId); - const rows = await this.executor - .db() - .select({ - conversationId: juniorConversations.conversationId, - parentConversationId: juniorConversations.parentConversationId, - rootConversationId: juniorConversations.rootConversationId, - }) - .from(juniorConversations) - .where(eq(juniorConversations.conversationId, currentId)); - const row = rows[0]; - if (!row) { - throw new Error(`Parent conversation does not exist: ${currentId}`); - } - if ( - row.rootConversationId && - declaredRootConversationId && - row.rootConversationId !== declaredRootConversationId - ) { - throw new SubagentLineageConflictError( - "Conversation parent lineage declares conflicting roots", - ); - } - if (row.rootConversationId) { - declaredRootConversationId = row.rootConversationId; - } - if (!row.parentConversationId) { - if ( - declaredRootConversationId && - declaredRootConversationId !== row.conversationId - ) { - throw new SubagentLineageConflictError( - "Conversation parent lineage does not resolve to its declared root", - ); - } - return row.conversationId; - } - currentId = row.parentConversationId; - } - throw new SubagentLineageConflictError( - "Conversation parent lineage contains a cycle", - ); - } -} diff --git a/packages/junior/src/cli/upgrade.ts b/packages/junior/src/cli/upgrade.ts index 215a2a2f3..18decfe05 100644 --- a/packages/junior/src/cli/upgrade.ts +++ b/packages/junior/src/cli/upgrade.ts @@ -15,7 +15,6 @@ import { agentTurnSessionActorMigration } from "./upgrade/migrations/agent-turn- import { conversationUsageRepairMigration } from "./upgrade/migrations/conversation-usage"; import { conversationEventDataMigration } from "./upgrade/migrations/conversation-event-data"; import { conversationVisibleMessageEventsMigration } from "./upgrade/migrations/conversation-visible-message-events"; -import { conversationLineageMigration } from "./upgrade/migrations/conversation-lineage"; import type { MigrationContext, MigrationResult, @@ -33,7 +32,6 @@ const MIGRATIONS: UpgradeMigration[] = [ agentTurnSessionActorMigration, redisConversationStateMigration, coreSqlSchemaMigration, - conversationLineageMigration, sqlConversationMigration, conversationEventDataMigration, sqlConversationHistoryMigration, diff --git a/packages/junior/src/cli/upgrade/migrations/conversation-history/legacy-history-import.ts b/packages/junior/src/cli/upgrade/migrations/conversation-history/legacy-history-import.ts index b45881f9f..3baac6efb 100644 --- a/packages/junior/src/cli/upgrade/migrations/conversation-history/legacy-history-import.ts +++ b/packages/junior/src/cli/upgrade/migrations/conversation-history/legacy-history-import.ts @@ -266,9 +266,6 @@ export function convertLegacySessionLog(args: { ? { parentToolCallId: entry.parentToolCallId } : {}), childConversationId, - // The legacy transcript records no exact parent event fork. Keep - // it isolated rather than guessing a shared-context boundary. - historyMode: "isolated", }, entry.createdAtMs, ); @@ -543,10 +540,6 @@ async function insertLegacyAdvisorChildRow( createdAt, lastActivityAt, ); - const rootConversationId = await resolveLegacyRootConversationId( - executor, - parentConversationId, - ); await executor .db() .insert(juniorConversations) @@ -554,7 +547,6 @@ async function insertLegacyAdvisorChildRow( conversationId: childConversationId, schemaVersion: 1, parentConversationId, - rootConversationId, createdAt, lastActivityAt, updatedAt: lastActivityAt, @@ -572,72 +564,13 @@ async function insertLegacyAdvisorChildRow( .db() .select({ parentConversationId: juniorConversations.parentConversationId, - rootConversationId: juniorConversations.rootConversationId, - parentTurnId: juniorConversations.parentTurnId, - parentEventSeq: juniorConversations.parentEventSeq, - contextForkSeq: juniorConversations.contextForkSeq, }) .from(juniorConversations) .where(eq(juniorConversations.conversationId, childConversationId)); const child = rows[0]; - if ( - !child || - child.parentConversationId !== parentConversationId || - child.rootConversationId !== rootConversationId || - child.parentTurnId !== null || - child.parentEventSeq !== null || - child.contextForkSeq !== null - ) { + if (!child || child.parentConversationId !== parentConversationId) { throw new Error( "Legacy advisor child conflicts with existing conversation lineage", ); } } - -async function resolveLegacyRootConversationId( - executor: JuniorSqlDatabase, - conversationId: string, -): Promise { - let currentId = conversationId; - const seen = new Set(); - let declaredRootConversationId: string | undefined; - while (!seen.has(currentId)) { - seen.add(currentId); - const rows = await executor - .db() - .select({ - conversationId: juniorConversations.conversationId, - parentConversationId: juniorConversations.parentConversationId, - rootConversationId: juniorConversations.rootConversationId, - }) - .from(juniorConversations) - .where(eq(juniorConversations.conversationId, currentId)); - const row = rows[0]; - if (!row) { - throw new Error(`Legacy advisor parent is missing: ${currentId}`); - } - if ( - row.rootConversationId && - declaredRootConversationId && - row.rootConversationId !== declaredRootConversationId - ) { - throw new Error("Legacy advisor parent declares conflicting roots"); - } - if (row.rootConversationId) { - declaredRootConversationId = row.rootConversationId; - } - if (!row.parentConversationId) { - if ( - declaredRootConversationId && - declaredRootConversationId !== row.conversationId - ) { - throw new Error( - "Legacy advisor parent does not resolve to its declared root", - ); - } - return row.conversationId; - } - currentId = row.parentConversationId; - } - throw new Error("Legacy advisor parent lineage contains a cycle"); -} diff --git a/packages/junior/src/cli/upgrade/migrations/conversation-lineage.ts b/packages/junior/src/cli/upgrade/migrations/conversation-lineage.ts deleted file mode 100644 index ed6359610..000000000 --- a/packages/junior/src/cli/upgrade/migrations/conversation-lineage.ts +++ /dev/null @@ -1,130 +0,0 @@ -import { and, asc, eq, isNotNull, isNull } from "drizzle-orm"; -import { getChatConfig } from "@/chat/config"; -import type { JuniorSqlExecutor } from "@/db/db"; -import { createJuniorSqlExecutor } from "@/db/executor"; -import { juniorConversations } from "@/db/schema"; -import type { MigrationContext, MigrationResult } from "../types"; - -const LINEAGE_BATCH_SIZE = 500; -const LINEAGE_BACKFILL_LOCK = "junior:upgrade:conversation-lineage"; - -async function resolveHistoricalRoot( - executor: JuniorSqlExecutor, - conversationId: string, -): Promise { - let currentId = conversationId; - const seen = new Set(); - let declaredRootConversationId: string | undefined; - while (!seen.has(currentId)) { - seen.add(currentId); - const rows = await executor - .db() - .select({ - conversationId: juniorConversations.conversationId, - parentConversationId: juniorConversations.parentConversationId, - rootConversationId: juniorConversations.rootConversationId, - }) - .from(juniorConversations) - .where(eq(juniorConversations.conversationId, currentId)); - const row = rows[0]; - if (!row) { - throw new Error( - `Conversation lineage references missing parent ${currentId}`, - ); - } - if ( - row.rootConversationId && - declaredRootConversationId && - row.rootConversationId !== declaredRootConversationId - ) { - throw new Error("Conversation lineage declares conflicting roots"); - } - if (row.rootConversationId) { - declaredRootConversationId = row.rootConversationId; - } - if (!row.parentConversationId) { - if ( - declaredRootConversationId && - declaredRootConversationId !== row.conversationId - ) { - throw new Error( - "Conversation lineage does not resolve to its declared root", - ); - } - return row.conversationId; - } - currentId = row.parentConversationId; - } - throw new Error(`Conversation lineage contains a cycle at ${currentId}`); -} - -/** Fill only historical root lineage; unknown fork/correlation stays null and isolated. */ -export async function migrateConversationLineage( - _context: MigrationContext, - options: { batchSize?: number; executor?: JuniorSqlExecutor } = {}, -): Promise { - let executor = options.executor; - let closeExecutor: (() => Promise) | undefined; - if (!executor) { - const { sql } = getChatConfig(); - executor = createJuniorSqlExecutor({ - connectionString: sql.databaseUrl, - driver: sql.driver, - }); - closeExecutor = () => executor!.close(); - } - const batchSize = Math.max( - 1, - Math.floor(options.batchSize ?? LINEAGE_BATCH_SIZE), - ); - let migrated = 0; - try { - while (true) { - const processed = await executor.withLock( - LINEAGE_BACKFILL_LOCK, - async () => { - const rows = await executor! - .db() - .select({ conversationId: juniorConversations.conversationId }) - .from(juniorConversations) - .where( - and( - isNotNull(juniorConversations.parentConversationId), - isNull(juniorConversations.rootConversationId), - ), - ) - .orderBy(asc(juniorConversations.conversationId)) - .limit(batchSize); - await executor!.transaction(async () => { - for (const row of rows) { - const rootConversationId = await resolveHistoricalRoot( - executor!, - row.conversationId, - ); - await executor! - .db() - .update(juniorConversations) - .set({ rootConversationId }) - .where( - eq(juniorConversations.conversationId, row.conversationId), - ); - } - }); - return rows.length; - }, - ); - if (processed === 0) { - break; - } - migrated += processed; - } - return { existing: 0, migrated, missing: 0, scanned: migrated }; - } finally { - await closeExecutor?.(); - } -} - -export const conversationLineageMigration = { - name: "backfill-conversation-lineage", - run: migrateConversationLineage, -}; diff --git a/packages/junior/src/db/schema/conversations.ts b/packages/junior/src/db/schema/conversations.ts index dcc01ee23..cbed75102 100644 --- a/packages/junior/src/db/schema/conversations.ts +++ b/packages/junior/src/db/schema/conversations.ts @@ -1,7 +1,6 @@ import { sql } from "drizzle-orm"; import { type AnyPgColumn, - check, index, integer, jsonb, @@ -66,12 +65,6 @@ export const juniorConversations = pgTable( executionUsage: jsonb("execution_usage_json").$type(), metricRunId: text("metric_run_id"), archivedAt: timestamptz("archived_at"), - rootConversationId: text("root_conversation_id").references( - (): AnyPgColumn => juniorConversations.conversationId, - ), - parentTurnId: text("parent_turn_id"), - parentEventSeq: integer("parent_event_seq"), - contextForkSeq: integer("context_fork_seq"), }, (table) => [ index("junior_conversations_last_activity_idx").on( @@ -99,34 +92,5 @@ export const juniorConversations = pgTable( table.lastActivityAt.desc(), ), index("junior_conversations_parent_idx").on(table.parentConversationId), - index("junior_conversations_parent_event_idx").on( - table.parentConversationId, - table.parentEventSeq, - ), - index("junior_conversations_root_idx").on(table.rootConversationId), - check( - "junior_conversations_not_own_parent_check", - sql`${table.parentConversationId} is null or ${table.parentConversationId} <> ${table.conversationId}`, - ), - check( - "junior_conversations_not_own_root_check", - sql`${table.rootConversationId} is null or ${table.rootConversationId} <> ${table.conversationId}`, - ), - check( - "junior_conversations_context_fork_check", - sql`${table.contextForkSeq} is null or (${table.parentEventSeq} is not null and ${table.contextForkSeq} <= ${table.parentEventSeq})`, - ), - check( - "junior_conversations_top_level_lineage_check", - sql`${table.parentConversationId} is not null or (${table.rootConversationId} is null and ${table.parentTurnId} is null and ${table.parentEventSeq} is null and ${table.contextForkSeq} is null)`, - ), - check( - "junior_conversations_parent_correlation_check", - sql`(${table.parentTurnId} is null) = (${table.parentEventSeq} is null)`, - ), - check( - "junior_conversations_correlated_root_check", - sql`${table.parentTurnId} is null or ${table.rootConversationId} is not null`, - ), ], ); diff --git a/packages/junior/tests/component/cli/conversation-event-migration.test.ts b/packages/junior/tests/component/cli/conversation-event-migration.test.ts index b3eebd47d..3ea56827a 100644 --- a/packages/junior/tests/component/cli/conversation-event-migration.test.ts +++ b/packages/junior/tests/component/cli/conversation-event-migration.test.ts @@ -186,9 +186,6 @@ describe("conversation event migration", () => { const visibleMessageEvents = migrationStatements( "0005_visible_message_events.sql", ); - const conversationLineage = migrationStatements( - "0007_conversation_lineage.sql", - ); const conversationId = "conversation-visible-events"; try { @@ -219,10 +216,6 @@ describe("conversation event migration", () => { (statement) => fixture.sql.execute(statement), visibleMessageEvents, ); - await executeStatements( - (statement) => fixture.sql.execute(statement), - conversationLineage, - ); const context = { io: { info: () => {} }, diff --git a/packages/junior/tests/component/cli/conversation-history-import.test.ts b/packages/junior/tests/component/cli/conversation-history-import.test.ts index 7bcd2e9b3..9f0a4a844 100644 --- a/packages/junior/tests/component/cli/conversation-history-import.test.ts +++ b/packages/junior/tests/component/cli/conversation-history-import.test.ts @@ -148,7 +148,6 @@ describe("operator conversation history import", () => { { conversationId: CONVERSATION_ID, parentConversationId: rootId, - rootConversationId: rootId, createdAt: at, lastActivityAt: at, updatedAt: at, @@ -257,13 +256,9 @@ describe("operator conversation history import", () => { .db() .select({ conversationId: juniorConversations.conversationId, - contextForkSeq: juniorConversations.contextForkSeq, createdAt: juniorConversations.createdAt, lastActivityAt: juniorConversations.lastActivityAt, parentConversationId: juniorConversations.parentConversationId, - parentEventSeq: juniorConversations.parentEventSeq, - parentTurnId: juniorConversations.parentTurnId, - rootConversationId: juniorConversations.rootConversationId, updatedAt: juniorConversations.updatedAt, }) .from(juniorConversations) @@ -282,13 +277,9 @@ describe("operator conversation history import", () => { }), expect.objectContaining({ conversationId: childId, - contextForkSeq: null, createdAt: new Date(960), lastActivityAt: new Date(961), parentConversationId: CONVERSATION_ID, - parentEventSeq: null, - parentTurnId: null, - rootConversationId: rootId, updatedAt: new Date(961), }), ]), diff --git a/packages/junior/tests/component/conversation-transcripts-sql.test.ts b/packages/junior/tests/component/conversation-transcripts-sql.test.ts index e5cc68b32..659fee0a7 100644 --- a/packages/junior/tests/component/conversation-transcripts-sql.test.ts +++ b/packages/junior/tests/component/conversation-transcripts-sql.test.ts @@ -557,7 +557,6 @@ describe("conversation transcript SQL stores", () => { subagentInvocationId: "future-subagent-call", subagentKind: "task", childConversationId: "subagent:future-child", - historyMode: "isolated" as const, }; await store.append(CONVERSATION_ID, [{ data, createdAtMs: 1_000 }]); diff --git a/packages/junior/tests/component/conversations/retention.test.ts b/packages/junior/tests/component/conversations/retention.test.ts index 23c3b5192..76d569a5d 100644 --- a/packages/junior/tests/component/conversations/retention.test.ts +++ b/packages/junior/tests/component/conversations/retention.test.ts @@ -251,7 +251,7 @@ describe("retention purge job", () => { expect(await eventCount(fixture.sql, "raced")).toBe(1); }); - it("rides children on the root window and purges them with the root", async () => { + it("rides descendants on the root window and purges the whole subtree", async () => { const dest = await seedDestination(fixture.sql, "public"); // Fresh public root, but its advisor child has old activity and content. await seedConversation(fixture.sql, { @@ -264,6 +264,11 @@ describe("retention purge job", () => { parentConversationId: "root", lastActivityAtMs: BASE_MS, }); + await seedConversation(fixture.sql, { + conversationId: "grandchild", + parentConversationId: "child", + lastActivityAtMs: BASE_MS, + }); // The child is never a purge candidate on its own: it rides the root. const result = await runRetentionPurge(fixture.sql, { @@ -271,6 +276,7 @@ describe("retention purge job", () => { }); expect(result.purged).toBe(0); expect(await eventCount(fixture.sql, "child")).toBe(1); + expect(await eventCount(fixture.sql, "grandchild")).toBe(1); // Erasing the root purges the child's content too. await purgeConversation(fixture.sql, "root", { @@ -278,6 +284,7 @@ describe("retention purge job", () => { }); expect(await eventCount(fixture.sql, "root")).toBe(0); expect(await eventCount(fixture.sql, "child")).toBe(0); + expect(await eventCount(fixture.sql, "grandchild")).toBe(0); }); it("uses the freshest activity in the tree for selection and destructive recheck", async () => { diff --git a/packages/junior/tests/component/conversations/shared-lineage-projection.test.ts b/packages/junior/tests/component/conversations/shared-lineage-projection.test.ts deleted file mode 100644 index 0d8dd4eea..000000000 --- a/packages/junior/tests/component/conversations/shared-lineage-projection.test.ts +++ /dev/null @@ -1,581 +0,0 @@ -import { eq } from "drizzle-orm"; -import { describe, expect, it } from "vitest"; -import { - commitMessages, - ConversationLineageProjectionError, - MAX_LINEAGE_PROJECTION_DEPTH, - loadConversationProjection, - loadProjection, - loadTurnProjection, - openConversationProjection, -} from "@/chat/conversations/projection"; -import { - getConversationEventStore, - getConversationStore, - getDb, - getSqlExecutor, - getSubagentLineageService, -} from "@/chat/db"; -import type { PiMessage } from "@/chat/pi/messages"; -import { juniorConversations } from "@/db/schema"; -import { resolveRootVisibility } from "@/chat/conversations/sql/privacy"; -import { purgeConversation } from "@/chat/conversations/retention"; -import { - getAgentTurnSessionRecord, - upsertAgentTurnSessionRecord, -} from "@/chat/state/turn-session"; -import { migrateSchema } from "@/chat/conversations/sql/migrations"; -import { createSqlConversationEventStore } from "@/chat/conversations/sql/history"; -import { projectConversationEvents } from "@/chat/pi/conversation-events"; -import { createLocalJuniorSqlFixture } from "../../fixtures/sql"; - -const MODEL_ID = "openai/gpt-5.4"; - -function message(role: "assistant" | "user", text: string): PiMessage { - return { - role, - content: [{ type: "text", text }], - timestamp: 1, - } as PiMessage; -} - -async function appendTurn(conversationId: string, turnId: string) { - await getConversationEventStore().append(conversationId, [ - { - data: { - type: "turn_started", - turnId, - inputMessageIds: [`${turnId}:input`], - surface: "internal", - }, - idempotencyKey: `turn:${turnId}:started`, - createdAtMs: 2, - }, - ]); -} - -async function startChild(args: { - childConversationId: string; - historyMode: "isolated" | "shared"; - parentConversationId: string; - parentTurnId: string; -}) { - return await getSubagentLineageService().start({ - ...args, - subagentInvocationId: `${args.childConversationId}:invocation`, - subagentKind: "task", - nowMs: 3, - }); -} - -describe("shared child Pi projection", () => { - it("pins direct shared context and commits only the child-local suffix", async () => { - const parent = message("user", "parent before fork"); - await commitMessages({ - conversationId: "shared-parent", - modelId: MODEL_ID, - messages: [parent], - }); - await appendTurn("shared-parent", "parent-turn"); - await startChild({ - childConversationId: "shared-child", - historyMode: "shared", - parentConversationId: "shared-parent", - parentTurnId: "parent-turn", - }); - - await expect( - openConversationProjection({ - conversationId: "shared-child", - modelId: MODEL_ID, - }), - ).resolves.toMatchObject({ messages: [parent] }); - await expect( - loadProjection({ conversationId: "shared-child" }), - ).resolves.toEqual([parent]); - - const child = message("assistant", "child result"); - const commit = await commitMessages({ - conversationId: "shared-child", - modelId: MODEL_ID, - messages: [parent, child], - }); - expect(commit).toMatchObject({ - localMessageStartIndex: 1, - provenance: expect.any(Array), - }); - expect(commit.messageSeqs).toHaveLength(1); - await expect( - loadConversationProjection({ conversationId: "shared-child" }), - ).resolves.toMatchObject({ messages: [parent, child] }); - await expect( - loadTurnProjection({ - conversationId: "shared-child", - committedSeq: commit.committedSeq, - includeTail: false, - }), - ).resolves.toMatchObject({ - localMessageStartIndex: 1, - messages: [parent, child], - seqs: commit.messageSeqs, - }); - await upsertAgentTurnSessionRecord({ - conversationId: "shared-child", - modelId: MODEL_ID, - piMessages: [parent, child], - sessionId: "shared-child-session", - sliceId: 1, - state: "running", - turnStartMessageIndex: 1, - }); - await expect( - getAgentTurnSessionRecord("shared-child", "shared-child-session"), - ).resolves.toMatchObject({ - piMessages: [parent, child], - turnStartMessageIndex: 1, - }); - - const childHistory = - await getConversationEventStore().loadHistory("shared-child"); - expect( - childHistory.flatMap((event) => - event.data.type === "message" ? [event.data.message] : [], - ), - ).toEqual([child]); - expect(childHistory).toContainEqual( - expect.objectContaining({ - data: expect.objectContaining({ - type: "context_epoch_started", - reason: "initial", - inheritsLineageContext: true, - }), - }), - ); - - const regeneratedChild = message("assistant", "regenerated child result"); - await commitMessages({ - conversationId: "shared-child", - modelId: MODEL_ID, - messages: [parent, regeneratedChild], - }); - const rollbackEpoch = - await getConversationEventStore().loadCurrentEpoch("shared-child"); - expect(rollbackEpoch).toContainEqual( - expect.objectContaining({ - data: expect.objectContaining({ - type: "context_epoch_started", - reason: "rollback", - inheritsLineageContext: true, - }), - }), - ); - expect( - rollbackEpoch.flatMap((event) => - event.data.type === "message" ? [event.data.message] : [], - ), - ).toEqual([regeneratedChild]); - - const parentAfterFork = message("assistant", "parent after fork"); - await commitMessages({ - conversationId: "shared-parent", - modelId: MODEL_ID, - messages: [parent, parentAfterFork], - }); - await getConversationEventStore().startEpoch("shared-parent", { - reason: "compaction", - modelProfile: "standard", - modelId: MODEL_ID, - messages: [ - { - message: message("user", "later parent compaction"), - createdAtMs: 4, - }, - ], - }); - await expect( - loadProjection({ conversationId: "shared-child" }), - ).resolves.toEqual([parent, regeneratedChild]); - - const historyLength = ( - await getConversationEventStore().loadHistory("shared-child") - ).length; - await expect( - commitMessages({ - conversationId: "shared-child", - modelId: MODEL_ID, - messages: [message("user", "mutated parent"), regeneratedChild], - }), - ).rejects.toBeInstanceOf(ConversationLineageProjectionError); - expect( - await getConversationEventStore().loadHistory("shared-child"), - ).toHaveLength(historyLength); - - await getConversationEventStore().startEpoch("shared-child", { - reason: "compaction", - modelProfile: "standard", - modelId: MODEL_ID, - messages: [ - { - message: message("user", "self-contained child summary"), - createdAtMs: 5, - }, - ], - }); - await expect( - loadProjection({ conversationId: "shared-child" }), - ).resolves.toEqual([message("user", "self-contained child summary")]); - }); - - it("recursively composes nested shared children at each immutable fork", async () => { - const root = message("user", "root context"); - await commitMessages({ - conversationId: "nested-root", - modelId: MODEL_ID, - messages: [root], - }); - await appendTurn("nested-root", "root-turn"); - await startChild({ - childConversationId: "nested-child", - historyMode: "shared", - parentConversationId: "nested-root", - parentTurnId: "root-turn", - }); - const child = message("assistant", "child context"); - await commitMessages({ - conversationId: "nested-child", - modelId: MODEL_ID, - messages: [root, child], - }); - await appendTurn("nested-child", "child-turn"); - await startChild({ - childConversationId: "nested-grandchild", - historyMode: "shared", - parentConversationId: "nested-child", - parentTurnId: "child-turn", - }); - - await expect( - loadProjection({ conversationId: "nested-grandchild" }), - ).resolves.toEqual([root, child]); - const lateChild = message("user", "child after grandchild fork"); - await commitMessages({ - conversationId: "nested-child", - modelId: MODEL_ID, - messages: [root, child, lateChild], - }); - await expect( - loadProjection({ conversationId: "nested-grandchild" }), - ).resolves.toEqual([root, child]); - - const grandchild = message("assistant", "grandchild result"); - await commitMessages({ - conversationId: "nested-grandchild", - modelId: MODEL_ID, - messages: [root, child, grandchild], - }); - const grandchildMessages = ( - await getConversationEventStore().loadHistory("nested-grandchild") - ).flatMap((event) => - event.data.type === "message" ? [event.data.message] : [], - ); - expect(grandchildMessages).toEqual([grandchild]); - }); - - it("keeps isolated and historical null-fork children child-local", async () => { - const parent = message("user", "private parent context"); - await commitMessages({ - conversationId: "isolated-parent", - modelId: MODEL_ID, - messages: [parent], - }); - await appendTurn("isolated-parent", "isolated-parent-turn"); - await startChild({ - childConversationId: "isolated-child", - historyMode: "isolated", - parentConversationId: "isolated-parent", - parentTurnId: "isolated-parent-turn", - }); - await expect( - loadProjection({ conversationId: "isolated-child" }), - ).resolves.toEqual([]); - - const local = message("user", "isolated instruction"); - await commitMessages({ - conversationId: "isolated-child", - modelId: MODEL_ID, - messages: [local], - }); - await expect( - loadProjection({ conversationId: "isolated-child" }), - ).resolves.toEqual([local]); - expect( - await getConversationEventStore().loadHistory("isolated-child"), - ).not.toContainEqual( - expect.objectContaining({ - data: expect.objectContaining({ inheritsLineageContext: true }), - }), - ); - - const sibling = await startChild({ - childConversationId: "isolated-sibling", - historyMode: "isolated", - parentConversationId: "isolated-parent", - parentTurnId: "isolated-parent-turn", - }); - await getDb() - .update(juniorConversations) - .set({ parentEventSeq: sibling.parentEventSeq }) - .where(eq(juniorConversations.conversationId, "isolated-child")); - await expect( - loadProjection({ conversationId: "isolated-child" }), - ).rejects.toBeInstanceOf(ConversationLineageProjectionError); - await getDb() - .update(juniorConversations) - .set({ parentEventSeq: null, parentTurnId: null }) - .where(eq(juniorConversations.conversationId, "isolated-sibling")); - await expect( - loadProjection({ conversationId: "isolated-sibling" }), - ).resolves.toEqual([]); - }); - - it("fails closed when shared correlation no longer identifies its fork", async () => { - await commitMessages({ - conversationId: "conflict-parent", - modelId: MODEL_ID, - messages: [message("user", "parent")], - }); - await appendTurn("conflict-parent", "conflict-turn"); - const lineage = await startChild({ - childConversationId: "conflict-child", - historyMode: "shared", - parentConversationId: "conflict-parent", - parentTurnId: "conflict-turn", - }); - await getDb() - .update(juniorConversations) - .set({ contextForkSeq: lineage.parentEventSeq - 1 }) - .where(eq(juniorConversations.conversationId, "conflict-child")); - - await expect( - loadProjection({ conversationId: "conflict-child" }), - ).rejects.toBeInstanceOf(ConversationLineageProjectionError); - }); - - it("rejects a shared reference whose durable fork was cleared", async () => { - await appendTurn("lost-fork-parent", "lost-fork-turn"); - await startChild({ - childConversationId: "lost-fork-child", - historyMode: "shared", - parentConversationId: "lost-fork-parent", - parentTurnId: "lost-fork-turn", - }); - await getDb() - .update(juniorConversations) - .set({ contextForkSeq: null }) - .where(eq(juniorConversations.conversationId, "lost-fork-child")); - await expect( - loadProjection({ conversationId: "lost-fork-child" }), - ).rejects.toBeInstanceOf(ConversationLineageProjectionError); - }); - - it("fails closed beyond the bounded shared-lineage projection depth", async () => { - let parentConversationId = "depth-root"; - let parentTurnId = "depth-turn-0"; - await appendTurn(parentConversationId, parentTurnId); - for (let depth = 1; depth <= MAX_LINEAGE_PROJECTION_DEPTH + 1; depth += 1) { - const childConversationId = `depth-child-${depth}`; - await startChild({ - childConversationId, - historyMode: "shared", - parentConversationId, - parentTurnId, - }); - parentConversationId = childConversationId; - parentTurnId = `depth-turn-${depth}`; - await appendTurn(parentConversationId, parentTurnId); - } - await expect( - loadProjection({ conversationId: parentConversationId }), - ).rejects.toThrow("exceeds maximum projection depth"); - }); - - it("fails closed when correlated shared lineage forms a cycle", async () => { - await appendTurn("cycle-root", "cycle-root-turn"); - await appendTurn("cycle-a", "cycle-a-turn"); - await startChild({ - childConversationId: "cycle-b", - historyMode: "shared", - parentConversationId: "cycle-a", - parentTurnId: "cycle-a-turn", - }); - await appendTurn("cycle-b", "cycle-b-turn"); - await getConversationEventStore().append("cycle-b", [ - { - data: { - type: "subagent_started", - subagentInvocationId: "cycle-a:invocation", - subagentKind: "task", - childConversationId: "cycle-a", - parentTurnId: "cycle-b-turn", - historyMode: "shared", - }, - createdAtMs: 4, - }, - ]); - const cycleAReference = ( - await getConversationEventStore().loadHistory("cycle-b") - ).find( - (event) => - event.data.type === "subagent_started" && - event.data.childConversationId === "cycle-a", - )!; - await getDb() - .update(juniorConversations) - .set({ rootConversationId: "cycle-root" }) - .where(eq(juniorConversations.conversationId, "cycle-b")); - await getDb() - .update(juniorConversations) - .set({ - parentConversationId: "cycle-b", - rootConversationId: "cycle-root", - parentTurnId: "cycle-b-turn", - parentEventSeq: cycleAReference.seq, - contextForkSeq: cycleAReference.seq, - }) - .where(eq(juniorConversations.conversationId, "cycle-a")); - await expect(loadProjection({ conversationId: "cycle-b" })).rejects.toThrow( - "contains a cycle", - ); - }); - - it("serializes concurrent commits into individually reproducible boundaries", async () => { - const fixture = await createLocalJuniorSqlFixture(); - try { - await migrateSchema(fixture.sql); - const base = message("user", "base"); - await commitMessages({ - conversationId: "concurrent-commit", - executor: fixture.sql, - modelId: MODEL_ID, - messages: [base], - }); - const left = [base, message("assistant", "left")]; - const right = [base, message("assistant", "right")]; - const [leftCommit, rightCommit] = await Promise.all([ - commitMessages({ - conversationId: "concurrent-commit", - executor: fixture.sql, - modelId: MODEL_ID, - messages: left, - }), - commitMessages({ - conversationId: "concurrent-commit", - executor: fixture.sql, - modelId: MODEL_ID, - messages: right, - }), - ]); - const history = await createSqlConversationEventStore( - fixture.sql, - ).loadHistory("concurrent-commit"); - const projectionAt = (committedSeq: number) => { - const committed = history.find((event) => event.seq === committedSeq)!; - return projectConversationEvents( - history.filter( - (event) => - event.contextEpoch === committed.contextEpoch && - event.seq <= committedSeq, - ), - ); - }; - expect(projectionAt(leftCommit.committedSeq)).toMatchObject({ - messages: left, - provenance: leftCommit.provenance, - }); - expect(projectionAt(rightCommit.committedSeq)).toMatchObject({ - messages: right, - provenance: rightCommit.provenance, - }); - } finally { - await fixture.close(); - } - }); - - it("inherits a pre-fork compaction or handoff as a self-contained parent epoch", async () => { - for (const [reason, profile] of [ - ["compaction", "standard"], - ["handoff", "handoff"], - ] as const) { - const parentConversationId = `pre-fork-${reason}-parent`; - const childConversationId = `pre-fork-${reason}-child`; - await commitMessages({ - conversationId: parentConversationId, - modelId: MODEL_ID, - messages: [message("user", "discarded parent context")], - }); - const replacement = message("user", `${reason} replacement`); - await getConversationEventStore().startEpoch(parentConversationId, { - reason, - modelProfile: profile, - modelId: MODEL_ID, - messages: [{ message: replacement, createdAtMs: 2 }], - }); - const turnId = `pre-fork-${reason}-turn`; - await appendTurn(parentConversationId, turnId); - await startChild({ - childConversationId, - historyMode: "shared", - parentConversationId, - parentTurnId: turnId, - }); - await expect( - loadProjection({ conversationId: childConversationId }), - ).resolves.toEqual([replacement]); - } - }); - - it("rejects inherited provenance mutation and follows private-root purge", async () => { - await getConversationStore().recordActivity({ - conversationId: "private-root", - destination: { - platform: "slack", - teamId: "T123", - channelId: "C123", - }, - source: "slack", - visibility: "private", - nowMs: 1, - }); - const parent = message("user", "private inherited context"); - await commitMessages({ - conversationId: "private-root", - modelId: MODEL_ID, - messages: [parent], - }); - await appendTurn("private-root", "private-turn"); - await startChild({ - childConversationId: "private-child", - historyMode: "shared", - parentConversationId: "private-root", - parentTurnId: "private-turn", - }); - await expect( - commitMessages({ - conversationId: "private-child", - modelId: MODEL_ID, - messages: [parent], - provenance: [{ authority: "instruction" }], - }), - ).rejects.toThrow("mutated inherited provenance"); - await expect( - resolveRootVisibility(getSqlExecutor(), "private-child"), - ).resolves.toEqual({ - rootConversationId: "private-root", - visibility: "private", - }); - await purgeConversation(getSqlExecutor(), "private-root", { nowMs: 10 }); - await expect( - loadProjection({ conversationId: "private-child" }), - ).rejects.toThrow("conversation lineage reference conflicts"); - }); -}); diff --git a/packages/junior/tests/component/services/subagent-lineage.test.ts b/packages/junior/tests/component/services/subagent-lineage.test.ts deleted file mode 100644 index 1a42bae21..000000000 --- a/packages/junior/tests/component/services/subagent-lineage.test.ts +++ /dev/null @@ -1,495 +0,0 @@ -import { describe, expect, it } from "vitest"; -import type { PiMessage } from "@/chat/pi/messages"; -import { eq } from "drizzle-orm"; -import { migrateSchema } from "@/chat/conversations/sql/migrations"; -import { createSqlConversationEventStore } from "@/chat/conversations/sql/history"; -import { createSqlStore } from "@/chat/conversations/sql/store"; -import { resolveRootVisibility } from "@/chat/conversations/sql/privacy"; -import { - SubagentLineageConflictError, - SubagentLineageService, -} from "@/chat/services/subagent-lineage"; -import { juniorConversations } from "@/db/schema"; -import { migrateConversationLineage } from "@/cli/upgrade/migrations/conversation-lineage"; -import { createLocalJuniorSqlFixture } from "../../fixtures/sql"; - -async function appendTurnStarted( - eventStore: ReturnType, - conversationId: string, - turnId: string, -) { - await eventStore.append(conversationId, [ - { - data: { - type: "turn_started", - turnId, - inputMessageIds: [`${turnId}:input`], - surface: "internal", - }, - idempotencyKey: `turn:${turnId}:started`, - createdAtMs: 1, - }, - ]); -} - -describe("subagent lineage service", () => { - it("records direct and nested references without copying child events", async () => { - const fixture = await createLocalJuniorSqlFixture(); - try { - await migrateSchema(fixture.sql); - const store = createSqlStore(fixture.sql); - const events = createSqlConversationEventStore(fixture.sql); - const lineage = new SubagentLineageService(fixture.sql); - await store.recordActivity({ - conversationId: "root", - destination: { - platform: "slack", - teamId: "T1", - channelId: "C1", - }, - source: "slack", - visibility: "private", - nowMs: 1, - }); - await appendTurnStarted(events, "root", "root-turn"); - - const child = await lineage.start({ - childConversationId: "child", - historyMode: "shared", - parentConversationId: "root", - parentTurnId: "root-turn", - subagentInvocationId: "child-call", - subagentKind: "task", - nowMs: 2, - }); - expect(child).toMatchObject({ - contextForkSeq: child.parentEventSeq, - rootConversationId: "root", - }); - await expect( - store.get({ conversationId: "child" }), - ).resolves.toMatchObject({ - lineage: { - contextForkSeq: child.parentEventSeq, - parentConversationId: "root", - parentEventSeq: child.parentEventSeq, - parentTurnId: "root-turn", - rootConversationId: "root", - }, - }); - await events.append("child", [ - { - data: { - type: "message", - message: { - role: "assistant", - content: [{ type: "text", text: "child-only" }], - timestamp: 3, - } as PiMessage, - }, - createdAtMs: 3, - }, - ]); - await appendTurnStarted(events, "child", "child-turn"); - const grandchild = await lineage.start({ - childConversationId: "grandchild", - historyMode: "isolated", - parentConversationId: "child", - parentTurnId: "child-turn", - subagentInvocationId: "grandchild-call", - subagentKind: "review", - nowMs: 4, - }); - expect(grandchild).toMatchObject({ - contextForkSeq: null, - rootConversationId: "root", - }); - await lineage.finish({ - parentConversationId: "child", - parentTurnId: "child-turn", - subagentInvocationId: "grandchild-call", - outcome: "success", - nowMs: 5, - }); - - const rootHistory = await events.loadHistory("root"); - const childHistory = await events.loadHistory("child"); - expect(rootHistory.map((event) => event.data.type)).toEqual([ - "turn_started", - "subagent_started", - ]); - expect(JSON.stringify(rootHistory)).not.toContain("child-only"); - expect(childHistory.map((event) => event.data.type)).toEqual([ - "message", - "turn_started", - "subagent_started", - "subagent_ended", - ]); - await expect( - resolveRootVisibility(fixture.sql, "grandchild"), - ).resolves.toEqual({ rootConversationId: "root", visibility: "private" }); - } finally { - await fixture.close(); - } - }); - - it("replays exact references and rejects mode, correlation, and reparenting conflicts", async () => { - const fixture = await createLocalJuniorSqlFixture(); - try { - await migrateSchema(fixture.sql); - const events = createSqlConversationEventStore(fixture.sql); - const lineage = new SubagentLineageService(fixture.sql); - await appendTurnStarted(events, "parent-a", "turn-a"); - await appendTurnStarted(events, "parent-b", "turn-b"); - await expect( - lineage.start({ - childConversationId: "orphaned-turn-child", - historyMode: "isolated", - parentConversationId: "parent-a", - parentTurnId: "missing-turn", - subagentInvocationId: "missing-turn-call", - subagentKind: "task", - }), - ).rejects.toBeInstanceOf(SubagentLineageConflictError); - const input = { - childConversationId: "child", - historyMode: "shared" as const, - parentConversationId: "parent-a", - parentTurnId: "turn-a", - subagentInvocationId: "call", - subagentKind: "task", - nowMs: 2, - }; - const first = await lineage.start(input); - await expect(lineage.start(input)).resolves.toEqual(first); - await lineage.finish({ - parentConversationId: "parent-a", - parentTurnId: "turn-a", - subagentInvocationId: "call", - outcome: "success", - nowMs: 3, - }); - await expect( - lineage.finish({ - parentConversationId: "parent-a", - parentTurnId: "turn-a", - subagentInvocationId: "call", - outcome: "success", - nowMs: 4, - }), - ).resolves.toBeUndefined(); - await expect( - lineage.start({ ...input, historyMode: "isolated" }), - ).rejects.toBeInstanceOf(SubagentLineageConflictError); - await expect( - lineage.start({ - ...input, - parentConversationId: "parent-b", - parentTurnId: "turn-b", - subagentInvocationId: "other-call", - }), - ).rejects.toBeInstanceOf(SubagentLineageConflictError); - expect( - (await events.loadHistory("parent-b")).map((event) => event.data.type), - ).toEqual(["turn_started"]); - expect( - (await events.loadHistory("parent-a")).filter( - (event) => event.data.type === "subagent_started", - ), - ).toHaveLength(1); - expect( - (await events.loadHistory("parent-a")).filter( - (event) => event.data.type === "subagent_ended", - ), - ).toHaveLength(1); - } finally { - await fixture.close(); - } - }); - - it("upgrades only metadata-bare children and rolls back a real-root conflict", async () => { - const fixture = await createLocalJuniorSqlFixture(); - try { - await migrateSchema(fixture.sql); - const store = createSqlStore(fixture.sql); - const events = createSqlConversationEventStore(fixture.sql); - const lineage = new SubagentLineageService(fixture.sql); - await appendTurnStarted(events, "parent", "turn"); - await events.append("bare-child", [ - { - data: { - type: "message", - message: { - role: "assistant", - content: [{ type: "text", text: "created before lineage" }], - timestamp: 1, - } as PiMessage, - }, - createdAtMs: 1, - }, - ]); - await expect( - lineage.start({ - childConversationId: "bare-child", - historyMode: "isolated", - parentConversationId: "parent", - parentTurnId: "turn", - subagentInvocationId: "bare-call", - subagentKind: "task", - nowMs: 2, - }), - ).resolves.toMatchObject({ rootConversationId: "parent" }); - - await store.recordActivity({ - conversationId: "real-root", - source: "local", - nowMs: 2, - }); - await expect( - lineage.start({ - childConversationId: "real-root", - historyMode: "isolated", - parentConversationId: "parent", - parentTurnId: "turn", - subagentInvocationId: "conflict-call", - subagentKind: "task", - nowMs: 3, - }), - ).rejects.toBeInstanceOf(SubagentLineageConflictError); - for (const [childConversationId, fields, invocationId] of [ - ["purged-root", { transcriptPurgedAt: new Date(3) }, "purged-call"], - [ - "execution-root", - { - executionStatus: "running" as const, - executionUpdatedAt: new Date(3), - lastCheckpointAt: new Date(3), - }, - "execution-call", - ], - ] as const) { - await events.append(childConversationId, [ - { - data: { - type: "message", - message: { - role: "assistant", - content: [{ type: "text", text: "existing root content" }], - timestamp: 2, - } as PiMessage, - }, - createdAtMs: 2, - }, - ]); - await fixture.sql - .db() - .update(juniorConversations) - .set(fields) - .where(eq(juniorConversations.conversationId, childConversationId)); - await expect( - lineage.start({ - childConversationId, - historyMode: "isolated", - parentConversationId: "parent", - parentTurnId: "turn", - subagentInvocationId: invocationId, - subagentKind: "task", - nowMs: 3, - }), - ).rejects.toBeInstanceOf(SubagentLineageConflictError); - } - expect( - (await events.loadHistory("parent")).filter( - (event) => - event.data.type === "subagent_started" && - event.data.subagentInvocationId === "conflict-call", - ), - ).toEqual([]); - } finally { - await fixture.close(); - } - }); - - it("atomically resolves different parents racing for one child", async () => { - const fixture = await createLocalJuniorSqlFixture(); - try { - await migrateSchema(fixture.sql); - const events = createSqlConversationEventStore(fixture.sql); - const lineage = new SubagentLineageService(fixture.sql); - await appendTurnStarted(events, "parent-a", "turn-a"); - await appendTurnStarted(events, "parent-b", "turn-b"); - const results = await Promise.allSettled([ - lineage.start({ - childConversationId: "contended-child", - historyMode: "isolated", - parentConversationId: "parent-a", - parentTurnId: "turn-a", - subagentInvocationId: "call-a", - subagentKind: "task", - }), - lineage.start({ - childConversationId: "contended-child", - historyMode: "isolated", - parentConversationId: "parent-b", - parentTurnId: "turn-b", - subagentInvocationId: "call-b", - subagentKind: "task", - }), - ]); - expect( - results.filter((result) => result.status === "fulfilled"), - ).toHaveLength(1); - expect( - results.filter((result) => result.status === "rejected"), - ).toHaveLength(1); - const childRows = await fixture.sql - .db() - .select() - .from(juniorConversations) - .where(eq(juniorConversations.conversationId, "contended-child")); - const winner = childRows[0]?.parentConversationId; - expect(winner === "parent-a" || winner === "parent-b").toBe(true); - for (const parent of ["parent-a", "parent-b"]) { - const starts = (await events.loadHistory(parent)).filter( - (event) => event.data.type === "subagent_started", - ); - expect(starts).toHaveLength(parent === winner ? 1 : 0); - } - } finally { - await fixture.close(); - } - }); - - it("backfills historical roots in bounded rerunnable batches without inventing forks", async () => { - const fixture = await createLocalJuniorSqlFixture(); - try { - await migrateSchema(fixture.sql); - const at = new Date(1); - await fixture.sql - .db() - .insert(juniorConversations) - .values([ - { - conversationId: "root", - createdAt: at, - lastActivityAt: at, - updatedAt: at, - executionStatus: "idle", - }, - { - conversationId: "child", - parentConversationId: "root", - createdAt: at, - lastActivityAt: at, - updatedAt: at, - executionStatus: "idle", - }, - { - conversationId: "grandchild", - parentConversationId: "child", - createdAt: at, - lastActivityAt: at, - updatedAt: at, - executionStatus: "idle", - }, - ]); - await expect( - migrateConversationLineage({} as never, { - batchSize: 1, - executor: fixture.sql, - }), - ).resolves.toMatchObject({ migrated: 2 }); - const rows = await fixture.sql - .db() - .select() - .from(juniorConversations) - .where(eq(juniorConversations.rootConversationId, "root")); - expect(rows).toHaveLength(2); - expect(rows).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - conversationId: "child", - contextForkSeq: null, - parentEventSeq: null, - parentTurnId: null, - }), - expect.objectContaining({ - conversationId: "grandchild", - contextForkSeq: null, - parentEventSeq: null, - parentTurnId: null, - }), - ]), - ); - await expect( - migrateConversationLineage({} as never, { executor: fixture.sql }), - ).resolves.toMatchObject({ migrated: 0 }); - } finally { - await fixture.close(); - } - }); - - it("rejects partial lineage while allowing historical uncorrelated children", async () => { - const fixture = await createLocalJuniorSqlFixture(); - try { - await migrateSchema(fixture.sql); - const at = new Date(1); - const base = { - createdAt: at, - lastActivityAt: at, - updatedAt: at, - executionStatus: "idle" as const, - }; - await fixture.sql - .db() - .insert(juniorConversations) - .values({ conversationId: "root", ...base }); - await expect( - fixture.sql - .db() - .insert(juniorConversations) - .values({ - conversationId: "poisoned-root", - rootConversationId: "root", - ...base, - }), - ).rejects.toThrow("Failed query"); - await expect( - fixture.sql - .db() - .insert(juniorConversations) - .values({ - conversationId: "half-correlated-child", - parentConversationId: "root", - rootConversationId: "root", - parentTurnId: "turn", - ...base, - }), - ).rejects.toThrow("Failed query"); - await expect( - fixture.sql - .db() - .insert(juniorConversations) - .values({ - conversationId: "rootless-correlated-child", - parentConversationId: "root", - parentTurnId: "turn", - parentEventSeq: 1, - ...base, - }), - ).rejects.toThrow("Failed query"); - await expect( - fixture.sql - .db() - .insert(juniorConversations) - .values({ - conversationId: "historical-child", - parentConversationId: "root", - ...base, - }), - ).resolves.toBeDefined(); - } finally { - await fixture.close(); - } - }); -}); diff --git a/packages/junior/tests/integration/api/conversations/list.test.ts b/packages/junior/tests/integration/api/conversations/list.test.ts index 611f9d7af..02583cdc3 100644 --- a/packages/junior/tests/integration/api/conversations/list.test.ts +++ b/packages/junior/tests/integration/api/conversations/list.test.ts @@ -3,7 +3,7 @@ import { eq } from "drizzle-orm"; import { readConversationFeedFromSql } from "@/api/conversations/list"; import { migrateSchema } from "@/chat/conversations/sql/migrations"; import { createSqlStore } from "@/chat/conversations/sql/store"; -import { juniorIdentities } from "@/db/schema"; +import { juniorConversations, juniorIdentities } from "@/db/schema"; import { createConfiguredJuniorSqlFixture } from "../../../fixtures/sql"; describe("conversation list API", () => { @@ -71,4 +71,34 @@ describe("conversation list API", () => { await fixture.close(); } }); + + test("excludes child conversations from the top-level feed", async () => { + const fixture = createConfiguredJuniorSqlFixture(); + const store = createSqlStore(fixture.sql); + try { + await migrateSchema(fixture.sql); + await store.recordActivity({ + conversationId: "slack:C1:root", + nowMs: 1_000, + source: "slack", + }); + const childAt = new Date(2_000); + await fixture.sql.db().insert(juniorConversations).values({ + conversationId: "advisor:child", + parentConversationId: "slack:C1:root", + createdAt: childAt, + lastActivityAt: childAt, + updatedAt: childAt, + executionStatus: "idle", + }); + + const feed = await readConversationFeedFromSql(); + + expect(feed.conversations.map((item) => item.conversationId)).toEqual([ + "slack:C1:root", + ]); + } finally { + await fixture.close(); + } + }); }); diff --git a/packages/junior/tests/integration/api/conversations/stats.test.ts b/packages/junior/tests/integration/api/conversations/stats.test.ts index 266b40361..37404f139 100644 --- a/packages/junior/tests/integration/api/conversations/stats.test.ts +++ b/packages/junior/tests/integration/api/conversations/stats.test.ts @@ -2,8 +2,6 @@ import { describe, expect, test, vi } from "vitest"; import { readConversationStatsFromSql } from "@/api/conversations/stats.query"; import { migrateSchema } from "@/chat/conversations/sql/migrations"; import { createSqlStore } from "@/chat/conversations/sql/store"; -import { createSqlConversationEventStore } from "@/chat/conversations/sql/history"; -import { SubagentLineageService } from "@/chat/services/subagent-lineage"; import { juniorConversations } from "@/db/schema"; import { buildJuniorSqlConversation, @@ -122,28 +120,14 @@ describe("conversation stats API", () => { visibility: "public", nowMs: Date.parse("2026-02-01T10:00:00.000Z"), }); - await createSqlConversationEventStore(fixture.sql).append( - "slack:C1:recent", - [ - { - data: { - type: "turn_started", - turnId: "turn-recent", - inputMessageIds: ["recent-input"], - surface: "slack", - }, - createdAtMs: Date.parse("2026-06-15T11:50:00.000Z"), - }, - ], - ); - await new SubagentLineageService(fixture.sql).start({ - childConversationId: "advisor:child", - historyMode: "isolated", + const childAt = new Date("2026-06-15T11:55:00.000Z"); + await fixture.sql.db().insert(juniorConversations).values({ + conversationId: "advisor:child", parentConversationId: "slack:C1:recent", - parentTurnId: "turn-recent", - subagentInvocationId: "advisor-child", - subagentKind: "advisor", - nowMs: Date.parse("2026-06-15T11:55:00.000Z"), + createdAt: childAt, + lastActivityAt: childAt, + updatedAt: childAt, + executionStatus: "idle", }); const report = await readConversationStatsFromSql(); diff --git a/packages/junior/tests/integration/dashboard-reporting.test.ts b/packages/junior/tests/integration/dashboard-reporting.test.ts index 54fae548c..ccf64f601 100644 --- a/packages/junior/tests/integration/dashboard-reporting.test.ts +++ b/packages/junior/tests/integration/dashboard-reporting.test.ts @@ -119,7 +119,6 @@ async function appendVisibleHistory( subagentInvocationId: `${conversationId}:subagent-call`, subagentKind: "review", childConversationId: `${conversationId}:child`, - historyMode: "isolated", }, createdAtMs: 17, }, @@ -150,28 +149,35 @@ async function createChild(args: { childConversationId: string; parentConversationId: string; }): Promise { - const { getConversationEventStore, getSubagentLineageService } = - await import("@/chat/db"); + const { getConversationEventStore, getDb } = await import("@/chat/db"); + const at = new Date(3); + await getDb().insert(juniorConversations).values({ + conversationId: args.childConversationId, + parentConversationId: args.parentConversationId, + createdAt: at, + lastActivityAt: at, + updatedAt: at, + executionStatus: "idle", + }); await getConversationEventStore().append(args.parentConversationId, [ { data: { - type: "turn_started", - turnId: `${args.parentConversationId}:turn`, - inputMessageIds: [`${args.parentConversationId}:input`], - surface: "internal", + type: "subagent_started", + childConversationId: args.childConversationId, + subagentInvocationId: `${args.childConversationId}:call`, + subagentKind: "advisor", }, createdAtMs: 2, }, + { + data: { + type: "subagent_ended", + subagentInvocationId: `${args.childConversationId}:call`, + outcome: "success", + }, + createdAtMs: 3, + }, ]); - await getSubagentLineageService().start({ - childConversationId: args.childConversationId, - historyMode: "isolated", - parentConversationId: args.parentConversationId, - parentTurnId: `${args.parentConversationId}:turn`, - subagentInvocationId: `${args.childConversationId}:call`, - subagentKind: "task", - nowMs: 3, - }); await appendVisibleHistory(args.childConversationId, "Child answer"); } @@ -289,13 +295,11 @@ describe("dashboard canonical event reporting", () => { type: "subagent_started", childConversationId: `${conversationId}:child`, subagentKind: "review", - historyMode: "isolated", }, { type: "subagent_ended", childConversationId: `${conversationId}:child`, subagentKind: "review", - historyMode: "isolated", outcome: "success", }, { type: "context_compacted" }, @@ -404,16 +408,6 @@ describe("dashboard canonical event reporting", () => { expect(JSON.stringify(privateChildDetail)).not.toContain( "forged-public-child-channel", ); - - const forgedRoot = "slack:C-reporting:forged-root"; - await recordRoot(forgedRoot, "public"); - await getDb() - .update(juniorConversations) - .set({ rootConversationId: forgedRoot }) - .where(eq(juniorConversations.conversationId, publicChild)); - expect((await requireDetail(publicChild)).eventHistory.status).toBe( - "redacted", - ); }); it("lets requested-row expiry win and stamps both root and child purges", async () => { @@ -654,114 +648,4 @@ describe("dashboard canonical event reporting", () => { ]); } }, 15_000); - - it("rediscovers a child attached while known descendants are being locked", async () => { - const rootConversationId = "slack:C-reporting:membership-race-root"; - const firstChildId = "child:a-reporting-membership-race"; - const secondChildId = "child:z-reporting-membership-race"; - const lateGrandchildId = "child:late-reporting-membership-race"; - await recordRoot(rootConversationId, "public"); - await createChild({ - childConversationId: firstChildId, - parentConversationId: rootConversationId, - }); - await createChild({ - childConversationId: secondChildId, - parentConversationId: rootConversationId, - }); - - const blocker = createPostgresJuniorSqlExecutor({ - applicationName: "junior-membership-race-blocker", - connectionString: TEST_DATABASE_URL, - }); - const observer = createPostgresJuniorSqlExecutor({ - applicationName: "junior-membership-race-observer", - connectionString: TEST_DATABASE_URL, - }); - const creator = createPostgresJuniorSqlExecutor({ - applicationName: "junior-membership-race-creator", - connectionString: TEST_DATABASE_URL, - }); - const purger = createPostgresJuniorSqlExecutor({ - applicationName: "junior-membership-race-purger", - connectionString: TEST_DATABASE_URL, - }); - const childLocked = deferred(); - const releaseChild = deferred(); - const blockerDone = blocker.transaction(async () => { - await blocker - .db() - .select() - .from(juniorConversations) - .where(eq(juniorConversations.conversationId, firstChildId)) - .for("update"); - childLocked.resolve(); - await releaseChild.promise; - }); - - try { - await childLocked.promise; - const purgePromise = purgeConversation(purger, rootConversationId, { - nowMs: Date.now(), - }); - await waitUntilApplicationWaitsOnLock( - observer, - "junior-membership-race-purger", - "junior_conversations", - ); - - const now = new Date(); - await creator.db().insert(juniorConversations).values({ - conversationId: lateGrandchildId, - parentConversationId: secondChildId, - createdAt: now, - lastActivityAt: now, - updatedAt: now, - executionStatus: "idle", - }); - await createSqlConversationMessageStore(creator).record( - lateGrandchildId, - [ - { - messageId: "late-grandchild-message", - role: "assistant", - text: "late grandchild payload", - createdAtMs: now.getTime(), - }, - ], - ); - - releaseChild.resolve(); - await blockerDone; - await purgePromise; - - const events = await observer - .db() - .select() - .from(juniorConversationEvents) - .where(eq(juniorConversationEvents.conversationId, lateGrandchildId)); - const messages = await observer - .db() - .select() - .from(juniorConversationMessages) - .where(eq(juniorConversationMessages.conversationId, lateGrandchildId)); - const [conversation] = await observer - .db() - .select({ transcriptPurgedAt: juniorConversations.transcriptPurgedAt }) - .from(juniorConversations) - .where(eq(juniorConversations.conversationId, lateGrandchildId)); - expect(events).toEqual([]); - expect(messages).toEqual([]); - expect(conversation?.transcriptPurgedAt).toBeInstanceOf(Date); - } finally { - releaseChild.resolve(); - await blockerDone.catch(() => undefined); - await Promise.all([ - blocker.close(), - observer.close(), - creator.close(), - purger.close(), - ]); - } - }, 15_000); }); diff --git a/packages/junior/tests/unit/api/conversation-events.test.ts b/packages/junior/tests/unit/api/conversation-events.test.ts index 207dcd55c..fcfe1ccd4 100644 --- a/packages/junior/tests/unit/api/conversation-events.test.ts +++ b/packages/junior/tests/unit/api/conversation-events.test.ts @@ -195,10 +195,7 @@ describe("conversation report event projection", () => { event(5, { type: "delivery_intended", deliveryId: "delivery:1", - correlation: { - kind: "authorization", - authorizationId: "private-authorization-id", - }, + correlation: { kind: "turn", turnId: "private-turn-id" }, messageId: "visible-private", deliveryKind: "assistant_reply", provider: "slack", @@ -262,6 +259,7 @@ describe("conversation report event projection", () => { "private provider error", "private tool argument", "model_execution_failed", + "private-turn-id", eventId, "providerMessageIds", "123.456", @@ -333,13 +331,10 @@ describe("conversation report event projection", () => { parentToolCallId: "private-parent-tool-id", reasoningLevel: "private-reasoning-level", childConversationId: "child-conversation-1", - parentTurnId: "turn-1", - historyMode: "shared", }), event(8, { type: "subagent_ended", subagentInvocationId: "private-invocation-id", - parentTurnId: "turn-1", outcome: "error", errorCode: "private-child-error-code", }), @@ -368,13 +363,11 @@ describe("conversation report event projection", () => { type: "subagent_started", childConversationId: "child-conversation-1", subagentKind: "advisor", - historyMode: "shared", }, { type: "subagent_ended", childConversationId: "child-conversation-1", subagentKind: "advisor", - historyMode: "shared", outcome: "error", }, { type: "turn_lifecycle", turnId: "turn-2", state: "no_reply" }, diff --git a/packages/junior/tests/unit/chat/pi/conversation-events.test.ts b/packages/junior/tests/unit/chat/pi/conversation-events.test.ts index 608a0187f..6befc191c 100644 --- a/packages/junior/tests/unit/chat/pi/conversation-events.test.ts +++ b/packages/junior/tests/unit/chat/pi/conversation-events.test.ts @@ -4,7 +4,6 @@ import { type ConversationEvent, type ConversationEventData, } from "@/chat/conversations/history"; -import { projectConversationEventHistory } from "@/chat/conversations/event-projection"; import { projectConversationEvents } from "@/chat/pi/conversation-events"; function event(seq: number, data: ConversationEventData): ConversationEvent { @@ -90,19 +89,21 @@ describe("projectConversationEvents", () => { ]; it("projects messages, authorization observations, provenance, and model binding", () => { - const genericProjection = projectConversationEventHistory(events); const projection = projectConversationEvents(events); - expect(projection).toEqual(genericProjection); - expect(projection).toEqual({ messages: [ firstMessage, - expect.objectContaining({ + { role: "user", - content: [expect.objectContaining({ type: "text" })], + content: [ + { + type: "text", + text: 'MCP authorization completed for provider "linear". Continue the blocked request and retry the provider operation if needed.', + }, + ], timestamp: 1_003, - }), + }, lastMessage, ], provenance: [ @@ -126,4 +127,16 @@ describe("projectConversationEvents", () => { }); expect(projection.messages).toHaveLength(2); }); + + it("validates opaque durable messages only at the Pi boundary", () => { + const invalid = { + schemaVersion: 1, + seq: 1, + contextEpoch: 2, + createdAtMs: 1_001, + data: { type: "message", message: {} }, + } as ConversationEvent; + + expect(() => projectConversationEvents([invalid])).toThrow(/role/); + }); }); diff --git a/packages/junior/tests/unit/conversations/model-messages.test.ts b/packages/junior/tests/unit/chat/pi/transcript.test.ts similarity index 59% rename from packages/junior/tests/unit/conversations/model-messages.test.ts rename to packages/junior/tests/unit/chat/pi/transcript.test.ts index 4b66c72b8..4438c2039 100644 --- a/packages/junior/tests/unit/conversations/model-messages.test.ts +++ b/packages/junior/tests/unit/chat/pi/transcript.test.ts @@ -1,9 +1,10 @@ import { describe, expect, it } from "vitest"; +import type { PiMessage } from "@/chat/pi/messages"; import { - hasRuntimeTurnContextMessages, - retainRuntimeTurnContextMessages, - stripRuntimeTurnContextMessages, -} from "@/chat/conversations/model-messages"; + hasRuntimeTurnContext, + retainRuntimeTurnContext, + stripRuntimeTurnContext, +} from "@/chat/pi/transcript"; const runtimePart = { type: "text", @@ -11,28 +12,32 @@ const runtimePart = { }; const instructionPart = { type: "text", text: "keep me" }; -describe("opaque conversation model messages", () => { +function asPiMessages(messages: unknown[]): PiMessage[] { + return messages as PiMessage[]; +} + +describe("Pi runtime turn context", () => { it("detects and retains only user runtime context", () => { - const messages = [ + const messages = asPiMessages([ { role: "assistant", content: [runtimePart] }, { role: "user", content: [runtimePart, instructionPart] }, - ]; + ]); - expect(hasRuntimeTurnContextMessages(messages)).toBe(true); - expect(retainRuntimeTurnContextMessages(messages)).toEqual([ + expect(hasRuntimeTurnContext(messages)).toBe(true); + expect(retainRuntimeTurnContext(messages)).toEqual([ { role: "user", content: [runtimePart] }, ]); }); it("strips runtime context and drops empty user messages", () => { const assistant = { role: "assistant", content: [runtimePart] }; - const messages = [ + const messages = asPiMessages([ assistant, { role: "user", content: [runtimePart] }, { role: "user", content: [runtimePart, instructionPart] }, - ]; + ]); - expect(stripRuntimeTurnContextMessages(messages)).toEqual([ + expect(stripRuntimeTurnContext(messages)).toEqual([ assistant, { role: "user", content: [instructionPart] }, ]); diff --git a/packages/junior/tests/unit/cli/conversation-history-conversion.test.ts b/packages/junior/tests/unit/cli/conversation-history-conversion.test.ts index d76672e50..df4e6615a 100644 --- a/packages/junior/tests/unit/cli/conversation-history-conversion.test.ts +++ b/packages/junior/tests/unit/cli/conversation-history-conversion.test.ts @@ -199,7 +199,6 @@ describe("operator legacy conversation history conversion", () => { subagentKind: "advisor", parentToolCallId: "call-1", childConversationId: `advisor:${CONVERSATION_ID}`, - historyMode: "isolated", }); expect(events[0]!.createdAtMs).toBe(50); expect(events[1]!.data).toEqual({ diff --git a/packages/junior/tests/unit/conversations/privacy.test.ts b/packages/junior/tests/unit/conversations/privacy.test.ts index fa09e87c0..72a376030 100644 --- a/packages/junior/tests/unit/conversations/privacy.test.ts +++ b/packages/junior/tests/unit/conversations/privacy.test.ts @@ -6,7 +6,6 @@ import { juniorConversations, juniorDestinations } from "@/db/schema"; interface ConversationRow { destinationId: string | null; parentId: string | null; - rootId: string | null; } function scriptedExecutor(args: { @@ -58,10 +57,10 @@ describe("resolveRootVisibility", () => { const result = await resolveRootVisibility( scriptedExecutor({ conversations: [ - { destinationId: null, parentId: "root", rootId: "root" }, - { destinationId: "destination", parentId: null, rootId: null }, - { destinationId: "destination", parentId: null, rootId: null }, - { destinationId: null, parentId: "root", rootId: "root" }, + { destinationId: null, parentId: "root" }, + { destinationId: "destination", parentId: null }, + { destinationId: "destination", parentId: null }, + { destinationId: null, parentId: "root" }, ], visibility: "public", }), @@ -81,22 +80,11 @@ describe("resolveRootVisibility", () => { }, { name: "a missing parent", - rows: [{ destinationId: null, parentId: "missing", rootId: "root" }], - }, - { - name: "a historical child without a declared root", - rows: [{ destinationId: null, parentId: "root", rootId: null }], - }, - { - name: "an inconsistent intermediate root", - rows: [ - { destinationId: null, parentId: "parent", rootId: "root" }, - { destinationId: null, parentId: "root", rootId: "other-root" }, - ], + rows: [{ destinationId: null, parentId: "missing" }], }, { name: "a missing root destination", - rows: [{ destinationId: null, parentId: null, rootId: null }], + rows: [{ destinationId: null, parentId: null }], }, ])("fails closed for $name", async ({ rows }) => { await expect( @@ -111,8 +99,8 @@ describe("resolveRootVisibility", () => { const result = await resolveRootVisibility( scriptedExecutor({ conversations: [ - { destinationId: null, parentId: "parent", rootId: "root" }, - { destinationId: null, parentId: "requested", rootId: "root" }, + { destinationId: null, parentId: "parent" }, + { destinationId: null, parentId: "requested" }, ], }), "requested", @@ -125,21 +113,18 @@ describe("resolveRootVisibility", () => { const result = await resolveRootVisibility( scriptedExecutor({ conversations: [ - { destinationId: null, parentId: "root", rootId: "root" }, + { destinationId: null, parentId: "root" }, { destinationId: "destination", parentId: null, - rootId: null, }, { destinationId: "destination", parentId: null, - rootId: null, }, { destinationId: null, parentId: "different-root", - rootId: "different-root", }, ], visibility: "public", @@ -154,7 +139,6 @@ describe("resolveRootVisibility", () => { const conversations = Array.from({ length: 32 }, (_, index) => ({ destinationId: null, parentId: `node-${index + 1}`, - rootId: "root", })); const result = await resolveRootVisibility( From 45992bccf1e236697f4b204c1955de9139c12187 Mon Sep 17 00:00:00 2001 From: David Cramer Date: Wed, 15 Jul 2026 11:21:29 -0700 Subject: [PATCH 18/58] ref(api): Hide internal conversation epochs Co-Authored-By: GPT-5 Codex --- .../junior-dashboard/e2e/dashboard.spec.ts | 1 - .../src/mock-conversations.ts | 31 ++++++------------- .../junior-dashboard/tests/format.test.ts | 1 - .../tests/markdownExport.test.ts | 1 - .../tests/telemetry-components.test.tsx | 2 +- .../tests/transcriptBottomPinning.test.ts | 3 -- .../tests/transcriptRenderModel.test.ts | 2 +- .../junior/src/api/conversations/events.ts | 1 - .../junior/src/api/conversations/schema.ts | 1 - .../unit/api/conversation-events.test.ts | 16 +++++----- 10 files changed, 19 insertions(+), 40 deletions(-) diff --git a/packages/junior-dashboard/e2e/dashboard.spec.ts b/packages/junior-dashboard/e2e/dashboard.spec.ts index 985a71203..c569d6437 100644 --- a/packages/junior-dashboard/e2e/dashboard.spec.ts +++ b/packages/junior-dashboard/e2e/dashboard.spec.ts @@ -317,7 +317,6 @@ test("scrolls long conversation and transcript panes independently", async ({ generatedAt, eventHistory: { status: "available" }, events: Array.from({ length: 60 }, (_, index) => ({ - contextEpoch: 0, createdAt: new Date( Date.parse(generatedAt) + index * 1_000, ).toISOString(), diff --git a/packages/junior-dashboard/src/mock-conversations.ts b/packages/junior-dashboard/src/mock-conversations.ts index 803b0d0df..e95fe3f27 100644 --- a/packages/junior-dashboard/src/mock-conversations.ts +++ b/packages/junior-dashboard/src/mock-conversations.ts @@ -51,9 +51,8 @@ function reportEvent( seq: number, createdAt: string, data: ConversationReportEventData, - contextEpoch = 0, ): ConversationReportEvent { - return { seq, contextEpoch, createdAt, data }; + return { seq, createdAt, data }; } type DetailOptions = Omit< @@ -244,25 +243,15 @@ function longConversation(nowMs: number): ConversationDetailReport { reportEvent(13, iso(Date.parse(startedAt), 53_000), { type: "context_compacted", }), - reportEvent( - 14, - iso(Date.parse(startedAt), 90_000), - { - type: "model_handoff", - }, - 1, - ), - reportEvent( - 15, - iso(Date.parse(startedAt), 166_000), - { - type: "visible_message", - messageId: "release-assistant", - role: "assistant", - text: "Released the package and opened the update pull request.", - }, - 1, - ), + reportEvent(14, iso(Date.parse(startedAt), 90_000), { + type: "model_handoff", + }), + reportEvent(15, iso(Date.parse(startedAt), 166_000), { + type: "visible_message", + messageId: "release-assistant", + role: "assistant", + text: "Released the package and opened the update pull request.", + }), ); return detail(nowMs, { conversationId: LONG_CONVERSATION_ID, diff --git a/packages/junior-dashboard/tests/format.test.ts b/packages/junior-dashboard/tests/format.test.ts index 9aabdee8e..83df3b98e 100644 --- a/packages/junior-dashboard/tests/format.test.ts +++ b/packages/junior-dashboard/tests/format.test.ts @@ -39,7 +39,6 @@ function event( ): ConversationReportEvent { return { seq, - contextEpoch: 0, createdAt: `2026-01-01T00:00:${String(seq).padStart(2, "0")}.000Z`, data, }; diff --git a/packages/junior-dashboard/tests/markdownExport.test.ts b/packages/junior-dashboard/tests/markdownExport.test.ts index b4aa351cf..e0fb1e87c 100644 --- a/packages/junior-dashboard/tests/markdownExport.test.ts +++ b/packages/junior-dashboard/tests/markdownExport.test.ts @@ -13,7 +13,6 @@ function event( ): ConversationReportEvent { return { seq, - contextEpoch: 0, createdAt: `2026-01-01T00:00:${String(seq).padStart(2, "0")}.000Z`, data, }; diff --git a/packages/junior-dashboard/tests/telemetry-components.test.tsx b/packages/junior-dashboard/tests/telemetry-components.test.tsx index 15a2cf7f1..b10b0e484 100644 --- a/packages/junior-dashboard/tests/telemetry-components.test.tsx +++ b/packages/junior-dashboard/tests/telemetry-components.test.tsx @@ -38,7 +38,7 @@ function event( data: ConversationReportEventData, createdAt = `2026-01-01T00:00:${String(seq).padStart(2, "0")}.000Z`, ): ConversationReportEvent { - return { seq, contextEpoch: 0, createdAt, data }; + return { seq, createdAt, data }; } function conversation( diff --git a/packages/junior-dashboard/tests/transcriptBottomPinning.test.ts b/packages/junior-dashboard/tests/transcriptBottomPinning.test.ts index 638197aa0..ced4cc8e0 100644 --- a/packages/junior-dashboard/tests/transcriptBottomPinning.test.ts +++ b/packages/junior-dashboard/tests/transcriptBottomPinning.test.ts @@ -25,7 +25,6 @@ function activeTurn( events: [ { seq: 0, - contextEpoch: 0, createdAt: "2026-01-01T00:00:01.000Z", data: { type: "visible_message", @@ -65,7 +64,6 @@ describe("transcript bottom pinning", () => { events: [ { seq: 0, - contextEpoch: 0, createdAt: "2026-01-01T00:00:01.000Z", data: { type: "visible_message", @@ -111,7 +109,6 @@ describe("transcript bottom pinning", () => { events: [ { seq: 0, - contextEpoch: 0, createdAt: "2026-01-01T00:00:01.000Z", data: { type: "turn_lifecycle", diff --git a/packages/junior-dashboard/tests/transcriptRenderModel.test.ts b/packages/junior-dashboard/tests/transcriptRenderModel.test.ts index 6e9c92e3e..048cebd79 100644 --- a/packages/junior-dashboard/tests/transcriptRenderModel.test.ts +++ b/packages/junior-dashboard/tests/transcriptRenderModel.test.ts @@ -17,7 +17,7 @@ function event( createdAt: string, data: ConversationReportEventData, ): ConversationReportEvent { - return { seq, contextEpoch: 0, createdAt, data }; + return { seq, createdAt, data }; } function conversation( diff --git a/packages/junior/src/api/conversations/events.ts b/packages/junior/src/api/conversations/events.ts index cd34366e0..86570e90b 100644 --- a/packages/junior/src/api/conversations/events.ts +++ b/packages/junior/src/api/conversations/events.ts @@ -150,7 +150,6 @@ export function projectConversationReportEvents(args: { projected.push( conversationReportEventSchema.parse({ seq: event.seq, - contextEpoch: event.contextEpoch, createdAt: new Date(event.createdAtMs).toISOString(), data, }), diff --git a/packages/junior/src/api/conversations/schema.ts b/packages/junior/src/api/conversations/schema.ts index 38b2f0294..c4869d591 100644 --- a/packages/junior/src/api/conversations/schema.ts +++ b/packages/junior/src/api/conversations/schema.ts @@ -157,7 +157,6 @@ export const conversationReportEventDataSchema = z.discriminatedUnion("type", [ export const conversationReportEventSchema = z .object({ seq: z.number().int().nonnegative(), - contextEpoch: z.number().int().nonnegative(), createdAt: z.string().datetime(), data: conversationReportEventDataSchema, }) diff --git a/packages/junior/tests/unit/api/conversation-events.test.ts b/packages/junior/tests/unit/api/conversation-events.test.ts index fcfe1ccd4..bb190caed 100644 --- a/packages/junior/tests/unit/api/conversation-events.test.ts +++ b/packages/junior/tests/unit/api/conversation-events.test.ts @@ -15,12 +15,11 @@ function event( seq: number, data: ConversationEventData, createdAtMs = seq * 1_000, - contextEpoch = 0, ): ConversationEvent { return conversationEventSchema.parse({ schemaVersion: 1, seq, - contextEpoch, + contextEpoch: 0, idempotencyKey: `private-idempotency-${seq}`, createdAtMs, data, @@ -83,7 +82,6 @@ describe("conversation report event projection", () => { expect(projected).toEqual([ { seq: 10, - contextEpoch: 0, createdAt: "1970-01-01T00:00:30.000Z", data: { type: "visible_message", @@ -94,7 +92,6 @@ describe("conversation report event projection", () => { }, { seq: 11, - contextEpoch: 0, createdAt: "1970-01-01T00:00:10.000Z", data: { type: "model_activity", @@ -103,13 +100,11 @@ describe("conversation report event projection", () => { }, { seq: 12, - contextEpoch: 0, createdAt: "1970-01-01T00:00:05.000Z", data: { type: "tool_started", name: "search" }, }, { seq: 13, - contextEpoch: 0, createdAt: "1970-01-01T00:00:01.000Z", data: { type: "visible_message_replied", @@ -394,7 +389,6 @@ describe("conversation report event projection", () => { it("owns a strict runtime boundary for envelope and data fields", () => { const valid = { seq: 1, - contextEpoch: 0, createdAt: "2026-07-15T12:00:00.000Z", data: { type: "turn_lifecycle", @@ -410,6 +404,12 @@ describe("conversation report event projection", () => { schemaVersion: 1, }).success, ).toBe(false); + expect( + conversationReportEventSchema.safeParse({ + ...valid, + contextEpoch: 0, + }).success, + ).toBe(false); expect( conversationReportEventSchema.safeParse({ ...valid, @@ -439,7 +439,6 @@ describe("conversation report event projection", () => { }; const reportEvent = (seq: number) => ({ seq, - contextEpoch: 0, createdAt: "2026-07-15T12:00:00.000Z", data: { type: "turn_lifecycle" as const, @@ -486,7 +485,6 @@ describe("conversation report event projection", () => { | { redacted: true; text?: never }, ) => ({ seq: 1, - contextEpoch: 0, createdAt: "2026-07-15T12:00:00.000Z", data: { type: "visible_message" as const, From e124305ae192a327bb53591388c68a2349364799 Mon Sep 17 00:00:00 2001 From: David Cramer Date: Wed, 15 Jul 2026 11:21:44 -0700 Subject: [PATCH 19/58] ref(conversations): Trim redundant event storage Co-Authored-By: GPT-5 Codex --- .../migrations/0005_conversation_events.sql | 3 +- .../junior/migrations/meta/0004_snapshot.json | 90 +++++-------------- .../junior/migrations/meta/0005_snapshot.json | 6 -- .../src/chat/conversations/sql/history.ts | 8 -- .../legacy-history-import.ts | 8 -- .../src/db/schema/conversation-events.ts | 1 - .../cli/conversation-event-migration.test.ts | 23 +++-- .../conversation-transcripts-sql.test.ts | 9 +- .../component/conversations/retention.test.ts | 1 - 9 files changed, 43 insertions(+), 106 deletions(-) diff --git a/packages/junior/migrations/0005_conversation_events.sql b/packages/junior/migrations/0005_conversation_events.sql index af9a1c706..13233ce69 100644 --- a/packages/junior/migrations/0005_conversation_events.sql +++ b/packages/junior/migrations/0005_conversation_events.sql @@ -2,4 +2,5 @@ ALTER TABLE "junior_agent_steps" RENAME TO "junior_conversation_events";--> stat ALTER TABLE "junior_conversation_events" RENAME CONSTRAINT "junior_agent_steps_conversation_id_seq_pk" TO "junior_conversation_events_conversation_id_seq_pk";--> statement-breakpoint ALTER TABLE "junior_conversation_events" RENAME CONSTRAINT "junior_agent_steps_conversation_id_junior_conversations_conversation_id_fk" TO "junior_conversation_events_conversation_id_junior_conversations_conversation_id_fk";--> statement-breakpoint ALTER INDEX "junior_agent_steps_epoch_idx" RENAME TO "junior_conversation_events_epoch_idx";--> statement-breakpoint -ALTER TABLE "junior_conversation_events" ADD COLUMN "schema_version" integer DEFAULT 1 NOT NULL; +ALTER TABLE "junior_conversation_events" ADD COLUMN "schema_version" integer DEFAULT 1 NOT NULL;--> statement-breakpoint +ALTER TABLE "junior_conversation_events" DROP COLUMN "role"; diff --git a/packages/junior/migrations/meta/0004_snapshot.json b/packages/junior/migrations/meta/0004_snapshot.json index c82e8f4fa..59a6b1177 100644 --- a/packages/junior/migrations/meta/0004_snapshot.json +++ b/packages/junior/migrations/meta/0004_snapshot.json @@ -32,12 +32,6 @@ "primaryKey": false, "notNull": true }, - "role": { - "name": "role", - "type": "text", - "primaryKey": false, - "notNull": false - }, "payload": { "name": "payload", "type": "jsonb", @@ -85,12 +79,8 @@ "name": "junior_agent_steps_conversation_id_junior_conversations_conversation_id_fk", "tableFrom": "junior_agent_steps", "tableTo": "junior_conversations", - "columnsFrom": [ - "conversation_id" - ], - "columnsTo": [ - "conversation_id" - ], + "columnsFrom": ["conversation_id"], + "columnsTo": ["conversation_id"], "onDelete": "no action", "onUpdate": "no action" } @@ -98,10 +88,7 @@ "compositePrimaryKeys": { "junior_agent_steps_conversation_id_seq_pk": { "name": "junior_agent_steps_conversation_id_seq_pk", - "columns": [ - "conversation_id", - "seq" - ] + "columns": ["conversation_id", "seq"] } }, "uniqueConstraints": {}, @@ -205,12 +192,8 @@ "name": "junior_conversation_messages_conversation_id_junior_conversations_conversation_id_fk", "tableFrom": "junior_conversation_messages", "tableTo": "junior_conversations", - "columnsFrom": [ - "conversation_id" - ], - "columnsTo": [ - "conversation_id" - ], + "columnsFrom": ["conversation_id"], + "columnsTo": ["conversation_id"], "onDelete": "no action", "onUpdate": "no action" }, @@ -218,12 +201,8 @@ "name": "junior_conversation_messages_author_identity_id_junior_identities_id_fk", "tableFrom": "junior_conversation_messages", "tableTo": "junior_identities", - "columnsFrom": [ - "author_identity_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["author_identity_id"], + "columnsTo": ["id"], "onDelete": "no action", "onUpdate": "no action" } @@ -231,10 +210,7 @@ "compositePrimaryKeys": { "junior_conversation_messages_conversation_id_message_id_pk": { "name": "junior_conversation_messages_conversation_id_message_id_pk", - "columns": [ - "conversation_id", - "message_id" - ] + "columns": ["conversation_id", "message_id"] } }, "uniqueConstraints": {}, @@ -564,12 +540,8 @@ "name": "junior_conversations_destination_id_junior_destinations_id_fk", "tableFrom": "junior_conversations", "tableTo": "junior_destinations", - "columnsFrom": [ - "destination_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["destination_id"], + "columnsTo": ["id"], "onDelete": "no action", "onUpdate": "no action" }, @@ -577,12 +549,8 @@ "name": "junior_conversations_actor_identity_id_junior_identities_id_fk", "tableFrom": "junior_conversations", "tableTo": "junior_identities", - "columnsFrom": [ - "actor_identity_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["actor_identity_id"], + "columnsTo": ["id"], "onDelete": "no action", "onUpdate": "no action" }, @@ -590,12 +558,8 @@ "name": "junior_conversations_creator_identity_id_junior_identities_id_fk", "tableFrom": "junior_conversations", "tableTo": "junior_identities", - "columnsFrom": [ - "creator_identity_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["creator_identity_id"], + "columnsTo": ["id"], "onDelete": "no action", "onUpdate": "no action" }, @@ -603,12 +567,8 @@ "name": "junior_conversations_credential_subject_identity_id_junior_identities_id_fk", "tableFrom": "junior_conversations", "tableTo": "junior_identities", - "columnsFrom": [ - "credential_subject_identity_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["credential_subject_identity_id"], + "columnsTo": ["id"], "onDelete": "no action", "onUpdate": "no action" }, @@ -616,12 +576,8 @@ "name": "junior_conversations_parent_conversation_id_junior_conversations_conversation_id_fk", "tableFrom": "junior_conversations", "tableTo": "junior_conversations", - "columnsFrom": [ - "parent_conversation_id" - ], - "columnsTo": [ - "conversation_id" - ], + "columnsFrom": ["parent_conversation_id"], + "columnsTo": ["conversation_id"], "onDelete": "no action", "onUpdate": "no action" } @@ -945,12 +901,8 @@ "name": "junior_identities_user_id_junior_users_id_fk", "tableFrom": "junior_identities", "tableTo": "junior_users", - "columnsFrom": [ - "user_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["user_id"], + "columnsTo": ["id"], "onDelete": "no action", "onUpdate": "no action" } @@ -1038,4 +990,4 @@ "schemas": {}, "tables": {} } -} \ No newline at end of file +} diff --git a/packages/junior/migrations/meta/0005_snapshot.json b/packages/junior/migrations/meta/0005_snapshot.json index d7dfee2b9..52e72fd8a 100644 --- a/packages/junior/migrations/meta/0005_snapshot.json +++ b/packages/junior/migrations/meta/0005_snapshot.json @@ -45,12 +45,6 @@ "primaryKey": false, "notNull": true }, - "role": { - "name": "role", - "type": "text", - "primaryKey": false, - "notNull": false - }, "payload": { "name": "payload", "type": "jsonb", diff --git a/packages/junior/src/chat/conversations/sql/history.ts b/packages/junior/src/chat/conversations/sql/history.ts index 5a52ad80d..f55c84b44 100644 --- a/packages/junior/src/chat/conversations/sql/history.ts +++ b/packages/junior/src/chat/conversations/sql/history.ts @@ -25,13 +25,6 @@ type PersistedConversationEvent = { createdAtMs: number; }; -function messageRole(data: ConversationEventData): string | null { - if (data.type === "message") { - return data.message.role; - } - return data.type === "visible_message_recorded" ? data.role : null; -} - /** Split validated event data into column-lifted and JSON payload fields. */ function insertFromEvent( conversationId: string, @@ -47,7 +40,6 @@ function insertFromEvent( schemaVersion: 1, idempotencyKey: event.idempotencyKey ?? null, type, - role: messageRole(event.data), payload: sanitizePostgresJson(payload), createdAt: new Date(event.createdAtMs), }; diff --git a/packages/junior/src/cli/upgrade/migrations/conversation-history/legacy-history-import.ts b/packages/junior/src/cli/upgrade/migrations/conversation-history/legacy-history-import.ts index 3baac6efb..7e8a4b67e 100644 --- a/packages/junior/src/cli/upgrade/migrations/conversation-history/legacy-history-import.ts +++ b/packages/junior/src/cli/upgrade/migrations/conversation-history/legacy-history-import.ts @@ -318,13 +318,6 @@ export function convertAdvisorMessages( type ConversationEventInsert = typeof juniorConversationEvents.$inferInsert; -function messageRole(entry: ConversationEventData): string | null { - if (entry.type === "message") { - return entry.message.role; - } - return entry.type === "visible_message_recorded" ? entry.role : null; -} - /** Encode one validated imported event as a physical SQL row. */ function insertRow( conversationId: string, @@ -338,7 +331,6 @@ function insertRow( schemaVersion: 1, idempotencyKey: event.idempotencyKey ?? null, type, - role: messageRole(event.data), payload: sanitizePostgresJson(payload), createdAt: new Date(event.createdAtMs), }; diff --git a/packages/junior/src/db/schema/conversation-events.ts b/packages/junior/src/db/schema/conversation-events.ts index 52ebf840c..988974c7d 100644 --- a/packages/junior/src/db/schema/conversation-events.ts +++ b/packages/junior/src/db/schema/conversation-events.ts @@ -25,7 +25,6 @@ export const juniorConversationEvents = pgTable( schemaVersion: integer("schema_version").default(1).notNull(), idempotencyKey: text("idempotency_key"), type: text("type").notNull(), - role: text("role"), payload: jsonb("payload").$type>().notNull(), createdAt: timestamptz("created_at").notNull(), }, diff --git a/packages/junior/tests/component/cli/conversation-event-migration.test.ts b/packages/junior/tests/component/cli/conversation-event-migration.test.ts index 3ea56827a..2a8e80bf2 100644 --- a/packages/junior/tests/component/cli/conversation-event-migration.test.ts +++ b/packages/junior/tests/component/cli/conversation-event-migration.test.ts @@ -400,14 +400,27 @@ describe("conversation event migration", () => { deleteFunction: string | null; insertFunction: string | null; relation: string | null; + roleColumn: boolean; }>( `SELECT to_regclass('public.junior_agent_steps')::text AS relation, to_regprocedure('public.junior_agent_steps_insert_compat()')::text AS "insertFunction", - to_regprocedure('public.junior_agent_steps_delete_compat()')::text AS "deleteFunction"`, + to_regprocedure('public.junior_agent_steps_delete_compat()')::text AS "deleteFunction", + EXISTS ( + SELECT 1 + FROM information_schema.columns + WHERE table_schema = 'public' + AND table_name = 'junior_conversation_events' + AND column_name = 'role' + ) AS "roleColumn"`, ), ).resolves.toEqual([ - { deleteFunction: null, insertFunction: null, relation: null }, + { + deleteFunction: null, + insertFunction: null, + relation: null, + roleColumn: false, + }, ]); await expect( @@ -415,7 +428,6 @@ describe("conversation event migration", () => { conversationId: string; contextEpoch: number; createdAtPreserved: boolean; - role: string | null; schemaVersion: number; seq: number; type: string; @@ -426,8 +438,7 @@ SELECT context_epoch AS "contextEpoch", created_at = '2026-07-14T10:01:00.000Z'::timestamptz AS "createdAtPreserved", schema_version AS "schemaVersion", - type, - role + type FROM junior_conversation_events ORDER BY seq `), @@ -436,7 +447,6 @@ ORDER BY seq contextEpoch: 2, conversationId: "conversation-one", createdAtPreserved: true, - role: "assistant", schemaVersion: 1, seq: 1, type: "pi_message", @@ -445,7 +455,6 @@ ORDER BY seq contextEpoch: 3, conversationId: "conversation-one", createdAtPreserved: true, - role: null, schemaVersion: 1, seq: 2, type: "authorization_completed", diff --git a/packages/junior/tests/component/conversation-transcripts-sql.test.ts b/packages/junior/tests/component/conversation-transcripts-sql.test.ts index 659fee0a7..6ee14f579 100644 --- a/packages/junior/tests/component/conversation-transcripts-sql.test.ts +++ b/packages/junior/tests/component/conversation-transcripts-sql.test.ts @@ -416,7 +416,6 @@ describe("conversation transcript SQL stores", () => { contextEpoch: 0, schemaVersion: 1, type: "message", - role: "user", payload: { message: userMessage("clobber") }, createdAt: new Date(4_000), }), @@ -628,8 +627,8 @@ describe("conversation transcript SQL stores", () => { await fixture.sql.execute( ` INSERT INTO junior_conversation_events ( - conversation_id, seq, context_epoch, schema_version, type, role, payload, created_at -) VALUES ($1, $2, $3, $4, $5, NULL, $6::jsonb, $7) + conversation_id, seq, context_epoch, schema_version, type, payload, created_at +) VALUES ($1, $2, $3, $4, $5, $6::jsonb, $7) `, [ CONVERSATION_ID, @@ -664,8 +663,8 @@ INSERT INTO junior_conversation_events ( await fixture.sql.execute( ` INSERT INTO junior_conversation_events ( - conversation_id, seq, context_epoch, schema_version, type, role, payload, created_at -) VALUES ($1, $2, $3, $4, $5, NULL, $6::jsonb, $7) + conversation_id, seq, context_epoch, schema_version, type, payload, created_at +) VALUES ($1, $2, $3, $4, $5, $6::jsonb, $7) `, [ CONVERSATION_ID, diff --git a/packages/junior/tests/component/conversations/retention.test.ts b/packages/junior/tests/component/conversations/retention.test.ts index 76d569a5d..a7b22abd3 100644 --- a/packages/junior/tests/component/conversations/retention.test.ts +++ b/packages/junior/tests/component/conversations/retention.test.ts @@ -86,7 +86,6 @@ async function seedConversation( contextEpoch: 0, schemaVersion: 1, type: "message", - role: "user", payload: { message: { role: "user", content: [] } }, createdAt: at, }); From b099bf04965194443dcd058b89064b6b642fa34f Mon Sep 17 00:00:00 2001 From: David Cramer Date: Wed, 15 Jul 2026 11:39:27 -0700 Subject: [PATCH 20/58] fix(migrations): Read usage role from event payload Co-Authored-By: GPT-5 Codex --- .../src/cli/upgrade/migrations/conversation-usage.ts | 2 +- .../junior/tests/component/cli/upgrade-cli.test.ts | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/packages/junior/src/cli/upgrade/migrations/conversation-usage.ts b/packages/junior/src/cli/upgrade/migrations/conversation-usage.ts index a97fd31bb..74416c538 100644 --- a/packages/junior/src/cli/upgrade/migrations/conversation-usage.ts +++ b/packages/junior/src/cli/upgrade/migrations/conversation-usage.ts @@ -49,7 +49,7 @@ message_occurrences AS ( INNER JOIN candidates AS candidate ON candidate.conversation_id = event.conversation_id WHERE event.type = 'message' - AND event.role = 'assistant' + AND event.payload -> 'message' ->> 'role' = 'assistant' AND jsonb_typeof(event.payload -> 'message' -> 'usage') = 'object' ), canonical_messages AS ( diff --git a/packages/junior/tests/component/cli/upgrade-cli.test.ts b/packages/junior/tests/component/cli/upgrade-cli.test.ts index 00affd590..e09d951ea 100644 --- a/packages/junior/tests/component/cli/upgrade-cli.test.ts +++ b/packages/junior/tests/component/cli/upgrade-cli.test.ts @@ -666,6 +666,16 @@ WHERE conversation_id = $1 }, createdAtMs: 3_000, }, + { + data: { + type: "message", + message: { + role: "user", + usage: { input: 1_000, output: 1_000 }, + } as unknown as PiMessage, + }, + createdAtMs: 3_500, + }, ]); await sqlStore.recordExecution({ conversationId: unsafeConversationId, From 415cdec889dc1289f933f2c6426fa4817a480b88 Mon Sep 17 00:00:00 2001 From: David Cramer Date: Wed, 15 Jul 2026 12:34:50 -0700 Subject: [PATCH 21/58] test(dashboard): Match profile activity horizon Keep the canonical-event mock route assertion aligned with the 365-day profile history introduced on main. Co-Authored-By: OpenAI Codex --- packages/junior-dashboard/tests/dashboard-mock-routes.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/junior-dashboard/tests/dashboard-mock-routes.test.ts b/packages/junior-dashboard/tests/dashboard-mock-routes.test.ts index 9618b20d3..8d53782ab 100644 --- a/packages/junior-dashboard/tests/dashboard-mock-routes.test.ts +++ b/packages/junior-dashboard/tests/dashboard-mock-routes.test.ts @@ -134,7 +134,7 @@ describe("dashboard canonical-event mock routes", () => { locations: unknown[]; surfaces: unknown[]; }; - expect(profile.activityDays).toHaveLength(90); + expect(profile.activityDays).toHaveLength(365); expect(profile.locations.length).toBeGreaterThan(0); expect(profile.surfaces.length).toBeGreaterThan(0); From 7e9aa4ee1cab8d2429f41cb6f33dc8825037b3d2 Mon Sep 17 00:00:00 2001 From: David Cramer Date: Wed, 15 Jul 2026 12:46:00 -0700 Subject: [PATCH 22/58] fix(dashboard): Align mock feed with production roots Exclude parent-linked mock conversations at the feed boundary and preserve public location links in projected summaries. Co-Authored-By: OpenAI Codex --- .../src/mock-conversations.ts | 34 +++++++++++++------ .../tests/dashboard-mock-routes.test.ts | 34 ++++++++++++++++++- 2 files changed, 57 insertions(+), 11 deletions(-) diff --git a/packages/junior-dashboard/src/mock-conversations.ts b/packages/junior-dashboard/src/mock-conversations.ts index e95fe3f27..5afaa3e9a 100644 --- a/packages/junior-dashboard/src/mock-conversations.ts +++ b/packages/junior-dashboard/src/mock-conversations.ts @@ -79,6 +79,10 @@ type DetailOptions = Omit< surface?: ConversationDetailReport["surface"]; }; +type MockConversation = ConversationDetailReport & { + parentConversationId?: string; +}; + function detail( nowMs: number, options: DetailOptions, @@ -191,9 +195,9 @@ function advisorConversation( nowMs: number, conversationId: string, text: string, -): ConversationDetailReport { +): MockConversation { const startedAt = iso(nowMs, -10 * 60_000); - return detail(nowMs, { + const conversation = detail(nowMs, { conversationId, displayTitle: "Advisor review", startedAt, @@ -215,6 +219,10 @@ function advisorConversation( }), ], }); + return { + ...conversation, + parentConversationId: DASHBOARD_QA_CONVERSATION_ID, + }; } function longConversation(nowMs: number): ConversationDetailReport { @@ -420,7 +428,7 @@ function usage(cost: number) { }; } -function mockConversations(nowMs: number): ConversationDetailReport[] { +function mockConversations(nowMs: number): MockConversation[] { return [ activeConversation(nowMs), dashboardQaConversation(nowMs), @@ -453,23 +461,28 @@ function mockConversations(nowMs: number): ConversationDetailReport[] { } function summaryFromConversation( - conversation: ConversationDetailReport, + conversation: MockConversation, ): ConversationSummaryReport { const { eventHistory: _eventHistory, events: _events, generatedAt: _generatedAt, + parentConversationId: _parentConversationId, sentryConversationUrl: _sentryConversationUrl, ...summary } = conversation; - return summary; + return summary.channel && PUBLIC_MOCK_CHANNEL_IDS.has(summary.channel) + ? { ...summary, locationId: `mock:${summary.channel}` } + : summary; } function mockConversationFeed(nowMs: number): ConversationFeed { return { source: "conversation_index", generatedAt: iso(nowMs), - conversations: mockConversations(nowMs).map(summaryFromConversation), + conversations: mockConversations(nowMs) + .filter((conversation) => !conversation.parentConversationId) + .map(summaryFromConversation), }; } @@ -549,10 +562,11 @@ export function readMockConversationDetail( (candidate) => candidate.conversationId === conversationId, ); if (!conversation) return undefined; - return conversation.channel && - PUBLIC_MOCK_CHANNEL_IDS.has(conversation.channel) - ? { ...conversation, locationId: `mock:${conversation.channel}` } - : conversation; + const { parentConversationId: _parentConversationId, ...detail } = + conversation; + return detail.channel && PUBLIC_MOCK_CHANNEL_IDS.has(detail.channel) + ? { ...detail, locationId: `mock:${detail.channel}` } + : detail; } /** Build mock dashboard stats from canonical-event mock conversations. */ diff --git a/packages/junior-dashboard/tests/dashboard-mock-routes.test.ts b/packages/junior-dashboard/tests/dashboard-mock-routes.test.ts index 8d53782ab..c0638e63b 100644 --- a/packages/junior-dashboard/tests/dashboard-mock-routes.test.ts +++ b/packages/junior-dashboard/tests/dashboard-mock-routes.test.ts @@ -11,6 +11,11 @@ import { DASHBOARD_QA_CONVERSATION_ID, } from "../src/mock-conversations"; +const DASHBOARD_QA_CHILD_IDS = [ + "junior:internal:dashboard-qa:advisor-plan", + "junior:internal:dashboard-qa:advisor-review", +]; + describe("dashboard canonical-event mock routes", () => { afterEach(() => vi.useRealTimers()); @@ -70,8 +75,10 @@ describe("dashboard canonical-event mock routes", () => { const body = (await conversations.json()) as { conversations: Array<{ actorIdentity?: { email?: string }; + channel?: string; conversationId: string; cumulativeDurationMs: number; + locationId?: string; }>; source: string; }; @@ -82,6 +89,14 @@ describe("dashboard canonical-event mock routes", () => { expect(body.conversations.map((item) => item.conversationId)).toContain( DASHBOARD_QA_CONVERSATION_ID, ); + expect(body.conversations.map((item) => item.conversationId)).not.toEqual( + expect.arrayContaining(DASHBOARD_QA_CHILD_IDS), + ); + expect( + body.conversations + .filter((item) => item.channel?.startsWith("C")) + .every((item) => item.locationId === `mock:${item.channel}`), + ).toBe(true); expect(body.conversations[0]).not.toHaveProperty("events"); const personal = await app.fetch( @@ -142,13 +157,23 @@ describe("dashboard canonical-event mock routes", () => { new Request("http://localhost/api/locations"), ); const locationBody = (await locations.json()) as { - locations: Array<{ id: string; label: string }>; + locations: Array<{ + conversations: number; + id: string; + label: string; + }>; privateActivity: { conversations: number }; }; expect(locationBody.locations.map((item) => item.label)).toContain( "#proj-checkout", ); expect(locationBody.privateActivity.conversations).toBeGreaterThan(0); + expect( + locationBody.locations.reduce( + (sum, item) => sum + item.conversations, + locationBody.privateActivity.conversations, + ), + ).toBe(body.conversations.length); const locationResponse = await app.fetch( new Request( `http://localhost/api/locations/${encodeURIComponent( @@ -159,9 +184,16 @@ describe("dashboard canonical-event mock routes", () => { const location = (await locationResponse.json()) as { activityDays: unknown[]; actors: unknown[]; + recentConversations: Array<{ locationId?: string }>; }; expect(location.activityDays).toHaveLength(90); expect(location.actors.length).toBeGreaterThan(0); + expect( + location.recentConversations.every( + (conversation) => + conversation.locationId === locationBody.locations[0]!.id, + ), + ).toBe(true); }); it("serves direct canonical event fixtures for every dashboard state", async () => { From 419874acd26c5f7c0584a2b84d7e46c88aef98e7 Mon Sep 17 00:00:00 2001 From: David Cramer Date: Wed, 15 Jul 2026 12:55:18 -0700 Subject: [PATCH 23/58] test(dashboard): Follow canonical child navigation Select root conversations by identity and verify imported advisor transcripts through the parent event and ordinary child detail route. Co-Authored-By: OpenAI Codex --- .../junior-dashboard/e2e/dashboard.spec.ts | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/packages/junior-dashboard/e2e/dashboard.spec.ts b/packages/junior-dashboard/e2e/dashboard.spec.ts index c569d6437..fcc928f62 100644 --- a/packages/junior-dashboard/e2e/dashboard.spec.ts +++ b/packages/junior-dashboard/e2e/dashboard.spec.ts @@ -163,6 +163,7 @@ test("hydrates the built dashboard client in a real browser", async ({ await expect( page.getByRole("heading", { name: "Conversations" }), ).toBeVisible(); + await page.getByRole("link", { name: /Checkout latency triage/ }).click(); await expect(page).toHaveURL( `${baseURL}/conversations/${encodeURIComponent("slack:CQA123:1770000000.000100")}`, ); @@ -414,6 +415,7 @@ test("groups the signed-in profile and session actions in the header", async ({ }); test("inspects and copies an advisor transcript", async ({ context, page }) => { + const childConversationId = "junior:internal:dashboard-qa:advisor-plan"; await context.grantPermissions(["clipboard-read", "clipboard-write"], { origin: baseURL, }); @@ -431,18 +433,26 @@ test("inspects and copies an advisor transcript", async ({ context, page }) => { await subagentRow.click(); const drawer = page.getByRole("dialog"); - await expect(drawer.getByRole("heading", { name: "advisor" })).toBeVisible(); - await expect(drawer.getByText("Conversation ID")).toBeVisible(); + await expect( + drawer.getByRole("heading", { name: "Advisor review" }), + ).toBeVisible(); + await expect( + drawer.getByText(childConversationId, { exact: true }), + ).toBeVisible(); + await expect( + drawer.getByRole("link", { name: "Open conversation" }), + ).toHaveAttribute( + "href", + `/conversations/${encodeURIComponent(childConversationId)}`, + ); const copy = drawer.getByRole("button", { name: "Copy as Markdown" }); await expect(copy).toBeEnabled(); await copy.click(); await expect(drawer.getByRole("button", { name: "Copied" })).toBeVisible(); const markdown = await page.evaluate(() => navigator.clipboard.readText()); - expect(markdown).toContain("# advisor"); + expect(markdown).toContain("# Advisor review"); expect(markdown).toContain("Review the dashboard plan before editing."); - expect(markdown).toContain( - "Actor identity email is a reasonable profile key", - ); + expect(markdown).toContain("Review complete; no blocking issues found."); await page.setViewportSize({ height: 844, width: 390 }); await expect(drawer).toBeVisible(); From f8f5fc2afe4d844d307076bf1e548a7c293d2341 Mon Sep 17 00:00:00 2001 From: David Cramer Date: Wed, 15 Jul 2026 13:22:02 -0700 Subject: [PATCH 24/58] fix(dashboard): Correlate imported child outcomes Pair repeated imported advisor lifecycles through public event sequence metadata. Keep private invocation identifiers internal, reduce the dashboard view to one authoritative status, and allow incomplete child histories to remain navigable. Co-Authored-By: OpenAI Codex --- .../components/SubagentTranscriptDrawer.tsx | 10 ++- .../components/TranscriptSubagentView.tsx | 11 +-- .../components/transcriptBottomPinning.ts | 1 - .../components/transcriptRenderModel.ts | 1 - .../client/components/transcriptSearch.tsx | 3 +- .../src/client/eventTranscript.ts | 9 +- .../src/client/markdownExport.ts | 2 +- .../src/client/pages/ConversationPage.tsx | 7 +- packages/junior-dashboard/src/client/types.ts | 5 +- .../src/mock-conversations.ts | 6 +- .../tests/telemetry-components.test.tsx | 89 +++++++++++++++++-- .../tests/transcriptRenderModel.test.ts | 36 +++++++- .../junior/src/api/conversations/events.ts | 19 ++-- .../junior/src/api/conversations/schema.ts | 3 +- .../integration/dashboard-reporting.test.ts | 3 +- .../unit/api/conversation-events.test.ts | 44 +++++++-- 16 files changed, 190 insertions(+), 59 deletions(-) diff --git a/packages/junior-dashboard/src/client/components/SubagentTranscriptDrawer.tsx b/packages/junior-dashboard/src/client/components/SubagentTranscriptDrawer.tsx index 2f19ef61b..5046c7f6c 100644 --- a/packages/junior-dashboard/src/client/components/SubagentTranscriptDrawer.tsx +++ b/packages/junior-dashboard/src/client/components/SubagentTranscriptDrawer.tsx @@ -39,7 +39,7 @@ export function SubagentTranscriptDrawer(props: { const detail = query.data; const label = detail?.displayTitle || props.target.part.subagentKind; const meta = [ - detail?.status ?? statusLabel(props.target.part), + statusLabel(props.target.part, detail?.status), detail ? formatMessageTimestamp(Date.parse(detail.startedAt)) : undefined, ].filter(isString); @@ -117,8 +117,12 @@ export function SubagentTranscriptDrawer(props: { ); } -function statusLabel(part: TranscriptViewSubagentPart): string { - return part.outcome ?? part.status; +function statusLabel( + part: TranscriptViewSubagentPart, + detailStatus?: string, +): string { + if (part.status !== "completed") return part.status; + return detailStatus ?? part.status; } function DrawerEmptyState(props: { diff --git a/packages/junior-dashboard/src/client/components/TranscriptSubagentView.tsx b/packages/junior-dashboard/src/client/components/TranscriptSubagentView.tsx index 4e79e88a5..d30741847 100644 --- a/packages/junior-dashboard/src/client/components/TranscriptSubagentView.tsx +++ b/packages/junior-dashboard/src/client/components/TranscriptSubagentView.tsx @@ -39,7 +39,7 @@ export function TranscriptSubagentView(props: { /> ); - if (!props.onOpenTranscript || props.part.status === "running") { + if (!props.onOpenTranscript) { return frame; } @@ -56,12 +56,9 @@ export function TranscriptSubagentView(props: { } function statusLabel(part: TranscriptViewSubagentPart): string | undefined { - if (part.outcome === "error") return "error"; - if (part.outcome === "aborted") return "aborted"; - if (part.status === "error" || part.status === "aborted") { - return part.status; - } - return undefined; + return part.status === "error" || part.status === "aborted" + ? part.status + : undefined; } function isString(value: string | undefined): value is string { diff --git a/packages/junior-dashboard/src/client/components/transcriptBottomPinning.ts b/packages/junior-dashboard/src/client/components/transcriptBottomPinning.ts index 075050547..121d3d50d 100644 --- a/packages/junior-dashboard/src/client/components/transcriptBottomPinning.ts +++ b/packages/junior-dashboard/src/client/components/transcriptBottomPinning.ts @@ -264,7 +264,6 @@ function transcriptPartVersion(part: TranscriptViewPart | undefined): string { part.name ?? "", part.subagentKind ?? "", part.status ?? "", - part.outcome ?? "", part.chars ?? part.text?.length ?? "", part.bytes ?? "", part.inputSizeChars ?? "", diff --git a/packages/junior-dashboard/src/client/components/transcriptRenderModel.ts b/packages/junior-dashboard/src/client/components/transcriptRenderModel.ts index b9748146d..a5ce46397 100644 --- a/packages/junior-dashboard/src/client/components/transcriptRenderModel.ts +++ b/packages/junior-dashboard/src/client/components/transcriptRenderModel.ts @@ -299,7 +299,6 @@ export function messageRawText(message: TranscriptViewMessage): string { `subagent ${part.subagentKind}`, stringifyPartValue({ id: part.id, - outcome: part.outcome, status: part.status, }), ] diff --git a/packages/junior-dashboard/src/client/components/transcriptSearch.tsx b/packages/junior-dashboard/src/client/components/transcriptSearch.tsx index 16556c469..e932171c8 100644 --- a/packages/junior-dashboard/src/client/components/transcriptSearch.tsx +++ b/packages/junior-dashboard/src/client/components/transcriptSearch.tsx @@ -155,8 +155,7 @@ export function entryMatchesSearch( return ( textContains(entry.part.subagentKind, normalizedQuery) || textContains(entry.part.id, normalizedQuery) || - textContains(entry.part.status, normalizedQuery) || - textContains(entry.part.outcome, normalizedQuery) + textContains(entry.part.status, normalizedQuery) ); } diff --git a/packages/junior-dashboard/src/client/eventTranscript.ts b/packages/junior-dashboard/src/client/eventTranscript.ts index 09444dfe0..20b9414e6 100644 --- a/packages/junior-dashboard/src/client/eventTranscript.ts +++ b/packages/junior-dashboard/src/client/eventTranscript.ts @@ -22,14 +22,14 @@ function eventMessage( function subagentOutcomes( events: ConversationReportEvent[], ): Map< - string, + number, Extract< ConversationReportEvent["data"], { type: "subagent_ended" } >["outcome"] > { const outcomes = new Map< - string, + number, Extract< ConversationReportEvent["data"], { type: "subagent_ended" } @@ -37,7 +37,7 @@ function subagentOutcomes( >(); for (const event of events) { if (event.data.type === "subagent_ended") { - outcomes.set(event.data.childConversationId, event.data.outcome); + outcomes.set(event.data.startedSeq, event.data.outcome); } } return outcomes; @@ -58,7 +58,6 @@ function subagentPart( : outcome === "success" ? "completed" : "running", - ...(outcome ? { outcome } : {}), }; } @@ -98,7 +97,7 @@ export function conversationTranscriptMessages( if (data.type === "subagent_started") { messages.push( eventMessage(event, "tool", [ - subagentPart(data, outcomes.get(data.childConversationId)), + subagentPart(data, outcomes.get(event.seq)), ]), ); continue; diff --git a/packages/junior-dashboard/src/client/markdownExport.ts b/packages/junior-dashboard/src/client/markdownExport.ts index 9765053e4..bc40e76b3 100644 --- a/packages/junior-dashboard/src/client/markdownExport.ts +++ b/packages/junior-dashboard/src/client/markdownExport.ts @@ -246,7 +246,7 @@ function appendSubagent( ): void { lines.push("", `### Subagent: ${headingText(part.subagentKind)}`); addEventMeta(lines, conversationTranscript, timestamp); - addMetaLine(lines, "Status", part.outcome ?? part.status); + addMetaLine(lines, "Status", part.status); } function appendTool( diff --git a/packages/junior-dashboard/src/client/pages/ConversationPage.tsx b/packages/junior-dashboard/src/client/pages/ConversationPage.tsx index 6aac44ae7..35b7dd172 100644 --- a/packages/junior-dashboard/src/client/pages/ConversationPage.tsx +++ b/packages/junior-dashboard/src/client/pages/ConversationPage.tsx @@ -138,9 +138,10 @@ export function ConversationPage(props: { } live={conversationIsLive(visualStatus, detail.data)} onOpenSubagentTranscript={({ part }) => { - const childConversationId = part.childConversationId; - if (!childConversationId) return; - setSubagentTarget({ conversationId: childConversationId, part }); + setSubagentTarget({ + conversationId: part.childConversationId, + part, + }); }} transcript={detail.data} /> diff --git a/packages/junior-dashboard/src/client/types.ts b/packages/junior-dashboard/src/client/types.ts index b7265ee25..9e648b97e 100644 --- a/packages/junior-dashboard/src/client/types.ts +++ b/packages/junior-dashboard/src/client/types.ts @@ -49,7 +49,7 @@ export type TranscriptViewToolCallPart = TranscriptViewReportingPart & { export type TranscriptViewSubagentPart = { bytes?: never; chars?: never; - childConversationId?: string; + childConversationId: string; id: string; input?: never; inputKeys?: never; @@ -57,14 +57,13 @@ export type TranscriptViewSubagentPart = { inputSizeChars?: never; inputType?: never; name?: never; - outcome?: "success" | "error" | "aborted"; output?: never; outputKeys?: never; outputSizeBytes?: never; outputSizeChars?: never; outputType?: never; redacted?: boolean; - status: TranscriptViewStatus; + status: "aborted" | "completed" | "error" | "running"; subagentKind: string; text?: never; type: "subagent"; diff --git a/packages/junior-dashboard/src/mock-conversations.ts b/packages/junior-dashboard/src/mock-conversations.ts index 5afaa3e9a..a433f5da6 100644 --- a/packages/junior-dashboard/src/mock-conversations.ts +++ b/packages/junior-dashboard/src/mock-conversations.ts @@ -163,8 +163,7 @@ function dashboardQaConversation(nowMs: number): ConversationDetailReport { }), reportEvent(3, iso(Date.parse(startedAt), 20_000), { type: "subagent_ended", - childConversationId: DASHBOARD_QA_PLAN_ID, - subagentKind: "advisor", + startedSeq: 2, outcome: "success", }), reportEvent(4, iso(Date.parse(startedAt), 25_000), { @@ -174,8 +173,7 @@ function dashboardQaConversation(nowMs: number): ConversationDetailReport { }), reportEvent(5, iso(Date.parse(startedAt), 44_000), { type: "subagent_ended", - childConversationId: DASHBOARD_QA_REVIEW_ID, - subagentKind: "advisor", + startedSeq: 4, outcome: "success", }), reportEvent(6, iso(Date.parse(startedAt), 50_000), { diff --git a/packages/junior-dashboard/tests/telemetry-components.test.tsx b/packages/junior-dashboard/tests/telemetry-components.test.tsx index b10b0e484..15c743d9d 100644 --- a/packages/junior-dashboard/tests/telemetry-components.test.tsx +++ b/packages/junior-dashboard/tests/telemetry-components.test.tsx @@ -258,8 +258,7 @@ describe("dashboard canonical-event components", () => { }), event(1, { type: "subagent_ended", - childConversationId: "child-1", - subagentKind: "advisor", + startedSeq: 0, outcome: "success", }), ])} @@ -294,7 +293,6 @@ describe("dashboard canonical-event components", () => { id: "child-1", childConversationId: "child-1", status: "completed", - outcome: "success", subagentKind: "advisor", }, }; @@ -312,6 +310,88 @@ describe("dashboard canonical-event components", () => { expect(html).toContain("Open conversation"); }); + it.each(["error", "aborted"] as const)( + "keeps the parent %s status when child detail says completed", + (status) => { + const child = conversation([], { + conversationId: "child-1", + displayTitle: "Advisor review", + status: "completed", + }); + client.setQueryData(["conversation", "child-1"], child); + const target: SubagentTranscriptTarget = { + conversationId: "child-1", + part: { + type: "subagent", + id: "child-1", + childConversationId: "child-1", + status, + subagentKind: "advisor", + }, + }; + + const html = renderToStaticMarkup( + + + {}} /> + + , + ); + expect(html).toContain(`${status} ·`); + expect(html).not.toContain("completed ·"); + }, + ); + + it("keeps a child transcript openable when its end event is missing", () => { + const parentHtml = renderToStaticMarkup( + + + {}} + view="rich" + /> + + , + ); + + expect(parentHtml).toContain('aria-label="Open advisor transcript"'); + + const child = conversation([], { + conversationId: "child-1", + displayTitle: "Advisor review", + status: "completed", + }); + client.setQueryData(["conversation", "child-1"], child); + const drawerHtml = renderToStaticMarkup( + + + {}} + /> + + , + ); + expect(drawerHtml).toContain("running ·"); + expect(drawerHtml).toContain("Open conversation"); + }); + it("announces child conversation load failures with an error tone", () => { const errorClient = new QueryClient({ defaultOptions: { @@ -343,8 +423,7 @@ describe("dashboard canonical-event components", () => { type: "subagent", id: "child-error", childConversationId: "child-error", - status: "completed", - outcome: "error", + status: "error", subagentKind: "advisor", }, }; diff --git a/packages/junior-dashboard/tests/transcriptRenderModel.test.ts b/packages/junior-dashboard/tests/transcriptRenderModel.test.ts index 048cebd79..c32a21339 100644 --- a/packages/junior-dashboard/tests/transcriptRenderModel.test.ts +++ b/packages/junior-dashboard/tests/transcriptRenderModel.test.ts @@ -120,8 +120,7 @@ describe("canonical event transcript reduction", () => { }), event(3, "2026-01-01T00:00:03.000Z", { type: "subagent_ended", - childConversationId: "child-1", - subagentKind: "advisor", + startedSeq: 2, outcome: "success", }), event(4, "2026-01-01T00:00:04.000Z", { @@ -142,12 +141,43 @@ describe("canonical event transcript reduction", () => { expect(entries[2]).toMatchObject({ part: { childConversationId: "child-1", - outcome: "success", status: "completed", }, }); }); + it("correlates repeated child outcomes by start sequence", () => { + const messages = conversationTranscriptMessages( + conversation([ + event(0, "2026-01-01T00:00:00.000Z", { + type: "subagent_started", + childConversationId: "child-1", + subagentKind: "advisor", + }), + event(1, "2026-01-01T00:00:01.000Z", { + type: "subagent_started", + childConversationId: "child-1", + subagentKind: "advisor", + }), + event(2, "2026-01-01T00:00:02.000Z", { + type: "subagent_ended", + startedSeq: 1, + outcome: "success", + }), + event(3, "2026-01-01T00:00:03.000Z", { + type: "subagent_ended", + startedSeq: 0, + outcome: "error", + }), + ]), + ); + + expect(messages.map((message) => message.parts[0])).toMatchObject([ + { status: "error" }, + { status: "completed" }, + ]); + }); + it("projects only failed deliveries as neutral failure history", () => { const entries = groupTranscriptMessages( conversationTranscriptMessages( diff --git a/packages/junior/src/api/conversations/events.ts b/packages/junior/src/api/conversations/events.ts index 86570e90b..83b495794 100644 --- a/packages/junior/src/api/conversations/events.ts +++ b/packages/junior/src/api/conversations/events.ts @@ -8,11 +8,6 @@ import { type ConversationReportEventData, } from "./schema"; -interface SubagentReference { - childConversationId: string; - subagentKind: string; -} - function isRecord(value: unknown): value is Record { return value !== null && typeof value === "object" && !Array.isArray(value); } @@ -118,24 +113,24 @@ export function projectConversationReportEvents(args: { canExposePayload: boolean; events: ConversationEvent[]; }): ConversationReportEvent[] { - const subagents = new Map(); + const subagentStarts = new Map(); const projected: ConversationReportEvent[] = []; for (const event of args.events) { let data: ConversationReportEventData | undefined; if (event.data.type === "subagent_started") { - const reference: SubagentReference = { + subagentStarts.set(event.data.subagentInvocationId, event.seq); + data = { + type: "subagent_started", childConversationId: event.data.childConversationId, subagentKind: event.data.subagentKind, }; - subagents.set(event.data.subagentInvocationId, reference); - data = { type: "subagent_started", ...reference }; } else if (event.data.type === "subagent_ended") { - const reference = subagents.get(event.data.subagentInvocationId); - if (reference) { + const startedSeq = subagentStarts.get(event.data.subagentInvocationId); + if (startedSeq !== undefined) { data = { type: "subagent_ended", - ...reference, + startedSeq, outcome: event.data.outcome, }; } diff --git a/packages/junior/src/api/conversations/schema.ts b/packages/junior/src/api/conversations/schema.ts index c4869d591..43d2b25e0 100644 --- a/packages/junior/src/api/conversations/schema.ts +++ b/packages/junior/src/api/conversations/schema.ts @@ -133,8 +133,7 @@ const conversationReportSubagentStartedEventDataSchema = z const conversationReportSubagentEndedEventDataSchema = z .object({ type: z.literal("subagent_ended"), - childConversationId: z.string().min(1), - subagentKind: z.string().min(1), + startedSeq: z.number().int().nonnegative(), outcome: z.enum(["success", "error", "aborted"]), }) .strict(); diff --git a/packages/junior/tests/integration/dashboard-reporting.test.ts b/packages/junior/tests/integration/dashboard-reporting.test.ts index ccf64f601..6992906e5 100644 --- a/packages/junior/tests/integration/dashboard-reporting.test.ts +++ b/packages/junior/tests/integration/dashboard-reporting.test.ts @@ -298,8 +298,7 @@ describe("dashboard canonical event reporting", () => { }, { type: "subagent_ended", - childConversationId: `${conversationId}:child`, - subagentKind: "review", + startedSeq: 7, outcome: "success", }, { type: "context_compacted" }, diff --git a/packages/junior/tests/unit/api/conversation-events.test.ts b/packages/junior/tests/unit/api/conversation-events.test.ts index bb190caed..f615851ba 100644 --- a/packages/junior/tests/unit/api/conversation-events.test.ts +++ b/packages/junior/tests/unit/api/conversation-events.test.ts @@ -320,7 +320,7 @@ describe("conversation report event projection", () => { }), event(7, { type: "subagent_started", - subagentInvocationId: "private-invocation-id", + subagentInvocationId: "subagent-invocation-1", subagentKind: "advisor", modelId: "private-child-model-id", parentToolCallId: "private-parent-tool-id", @@ -329,7 +329,7 @@ describe("conversation report event projection", () => { }), event(8, { type: "subagent_ended", - subagentInvocationId: "private-invocation-id", + subagentInvocationId: "subagent-invocation-1", outcome: "error", errorCode: "private-child-error-code", }), @@ -361,8 +361,7 @@ describe("conversation report event projection", () => { }, { type: "subagent_ended", - childConversationId: "child-conversation-1", - subagentKind: "advisor", + startedSeq: 7, outcome: "error", }, { type: "turn_lifecycle", turnId: "turn-2", state: "no_reply" }, @@ -374,7 +373,7 @@ describe("conversation report event projection", () => { "private-handoff-model-id", "private-rollback-model-id", "123.456", - "private-invocation-id", + "subagent-invocation-1", "private-child-model-id", "private-parent-tool-id", "private-reasoning-level", @@ -422,6 +421,41 @@ describe("conversation report event projection", () => { data: { type: "visible_message", messageId: "m1", role: "user" }, }).success, ).toBe(false); + + expect( + conversationReportEventSchema.safeParse({ + ...valid, + data: { + type: "subagent_started", + childConversationId: "child-1", + subagentKind: "advisor", + }, + }).success, + ).toBe(true); + + const subagentEnded = { + ...valid, + data: { + type: "subagent_ended", + startedSeq: 1, + outcome: "success", + }, + }; + expect(conversationReportEventSchema.safeParse(subagentEnded).success).toBe( + true, + ); + expect( + conversationReportEventSchema.safeParse({ + ...subagentEnded, + data: { ...subagentEnded.data, startedSeq: undefined }, + }).success, + ).toBe(false); + expect( + conversationReportEventSchema.safeParse({ + ...subagentEnded, + data: { ...subagentEnded.data, childConversationId: "child-1" }, + }).success, + ).toBe(false); }); it("rejects non-increasing event sequences at the detail boundary", () => { From 6df7502a7f04d741c84db33c9966ef1b7903a336 Mon Sep 17 00:00:00 2001 From: David Cramer Date: Wed, 15 Jul 2026 13:22:34 -0700 Subject: [PATCH 25/58] ref(runtime): Remove obsolete child projection offset Treat every model projection as conversation-local now that shared child history no longer exists. Simplify turn cursor conversion around the canonical event sequence. Co-Authored-By: OpenAI Codex --- .../src/chat/conversations/projection.ts | 30 ++++--------------- .../junior/src/chat/state/turn-session.ts | 13 ++++---- 2 files changed, 10 insertions(+), 33 deletions(-) diff --git a/packages/junior/src/chat/conversations/projection.ts b/packages/junior/src/chat/conversations/projection.ts index 99d8eb739..3a86bc6e6 100644 --- a/packages/junior/src/chat/conversations/projection.ts +++ b/packages/junior/src/chat/conversations/projection.ts @@ -21,6 +21,7 @@ import { createSqlConversationEventStore } from "@/chat/conversations/sql/histor import { withConversationEventLock } from "@/chat/conversations/sql/event-lock"; import { projectConversationEvents, + type PiConversationEventProjection, type PiConversationProjection, } from "@/chat/pi/conversation-events"; @@ -177,27 +178,14 @@ export async function loadTurnProjection(args: { conversationId: string; committedSeq: number; includeTail: boolean; -}): Promise< - | (PiConversationProjection & { - /** Number of inherited messages before the child-local projection. */ - localMessageStartIndex: number; - /** Child-local message event sequences only. */ - seqs: number[]; - }) - | undefined -> { +}): Promise { const eventStore = getConversationEventStore(); // A record that committed no messages materializes the live projection, the // same way count-based records with a zero cursor did. if (args.committedSeq < 0) { - const projection = projectConversationEvents( + return projectConversationEvents( await eventStore.loadCurrentEpoch(args.conversationId), ); - return { - ...projection, - localMessageStartIndex: 0, - seqs: projection.seqs, - }; } const history = await eventStore.loadHistory(args.conversationId); const committedEvent = history.find( @@ -212,12 +200,7 @@ export async function loadTurnProjection(args: { const localEvents = args.includeTail ? epochEvents : epochEvents.filter((event) => event.seq <= args.committedSeq); - const projection = projectConversationEvents(localEvents); - return { - ...projection, - localMessageStartIndex: 0, - seqs: projection.seqs, - }; + return projectConversationEvents(localEvents); } /** Load MCP providers durably connected in this conversation's current epoch. */ @@ -259,9 +242,7 @@ export async function commitMessages(args: { executor?: JuniorSqlDatabase; }): Promise<{ committedSeq: number; - /** Index where child-local messages begin in the returned provenance. */ - localMessageStartIndex: number; - /** Child-local message event sequences only. */ + /** Event sequence for every projected model message. */ messageSeqs: number[]; provenance: ConversationMessageProvenance[]; }> { @@ -348,7 +329,6 @@ async function commitMessagesLocked( ); return { committedSeq: committed.seqs.at(-1) ?? -1, - localMessageStartIndex: 0, messageSeqs: committed.seqs, provenance: nextLocalProvenance, }; diff --git a/packages/junior/src/chat/state/turn-session.ts b/packages/junior/src/chat/state/turn-session.ts index 3a5ab32b8..08ff78589 100644 --- a/packages/junior/src/chat/state/turn-session.ts +++ b/packages/junior/src/chat/state/turn-session.ts @@ -375,8 +375,7 @@ export async function getAgentTurnSessionRecord( const turnStartMessageIndex = parsed.turnStartSeq === undefined ? undefined - : piProjection.localMessageStartIndex + - piProjection.seqs.filter((seq) => seq <= parsed.turnStartSeq!).length; + : piProjection.seqs.filter((seq) => seq <= parsed.turnStartSeq!).length; return materializeAgentTurnSessionRecord( parsed, @@ -615,17 +614,15 @@ export async function upsertAgentTurnSessionRecord(args: { const turnStartSeq = args.turnStartMessageIndex === undefined ? existingRecord?.turnStartSeq - : args.turnStartMessageIndex <= commit.localMessageStartIndex + : args.turnStartMessageIndex <= 0 ? -1 - : (commit.messageSeqs[ - args.turnStartMessageIndex - commit.localMessageStartIndex - 1 - ] ?? commit.committedSeq); + : (commit.messageSeqs[args.turnStartMessageIndex - 1] ?? + commit.committedSeq); const turnStartMessageIndex = args.turnStartMessageIndex ?? (turnStartSeq === undefined ? undefined - : commit.localMessageStartIndex + - commit.messageSeqs.filter((seq) => seq <= turnStartSeq).length); + : commit.messageSeqs.filter((seq) => seq <= turnStartSeq).length); return await setStoredRecord({ conversationStore: args.conversationStore, From 552ef1c637b1b344d41de18d4e68b8818c78da7c Mon Sep 17 00:00:00 2001 From: David Cramer Date: Wed, 15 Jul 2026 13:34:52 -0700 Subject: [PATCH 26/58] ref(conversations): Simplify immutable lineage locking Trace immutable parent links once and remove duplicate topology validation. Preserve root-derived privacy and event-lock fencing, then reread activity after locks so retention never acts on a stale child timestamp. Co-Authored-By: OpenAI Codex --- .../src/chat/conversations/sql/privacy.ts | 71 +------ .../src/chat/conversations/sql/purge.ts | 194 +++++++++--------- .../component/conversations/retention.test.ts | 52 +++++ .../tests/unit/conversations/privacy.test.ts | 30 +-- 4 files changed, 162 insertions(+), 185 deletions(-) diff --git a/packages/junior/src/chat/conversations/sql/privacy.ts b/packages/junior/src/chat/conversations/sql/privacy.ts index 73d56c101..c354fd4f9 100644 --- a/packages/junior/src/chat/conversations/sql/privacy.ts +++ b/packages/junior/src/chat/conversations/sql/privacy.ts @@ -11,16 +11,15 @@ interface LineageRow { } interface LineageCandidate { - path: string[]; + destinationId: string; rootConversationId: string; } async function readLineageRow( executor: JuniorSqlDatabase, conversationId: string, - lock: boolean, ): Promise { - const query = executor + const rows = await executor .db() .select({ parentId: juniorConversations.parentConversationId, @@ -28,7 +27,6 @@ async function readLineageRow( }) .from(juniorConversations) .where(eq(juniorConversations.conversationId, conversationId)); - const rows = lock ? await query.for("share") : await query; return rows[0]; } @@ -37,13 +35,11 @@ async function traceLineage( conversationId: string, ): Promise { let currentId = conversationId; - const path: string[] = []; const seen = new Set(); while (!seen.has(currentId) && seen.size < MAX_LINEAGE_DEPTH) { seen.add(currentId); - path.push(currentId); - const row = await readLineageRow(executor, currentId, false); + const row = await readLineageRow(executor, currentId); if (!row) return undefined; if (row.parentId) { @@ -54,35 +50,21 @@ async function traceLineage( if (!row.destinationId) { return undefined; } - return { path, rootConversationId: currentId }; + return { + destinationId: row.destinationId, + rootConversationId: currentId, + }; } return undefined; } -function lockedLineageIsConsistent( - candidate: LineageCandidate, - rows: Map, -): boolean { - for (const [index, conversationId] of candidate.path.entries()) { - const row = rows.get(conversationId); - if (!row) return false; - const expectedParentId = candidate.path[index + 1] ?? null; - if (row.parentId !== expectedParentId) return false; - if (!expectedParentId && !row.destinationId) { - return false; - } - } - return true; -} - /** * Resolve a conversation's privacy authority from its persisted root. * - * The lineage is discovered without locks, then locked and revalidated from - * root to requested conversation. Root-first ordering matches tree purges; - * callers that keep the transaction open receive a stable privacy decision. - * Missing, cyclic, over-depth, or concurrently changed lineage fails closed. + * Parent links are immutable after insertion. Missing, cyclic, or over-depth + * lineage fails closed, while the root destination is locked so callers that + * keep the transaction open receive a stable visibility decision. */ export async function resolveRootVisibility( executor: JuniorSqlDatabase, @@ -96,43 +78,12 @@ export async function resolveRootVisibility( return { rootConversationId: conversationId, visibility: null }; } - const rootRow = await readLineageRow( - executor, - candidate.rootConversationId, - true, - ); - if (!rootRow?.destinationId) { - return { - rootConversationId: candidate.rootConversationId, - visibility: null, - }; - } const destinations = await executor .db() .select({ visibility: juniorDestinations.visibility }) .from(juniorDestinations) - .where(eq(juniorDestinations.id, rootRow.destinationId)) + .where(eq(juniorDestinations.id, candidate.destinationId)) .for("share"); - - const lockedRows = new Map([ - [candidate.rootConversationId, rootRow], - ]); - for (const id of [...candidate.path].reverse().slice(1)) { - const row = await readLineageRow(executor, id, true); - if (!row) { - return { - rootConversationId: candidate.rootConversationId, - visibility: null, - }; - } - lockedRows.set(id, row); - } - if (!lockedLineageIsConsistent(candidate, lockedRows)) { - return { - rootConversationId: candidate.rootConversationId, - visibility: null, - }; - } return { rootConversationId: candidate.rootConversationId, visibility: destinations[0]?.visibility ?? null, diff --git a/packages/junior/src/chat/conversations/sql/purge.ts b/packages/junior/src/chat/conversations/sql/purge.ts index 3ff6e3e7d..b7c9ce903 100644 --- a/packages/junior/src/chat/conversations/sql/purge.ts +++ b/packages/junior/src/chat/conversations/sql/purge.ts @@ -6,6 +6,7 @@ import { juniorConversationMessages, juniorConversations, juniorDestinations, + juniorPendingDeliveries, } from "@/db/schema"; import { withConversationEventLock } from "./event-lock"; import { resolveRootVisibility } from "./privacy"; @@ -26,27 +27,23 @@ export interface PurgeTreeResult { interface ConversationTreeRow { conversationId: string; - depth: number; - lastActivityAt: Date; parentConversationId: string | null; } /** Discover a root and its current descendants via `parent_conversation_id`. */ async function discoverConversationTree( executor: JuniorSqlDatabase, - root: Omit, + root: ConversationTreeRow, ): Promise { const all = new Map([ - [root.conversationId, { ...root, depth: 0 }], + [root.conversationId, root], ]); let frontier = [root.conversationId]; - let depth = 1; while (frontier.length > 0) { const children = await executor .db() .select({ conversationId: juniorConversations.conversationId, - lastActivityAt: juniorConversations.lastActivityAt, parentConversationId: juniorConversations.parentConversationId, }) .from(juniorConversations) @@ -55,11 +52,10 @@ async function discoverConversationTree( frontier = []; for (const child of children) { if (!all.has(child.conversationId)) { - all.set(child.conversationId, { ...child, depth }); + all.set(child.conversationId, child); frontier.push(child.conversationId); } } - depth += 1; } return [...all.values()]; } @@ -178,8 +174,6 @@ export async function purgeConversationTree( .db() .select({ conversationId: juniorConversations.conversationId, - destinationId: juniorConversations.destinationId, - lastActivityAt: juniorConversations.lastActivityAt, parentConversationId: juniorConversations.parentConversationId, }) .from(juniorConversations) @@ -191,94 +185,102 @@ export async function purgeConversationTree( ) { return { purged: false, conversations: 0 }; } - const initialTree = await discoverConversationTree(executor, initialRoot); - for (const conversation of initialTree) { - await withConversationEventLock( - executor, - conversation.conversationId, - async () => undefined, - ); - } - const resolvedScrubMetadata = args.scrubMetadataFromRootVisibility - ? (await resolveRootVisibility(executor, args.rootConversationId)) - .visibility !== "public" - : args.scrubMetadata; + const tree = await discoverConversationTree(executor, initialRoot); - const roots = await executor - .db() - .select({ - conversationId: juniorConversations.conversationId, - destinationId: juniorConversations.destinationId, - lastActivityAt: juniorConversations.lastActivityAt, - parentConversationId: juniorConversations.parentConversationId, - }) - .from(juniorConversations) - .where(eq(juniorConversations.conversationId, args.rootConversationId)) - .for("update"); - const root = roots[0]; - if (!root || (args.retention && root.parentConversationId !== null)) { - return { purged: false, conversations: 0 }; - } - const destinations = root.destinationId - ? await executor + return await withConversationEventLock( + executor, + args.rootConversationId, + async () => { + // Parent links are immutable and child creation is migration-only, so + // one unlocked traversal is authoritative. Acquire every event lock + // before row locks so a child writer can finish without deadlocking on + // its parent relationship. + for (const conversation of tree.slice(1)) { + await withConversationEventLock( + executor, + conversation.conversationId, + async () => undefined, + ); + } + + const roots = await executor + .db() + .select({ + conversationId: juniorConversations.conversationId, + destinationId: juniorConversations.destinationId, + parentConversationId: juniorConversations.parentConversationId, + }) + .from(juniorConversations) + .where( + eq(juniorConversations.conversationId, args.rootConversationId), + ) + .for("update"); + const root = roots[0]; + if (!root || (args.retention && root.parentConversationId !== null)) { + return { purged: false, conversations: 0 }; + } + const resolvedScrubMetadata = args.scrubMetadataFromRootVisibility + ? (await resolveRootVisibility(executor, args.rootConversationId)) + .visibility !== "public" + : args.scrubMetadata; + const destinations = root.destinationId + ? await executor + .db() + .select({ visibility: juniorDestinations.visibility }) + .from(juniorDestinations) + .where(eq(juniorDestinations.id, root.destinationId)) + .for("share") + : []; + const isPublic = destinations[0]?.visibility === "public"; + const ids = tree.map((conversation) => conversation.conversationId); + if (args.retention) { + const currentActivity = await executor + .db() + .select({ + conversationId: juniorConversations.conversationId, + lastActivityAt: juniorConversations.lastActivityAt, + }) + .from(juniorConversations) + .where(inArray(juniorConversations.conversationId, ids)); + if (currentActivity.length !== ids.length) { + return { purged: false, conversations: 0 }; + } + const windowMs = isPublic + ? args.retention.publicWindowMs + : args.retention.privateWindowMs; + const effectiveLastActivityAt = Math.max( + ...currentActivity.map((conversation) => + conversation.lastActivityAt.getTime(), + ), + ); + if (effectiveLastActivityAt >= args.nowMs - windowMs) { + return { purged: false, conversations: 0 }; + } + } + await executor + .db() + .delete(juniorPendingDeliveries) + .where(inArray(juniorPendingDeliveries.conversationId, ids)); + await executor + .db() + .delete(juniorConversationEvents) + .where(inArray(juniorConversationEvents.conversationId, ids)); + await executor .db() - .select({ visibility: juniorDestinations.visibility }) - .from(juniorDestinations) - .where(eq(juniorDestinations.id, root.destinationId)) - .for("share") - : []; - const isPublic = destinations[0]?.visibility === "public"; - const tree = await discoverConversationTree(executor, { - conversationId: root.conversationId, - lastActivityAt: root.lastActivityAt, - parentConversationId: root.parentConversationId, - }); - const initiallyDiscovered = new Map( - initialTree.map((conversation) => [ - conversation.conversationId, - conversation.parentConversationId, - ]), + .delete(juniorConversationMessages) + .where(inArray(juniorConversationMessages.conversationId, ids)); + await executor + .db() + .update(juniorConversations) + .set({ + transcriptPurgedAt: new Date(args.nowMs), + ...((args.retention ? !isPublic : resolvedScrubMetadata) + ? { title: null, channelName: null, actor: null } + : {}), + }) + .where(inArray(juniorConversations.conversationId, ids)); + return { purged: true, conversations: ids.length }; + }, ); - if ( - tree.length !== initiallyDiscovered.size || - tree.some( - (conversation) => - initiallyDiscovered.get(conversation.conversationId) !== - conversation.parentConversationId, - ) - ) { - return { purged: false, conversations: 0 }; - } - if (args.retention) { - const windowMs = isPublic - ? args.retention.publicWindowMs - : args.retention.privateWindowMs; - const effectiveLastActivityAt = Math.max( - ...tree.map((conversation) => conversation.lastActivityAt.getTime()), - ); - if (effectiveLastActivityAt >= args.nowMs - windowMs) { - return { purged: false, conversations: 0 }; - } - } - const ids = tree.map((conversation) => conversation.conversationId); - await executor - .db() - .delete(juniorConversationEvents) - .where(inArray(juniorConversationEvents.conversationId, ids)); - await executor - .db() - .delete(juniorConversationMessages) - .where(inArray(juniorConversationMessages.conversationId, ids)); - await executor - .db() - .update(juniorConversations) - .set({ - transcriptPurgedAt: new Date(args.nowMs), - ...((args.retention ? !isPublic : resolvedScrubMetadata) - ? { title: null, channelName: null, actor: null } - : {}), - }) - .where(inArray(juniorConversations.conversationId, ids)); - return { purged: true, conversations: ids.length }; }); } diff --git a/packages/junior/tests/component/conversations/retention.test.ts b/packages/junior/tests/component/conversations/retention.test.ts index a7b22abd3..490074f84 100644 --- a/packages/junior/tests/component/conversations/retention.test.ts +++ b/packages/junior/tests/component/conversations/retention.test.ts @@ -322,6 +322,58 @@ describe("retention purge job", () => { expect(await eventCount(fixture.sql, "fresh-child")).toBe(1); }); + it("keeps a tree when a child write finishes before purge locks the child", async () => { + const destinationId = await seedDestination(fixture.sql, "public"); + const nowMs = BASE_MS + 100 * DAY_MS; + await seedConversation(fixture.sql, { + conversationId: "racing-root", + destinationId, + lastActivityAtMs: BASE_MS, + }); + await seedConversation(fixture.sql, { + conversationId: "racing-child", + parentConversationId: "racing-root", + lastActivityAtMs: BASE_MS, + }); + + let childWriteCompleted = false; + const racingExecutor: JuniorSqlDatabase = { + db: () => fixture.sql.db(), + transaction: (callback) => fixture.sql.transaction(callback), + withLock: async (name, callback) => { + if ( + !childWriteCompleted && + name === "junior_conversation:event:racing-child" + ) { + await fixture.sql + .db() + .update(juniorConversations) + .set({ + lastActivityAt: new Date(nowMs), + updatedAt: new Date(nowMs), + }) + .where(eq(juniorConversations.conversationId, "racing-child")); + childWriteCompleted = true; + } + return await fixture.sql.withLock(name, callback); + }, + }; + + await expect( + purgeConversationTree(racingExecutor, { + rootConversationId: "racing-root", + nowMs, + retention: { + publicWindowMs: 90 * DAY_MS, + privateWindowMs: 14 * DAY_MS, + }, + }), + ).resolves.toEqual({ purged: false, conversations: 0 }); + expect(childWriteCompleted).toBe(true); + expect(await eventCount(fixture.sql, "racing-root")).toBe(1); + expect(await eventCount(fixture.sql, "racing-child")).toBe(1); + }); + it("purges an expired root whose remaining content exists only on a child", async () => { const dest = await seedDestination(fixture.sql, "public"); await seedConversation(fixture.sql, { diff --git a/packages/junior/tests/unit/conversations/privacy.test.ts b/packages/junior/tests/unit/conversations/privacy.test.ts index 72a376030..919737629 100644 --- a/packages/junior/tests/unit/conversations/privacy.test.ts +++ b/packages/junior/tests/unit/conversations/privacy.test.ts @@ -53,14 +53,12 @@ function scriptedExecutor(args: { } describe("resolveRootVisibility", () => { - it("resolves visibility only from a consistent root destination", async () => { + it("resolves visibility from the persisted root destination", async () => { const result = await resolveRootVisibility( scriptedExecutor({ conversations: [ { destinationId: null, parentId: "root" }, { destinationId: "destination", parentId: null }, - { destinationId: "destination", parentId: null }, - { destinationId: null, parentId: "root" }, ], visibility: "public", }), @@ -109,32 +107,6 @@ describe("resolveRootVisibility", () => { expect(result.visibility).toBeNull(); }); - it("fails closed when lineage changes between discovery and locking", async () => { - const result = await resolveRootVisibility( - scriptedExecutor({ - conversations: [ - { destinationId: null, parentId: "root" }, - { - destinationId: "destination", - parentId: null, - }, - { - destinationId: "destination", - parentId: null, - }, - { - destinationId: null, - parentId: "different-root", - }, - ], - visibility: "public", - }), - "child", - ); - - expect(result.visibility).toBeNull(); - }); - it("fails closed when lineage exceeds the traversal bound", async () => { const conversations = Array.from({ length: 32 }, (_, index) => ({ destinationId: null, From 86d9f4cf553758eb0618855b37e36caf203e73c4 Mon Sep 17 00:00:00 2001 From: David Cramer Date: Wed, 15 Jul 2026 14:12:15 -0700 Subject: [PATCH 27/58] ref(dashboard): Remove unreachable transcript shapes Consume only the privacy-safe semantic events exposed by the conversation API, and delete the legacy Pi payload rendering universe. Keep canonical subagent lifecycle status visible and searchable. Co-Authored-By: OpenAI Codex --- .../components/ConversationTranscript.tsx | 390 ++--------------- .../client/components/TelemetryMetrics.tsx | 12 +- .../client/components/ToolValueInspector.tsx | 394 ------------------ .../components/TranscriptHeadingRow.tsx | 14 - .../components/TranscriptSubagentView.tsx | 8 +- .../components/TranscriptThinkingView.tsx | 87 ---- .../client/components/TranscriptToolRun.tsx | 89 +--- .../client/components/TranscriptToolView.tsx | 250 +---------- .../components/transcriptBottomPinning.ts | 39 +- .../client/components/transcriptPreview.ts | 20 - .../components/transcriptRenderModel.ts | 309 ++------------ .../client/components/transcriptSearch.tsx | 21 +- .../src/client/eventTranscript.ts | 2 +- .../junior-dashboard/src/client/format.ts | 89 +--- .../src/client/markdownExport.ts | 170 +------- .../src/client/toolInvocations.ts | 16 - packages/junior-dashboard/src/client/types.ts | 76 +--- .../tests/markdownExport.test.ts | 8 +- .../tests/telemetry-components.test.tsx | 116 ++---- .../tests/transcriptRenderModel.test.ts | 27 +- .../junior/src/api/conversations/events.ts | 35 +- .../junior/src/api/conversations/schema.ts | 18 - .../unit/api/conversation-events.test.ts | 30 +- 23 files changed, 217 insertions(+), 2003 deletions(-) delete mode 100644 packages/junior-dashboard/src/client/components/ToolValueInspector.tsx delete mode 100644 packages/junior-dashboard/src/client/components/TranscriptThinkingView.tsx delete mode 100644 packages/junior-dashboard/src/client/components/transcriptPreview.ts delete mode 100644 packages/junior-dashboard/src/client/toolInvocations.ts diff --git a/packages/junior-dashboard/src/client/components/ConversationTranscript.tsx b/packages/junior-dashboard/src/client/components/ConversationTranscript.tsx index 8d9ac65e7..9a8f32e15 100644 --- a/packages/junior-dashboard/src/client/components/ConversationTranscript.tsx +++ b/packages/junior-dashboard/src/client/components/ConversationTranscript.tsx @@ -1,9 +1,4 @@ -import { - Fragment, - useState, - type ClipboardEventHandler, - type ReactNode, -} from "react"; +import { Fragment, type ClipboardEventHandler, type ReactNode } from "react"; import { Bot, CircleAlert, @@ -12,20 +7,18 @@ import { type LucideIcon, } from "lucide-react"; -import { HighlightedCode } from "../code"; +import { countStructuredBlockChildren, HighlightedCode } from "../code"; import { detectLanguage, transcriptRoleKind, - formatBytes, formatMessageTimestamp, - formatMs, formatTranscriptDuration, actorLabel, + parseMarkdownBlocks, summarizeCost, summarizeTurns, summarizeToolCalls, summarizeUsage, - stringifyPartValue, unavailableTranscriptLabel, visualStatusForSummary, } from "../format"; @@ -34,15 +27,12 @@ import { conversationTranscriptMessages } from "../eventTranscript"; import type { ConversationTranscript, TranscriptViewMessage, - TranscriptViewPart, TranscriptViewSubagentPart, } from "../types"; import { StatusBadge } from "./StatusBadge"; -import { ToolFrame, toolFrameClass } from "./ToolFrame"; import { TranscriptHeadingMeta, TranscriptHeadingRow, - TranscriptThoughtLabel, } from "./TranscriptHeadingRow"; import { MetricList, type MetricListItem } from "./Metric"; import { @@ -53,26 +43,21 @@ import { ToolCallsMetric, } from "./TelemetryMetrics"; import { TranscriptText } from "./TranscriptText"; -import { TranscriptThinkingView } from "./TranscriptThinkingView"; import { TranscriptSubagentView } from "./TranscriptSubagentView"; import { TranscriptContextEventView } from "./TranscriptContextEventView"; import { TranscriptToolRun } from "./TranscriptToolRun"; import { TranscriptToolView } from "./TranscriptToolView"; import { shouldCopyRawTranscript } from "./transcriptCopy"; import { - countRenderedTranscriptChildren, groupTranscriptMessages, - groupTranscriptParts, messageRawText, - type RenderedToolRunEntry, - type RenderedTranscriptPart, + type RenderedToolEntry, type TranscriptViewMode, } from "./transcriptRenderModel"; import { transcriptEmptyClass, mutedTranscriptMetaClass, } from "./transcriptStyles"; -import { previewToolValue } from "./transcriptPreview"; import { entryMatchesSearch, useTranscriptSearch } from "./transcriptSearch"; type TranscriptEntry = ReturnType[number]; @@ -80,7 +65,6 @@ type TranscriptContextEntry = Extract; type TranscriptFailureEntry = Extract; type TranscriptMessageEntry = Extract; type TranscriptSubagentEntry = Extract; -type TranscriptThinkingEntry = Extract; type TranscriptToolEntry = Extract; /** Render one conversation transcript segment as actor messages and tool events. */ @@ -334,21 +318,11 @@ function VisibleTranscriptEntries(props: { /> )} - renderThinking={(entry, index) => ( - - )} renderTool={(entry, index) => ( )} /> @@ -362,7 +336,6 @@ function TranscriptEntryList(props: { renderFailure: (entry: TranscriptFailureEntry, index: number) => ReactNode; renderMessage: (entry: TranscriptMessageEntry, index: number) => ReactNode; renderSubagent: (entry: TranscriptSubagentEntry, index: number) => ReactNode; - renderThinking: (entry: TranscriptThinkingEntry, index: number) => ReactNode; renderTool: (entry: TranscriptToolEntry, index: number) => ReactNode; }) { const search = useTranscriptSearch(); @@ -371,14 +344,11 @@ function TranscriptEntryList(props: { for (let index = 0; index < props.entries.length; ) { const entry = props.entries[index]!; - if (entry.kind === "tool" || entry.kind === "thinking") { + if (entry.kind === "tool") { const startIndex = index; - const runEntries: RenderedToolRunEntry[] = []; - while ( - props.entries[index]?.kind === "tool" || - props.entries[index]?.kind === "thinking" - ) { - runEntries.push(props.entries[index] as RenderedToolRunEntry); + const runEntries: RenderedToolEntry[] = []; + while (props.entries[index]?.kind === "tool") { + runEntries.push(props.entries[index] as RenderedToolEntry); index += 1; } const visibleEntries = search.active @@ -393,7 +363,6 @@ function TranscriptEntryList(props: { entries={visibleEntries} key={`${props.keyPrefix}:tool-run:${startIndex}`} keyPrefix={props.keyPrefix} - renderThinking={props.renderThinking} renderTool={props.renderTool} startIndex={startIndex} />, @@ -428,57 +397,35 @@ function TranscriptEntryList(props: { } function TranscriptFailureView(props: { - outcome: "error" | "aborted" | "delivery_failed"; + outcome: "error" | "delivery_failed"; timestamp?: number; }) { const timestamp = formatMessageTimestamp(props.timestamp); - const isError = props.outcome !== "aborted"; const deliveryFailed = props.outcome === "delivery_failed"; return (