feat(migrations): Unify schema and data journals

.craft.yml

@@ -5,6 +5,9 @@ targets:
   - name: npm
     id: "@sentry/junior"
     includeNames: /^sentry-junior-\d.*\.tgz$/
+  - name: npm
+    id: "@sentry/junior-migrations"
+    includeNames: /^sentry-junior-migrations-\d.*\.tgz$/
   - name: npm
     id: "@sentry/junior-plugin-api"
     includeNames: /^sentry-junior-plugin-api-\d.*\.tgz$/

.github/workflows/ci.yml

@@ -49,6 +49,7 @@ jobs:
               - .github/actions/**
             core:
               - packages/junior/**
+              - packages/junior-migrations/**
               - packages/junior-plugin-api/**
               - packages/junior-scheduler/**
               - packages/junior-testing/**
@@ -215,6 +216,7 @@ jobs:
       - uses: ./.github/actions/setup-node-pnpm
       - run: pnpm --filter @sentry/junior-memory build
       - run: pnpm --filter @sentry/junior-github build
+      - run: pnpm --filter @sentry/junior-migrations test
       - run: pnpm --filter @sentry/junior-memory test
       - run: pnpm --filter @sentry/junior-github test
 
@@ -341,6 +343,7 @@ jobs:
         run: |
           mkdir -p artifacts
           pnpm --filter @sentry/junior pack --pack-destination artifacts
+          pnpm --filter @sentry/junior-migrations pack --pack-destination artifacts
           pnpm --filter @sentry/junior-plugin-api pack --pack-destination artifacts
           pnpm --filter @sentry/junior-agent-browser pack --pack-destination artifacts
           pnpm --filter @sentry/junior-amplitude pack --pack-destination artifacts

CONTRIBUTING.md

@@ -119,6 +119,7 @@ pnpm build:pkg
 This repo uses Craft for manual lockstep npm releases of:
 
 - `@sentry/junior`
+- `@sentry/junior-migrations`
 - `@sentry/junior-plugin-api`
 - `@sentry/junior-agent-browser`
 - `@sentry/junior-amplitude`

README.md

@@ -23,6 +23,7 @@ Start here:
 | Package                        | Purpose                                                                      |
 | ------------------------------ | ---------------------------------------------------------------------------- |
 | `@sentry/junior`               | Core Slack bot runtime                                                       |
+| `@sentry/junior-migrations`    | Mixed Drizzle SQL and TypeScript migration format and runner                 |
 | `@sentry/junior-plugin-api`    | Lightweight plugin API types and helpers                                     |
 | `@sentry/junior-agent-browser` | Agent Browser plugin package for browser automation                          |
 | `@sentry/junior-amplitude`     | Read-only Amplitude product analytics through Amplitude's hosted MCP server  |

ast-grep/rules/no-core-plugin-dynamic-imports.yml

@@ -12,4 +12,4 @@ constraints:
     all:
       - regex: '^["@'']@sentry/junior-.+["@'']$'
       - not:
-          regex: '^["@'']@sentry/junior-plugin-api["@'']$'
+          regex: '^["@'']@sentry/junior-(?:plugin-api|migrations)["@'']$'

ast-grep/rules/no-core-plugin-reexports.yml

@@ -14,4 +14,4 @@ rule:
     - not:
         has:
           kind: string
-          regex: '^["@'']@sentry/junior-plugin-api["@'']$'
+          regex: '^["@'']@sentry/junior-(?:plugin-api|migrations)["@'']$'

ast-grep/rules/no-core-plugin-static-imports.yml

@@ -14,4 +14,4 @@ rule:
     - not:
         has:
           kind: string
-          regex: '^["@'']@sentry/junior-plugin-api["@'']$'
+          regex: '^["@'']@sentry/junior-(?:plugin-api|migrations)["@'']$'

package.json

@@ -12,19 +12,19 @@
     "worktree:setup": "node scripts/worktree.mjs setup",
     "cloudflare:token": "node scripts/refresh-cloudflare-tunnel-token.mjs",
     "prepare": "simple-git-hooks",
-    "lint": "pnpm --filter @sentry/junior lint && pnpm --filter @sentry/junior-memory lint && pnpm --filter @sentry/junior-github lint && pnpm ast-grep:lint && pnpm package:lint",
+    "lint": "pnpm --filter @sentry/junior-migrations lint && pnpm --filter @sentry/junior lint && pnpm --filter @sentry/junior-memory lint && pnpm --filter @sentry/junior-github lint && pnpm ast-grep:lint && pnpm package:lint",
     "lint:fix": "pnpm --filter @sentry/junior lint:fix",
     "ast-grep:lint": "ast-grep scan",
     "lint-staged": "lint-staged",
-    "build": "pnpm --filter @sentry/junior build && pnpm --filter @sentry/junior-memory build && pnpm --filter @sentry/junior-github build && pnpm --filter @sentry/junior-dashboard build",
+    "build": "pnpm --filter @sentry/junior-migrations build && pnpm --filter @sentry/junior build && pnpm --filter @sentry/junior-memory build && pnpm --filter @sentry/junior-github build && pnpm --filter @sentry/junior-dashboard build",
     "build:example": "pnpm --filter @sentry/junior-example build",
     "docs:dev": "pnpm --filter @sentry/junior-docs dev",
     "docs:build": "pnpm --filter @sentry/junior-docs build",
     "docs:check": "pnpm --filter @sentry/junior-docs check",
-    "package:lint": "for pkg in packages/junior packages/junior-plugin-api packages/junior-scheduler packages/junior-memory packages/junior-dashboard packages/junior-github packages/junior-agent-browser packages/junior-amplitude packages/junior-cloudflare packages/junior-datadog packages/junior-hex packages/junior-linear packages/junior-maintenance packages/junior-notion packages/junior-sentry packages/junior-vercel; do pnpm exec publint \"$pkg\" || exit $?; done",
+    "package:lint": "for pkg in packages/junior packages/junior-migrations packages/junior-plugin-api packages/junior-scheduler packages/junior-memory packages/junior-dashboard packages/junior-github packages/junior-agent-browser packages/junior-amplitude packages/junior-cloudflare packages/junior-datadog packages/junior-hex packages/junior-linear packages/junior-maintenance packages/junior-notion packages/junior-sentry packages/junior-vercel; do pnpm exec publint \"$pkg\" || exit $?; done",
     "release:check": "node scripts/check-release-config.mjs",
     "start": "pnpm --filter @sentry/junior-example dev",
-    "test": "pnpm --filter @sentry/junior build && pnpm --filter @sentry/junior-memory build && pnpm --filter @sentry/junior-github build && pnpm --filter @sentry/junior-dashboard build && pnpm --filter @sentry/junior test && pnpm --filter @sentry/junior-memory test && pnpm --filter @sentry/junior-github test && pnpm --filter @sentry/junior-dashboard test",
+    "test": "pnpm --filter @sentry/junior-migrations test && pnpm --filter @sentry/junior build && pnpm --filter @sentry/junior-memory build && pnpm --filter @sentry/junior-github build && pnpm --filter @sentry/junior-dashboard build && pnpm --filter @sentry/junior test && pnpm --filter @sentry/junior-memory test && pnpm --filter @sentry/junior-github test && pnpm --filter @sentry/junior-dashboard test",
     "test:e2e:dashboard": "pnpm --filter @sentry/junior-dashboard build && playwright test -c packages/junior-dashboard/playwright.config.ts",
     "test:watch": "pnpm --filter @sentry/junior test:watch",
     "evals": "pnpm --filter @sentry/junior-evals evals",

packages/docs/src/content/docs/cli/upgrade.md

@@ -1,8 +1,8 @@
 ---
 title: "junior upgrade"
-description: "Run one-shot Junior state upgrade migrations."
+description: "Apply Junior schema and data migrations."
 type: reference
-summary: Move persisted Junior state forward after upgrading packages.
+summary: Move configured SQL schemas and persisted state forward after upgrades.
 prerequisites:
   - /start-here/quickstart/
 related:
@@ -11,7 +11,10 @@ related:
   - /cli/snapshot-create/
 ---
 
-Use `junior upgrade` after installing a Junior release that includes a one-shot state migration. The command mutates the configured state stores, so run it from the same app environment that has the production state and SQL environment variables configured for the deployment you are upgrading.
+Use `junior upgrade` after installing a Junior release that includes schema or
+data migrations. The command mutates the configured SQL database and state
+stores, so run it from the same app environment that has the production state
+and SQL environment variables configured for the deployment you are upgrading.
 
 ## Usage
 
@@ -25,13 +28,25 @@ The command takes no extra arguments.
 
 ## What it does
 
-`junior upgrade` runs registered migrations sequentially. Current migrations:
+`junior upgrade` runs migrations sequentially. Core and plugin migration
+directories use Drizzle Kit's ordered journal and may contain either generated
+SQL schema migrations or TypeScript data migrations targeting a versioned
+host capability API. Current
+upgrade work includes:
 
+- Apply core and enabled-plugin schema and data journal entries.
+- Rewrite retained turn-session records from legacy storage shapes before the
+  new runtime reads them.
 - Move legacy `junior:conversation-work:*` Redis state into the newer conversation record and index state used by the durable worker and dashboard feed.
 - Backfill retained conversation records into the shared Junior SQL database. The upgrade requires `DATABASE_URL`.
-- Repair legacy token and estimated-cost rollups from durable SQL agent steps in bounded batches. Conversations that are active during the repair are left unchanged and can be repaired by rerunning the command after they become idle.
 
-The migrations are idempotent: rerunning them skips records that were already moved, removes stale legacy index entries that no longer have a record, and upserts SQL conversation rows. The SQL conversation backfill copies a bounded legacy slice of Redis conversation metadata; after cutover, durable conversation metadata is written to SQL while Redis remains the transcript and execution/cache store.
+Completed journal entries are tracked individually, and TypeScript migrations
+can checkpoint progress for safe retries. Legacy backfills remain idempotent:
+rerunning them skips records that were already moved, removes stale legacy
+index entries that no longer have a record, and upserts SQL conversation rows.
+The SQL conversation backfill copies a bounded legacy slice of Redis
+conversation metadata; after cutover, durable conversation metadata is written
+to SQL while Redis remains the transcript and execution/cache store.
 
 ## Vercel deploys
 
@@ -57,12 +72,10 @@ Typical logs look like this:
 
 ```text
 Running Junior upgrade migrations...
-Running migration migrate-redis-conversation-state...
-Finished migration migrate-redis-conversation-state: scanned=2 migrated=1 existing=0 missing=1
-Running migration backfill-conversations-sql...
-Finished migration backfill-conversations-sql: scanned=2 migrated=2 existing=0 missing=0
-Running migration repair-conversation-usage...
-Finished migration repair-conversation-usage: scanned=2 migrated=1 existing=1 missing=0
+Running migration core-migrations...
+Finished migration core-migrations: scanned=10 migrated=6 existing=4 missing=0 skipped=0
+Running migration plugin-migrations...
+Finished migration plugin-migrations: scanned=8 migrated=8 existing=0 missing=0 skipped=0
 Junior upgrade complete.
 ```
 
@@ -84,10 +97,6 @@ After running the command:
 2. Confirm the migration summary has the expected `scanned` and `migrated` counts.
 3. Run `pnpm exec junior check` before building or deploying the app.
 
-A nonzero `missing` count for `repair-conversation-usage` means retained SQL assistant messages did not contain usable, schema-safe usage values. Junior leaves those totals unchanged.
-
-The command does not rewrite legacy duration totals. Run summaries are TTL-bound and do not carry an authoritative completeness marker, so even a non-empty retained index may omit a run. Replacing a total from that evidence could silently undercount it.
-
 ## Next step
 
 Run [junior check](/cli/check/) after the upgrade, then continue with [junior snapshot create](/cli/snapshot-create/) if your plugins need sandbox dependencies.

packages/docs/src/content/docs/contribute/releasing.md

@@ -12,6 +12,7 @@ related:
 Junior uses lockstep package releases for:
 
 - `@sentry/junior`
+- `@sentry/junior-migrations`
 - `@sentry/junior-plugin-api`
 - `@sentry/junior-agent-browser`
 - `@sentry/junior-amplitude`

packages/junior-evals/postgres-global-setup.ts

@@ -2,7 +2,7 @@ import path from "node:path";
 import { fileURLToPath } from "node:url";
 import type { TestProject } from "vitest/node";
 import { setupJuniorPostgresHarness } from "@junior-tests/fixtures/postgres/global-setup";
-import { migratePluginSchemas } from "@/chat/plugins/migrations";
+import { bootstrapPluginSchemas } from "@/chat/plugins/migrations";
 
 const workspaceRoot = path.resolve(
   path.dirname(fileURLToPath(import.meta.url)),
@@ -14,7 +14,7 @@ export default async function setup(
 ): Promise<() => Promise<void>> {
   return await setupJuniorPostgresHarness(project, {
     migrateTemplate: async (executor) => {
-      await migratePluginSchemas(executor, [
+      await bootstrapPluginSchemas(executor, [
         {
           dir: path.resolve(workspaceRoot, "packages/junior-memory/migrations"),
           pluginName: "memory",

packages/junior-memory/README.md

@@ -55,6 +55,8 @@ exported types, tools, and tests are authoritative.
 - `MEMORY_RECALL_MAX_VECTOR_DISTANCE` or
   `recallMaxVectorDistance` configures the vector candidate threshold.
 - Generate schema changes with `pnpm --filter @sentry/junior-memory db:generate`.
+- Generate self-contained data migrations with
+  `pnpm --filter @sentry/junior-memory db:generate:data --name <name>`.
 
 Follow `../../policies/data-redaction.md`, `../../policies/security.md`, and the
 plugin contract in `../junior-plugin-api/README.md`.

packages/junior-memory/package.json

@@ -25,6 +25,7 @@
   "scripts": {
     "build": "tsup && tsc -p tsconfig.build.json --emitDeclarationOnly",
     "db:generate": "pnpm exec drizzle-kit generate --config drizzle.config.ts",
+    "db:generate:data": "pnpm exec junior-migrations generate --config drizzle.config.ts --out migrations",
     "lint": "oxlint --config ../junior/.oxlintrc.json --deny-warnings src tests tsup.config.ts",
     "prepare": "pnpm run build",
     "prepack": "pnpm run build",
@@ -39,6 +40,7 @@
     "zod": "catalog:"
   },
   "devDependencies": {
+    "@sentry/junior-migrations": "workspace:*",
     "@sentry/junior-testing": "workspace:*",
     "@types/node": "^25.9.1",
     "drizzle-kit": "catalog:",

packages/junior-migrations/README.md

@@ -0,0 +1,53 @@
+# `@sentry/junior-migrations`
+
+Junior's migration format extends a Drizzle Kit journal with TypeScript data
+migrations. Every journal entry resolves to exactly one `<tag>.sql` or
+`<tag>.ts` file. Drizzle Kit continues to own numbering and schema snapshots;
+the Junior migration runner owns execution and exact per-entry tracking.
+
+TypeScript migrations target a versioned capability API and must not import
+Junior runtime internals. Migration-specific parsers and transforms belong
+permanently in the migration file so application refactors or deletions cannot
+break pending upgrades.
+
+## Authoring
+
+Generate schema migrations with the owning package's normal Drizzle command.
+Generate data migrations through this package's wrapper:
+
+```bash
+junior-migrations generate \
+  --config drizzle.config.ts \
+  --out migrations \
+  --name backfill_actor
+```
+
+The wrapper asks Drizzle Kit to create a custom journal entry, replaces the
+empty SQL file with a `MigrationV1` TypeScript scaffold, and removes the
+unchanged custom snapshot. Schema generation continues from the latest real
+schema snapshot while preserving the mixed journal order.
+
+## Compatibility
+
+Drizzle Kit remains the supported authoring tool. Drizzle ORM's stock
+`migrate()` function is not a supported executor for mixed journals because it
+requires every entry to have a SQL file. Call `runMigrationJournal` instead and
+provide the host database adapter, state adapter, and TypeScript loader. The
+same database adapter drives the journal ledger and is exposed as
+`context.database`, so migration files never own connection or driver setup.
+
+Normal upgrades run with `mode: "all"` and execute every SQL and TypeScript
+entry in journal order. `mode: "schema-bootstrap"` is reserved for constructing
+an empty database at the latest schema in tests or bootstrap tooling. It skips
+TypeScript data migrations while executing SQL entries across the full journal,
+so it must not be used to upgrade an existing installation.
+
+The runner rejects runtime imports of application source, relative modules,
+and unversioned `@sentry/junior` modules. Migrations may import the append-only
+`@sentry/junior/migration-helpers/v1` surface for parsing primitives, adapters,
+stores, and key resolution. One-off migration decisions and data transforms
+must still remain in the journal entry. Add a new helper or capability version
+rather than changing an existing contract.
+
+This source validation is an authoring guard for trusted packaged migration
+code, not a security sandbox for untrusted scripts.

packages/junior-migrations/package.json

@@ -0,0 +1,47 @@
+{
+  "name": "@sentry/junior-migrations",
+  "version": "0.104.2",
+  "private": false,
+  "publishConfig": {
+    "access": "public"
+  },
+  "type": "module",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/getsentry/junior.git",
+    "directory": "packages/junior-migrations"
+  },
+  "bin": {
+    "junior-migrations": "dist/cli.js"
+  },
+  "exports": {
+    ".": {
+      "types": "./src/index.ts",
+      "default": "./dist/index.js"
+    }
+  },
+  "files": [
+    "dist",
+    "src"
+  ],
+  "scripts": {
+    "build": "tsup && tsc -p tsconfig.build.json --emitDeclarationOnly",
+    "prepare": "pnpm run build",
+    "prepack": "pnpm run build",
+    "lint": "oxlint --config ../junior/.oxlintrc.json --deny-warnings src tests tsup.config.ts",
+    "test": "vitest run",
+    "typecheck": "tsc --noEmit"
+  },
+  "peerDependencies": {
+    "drizzle-kit": "catalog:"
+  },
+  "devDependencies": {
+    "@types/node": "^25.9.1",
+    "drizzle-kit": "catalog:",
+    "drizzle-orm": "catalog:",
+    "oxlint": "^1.66.0",
+    "tsup": "^8.5.1",
+    "typescript": "^6.0.3",
+    "vitest": "^4.1.7"
+  }
+}

packages/junior-migrations/src/cli.ts

@@ -0,0 +1,24 @@
+import { parseArgs } from "node:util";
+import { generateTypeScriptMigration } from "./generate";
+
+const { positionals, values } = parseArgs({
+  allowPositionals: true,
+  options: {
+    config: { type: "string" },
+    name: { type: "string" },
+    out: { type: "string", default: "./migrations" },
+  },
+});
+
+if (positionals[0] !== "generate" || !values.config || !values.name) {
+  throw new Error(
+    "Usage: junior-migrations generate --config <path> --name <name> [--out <dir>]",
+  );
+}
+
+const path = await generateTypeScriptMigration({
+  configPath: values.config,
+  migrationsFolder: values.out,
+  name: values.name,
+});
+console.log(`Created TypeScript migration ${path}`);