diff --git a/.craft.yml b/.craft.yml index 2b05578d7..e4c71a09a 100644 --- a/.craft.yml +++ b/.craft.yml @@ -5,6 +5,9 @@ targets: - name: npm id: "@sentry/junior" includeNames: /^sentry-junior-\d.*\.tgz$/ + - name: npm + id: "@sentry/junior-migrations" + includeNames: /^sentry-junior-migrations-\d.*\.tgz$/ - name: npm id: "@sentry/junior-plugin-api" includeNames: /^sentry-junior-plugin-api-\d.*\.tgz$/ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2a639ce61..c05d85642 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,6 +49,7 @@ jobs: - .github/actions/** core: - packages/junior/** + - packages/junior-migrations/** - packages/junior-plugin-api/** - packages/junior-scheduler/** - packages/junior-testing/** @@ -215,6 +216,7 @@ jobs: - uses: ./.github/actions/setup-node-pnpm - run: pnpm --filter @sentry/junior-memory build - run: pnpm --filter @sentry/junior-github build + - run: pnpm --filter @sentry/junior-migrations test - run: pnpm --filter @sentry/junior-memory test - run: pnpm --filter @sentry/junior-github test @@ -341,6 +343,7 @@ jobs: run: | mkdir -p artifacts pnpm --filter @sentry/junior pack --pack-destination artifacts + pnpm --filter @sentry/junior-migrations pack --pack-destination artifacts pnpm --filter @sentry/junior-plugin-api pack --pack-destination artifacts pnpm --filter @sentry/junior-agent-browser pack --pack-destination artifacts pnpm --filter @sentry/junior-amplitude pack --pack-destination artifacts diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0aed6be2b..25619e00f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -119,6 +119,7 @@ pnpm build:pkg This repo uses Craft for manual lockstep npm releases of: - `@sentry/junior` +- `@sentry/junior-migrations` - `@sentry/junior-plugin-api` - `@sentry/junior-agent-browser` - `@sentry/junior-amplitude` diff --git a/README.md b/README.md index a24241acd..7dfe164c3 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,7 @@ Start here: | Package | Purpose | | ------------------------------ | ---------------------------------------------------------------------------- | | `@sentry/junior` | Core Slack bot runtime | +| `@sentry/junior-migrations` | Mixed Drizzle SQL and TypeScript migration format and runner | | `@sentry/junior-plugin-api` | Lightweight plugin API types and helpers | | `@sentry/junior-agent-browser` | Agent Browser plugin package for browser automation | | `@sentry/junior-amplitude` | Read-only Amplitude product analytics through Amplitude's hosted MCP server | diff --git a/ast-grep/rules/no-core-plugin-dynamic-imports.yml b/ast-grep/rules/no-core-plugin-dynamic-imports.yml index 9ae0f5e4d..a36ee7071 100644 --- a/ast-grep/rules/no-core-plugin-dynamic-imports.yml +++ b/ast-grep/rules/no-core-plugin-dynamic-imports.yml @@ -12,4 +12,4 @@ constraints: all: - regex: '^["@'']@sentry/junior-.+["@'']$' - not: - regex: '^["@'']@sentry/junior-plugin-api["@'']$' + regex: '^["@'']@sentry/junior-(?:plugin-api|migrations)["@'']$' diff --git a/ast-grep/rules/no-core-plugin-reexports.yml b/ast-grep/rules/no-core-plugin-reexports.yml index adb19e779..c4504cd33 100644 --- a/ast-grep/rules/no-core-plugin-reexports.yml +++ b/ast-grep/rules/no-core-plugin-reexports.yml @@ -14,4 +14,4 @@ rule: - not: has: kind: string - regex: '^["@'']@sentry/junior-plugin-api["@'']$' + regex: '^["@'']@sentry/junior-(?:plugin-api|migrations)["@'']$' diff --git a/ast-grep/rules/no-core-plugin-static-imports.yml b/ast-grep/rules/no-core-plugin-static-imports.yml index c2af1d8e9..c26903f92 100644 --- a/ast-grep/rules/no-core-plugin-static-imports.yml +++ b/ast-grep/rules/no-core-plugin-static-imports.yml @@ -14,4 +14,4 @@ rule: - not: has: kind: string - regex: '^["@'']@sentry/junior-plugin-api["@'']$' + regex: '^["@'']@sentry/junior-(?:plugin-api|migrations)["@'']$' diff --git a/package.json b/package.json index bd59a8af1..99e5fd539 100644 --- a/package.json +++ b/package.json @@ -12,19 +12,19 @@ "worktree:setup": "node scripts/worktree.mjs setup", "cloudflare:token": "node scripts/refresh-cloudflare-tunnel-token.mjs", "prepare": "simple-git-hooks", - "lint": "pnpm --filter @sentry/junior lint && pnpm --filter @sentry/junior-memory lint && pnpm --filter @sentry/junior-github lint && pnpm ast-grep:lint && pnpm package:lint", + "lint": "pnpm --filter @sentry/junior-migrations lint && pnpm --filter @sentry/junior lint && pnpm --filter @sentry/junior-memory lint && pnpm --filter @sentry/junior-github lint && pnpm ast-grep:lint && pnpm package:lint", "lint:fix": "pnpm --filter @sentry/junior lint:fix", "ast-grep:lint": "ast-grep scan", "lint-staged": "lint-staged", - "build": "pnpm --filter @sentry/junior build && pnpm --filter @sentry/junior-memory build && pnpm --filter @sentry/junior-github build && pnpm --filter @sentry/junior-dashboard build", + "build": "pnpm --filter @sentry/junior-migrations build && pnpm --filter @sentry/junior build && pnpm --filter @sentry/junior-memory build && pnpm --filter @sentry/junior-github build && pnpm --filter @sentry/junior-dashboard build", "build:example": "pnpm --filter @sentry/junior-example build", "docs:dev": "pnpm --filter @sentry/junior-docs dev", "docs:build": "pnpm --filter @sentry/junior-docs build", "docs:check": "pnpm --filter @sentry/junior-docs check", - "package:lint": "for pkg in packages/junior packages/junior-plugin-api packages/junior-scheduler packages/junior-memory packages/junior-dashboard packages/junior-github packages/junior-agent-browser packages/junior-amplitude packages/junior-cloudflare packages/junior-datadog packages/junior-hex packages/junior-linear packages/junior-maintenance packages/junior-notion packages/junior-sentry packages/junior-vercel; do pnpm exec publint \"$pkg\" || exit $?; done", + "package:lint": "for pkg in packages/junior packages/junior-migrations packages/junior-plugin-api packages/junior-scheduler packages/junior-memory packages/junior-dashboard packages/junior-github packages/junior-agent-browser packages/junior-amplitude packages/junior-cloudflare packages/junior-datadog packages/junior-hex packages/junior-linear packages/junior-maintenance packages/junior-notion packages/junior-sentry packages/junior-vercel; do pnpm exec publint \"$pkg\" || exit $?; done", "release:check": "node scripts/check-release-config.mjs", "start": "pnpm --filter @sentry/junior-example dev", - "test": "pnpm --filter @sentry/junior build && pnpm --filter @sentry/junior-memory build && pnpm --filter @sentry/junior-github build && pnpm --filter @sentry/junior-dashboard build && pnpm --filter @sentry/junior test && pnpm --filter @sentry/junior-memory test && pnpm --filter @sentry/junior-github test && pnpm --filter @sentry/junior-dashboard test", + "test": "pnpm --filter @sentry/junior-migrations test && pnpm --filter @sentry/junior build && pnpm --filter @sentry/junior-memory build && pnpm --filter @sentry/junior-github build && pnpm --filter @sentry/junior-dashboard build && pnpm --filter @sentry/junior test && pnpm --filter @sentry/junior-memory test && pnpm --filter @sentry/junior-github test && pnpm --filter @sentry/junior-dashboard test", "test:e2e:dashboard": "pnpm --filter @sentry/junior-dashboard build && playwright test -c packages/junior-dashboard/playwright.config.ts", "test:watch": "pnpm --filter @sentry/junior test:watch", "evals": "pnpm --filter @sentry/junior-evals evals", diff --git a/packages/docs/src/content/docs/cli/upgrade.md b/packages/docs/src/content/docs/cli/upgrade.md index f18765d37..768b8c2d9 100644 --- a/packages/docs/src/content/docs/cli/upgrade.md +++ b/packages/docs/src/content/docs/cli/upgrade.md @@ -1,8 +1,8 @@ --- title: "junior upgrade" -description: "Run one-shot Junior state upgrade migrations." +description: "Apply Junior schema and data migrations." type: reference -summary: Move persisted Junior state forward after upgrading packages. +summary: Move configured SQL schemas and persisted state forward after upgrades. prerequisites: - /start-here/quickstart/ related: @@ -11,7 +11,10 @@ related: - /cli/snapshot-create/ --- -Use `junior upgrade` after installing a Junior release that includes a one-shot state migration. The command mutates the configured state stores, so run it from the same app environment that has the production state and SQL environment variables configured for the deployment you are upgrading. +Use `junior upgrade` after installing a Junior release that includes schema or +data migrations. The command mutates the configured SQL database and state +stores, so run it from the same app environment that has the production state +and SQL environment variables configured for the deployment you are upgrading. ## Usage @@ -25,13 +28,25 @@ The command takes no extra arguments. ## What it does -`junior upgrade` runs registered migrations sequentially. Current migrations: +`junior upgrade` runs migrations sequentially. Core and plugin migration +directories use Drizzle Kit's ordered journal and may contain either generated +SQL schema migrations or TypeScript data migrations targeting a versioned +host capability API. Current +upgrade work includes: +- Apply core and enabled-plugin schema and data journal entries. +- Rewrite retained turn-session records from legacy storage shapes before the + new runtime reads them. - Move legacy `junior:conversation-work:*` Redis state into the newer conversation record and index state used by the durable worker and dashboard feed. - Backfill retained conversation records into the shared Junior SQL database. The upgrade requires `DATABASE_URL`. -- Repair legacy token and estimated-cost rollups from durable SQL agent steps in bounded batches. Conversations that are active during the repair are left unchanged and can be repaired by rerunning the command after they become idle. -The migrations are idempotent: rerunning them skips records that were already moved, removes stale legacy index entries that no longer have a record, and upserts SQL conversation rows. The SQL conversation backfill copies a bounded legacy slice of Redis conversation metadata; after cutover, durable conversation metadata is written to SQL while Redis remains the transcript and execution/cache store. +Completed journal entries are tracked individually, and TypeScript migrations +can checkpoint progress for safe retries. Legacy backfills remain idempotent: +rerunning them skips records that were already moved, removes stale legacy +index entries that no longer have a record, and upserts SQL conversation rows. +The SQL conversation backfill copies a bounded legacy slice of Redis +conversation metadata; after cutover, durable conversation metadata is written +to SQL while Redis remains the transcript and execution/cache store. ## Vercel deploys @@ -57,12 +72,10 @@ Typical logs look like this: ```text Running Junior upgrade migrations... -Running migration migrate-redis-conversation-state... -Finished migration migrate-redis-conversation-state: scanned=2 migrated=1 existing=0 missing=1 -Running migration backfill-conversations-sql... -Finished migration backfill-conversations-sql: scanned=2 migrated=2 existing=0 missing=0 -Running migration repair-conversation-usage... -Finished migration repair-conversation-usage: scanned=2 migrated=1 existing=1 missing=0 +Running migration core-migrations... +Finished migration core-migrations: scanned=10 migrated=6 existing=4 missing=0 skipped=0 +Running migration plugin-migrations... +Finished migration plugin-migrations: scanned=8 migrated=8 existing=0 missing=0 skipped=0 Junior upgrade complete. ``` @@ -84,10 +97,6 @@ After running the command: 2. Confirm the migration summary has the expected `scanned` and `migrated` counts. 3. Run `pnpm exec junior check` before building or deploying the app. -A nonzero `missing` count for `repair-conversation-usage` means retained SQL assistant messages did not contain usable, schema-safe usage values. Junior leaves those totals unchanged. - -The command does not rewrite legacy duration totals. Run summaries are TTL-bound and do not carry an authoritative completeness marker, so even a non-empty retained index may omit a run. Replacing a total from that evidence could silently undercount it. - ## Next step Run [junior check](/cli/check/) after the upgrade, then continue with [junior snapshot create](/cli/snapshot-create/) if your plugins need sandbox dependencies. diff --git a/packages/docs/src/content/docs/contribute/releasing.md b/packages/docs/src/content/docs/contribute/releasing.md index 5556fcb5c..9950005a3 100644 --- a/packages/docs/src/content/docs/contribute/releasing.md +++ b/packages/docs/src/content/docs/contribute/releasing.md @@ -12,6 +12,7 @@ related: Junior uses lockstep package releases for: - `@sentry/junior` +- `@sentry/junior-migrations` - `@sentry/junior-plugin-api` - `@sentry/junior-agent-browser` - `@sentry/junior-amplitude` diff --git a/packages/junior-evals/postgres-global-setup.ts b/packages/junior-evals/postgres-global-setup.ts index b02a88587..0d1ab3336 100644 --- a/packages/junior-evals/postgres-global-setup.ts +++ b/packages/junior-evals/postgres-global-setup.ts @@ -2,7 +2,7 @@ import path from "node:path"; import { fileURLToPath } from "node:url"; import type { TestProject } from "vitest/node"; import { setupJuniorPostgresHarness } from "@junior-tests/fixtures/postgres/global-setup"; -import { migratePluginSchemas } from "@/chat/plugins/migrations"; +import { bootstrapPluginSchemas } from "@/chat/plugins/migrations"; const workspaceRoot = path.resolve( path.dirname(fileURLToPath(import.meta.url)), @@ -14,7 +14,7 @@ export default async function setup( ): Promise<() => Promise> { return await setupJuniorPostgresHarness(project, { migrateTemplate: async (executor) => { - await migratePluginSchemas(executor, [ + await bootstrapPluginSchemas(executor, [ { dir: path.resolve(workspaceRoot, "packages/junior-memory/migrations"), pluginName: "memory", diff --git a/packages/junior-memory/README.md b/packages/junior-memory/README.md index f6c634a13..1eb814815 100644 --- a/packages/junior-memory/README.md +++ b/packages/junior-memory/README.md @@ -55,6 +55,8 @@ exported types, tools, and tests are authoritative. - `MEMORY_RECALL_MAX_VECTOR_DISTANCE` or `recallMaxVectorDistance` configures the vector candidate threshold. - Generate schema changes with `pnpm --filter @sentry/junior-memory db:generate`. +- Generate self-contained data migrations with + `pnpm --filter @sentry/junior-memory db:generate:data --name `. Follow `../../policies/data-redaction.md`, `../../policies/security.md`, and the plugin contract in `../junior-plugin-api/README.md`. diff --git a/packages/junior-memory/package.json b/packages/junior-memory/package.json index f98388ae6..e0192171d 100644 --- a/packages/junior-memory/package.json +++ b/packages/junior-memory/package.json @@ -25,6 +25,7 @@ "scripts": { "build": "tsup && tsc -p tsconfig.build.json --emitDeclarationOnly", "db:generate": "pnpm exec drizzle-kit generate --config drizzle.config.ts", + "db:generate:data": "pnpm exec junior-migrations generate --config drizzle.config.ts --out migrations", "lint": "oxlint --config ../junior/.oxlintrc.json --deny-warnings src tests tsup.config.ts", "prepare": "pnpm run build", "prepack": "pnpm run build", @@ -39,6 +40,7 @@ "zod": "catalog:" }, "devDependencies": { + "@sentry/junior-migrations": "workspace:*", "@sentry/junior-testing": "workspace:*", "@types/node": "^25.9.1", "drizzle-kit": "catalog:", diff --git a/packages/junior-migrations/README.md b/packages/junior-migrations/README.md new file mode 100644 index 000000000..5b652b25d --- /dev/null +++ b/packages/junior-migrations/README.md @@ -0,0 +1,53 @@ +# `@sentry/junior-migrations` + +Junior's migration format extends a Drizzle Kit journal with TypeScript data +migrations. Every journal entry resolves to exactly one `.sql` or +`.ts` file. Drizzle Kit continues to own numbering and schema snapshots; +the Junior migration runner owns execution and exact per-entry tracking. + +TypeScript migrations target a versioned capability API and must not import +Junior runtime internals. Migration-specific parsers and transforms belong +permanently in the migration file so application refactors or deletions cannot +break pending upgrades. + +## Authoring + +Generate schema migrations with the owning package's normal Drizzle command. +Generate data migrations through this package's wrapper: + +```bash +junior-migrations generate \ + --config drizzle.config.ts \ + --out migrations \ + --name backfill_actor +``` + +The wrapper asks Drizzle Kit to create a custom journal entry, replaces the +empty SQL file with a `MigrationV1` TypeScript scaffold, and removes the +unchanged custom snapshot. Schema generation continues from the latest real +schema snapshot while preserving the mixed journal order. + +## Compatibility + +Drizzle Kit remains the supported authoring tool. Drizzle ORM's stock +`migrate()` function is not a supported executor for mixed journals because it +requires every entry to have a SQL file. Call `runMigrationJournal` instead and +provide the host database adapter, state adapter, and TypeScript loader. The +same database adapter drives the journal ledger and is exposed as +`context.database`, so migration files never own connection or driver setup. + +Normal upgrades run with `mode: "all"` and execute every SQL and TypeScript +entry in journal order. `mode: "schema-bootstrap"` is reserved for constructing +an empty database at the latest schema in tests or bootstrap tooling. It skips +TypeScript data migrations while executing SQL entries across the full journal, +so it must not be used to upgrade an existing installation. + +The runner rejects runtime imports of application source, relative modules, +and unversioned `@sentry/junior` modules. Migrations may import the append-only +`@sentry/junior/migration-helpers/v1` surface for parsing primitives, adapters, +stores, and key resolution. One-off migration decisions and data transforms +must still remain in the journal entry. Add a new helper or capability version +rather than changing an existing contract. + +This source validation is an authoring guard for trusted packaged migration +code, not a security sandbox for untrusted scripts. diff --git a/packages/junior-migrations/package.json b/packages/junior-migrations/package.json new file mode 100644 index 000000000..36af03dbf --- /dev/null +++ b/packages/junior-migrations/package.json @@ -0,0 +1,47 @@ +{ + "name": "@sentry/junior-migrations", + "version": "0.104.2", + "private": false, + "publishConfig": { + "access": "public" + }, + "type": "module", + "repository": { + "type": "git", + "url": "git+https://github.com/getsentry/junior.git", + "directory": "packages/junior-migrations" + }, + "bin": { + "junior-migrations": "dist/cli.js" + }, + "exports": { + ".": { + "types": "./src/index.ts", + "default": "./dist/index.js" + } + }, + "files": [ + "dist", + "src" + ], + "scripts": { + "build": "tsup && tsc -p tsconfig.build.json --emitDeclarationOnly", + "prepare": "pnpm run build", + "prepack": "pnpm run build", + "lint": "oxlint --config ../junior/.oxlintrc.json --deny-warnings src tests tsup.config.ts", + "test": "vitest run", + "typecheck": "tsc --noEmit" + }, + "peerDependencies": { + "drizzle-kit": "catalog:" + }, + "devDependencies": { + "@types/node": "^25.9.1", + "drizzle-kit": "catalog:", + "drizzle-orm": "catalog:", + "oxlint": "^1.66.0", + "tsup": "^8.5.1", + "typescript": "^6.0.3", + "vitest": "^4.1.7" + } +} diff --git a/packages/junior-migrations/src/cli.ts b/packages/junior-migrations/src/cli.ts new file mode 100644 index 000000000..22714b788 --- /dev/null +++ b/packages/junior-migrations/src/cli.ts @@ -0,0 +1,24 @@ +import { parseArgs } from "node:util"; +import { generateTypeScriptMigration } from "./generate"; + +const { positionals, values } = parseArgs({ + allowPositionals: true, + options: { + config: { type: "string" }, + name: { type: "string" }, + out: { type: "string", default: "./migrations" }, + }, +}); + +if (positionals[0] !== "generate" || !values.config || !values.name) { + throw new Error( + "Usage: junior-migrations generate --config --name [--out ]", + ); +} + +const path = await generateTypeScriptMigration({ + configPath: values.config, + migrationsFolder: values.out, + name: values.name, +}); +console.log(`Created TypeScript migration ${path}`); diff --git a/packages/junior-migrations/src/generate.ts b/packages/junior-migrations/src/generate.ts new file mode 100644 index 000000000..97beca0c7 --- /dev/null +++ b/packages/junior-migrations/src/generate.ts @@ -0,0 +1,87 @@ +import { spawn } from "node:child_process"; +import { readFile, rename, rm, writeFile } from "node:fs/promises"; +import { resolve } from "node:path"; +import { readMigrationJournal } from "./journal"; + +/** Drizzle Kit inputs used to create one TypeScript migration entry. */ +export interface GenerateTypeScriptMigrationOptions { + configPath: string; + cwd?: string; + migrationsFolder: string; + name: string; +} + +function run(command: string, args: string[], cwd: string): Promise { + return new Promise((resolvePromise, reject) => { + const child = spawn(command, args, { cwd, stdio: "inherit" }); + child.on("error", reject); + child.on("exit", (code) => { + if (code === 0) { + resolvePromise(); + } else { + reject(new Error(`${command} exited with code ${code ?? "unknown"}`)); + } + }); + }); +} + +function scaffold(): string { + return `import type { MigrationV1 } from "@sentry/junior-migrations";\n\nconst migration = {\n apiVersion: 1,\n async up(context) {\n void context;\n },\n} satisfies MigrationV1;\n\nexport default migration;\n`; +} + +function isMissingJournal(error: unknown): boolean { + return ( + error instanceof Error && + (error.cause as NodeJS.ErrnoException | undefined)?.code === "ENOENT" + ); +} + +/** Create a journaled TypeScript migration through Drizzle Kit's custom generator. */ +export async function generateTypeScriptMigration( + options: GenerateTypeScriptMigrationOptions, +): Promise { + const cwd = resolve(options.cwd ?? process.cwd()); + const folder = resolve(cwd, options.migrationsFolder); + let before; + try { + before = await readMigrationJournal(folder); + } catch (error) { + if (!isMissingJournal(error)) throw error; + before = []; + } + await run( + "drizzle-kit", + [ + "generate", + "--custom", + "--config", + options.configPath, + "--name", + options.name, + ], + cwd, + ); + const after = await readMigrationJournal(folder); + if (after.length !== before.length + 1) { + throw new Error("Drizzle Kit did not create exactly one migration entry"); + } + const entry = after.at(-1); + if (!entry) { + throw new Error("Drizzle Kit did not create a migration entry"); + } + const sqlPath = resolve(folder, `${entry.tag}.sql`); + const source = await readFile(sqlPath, "utf8"); + if (!source.includes("Custom SQL migration file")) { + throw new Error(`Refusing to replace non-custom migration ${entry.tag}`); + } + const typescriptPath = resolve(folder, `${entry.tag}.ts`); + const snapshotPath = resolve( + folder, + "meta", + `${String(entry.index).padStart(4, "0")}_snapshot.json`, + ); + await rename(sqlPath, typescriptPath); + await rm(snapshotPath, { force: true }); + await writeFile(typescriptPath, scaffold(), "utf8"); + return typescriptPath; +} diff --git a/packages/junior-migrations/src/index.ts b/packages/junior-migrations/src/index.ts new file mode 100644 index 000000000..1770006de --- /dev/null +++ b/packages/junior-migrations/src/index.ts @@ -0,0 +1,20 @@ +export { generateTypeScriptMigration } from "./generate"; +export { readMigrationJournal, resolveMigrations } from "./journal"; +export { runMigrationJournal } from "./runner"; +export type { GenerateTypeScriptMigrationOptions } from "./generate"; +export type { RunMigrationJournalOptions } from "./runner"; +export type { + MigrationContextV1, + MigrationDatabaseAdapter, + MigrationJsonValue, + MigrationJournalEntry, + MigrationJournalExecutor, + MigrationLockV1, + MigrationProgressV1, + MigrationRedisV1, + MigrationRunResult, + MigrationStateV1, + MigrationV1, + ResolvedMigration, + TypeScriptMigrationLoader, +} from "./types"; diff --git a/packages/junior-migrations/src/journal.ts b/packages/junior-migrations/src/journal.ts new file mode 100644 index 000000000..ca1b92847 --- /dev/null +++ b/packages/junior-migrations/src/journal.ts @@ -0,0 +1,182 @@ +import { createHash } from "node:crypto"; +import { lstat, readFile, realpath } from "node:fs/promises"; +import { isAbsolute, join, relative, sep } from "node:path"; +import type { MigrationJournalEntry, ResolvedMigration } from "./types"; + +interface DrizzleJournalEntry { + breakpoints?: unknown; + idx?: unknown; + tag?: unknown; + when?: unknown; +} + +interface DrizzleJournal { + dialect?: unknown; + entries?: unknown; +} + +function parseJournalEntry( + value: DrizzleJournalEntry, + position: number, +): MigrationJournalEntry { + if ( + typeof value.idx !== "number" || + !Number.isInteger(value.idx) || + value.idx !== position || + typeof value.when !== "number" || + !Number.isFinite(value.when) || + typeof value.tag !== "string" || + !/^[a-z0-9][a-z0-9_-]*$/.test(value.tag) || + typeof value.breakpoints !== "boolean" + ) { + throw new Error(`Invalid Drizzle journal entry at index ${position}`); + } + return { + breakpoints: value.breakpoints, + index: value.idx, + tag: value.tag, + when: value.when, + }; +} + +async function optionalFile( + path: string, + migrationsRoot: string, +): Promise { + try { + const stat = await lstat(path); + if (!stat.isFile() || stat.isSymbolicLink()) { + throw new Error(`Migration source must be a regular file: ${path}`); + } + const canonical = await realpath(path); + const fromRoot = relative(migrationsRoot, canonical); + if ( + fromRoot === ".." || + fromRoot.startsWith(`..${sep}`) || + isAbsolute(fromRoot) + ) { + throw new Error(`Migration source escapes its journal root: ${path}`); + } + return await readFile(path, "utf8"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + return undefined; + } + throw error; + } +} + +function validateTypeScriptSource(tag: string, source: string): void { + const allowedRuntimeImports = new Set([ + "@sentry/junior/migration-helpers/v1", + ]); + const validateRuntimeSpecifier = (specifier: string | undefined): void => { + if ( + !specifier || + specifier.startsWith(".") || + specifier.startsWith("/") || + specifier.startsWith("@/") || + ((specifier === "@sentry/junior" || + specifier.startsWith("@sentry/junior/")) && + !allowedRuntimeImports.has(specifier)) + ) { + throw new Error( + `TypeScript migration ${tag} cannot import application runtime code`, + ); + } + }; + const imports = source.matchAll( + /^\s*import\s+([\s\S]*?)\s+from\s+["']([^"']+)["'];?/gm, + ); + for (const match of imports) { + const clause = match[1]?.trim(); + const specifier = match[2]; + if (clause?.startsWith("type ")) { + continue; + } + validateRuntimeSpecifier(specifier); + } + const exports = source.matchAll( + /^\s*export\s+([\s\S]*?)\s+from\s+["']([^"']+)["'];?/gm, + ); + for (const match of exports) { + const clause = match[1]?.trim(); + if (clause?.startsWith("type ")) { + continue; + } + validateRuntimeSpecifier(match[2]); + } + if ( + /^\s*import\s*["']/m.test(source) || + /\b(?:import\s*\(|require\s*\()/.test(source) + ) { + throw new Error(`TypeScript migration ${tag} cannot load runtime modules`); + } +} + +/** Read and validate the ordered Drizzle journal entries in one migration directory. */ +export async function readMigrationJournal( + migrationsFolder: string, +): Promise { + const journalPath = join(migrationsFolder, "meta", "_journal.json"); + let source: string; + try { + source = await readFile(journalPath, "utf8"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + throw new Error("Can't find meta/_journal.json file", { cause: error }); + } + throw error; + } + const parsed = JSON.parse(source) as DrizzleJournal; + if (parsed.dialect !== "postgresql" || !Array.isArray(parsed.entries)) { + throw new Error(`Unsupported Drizzle journal in ${journalPath}`); + } + const entries = parsed.entries.map((entry, index) => + parseJournalEntry(entry as DrizzleJournalEntry, index), + ); + const timestamps = new Set(); + for (const entry of entries) { + if (timestamps.has(entry.when)) { + throw new Error(`Duplicate migration timestamp ${entry.when}`); + } + timestamps.add(entry.when); + } + return entries; +} + +/** Resolve every journal entry to exactly one immutable SQL or TypeScript source file. */ +export async function resolveMigrations( + migrationsFolder: string, +): Promise { + const migrationsRoot = await realpath(migrationsFolder); + const entries = await readMigrationJournal(migrationsFolder); + return await Promise.all( + entries.map(async (entry) => { + const sqlPath = join(migrationsFolder, `${entry.tag}.sql`); + const typescriptPath = join(migrationsFolder, `${entry.tag}.ts`); + const [sqlSource, typescriptSource] = await Promise.all([ + optionalFile(sqlPath, migrationsRoot), + optionalFile(typescriptPath, migrationsRoot), + ]); + if ((sqlSource === undefined) === (typescriptSource === undefined)) { + throw new Error( + `Migration ${entry.tag} must have exactly one .sql or .ts file`, + ); + } + const kind = sqlSource === undefined ? "typescript" : "sql"; + const source = sqlSource ?? typescriptSource ?? ""; + const path = kind === "sql" ? sqlPath : typescriptPath; + if (kind === "typescript") { + validateTypeScriptSource(entry.tag, source); + } + return { + ...entry, + hash: createHash("sha256").update(source).digest("hex"), + kind, + path, + source, + }; + }), + ); +} diff --git a/packages/junior-migrations/src/runner.ts b/packages/junior-migrations/src/runner.ts new file mode 100644 index 000000000..dd9ae0a83 --- /dev/null +++ b/packages/junior-migrations/src/runner.ts @@ -0,0 +1,378 @@ +import { createHash } from "node:crypto"; +import { readFile } from "node:fs/promises"; +import type { + MigrationContextV1, + MigrationDatabaseAdapter, + MigrationJournalExecutor, + MigrationJsonValue, + MigrationRunResult, + MigrationV1, + ResolvedMigration, + TypeScriptMigrationLoader, +} from "./types"; +import { resolveMigrations } from "./journal"; + +interface MigrationRow { + createdAt: string; + hash: string; + progress: unknown; + status: string | null; +} + +function migrationJsonValue( + value: unknown, + label: string, + seen = new WeakSet(), +): MigrationJsonValue { + if ( + value === null || + typeof value === "string" || + typeof value === "boolean" + ) { + return value; + } + if (typeof value === "number") { + if (Number.isFinite(value)) return value; + throw new Error(`${label} must contain only finite JSON numbers`); + } + if (typeof value !== "object") { + throw new Error(`${label} must be JSON-compatible`); + } + if (seen.has(value)) { + throw new Error(`${label} must not contain circular references`); + } + seen.add(value); + if (Array.isArray(value)) { + const result = value.map((item) => migrationJsonValue(item, label, seen)); + seen.delete(value); + return result; + } + const prototype = Object.getPrototypeOf(value); + if (prototype !== Object.prototype && prototype !== null) { + throw new Error(`${label} must contain only plain JSON objects`); + } + const result = Object.fromEntries( + Object.entries(value).map(([key, item]) => [ + key, + migrationJsonValue(item, label, seen), + ]), + ); + seen.delete(value); + return result; +} + +interface RunMigrationJournalBaseOptions { + beforeRun?: () => Promise; + executor: MigrationJournalExecutor; + migrationsFolder: string; + migrationsTable: string; +} + +interface RunAllMigrationJournalOptions extends RunMigrationJournalBaseOptions { + createContext: (args: { + migration: ResolvedMigration; + progress: MigrationContextV1["progress"]; + }) => MigrationContextV1; + loadTypeScript: TypeScriptMigrationLoader; + mode?: "all"; +} + +interface RunSchemaBootstrapMigrationJournalOptions extends RunMigrationJournalBaseOptions { + createContext?: never; + loadTypeScript?: never; + mode: "schema-bootstrap"; +} + +/** Host capabilities and execution mode for one mixed migration journal. */ +export type RunMigrationJournalOptions = + | RunAllMigrationJournalOptions + | RunSchemaBootstrapMigrationJournalOptions; + +function identifier(value: string): string { + if (!/^[a-z_][a-z0-9_]*$/.test(value)) { + throw new Error(`Invalid migration table identifier: ${value}`); + } + return value; +} + +function qualifiedTable(table: string): string { + return `drizzle.${identifier(table)}`; +} + +async function ensureMigrationTable( + executor: MigrationJournalExecutor, + table: string, +): Promise { + const qualified = qualifiedTable(table); + await executor.execute("CREATE SCHEMA IF NOT EXISTS drizzle"); + await executor.execute(` +CREATE TABLE IF NOT EXISTS ${qualified} ( + id SERIAL PRIMARY KEY, + hash TEXT NOT NULL, + created_at BIGINT +) +`); + await executor.execute( + `ALTER TABLE ${qualified} ADD COLUMN IF NOT EXISTS name TEXT`, + ); + await executor.execute( + `ALTER TABLE ${qualified} ADD COLUMN IF NOT EXISTS kind TEXT`, + ); + await executor.execute( + `ALTER TABLE ${qualified} ADD COLUMN IF NOT EXISTS status TEXT`, + ); + await executor.execute( + `ALTER TABLE ${qualified} ADD COLUMN IF NOT EXISTS progress JSONB`, + ); + await executor.execute( + `ALTER TABLE ${qualified} ADD COLUMN IF NOT EXISTS result JSONB`, + ); + await executor.execute( + `ALTER TABLE ${qualified} ADD COLUMN IF NOT EXISTS started_at TIMESTAMPTZ`, + ); + await executor.execute( + `ALTER TABLE ${qualified} ADD COLUMN IF NOT EXISTS completed_at TIMESTAMPTZ`, + ); +} + +async function migrationRows( + executor: MigrationJournalExecutor, + table: string, +): Promise> { + const rows = await executor.query(` +SELECT + created_at::text AS "createdAt", + hash, + progress, + status +FROM ${qualifiedTable(table)} +WHERE created_at IS NOT NULL +ORDER BY created_at ASC, id ASC +`); + return new Map(rows.map((row) => [Number(row.createdAt), row])); +} + +async function adoptLegacySqlPrefix(args: { + executor: MigrationJournalExecutor; + migrations: readonly ResolvedMigration[]; + rows: Map; + table: string; +}): Promise { + const latestAppliedAt = Math.max( + ...args.rows.keys(), + Number.NEGATIVE_INFINITY, + ); + if (!Number.isFinite(latestAppliedAt)) { + return; + } + for (const migration of args.migrations) { + if ( + migration.when >= latestAppliedAt || + migration.kind !== "sql" || + args.rows.has(migration.when) + ) { + continue; + } + await args.executor.execute( + `INSERT INTO ${qualifiedTable(args.table)} + (hash, created_at, name, kind, status, completed_at) + VALUES ($1, $2, $3, 'sql', 'completed', NOW())`, + [migration.hash, migration.when, migration.tag], + ); + } +} + +function sqlStatements(migration: ResolvedMigration): string[] { + return migration.source + .split("--> statement-breakpoint") + .map((statement) => statement.trim()) + .filter(Boolean); +} + +function migrationV1(value: unknown, tag: string): MigrationV1 { + const candidate = + typeof value === "object" && value !== null && "default" in value + ? (value as { default?: unknown }).default + : value; + if ( + typeof candidate !== "object" || + candidate === null || + (candidate as { apiVersion?: unknown }).apiVersion !== 1 || + typeof (candidate as { up?: unknown }).up !== "function" + ) { + throw new Error(`TypeScript migration ${tag} does not export MigrationV1`); + } + return candidate as MigrationV1; +} + +async function runSqlMigration(args: { + executor: MigrationJournalExecutor; + migration: ResolvedMigration; + table: string; +}): Promise { + await args.executor.transaction(async () => { + for (const statement of sqlStatements(args.migration)) { + await args.executor.execute(statement); + } + await args.executor.execute( + `INSERT INTO ${qualifiedTable(args.table)} + (hash, created_at, name, kind, status, started_at, completed_at) + VALUES ($1, $2, $3, 'sql', 'completed', NOW(), NOW())`, + [args.migration.hash, args.migration.when, args.migration.tag], + ); + }); +} + +async function runTypeScriptMigration(args: { + createContext: RunAllMigrationJournalOptions["createContext"]; + executor: MigrationDatabaseAdapter; + loadTypeScript: TypeScriptMigrationLoader; + migration: ResolvedMigration; + row: MigrationRow | undefined; + table: string; +}): Promise { + const qualified = qualifiedTable(args.table); + if (args.row && args.row.hash !== args.migration.hash) { + throw new Error(`Migration ${args.migration.tag} changed after it started`); + } + if (args.row) { + await args.executor.execute( + `UPDATE ${qualified} + SET name = $1, kind = 'typescript', status = 'running', started_at = NOW() + WHERE created_at = $2`, + [args.migration.tag, args.migration.when], + ); + } else { + await args.executor.execute( + `INSERT INTO ${qualified} + (hash, created_at, name, kind, status, started_at) + VALUES ($1, $2, $3, 'typescript', 'running', NOW())`, + [args.migration.hash, args.migration.when, args.migration.tag], + ); + } + const progress: MigrationContextV1["progress"] = { + async load() { + const [current] = await args.executor.query<{ + progress: Awaited>; + }>(`SELECT progress FROM ${qualified} WHERE created_at = $1 LIMIT 1`, [ + args.migration.when, + ]); + return current?.progress == null + ? undefined + : migrationJsonValue(current.progress, "Migration progress"); + }, + async save(value) { + const progressValue = migrationJsonValue(value, "Migration progress"); + await args.executor.execute( + `UPDATE ${qualified} SET progress = $1, status = 'running' WHERE created_at = $2`, + [progressValue, args.migration.when], + ); + }, + }; + try { + const context = args.createContext({ migration: args.migration, progress }); + const currentSource = await readFile(args.migration.path, "utf8"); + const currentHash = createHash("sha256") + .update(currentSource) + .digest("hex"); + if (currentHash !== args.migration.hash) { + throw new Error(`Migration ${args.migration.tag} changed before loading`); + } + const migration = migrationV1( + await args.loadTypeScript(args.migration.path), + args.migration.tag, + ); + const result = await migration.up(context); + const resultValue = + result === undefined + ? null + : migrationJsonValue(result, "Migration result"); + await args.executor.execute( + `UPDATE ${qualified} + SET status = 'completed', result = $1, completed_at = NOW() + WHERE created_at = $2`, + [resultValue, args.migration.when], + ); + } catch (error) { + try { + await args.executor.execute( + `UPDATE ${qualified} SET status = 'failed' WHERE created_at = $1`, + [args.migration.when], + ); + } catch (statusError) { + throw new AggregateError( + [error, statusError], + `Migration ${args.migration.tag} failed and its failure status could not be persisted`, + ); + } + throw error; + } +} + +/** Execute one mixed Drizzle journal with exact SQL and TypeScript entry tracking. */ +export async function runMigrationJournal( + options: RunMigrationJournalOptions, +): Promise { + const migrations = await resolveMigrations(options.migrationsFolder); + const mode = options.mode ?? "all"; + return await options.executor.withMigrationLock( + options.migrationsTable, + async () => { + await options.beforeRun?.(); + await ensureMigrationTable(options.executor, options.migrationsTable); + let rows = await migrationRows(options.executor, options.migrationsTable); + await adoptLegacySqlPrefix({ + executor: options.executor, + migrations, + rows, + table: options.migrationsTable, + }); + rows = await migrationRows(options.executor, options.migrationsTable); + const result: MigrationRunResult = { + existing: 0, + migrated: 0, + scanned: migrations.length, + skipped: 0, + }; + for (const migration of migrations) { + const row = rows.get(migration.when); + if (row && row.hash !== migration.hash) { + throw new Error( + `Migration ${migration.tag} changed after it started`, + ); + } + if (row?.status === "completed" || (row && row.status === null)) { + result.existing += 1; + continue; + } + if (mode === "schema-bootstrap" && migration.kind === "typescript") { + result.skipped += 1; + continue; + } + if (migration.kind === "sql") { + await runSqlMigration({ + executor: options.executor, + migration, + table: options.migrationsTable, + }); + } else { + if (!options.createContext || !options.loadTypeScript) { + throw new Error( + `TypeScript migration ${migration.tag} requires a loader and context`, + ); + } + await runTypeScriptMigration({ + createContext: options.createContext, + executor: options.executor, + loadTypeScript: options.loadTypeScript, + migration, + row, + table: options.migrationsTable, + }); + } + result.migrated += 1; + } + return result; + }, + ); +} diff --git a/packages/junior-migrations/src/types.ts b/packages/junior-migrations/src/types.ts new file mode 100644 index 000000000..e3b3b2528 --- /dev/null +++ b/packages/junior-migrations/src/types.ts @@ -0,0 +1,107 @@ +/** Database capabilities exposed to v1 TypeScript migrations. */ +export interface MigrationDatabaseAdapter { + db(): unknown; + execute(statement: string, parameters?: readonly unknown[]): Promise; + query( + statement: string, + parameters?: readonly unknown[], + ): Promise; + transaction(callback: () => Promise): Promise; + withLock(lockName: string, callback: () => Promise): Promise; +} + +/** Host executor used to serialize and persist one migration journal. */ +export interface MigrationJournalExecutor extends MigrationDatabaseAdapter { + withMigrationLock( + migrationTable: string, + callback: () => Promise, + ): Promise; +} + +/** Stable state-store capabilities exposed to v1 TypeScript migrations. */ +export interface MigrationLockV1 { + expiresAt: number; + threadId: string; + token: string; +} + +/** Stable state-store capabilities exposed to v1 TypeScript migrations. */ +export interface MigrationStateV1 { + acquireLock(threadId: string, ttlMs: number): Promise; + appendToList( + key: string, + value: unknown, + options?: { maxLength?: number; ttlMs?: number }, + ): Promise; + connect(): Promise; + delete(key: string): Promise; + get(key: string): Promise; + getList(key: string): Promise; + releaseLock(lock: MigrationLockV1): Promise; + set(key: string, value: unknown, ttlMs?: number): Promise; + setIfNotExists(key: string, value: unknown, ttlMs?: number): Promise; +} + +/** Optional raw Redis capability for migrations preserving Redis indexes. */ +export interface MigrationRedisV1 { + sendCommand(args: readonly string[]): Promise; +} + +/** Resumable progress storage scoped to one TypeScript migration. */ +export interface MigrationProgressV1 { + load(): Promise; + save(value: MigrationJsonValue): Promise; +} + +/** JSON-compatible value persisted in migration progress and result columns. */ +export type MigrationJsonValue = + | boolean + | number + | string + | null + | MigrationJsonValue[] + | { [key: string]: MigrationJsonValue }; + +/** Permanent capability contract for apiVersion 1 migrations. */ +export interface MigrationContextV1 { + database: MigrationDatabaseAdapter; + log(message: string): void; + progress: MigrationProgressV1; + redis?: MigrationRedisV1; + state: MigrationStateV1; +} + +/** Isolated TypeScript data migration targeting the v1 ABI. */ +export interface MigrationV1 { + apiVersion: 1; + up(context: MigrationContextV1): Promise; +} + +/** One ordered entry from Drizzle Kit's journal metadata. */ +export interface MigrationJournalEntry { + breakpoints: boolean; + index: number; + tag: string; + when: number; +} + +/** Journal entry paired with its unique SQL or TypeScript source file. */ +export interface ResolvedMigration extends MigrationJournalEntry { + hash: string; + kind: "sql" | "typescript"; + path: string; + source: string; +} + +/** Aggregate counts returned after one journal execution. */ +export interface MigrationRunResult { + existing: number; + migrated: number; + scanned: number; + skipped: number; +} + +/** Host-provided loader for an isolated TypeScript migration module. */ +export interface TypeScriptMigrationLoader { + (path: string): Promise; +} diff --git a/packages/junior-migrations/tests/generate.test.ts b/packages/junior-migrations/tests/generate.test.ts new file mode 100644 index 000000000..d594a1a67 --- /dev/null +++ b/packages/junior-migrations/tests/generate.test.ts @@ -0,0 +1,122 @@ +import { spawn } from "node:child_process"; +import { access, mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { afterEach, expect, it } from "vitest"; +import { generateTypeScriptMigration } from "../src/generate"; +import { readMigrationJournal } from "../src/journal"; + +const temporaryDirectories: string[] = []; + +function runDrizzle(args: string[], cwd: string): Promise { + return new Promise((resolve, reject) => { + const child = spawn("drizzle-kit", args, { cwd, stdio: "pipe" }); + let output = ""; + child.stdout.on("data", (chunk) => { + output += String(chunk); + }); + child.stderr.on("data", (chunk) => { + output += String(chunk); + }); + child.on("error", reject); + child.on("exit", (code) => { + if (code === 0) { + resolve(); + } else { + reject(new Error(output)); + } + }); + }); +} + +afterEach(async () => { + await Promise.all( + temporaryDirectories + .splice(0) + .map((directory) => rm(directory, { recursive: true, force: true })), + ); +}); + +it("keeps Drizzle schema generation working across a TypeScript entry", async () => { + const relativeRoot = `.tmp-mixed-migrations-${Date.now()}`; + const root = join(process.cwd(), relativeRoot); + temporaryDirectories.push(root); + await mkdir(root); + await writeFile( + join(root, "schema.ts"), + 'import { pgTable, text } from "drizzle-orm/pg-core";\nexport const users = pgTable("users", { id: text("id").primaryKey() });\n', + ); + await writeFile( + join(root, "drizzle.config.ts"), + `import { defineConfig } from "drizzle-kit";\nexport default defineConfig({ dialect: "postgresql", schema: "./${relativeRoot}/schema.ts", out: "./${relativeRoot}/migrations" });\n`, + ); + + await runDrizzle( + [ + "generate", + "--config", + `${relativeRoot}/drizzle.config.ts`, + "--name", + "initial", + ], + process.cwd(), + ); + const typescriptPath = await generateTypeScriptMigration({ + configPath: `${relativeRoot}/drizzle.config.ts`, + cwd: process.cwd(), + migrationsFolder: `${relativeRoot}/migrations`, + name: "backfill", + }); + await expect(access(typescriptPath)).resolves.toBeUndefined(); + await expect( + access(join(root, "migrations", "0001_backfill.sql")), + ).rejects.toThrow("ENOENT"); + await expect( + access(join(root, "migrations", "meta", "0001_snapshot.json")), + ).rejects.toThrow("ENOENT"); + + await writeFile( + join(root, "schema.ts"), + 'import { pgTable, text } from "drizzle-orm/pg-core";\nexport const users = pgTable("users", { id: text("id").primaryKey(), name: text("name") });\n', + ); + await runDrizzle( + [ + "generate", + "--config", + `${relativeRoot}/drizzle.config.ts`, + "--name", + "add_name", + ], + process.cwd(), + ); + + await expect( + readMigrationJournal(join(root, "migrations")), + ).resolves.toMatchObject([ + { index: 0, tag: "0000_initial" }, + { index: 1, tag: "0001_backfill" }, + { index: 2, tag: "0002_add_name" }, + ]); + await expect( + readFile(join(root, "migrations", "0002_add_name.sql"), "utf8"), + ).resolves.toContain('ADD COLUMN "name" text'); +}); + +it("does not invoke Drizzle when an existing journal is invalid", async () => { + const relativeRoot = `.tmp-invalid-migrations-${Date.now()}`; + const root = join(process.cwd(), relativeRoot); + temporaryDirectories.push(root); + await mkdir(join(root, "migrations", "meta"), { recursive: true }); + await writeFile( + join(root, "migrations", "meta", "_journal.json"), + JSON.stringify({ dialect: "sqlite", entries: [] }), + ); + + await expect( + generateTypeScriptMigration({ + configPath: `${relativeRoot}/drizzle.config.ts`, + cwd: process.cwd(), + migrationsFolder: `${relativeRoot}/migrations`, + name: "backfill", + }), + ).rejects.toThrow("Unsupported Drizzle journal"); +}); diff --git a/packages/junior-migrations/tests/journal.test.ts b/packages/junior-migrations/tests/journal.test.ts new file mode 100644 index 000000000..38a2c9875 --- /dev/null +++ b/packages/junior-migrations/tests/journal.test.ts @@ -0,0 +1,135 @@ +import { mkdtemp, mkdir, rm, symlink, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { resolveMigrations } from "../src/journal"; + +const temporaryDirectories: string[] = []; + +async function migrationFolder(entries: string[]): Promise { + const root = await mkdtemp(join(tmpdir(), "junior-migrations-")); + temporaryDirectories.push(root); + await mkdir(join(root, "meta")); + await writeFile( + join(root, "meta", "_journal.json"), + JSON.stringify({ + version: "7", + dialect: "postgresql", + entries: entries.map((tag, index) => ({ + idx: index, + version: "7", + when: 1_000 + index, + tag, + breakpoints: true, + })), + }), + ); + return root; +} + +afterEach(async () => { + await Promise.all( + temporaryDirectories + .splice(0) + .map((directory) => rm(directory, { recursive: true, force: true })), + ); +}); + +describe("resolveMigrations", () => { + it("resolves one ordered SQL or TypeScript file per journal entry", async () => { + const folder = await migrationFolder(["0000_initial", "0001_backfill"]); + await writeFile(join(folder, "0000_initial.sql"), "SELECT 1;"); + await writeFile( + join(folder, "0001_backfill.ts"), + 'import type { MigrationV1 } from "@sentry/junior-migrations";\nexport default { apiVersion: 1, async up() {} } satisfies MigrationV1;\n', + ); + + await expect(resolveMigrations(folder)).resolves.toMatchObject([ + { index: 0, kind: "sql", tag: "0000_initial", when: 1_000 }, + { index: 1, kind: "typescript", tag: "0001_backfill", when: 1_001 }, + ]); + }); + + it("rejects migrations that import runtime modules", async () => { + const folder = await migrationFolder(["0000_unsafe"]); + await writeFile( + join(folder, "0000_unsafe.ts"), + 'import { getDb } from "@/db";\nexport default { apiVersion: 1, async up() { getDb(); } };\n', + ); + + await expect(resolveMigrations(folder)).rejects.toThrow( + "cannot import application runtime code", + ); + }); + + it("rejects multiline imports of Junior runtime modules", async () => { + const folder = await migrationFolder(["0000_multiline"]); + await writeFile( + join(folder, "0000_multiline.ts"), + 'import {\n getChatConfig,\n getDb,\n} from "@sentry/junior/internal";\nexport default { apiVersion: 1, async up() {} };\n', + ); + + await expect(resolveMigrations(folder)).rejects.toThrow( + "cannot import application runtime code", + ); + }); + + it("allows versioned Junior migration helpers", async () => { + const folder = await migrationFolder(["0000_helpers"]); + await writeFile( + join(folder, "0000_helpers.ts"), + 'import { isRecord } from "@sentry/junior/migration-helpers/v1";\nexport default { apiVersion: 1, async up() { isRecord({}); } };\n', + ); + + await expect(resolveMigrations(folder)).resolves.toMatchObject([ + { kind: "typescript", tag: "0000_helpers" }, + ]); + }); + + it("rejects runtime re-exports of Junior internals", async () => { + const folder = await migrationFolder(["0000_reexport"]); + await writeFile( + join(folder, "0000_reexport.ts"), + 'export { getChatConfig } from "@sentry/junior/internal";\nexport default { apiVersion: 1, async up() {} };\n', + ); + + await expect(resolveMigrations(folder)).rejects.toThrow( + "cannot import application runtime code", + ); + }); + + it("rejects ambiguous migration files", async () => { + const folder = await migrationFolder(["0000_ambiguous"]); + await writeFile(join(folder, "0000_ambiguous.sql"), "SELECT 1;"); + await writeFile( + join(folder, "0000_ambiguous.ts"), + "export default { apiVersion: 1, async up() {} };\n", + ); + + await expect(resolveMigrations(folder)).rejects.toThrow( + "exactly one .sql or .ts file", + ); + }); + + it("rejects journal tags that escape the migration directory", async () => { + const folder = await migrationFolder(["../outside"]); + + await expect(resolveMigrations(folder)).rejects.toThrow( + "Invalid Drizzle journal entry", + ); + }); + + it("rejects symlinked migration sources", async () => { + const folder = await migrationFolder(["0000_symlink"]); + const outside = join(folder, "..", "outside-migration.ts"); + await writeFile( + outside, + "export default { apiVersion: 1, async up() {} };\n", + ); + await symlink(outside, join(folder, "0000_symlink.ts")); + + await expect(resolveMigrations(folder)).rejects.toThrow( + "Migration source must be a regular file", + ); + }); +}); diff --git a/packages/junior-migrations/tests/runner.test.ts b/packages/junior-migrations/tests/runner.test.ts new file mode 100644 index 000000000..7c1baa0cc --- /dev/null +++ b/packages/junior-migrations/tests/runner.test.ts @@ -0,0 +1,327 @@ +import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { runMigrationJournal } from "../src/runner"; +import type { + MigrationContextV1, + MigrationDatabaseAdapter, + MigrationV1, +} from "../src/types"; + +interface StoredRow { + createdAt: number; + hash: string; + progress?: unknown; + status: string | null; +} + +class FakeExecutor implements MigrationDatabaseAdapter { + readonly rows = new Map(); + readonly statements: string[] = []; + + db(): undefined { + return undefined; + } + + async execute(statement: string, parameters: readonly unknown[] = []) { + const normalized = statement.trim(); + if ( + normalized.startsWith("CREATE SCHEMA") || + normalized.startsWith("CREATE TABLE IF NOT EXISTS drizzle.") || + normalized.startsWith("ALTER TABLE drizzle.") + ) { + return; + } + if (normalized.startsWith("INSERT INTO drizzle.")) { + const hash = String(parameters[0]); + const createdAt = Number(parameters[1]); + const kind = normalized.includes("'typescript'") + ? "running" + : "completed"; + this.rows.set(createdAt, { createdAt, hash, status: kind }); + return; + } + if (normalized.startsWith("UPDATE drizzle.")) { + const createdAt = Number(parameters.at(-1)); + const row = this.rows.get(createdAt); + if (!row) { + throw new Error(`Missing row ${createdAt}`); + } + if (normalized.includes("progress = $1")) { + row.progress = parameters[0]; + } + if (normalized.includes("status = 'running'")) { + row.status = "running"; + } else if (normalized.includes("status = 'completed'")) { + row.status = "completed"; + } else if (normalized.includes("status = 'failed'")) { + row.status = "failed"; + } + return; + } + this.statements.push(normalized); + } + + async query(statement: string, parameters: readonly unknown[] = []) { + if (statement.includes('created_at::text AS "createdAt"')) { + return [...this.rows.values()].map((row) => ({ + createdAt: String(row.createdAt), + hash: row.hash, + progress: row.progress, + status: row.status, + })) as T[]; + } + if (statement.includes("SELECT progress")) { + const row = this.rows.get(Number(parameters[0])); + return (row ? [{ progress: row.progress ?? null }] : []) as T[]; + } + return []; + } + + async transaction(callback: () => Promise): Promise { + return await callback(); + } + + async withMigrationLock( + _migrationTable: string, + callback: () => Promise, + ): Promise { + return await callback(); + } + + async withLock(_lockName: string, callback: () => Promise): Promise { + return await callback(); + } +} + +function fakeMigrationState(): MigrationContextV1["state"] { + return { + acquireLock: async () => null, + appendToList: async () => {}, + connect: async () => {}, + delete: async () => {}, + get: async () => undefined, + getList: async () => [], + releaseLock: async () => {}, + set: async () => {}, + setIfNotExists: async () => true, + }; +} + +const temporaryDirectories: string[] = []; + +async function mixedFolder(): Promise { + const root = await mkdtemp(join(tmpdir(), "junior-migrations-runner-")); + temporaryDirectories.push(root); + await mkdir(join(root, "meta")); + await writeFile( + join(root, "meta", "_journal.json"), + JSON.stringify({ + version: "7", + dialect: "postgresql", + entries: ["0000_schema", "0001_data", "0002_schema"].map( + (tag, index) => ({ + idx: index, + version: "7", + when: 2_000 + index, + tag, + breakpoints: true, + }), + ), + }), + ); + await writeFile(join(root, "0000_schema.sql"), "SELECT 'schema-zero';"); + await writeFile( + join(root, "0001_data.ts"), + 'import type { MigrationV1 } from "@sentry/junior-migrations";\nexport default {} satisfies MigrationV1;\n', + ); + await writeFile(join(root, "0002_schema.sql"), "SELECT 'schema-two';"); + return root; +} + +afterEach(async () => { + await Promise.all( + temporaryDirectories + .splice(0) + .map((directory) => rm(directory, { recursive: true, force: true })), + ); +}); + +describe("runMigrationJournal", () => { + it("runs mixed entries in journal order and is a no-op on rerun", async () => { + const folder = await mixedFolder(); + const executor = new FakeExecutor(); + const order: string[] = []; + const migration: MigrationV1 = { + apiVersion: 1, + async up(context) { + order.push("typescript"); + await context.progress.save({ cursor: 1 }); + }, + }; + + await expect( + runMigrationJournal({ + executor, + migrationsFolder: folder, + migrationsTable: "__drizzle_test", + loadTypeScript: async () => ({ default: migration }), + createContext: ({ progress }) => ({ + log: () => {}, + progress, + database: executor, + state: fakeMigrationState(), + }), + }), + ).resolves.toEqual({ existing: 0, migrated: 3, scanned: 3, skipped: 0 }); + expect(executor.statements).toEqual([ + "SELECT 'schema-zero';", + "SELECT 'schema-two';", + ]); + expect(order).toEqual(["typescript"]); + + await expect( + runMigrationJournal({ + executor, + migrationsFolder: folder, + migrationsTable: "__drizzle_test", + loadTypeScript: async () => ({ default: migration }), + createContext: ({ progress }) => ({ + log: () => {}, + progress, + database: executor, + state: fakeMigrationState(), + }), + }), + ).resolves.toEqual({ existing: 3, migrated: 0, scanned: 3, skipped: 0 }); + }); + + it("bootstraps the latest schema while leaving TypeScript entries pending", async () => { + const folder = await mixedFolder(); + const executor = new FakeExecutor(); + + await expect( + runMigrationJournal({ + executor, + migrationsFolder: folder, + migrationsTable: "__drizzle_test", + mode: "schema-bootstrap", + }), + ).resolves.toEqual({ existing: 0, migrated: 2, scanned: 3, skipped: 1 }); + expect([...executor.rows.keys()].sort()).toEqual([2_000, 2_002]); + expect(executor.statements).toEqual([ + "SELECT 'schema-zero';", + "SELECT 'schema-two';", + ]); + + const migration: MigrationV1 = { + apiVersion: 1, + async up() { + return { backfilled: true }; + }, + }; + await expect( + runMigrationJournal({ + executor, + migrationsFolder: folder, + migrationsTable: "__drizzle_test", + loadTypeScript: async () => ({ default: migration }), + createContext: ({ progress }) => ({ + database: executor, + log: () => {}, + progress, + state: fakeMigrationState(), + }), + }), + ).resolves.toEqual({ existing: 2, migrated: 1, scanned: 3, skipped: 0 }); + expect(executor.statements).toEqual([ + "SELECT 'schema-zero';", + "SELECT 'schema-two';", + ]); + }); + + it("resumes a failed TypeScript migration from saved progress", async () => { + const folder = await mixedFolder(); + const executor = new FakeExecutor(); + let attempts = 0; + const migration: MigrationV1 = { + apiVersion: 1, + async up(context) { + attempts += 1; + const progress = await context.progress.load(); + if (!progress) { + await context.progress.save({ cursor: 1 }); + throw new Error("interrupted"); + } + return { resumed: true }; + }, + }; + const options = { + executor, + migrationsFolder: folder, + migrationsTable: "__drizzle_test", + loadTypeScript: async () => ({ default: migration }), + createContext: ({ + progress, + }: { + progress: MigrationContextV1["progress"]; + }) => ({ + log: () => {}, + progress, + database: executor, + state: fakeMigrationState(), + }), + }; + + await expect(runMigrationJournal(options)).rejects.toThrow("interrupted"); + expect(executor.rows.get(2_001)).toMatchObject({ + progress: { cursor: 1 }, + status: "failed", + }); + expect(executor.statements).toEqual(["SELECT 'schema-zero';"]); + + await expect(runMigrationJournal(options)).resolves.toEqual({ + existing: 1, + migrated: 2, + scanned: 3, + skipped: 0, + }); + expect(attempts).toBe(2); + expect(executor.rows.get(2_001)).toMatchObject({ + progress: { cursor: 1 }, + status: "completed", + }); + expect(executor.statements).toEqual([ + "SELECT 'schema-zero';", + "SELECT 'schema-two';", + ]); + }); + + it("rejects non-JSON migration results before completing the ledger row", async () => { + const folder = await mixedFolder(); + const executor = new FakeExecutor(); + const migration = { + apiVersion: 1, + async up() { + return Number.NaN as never; + }, + } satisfies MigrationV1; + + await expect( + runMigrationJournal({ + executor, + migrationsFolder: folder, + migrationsTable: "__drizzle_test", + loadTypeScript: async () => ({ default: migration }), + createContext: ({ progress }) => ({ + database: executor, + log: () => {}, + progress, + state: fakeMigrationState(), + }), + }), + ).rejects.toThrow("Migration result must contain only finite JSON numbers"); + expect(executor.rows.get(2_001)?.status).toBe("failed"); + }); +}); diff --git a/packages/junior-migrations/tsconfig.build.json b/packages/junior-migrations/tsconfig.build.json new file mode 100644 index 000000000..7d583629c --- /dev/null +++ b/packages/junior-migrations/tsconfig.build.json @@ -0,0 +1,11 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "declaration": true, + "emitDeclarationOnly": true, + "noEmit": false, + "rootDir": "src", + "outDir": "dist" + }, + "include": ["src/**/*.ts"] +} diff --git a/packages/junior-migrations/tsconfig.json b/packages/junior-migrations/tsconfig.json new file mode 100644 index 000000000..165b4315c --- /dev/null +++ b/packages/junior-migrations/tsconfig.json @@ -0,0 +1,14 @@ +{ + "compilerOptions": { + "target": "ES2022", + "lib": ["ES2022", "DOM"], + "module": "ESNext", + "moduleResolution": "Bundler", + "strict": true, + "skipLibCheck": true, + "isolatedModules": true, + "noEmit": true, + "types": ["node", "vitest/globals"] + }, + "include": ["src/**/*.ts", "tests/**/*.ts"] +} diff --git a/packages/junior-migrations/tsup.config.ts b/packages/junior-migrations/tsup.config.ts new file mode 100644 index 000000000..254c4d2c6 --- /dev/null +++ b/packages/junior-migrations/tsup.config.ts @@ -0,0 +1,22 @@ +import { defineConfig } from "tsup"; + +const shared = { + format: "esm" as const, + tsconfig: "tsconfig.build.json", + dts: false, + outDir: "dist", +}; + +export default defineConfig([ + { + ...shared, + entry: { index: "src/index.ts" }, + clean: true, + }, + { + ...shared, + entry: { cli: "src/cli.ts" }, + clean: false, + banner: { js: "#!/usr/bin/env node" }, + }, +]); diff --git a/packages/junior-plugin-api/README.md b/packages/junior-plugin-api/README.md index 7074f6222..407d7925b 100644 --- a/packages/junior-plugin-api/README.md +++ b/packages/junior-plugin-api/README.md @@ -5,12 +5,13 @@ exported TypeScript types and runtime validators are authoritative. ## Registration -Use `defineJuniorPlugin({ manifest, hooks, tasks, cli, model })`. A plugin name -is a lowercase identifier and is unique within the enabled app plugin set. +Use `defineJuniorPlugin({ manifest, hooks, tasks, cli, model })`. +A plugin name is a lowercase identifier and is unique within the enabled app +plugin set. A plugin may instead be a declarative `plugin.yaml` package when it has no -host-executed hooks. Do not combine an inline manifest with a second YAML -definition for the same plugin. +host-executed hooks, tasks, CLI, or model implementation. Do not combine an +inline manifest with a second YAML definition for the same plugin. ## Manifest @@ -30,7 +31,7 @@ safe. ## Hooks Plugins may contribute tools, prompt messages, lifecycle work, operational -reports, migrations, and other typed hook surfaces exported by this package. +reports, and other typed hook surfaces exported by this package. - Hook context carries the active source, actor, conversation, plugin metadata, database, logging, and only the host capabilities required by that hook. @@ -58,6 +59,8 @@ reports, migrations, and other typed hook surfaces exported by this package. - Packaged migrations create plugin-owned tables through the host migration runner. +- TypeScript journal entries contain their complete durable implementation and + execute through the versioned migration capability API. - Generate migration artifacts from the package schema; do not hand-maintain a second schema contract. - Runtime hooks and CLI actions use host-provided `ctx.db`. diff --git a/packages/junior-plugin-api/src/hooks.ts b/packages/junior-plugin-api/src/hooks.ts index b4d05ad16..a787a3f56 100644 --- a/packages/junior-plugin-api/src/hooks.ts +++ b/packages/junior-plugin-api/src/hooks.ts @@ -18,8 +18,6 @@ import type { RouteRegistrationHookContext, SlackConversationLink, SlackConversationLinkHookContext, - StorageMigrationContext, - StorageMigrationResult, } from "./operations"; import type { BeforeToolExecuteHookContext, @@ -73,10 +71,4 @@ export interface PluginHooks { tools?( ctx: ToolRegistrationHookContext, ): Record; - migrateStorage?( - ctx: StorageMigrationContext, - ): - | Promise - | StorageMigrationResult - | undefined; } diff --git a/packages/junior-plugin-api/src/operations.ts b/packages/junior-plugin-api/src/operations.ts index 3e3cb8a3f..e57ffff1d 100644 --- a/packages/junior-plugin-api/src/operations.ts +++ b/packages/junior-plugin-api/src/operations.ts @@ -17,18 +17,6 @@ export interface HeartbeatResult { dispatchCount?: number; } -export interface StorageMigrationResult { - existing: number; - migrated: number; - missing: number; - scanned: number; - skipped?: number; -} - -export interface StorageMigrationContext extends PluginContext { - state: PluginState; -} - export type PluginOperationalTone = "danger" | "good" | "neutral" | "warning"; export interface PluginOperationalMetric { diff --git a/packages/junior-scheduler/README.md b/packages/junior-scheduler/README.md index a952c07bd..4681e43f4 100644 --- a/packages/junior-scheduler/README.md +++ b/packages/junior-scheduler/README.md @@ -46,6 +46,10 @@ Junior's durable agent runtime. The plugin exposes create, update, delete, list, and run-now tools plus bounded operational reporting. Generate schema changes with `pnpm --filter @sentry/junior-scheduler db:generate`. +Use `pnpm --filter @sentry/junior-scheduler db:generate:data --name ` +for a TypeScript data migration in the same journal. New migrations should use +the stable migration capabilities directly and keep their complete +implementation in the migration file. Follow `../../policies/serverless-background-work.md`, `../../policies/context-bound-systems.md`, and diff --git a/packages/junior-scheduler/migrations/0002_scheduler_state_to_sql.ts b/packages/junior-scheduler/migrations/0002_scheduler_state_to_sql.ts new file mode 100644 index 000000000..2c77ee189 --- /dev/null +++ b/packages/junior-scheduler/migrations/0002_scheduler_state_to_sql.ts @@ -0,0 +1,306 @@ +import type { + MigrationJsonValue, + MigrationV1, +} from "@sentry/junior-migrations"; +import { z } from "zod"; + +const TASK_INDEX_KEY = "junior:scheduler:tasks"; +const TASK_KEY_PREFIX = "junior:scheduler:task:"; +const ACTIVE_RUN_KEY_PREFIX = "junior:scheduler:active:"; +const RUN_KEY_PREFIX = "junior:scheduler:run:"; +const nonBlankStringSchema = z + .string() + .refine((value) => value.trim().length > 0); +const exactActorUserIdSchema = z + .string() + .min(1) + .refine( + (value) => value === value.trim() && value.toLowerCase() !== "unknown", + ); +const slackTeamIdSchema = z.string().regex(/^T[A-Z0-9]+$/); +const slackConversationIdSchema = z.string().regex(/^(C|G|D)[A-Z0-9]+$/); +const slackDestinationSchema = z + .object({ + platform: z.literal("slack"), + teamId: slackTeamIdSchema, + channelId: slackConversationIdSchema, + }) + .strict(); +const pluginCredentialSubjectSchema = z.discriminatedUnion("allowedWhen", [ + z + .object({ + type: z.literal("user"), + userId: exactActorUserIdSchema, + allowedWhen: z.literal("private-direct-conversation"), + }) + .strict(), + z + .object({ + type: z.literal("user"), + userId: exactActorUserIdSchema, + allowedWhen: z.literal("scheduled-task"), + taskId: nonBlankStringSchema.refine((value) => value === value.trim()), + }) + .strict(), +]); +const taskPrincipalSchema = z + .object({ + slackUserId: exactActorUserIdSchema, + fullName: z.string().optional(), + userName: z.string().optional(), + }) + .strict(); +const recurrenceSchema = z + .object({ + dayOfMonth: z.number().optional(), + frequency: z.enum(["daily", "weekly", "monthly", "yearly"]), + interval: z.number(), + month: z.number().optional(), + startDate: z.string(), + time: z.object({ hour: z.number(), minute: z.number() }).strict(), + weekdays: z.array(z.number()).optional(), + }) + .strict(); +const taskFields = { + id: z.string(), + conversationAccess: z + .object({ + audience: z.enum(["direct", "group", "channel"]), + visibility: z.enum(["private", "public", "unknown"]), + }) + .strict() + .optional(), + createdAtMs: z.number(), + createdBy: taskPrincipalSchema, + destination: slackDestinationSchema, + executionActor: z + .object({ platform: z.literal("system"), name: z.string() }) + .strict() + .optional(), + lastRunAtMs: z.number().optional(), + nextRunAtMs: z.number().optional(), + originalRequest: z.string().optional(), + runNowAtMs: z.number().optional(), + schedule: z + .object({ + description: z.string(), + kind: z.enum(["one_off", "recurring"]), + recurrence: recurrenceSchema.optional(), + timezone: z.string(), + }) + .strict(), + status: z.enum(["active", "paused", "blocked", "deleted"]), + statusReason: z.string().optional(), + task: z.object({ text: z.string() }).strict(), + updatedAtMs: z.number(), + version: z.number().optional(), +}; +const currentTaskSchema = z + .object({ + ...taskFields, + credentialMode: z.enum(["system", "creator"]), + }) + .strict(); +const legacyTaskSchema = z + .object({ + ...taskFields, + credentialSubject: pluginCredentialSubjectSchema.optional(), + }) + .strict(); +const runSchema = z + .object({ + id: z.string(), + attempt: z.number(), + claimedAtMs: z.number(), + completedAtMs: z.number().optional(), + dispatchId: z.string().optional(), + errorMessage: z.string().optional(), + idempotencyKey: z.string().optional(), + resultMessageTs: z.string().optional(), + scheduledForMs: z.number(), + startedAtMs: z.number().optional(), + status: z.enum([ + "pending", + "running", + "completed", + "failed", + "blocked", + "skipped", + ]), + taskId: z.string(), + taskVersion: z.number().optional(), + }) + .strict(); + +type JsonRecord = Record; + +function record(value: unknown): JsonRecord | undefined { + if (typeof value === "string") { + try { + return record(JSON.parse(value)); + } catch { + return undefined; + } + } + return value && typeof value === "object" && !Array.isArray(value) + ? (value as JsonRecord) + : undefined; +} + +function strings(value: unknown): string[] { + return Array.isArray(value) + ? [ + ...new Set( + value.filter((item): item is string => typeof item === "string"), + ), + ] + : []; +} + +function finiteNumber( + value: MigrationJsonValue | undefined, +): number | undefined { + return typeof value === "number" && Number.isFinite(value) + ? value + : undefined; +} + +function taskRecord(value: unknown): JsonRecord | undefined { + const current = currentTaskSchema.safeParse(record(value)); + if (current.success) { + const { version: _version, ...task } = current.data; + return task as JsonRecord; + } + const legacy = legacyTaskSchema.safeParse(record(value)); + if (!legacy.success) return undefined; + const { + credentialSubject: _credentialSubject, + version: _version, + ...task + } = legacy.data; + return { + ...task, + credentialMode: "system", + } as JsonRecord; +} + +function runRecord(value: unknown): JsonRecord | undefined { + const parsed = runSchema.safeParse(record(value)); + if (!parsed.success) return undefined; + const { + idempotencyKey: _idempotencyKey, + taskVersion: _taskVersion, + ...run + } = parsed.data; + return run as JsonRecord; +} + +const migration = { + apiVersion: 1, + async up(context) { + const ids = strings(await context.state.get(TASK_INDEX_KEY)); + let existing = 0; + let migrated = 0; + let missing = 0; + const tasks: JsonRecord[] = []; + + for (const id of ids) { + const task = taskRecord( + await context.state.get(`${TASK_KEY_PREFIX}${id}`), + ); + if (!task) { + missing += 1; + continue; + } + tasks.push(task); + const [stored] = await context.database.query<{ record: unknown }>( + "SELECT record FROM junior_scheduler_tasks WHERE id = $1 LIMIT 1", + [id], + ); + if ( + stored && + currentTaskSchema.safeParse(record(stored.record)).success + ) { + existing += 1; + continue; + } + const destination = record(task.destination)!; + await context.database.execute( + `INSERT INTO junior_scheduler_tasks + (id, team_id, status, next_run_at_ms, run_now_at_ms, created_at_ms, record) + VALUES ($1, $2, $3, $4, $5, $6, $7::jsonb) + ON CONFLICT (id) DO UPDATE SET + team_id = excluded.team_id, + status = excluded.status, + next_run_at_ms = excluded.next_run_at_ms, + run_now_at_ms = excluded.run_now_at_ms, + created_at_ms = excluded.created_at_ms, + record = excluded.record`, + [ + id, + destination.teamId, + task.status, + finiteNumber(task.nextRunAtMs) ?? null, + finiteNumber(task.runNowAtMs) ?? null, + finiteNumber(task.createdAtMs), + JSON.stringify(task), + ], + ); + migrated += 1; + } + + const runs: JsonRecord[] = []; + for (const task of tasks) { + const active = record( + await context.state.get(`${ACTIVE_RUN_KEY_PREFIX}${task.id}`), + ); + if (typeof active?.runId !== "string") { + continue; + } + const run = runRecord( + await context.state.get(`${RUN_KEY_PREFIX}${active.runId}`), + ); + if (run && (run.status === "pending" || run.status === "running")) { + runs.push(run); + } + } + + for (const run of runs) { + const [stored] = await context.database.query<{ record: unknown }>( + "SELECT record FROM junior_scheduler_runs WHERE id = $1 LIMIT 1", + [run.id], + ); + if (stored && runRecord(stored.record)) { + existing += 1; + continue; + } + await context.database.execute( + `INSERT INTO junior_scheduler_runs + (id, task_id, status, scheduled_for_ms, record) + VALUES ($1, $2, $3, $4, $5::jsonb) + ON CONFLICT (id) DO UPDATE SET + task_id = excluded.task_id, + status = excluded.status, + scheduled_for_ms = excluded.scheduled_for_ms, + record = excluded.record`, + [ + run.id, + run.taskId, + run.status, + finiteNumber(run.scheduledForMs), + JSON.stringify(run), + ], + ); + migrated += 1; + } + + return { + existing, + migrated, + missing, + scanned: ids.length + runs.length, + }; + }, +} satisfies MigrationV1; + +export default migration; diff --git a/packages/junior-scheduler/migrations/meta/_journal.json b/packages/junior-scheduler/migrations/meta/_journal.json index bf8784579..5552ed530 100644 --- a/packages/junior-scheduler/migrations/meta/_journal.json +++ b/packages/junior-scheduler/migrations/meta/_journal.json @@ -15,6 +15,13 @@ "when": 1783964295816, "tag": "0001_explicit_credential_mode", "breakpoints": true + }, + { + "idx": 2, + "version": "7", + "when": 1783977988234, + "tag": "0002_scheduler_state_to_sql", + "breakpoints": true } ] } diff --git a/packages/junior-scheduler/package.json b/packages/junior-scheduler/package.json index e44822d10..01b2884aa 100644 --- a/packages/junior-scheduler/package.json +++ b/packages/junior-scheduler/package.json @@ -25,6 +25,7 @@ "scripts": { "build": "tsup && tsc -p tsconfig.build.json --emitDeclarationOnly", "db:generate": "pnpm exec drizzle-kit generate --config drizzle.config.ts", + "db:generate:data": "pnpm exec junior-migrations generate --config drizzle.config.ts --out migrations", "prepare": "pnpm run build", "prepack": "pnpm run build", "typecheck": "tsc --noEmit" @@ -35,6 +36,7 @@ "zod": "catalog:" }, "devDependencies": { + "@sentry/junior-migrations": "workspace:*", "@types/node": "^25.9.1", "drizzle-kit": "catalog:", "tsup": "^8.5.1", diff --git a/packages/junior-scheduler/src/index.ts b/packages/junior-scheduler/src/index.ts index eec29d6ac..a145e9d74 100644 --- a/packages/junior-scheduler/src/index.ts +++ b/packages/junior-scheduler/src/index.ts @@ -10,7 +10,6 @@ export { export { createSchedulerOperationalSqlStore, createSchedulerSqlStore, - migrateSchedulerStateToSql, type SchedulerDb, } from "./store"; export type { diff --git a/packages/junior-scheduler/src/plugin.ts b/packages/junior-scheduler/src/plugin.ts index 108b7ce97..6f8d9ab95 100644 --- a/packages/junior-scheduler/src/plugin.ts +++ b/packages/junior-scheduler/src/plugin.ts @@ -13,7 +13,6 @@ import { import { createSchedulerOperationalSqlStore, createSchedulerSqlStore, - migrateSchedulerStateToSql, type SchedulerDb, type SchedulerOperationalStore, type SchedulerStore, @@ -578,12 +577,6 @@ export function createSchedulerPlugin() { store: schedulerOperationalStore(ctx), }); }, - async migrateStorage(ctx) { - return await migrateSchedulerStateToSql({ - db: ctx.db as SchedulerDb, - state: ctx.state, - }); - }, }, }); } diff --git a/packages/junior-scheduler/src/store.ts b/packages/junior-scheduler/src/store.ts index 7c0978be2..da310cf49 100644 --- a/packages/junior-scheduler/src/store.ts +++ b/packages/junior-scheduler/src/store.ts @@ -1692,58 +1692,3 @@ export function createSchedulerOperationalSqlStore( ): SchedulerOperationalStore { return new SqlSchedulerStore(db); } - -/** Copy retained scheduler plugin-state records into the scheduler SQL tables. */ -export async function migrateSchedulerStateToSql(args: { - db: SchedulerDb; - state: PluginState; -}): Promise<{ - existing: number; - migrated: number; - missing: number; - scanned: number; -}> { - const store = createSchedulerSqlStore(args.db); - const ids = await getIndex(args.state, globalTaskIndexKey()); - let existing = 0; - let migrated = 0; - let missing = 0; - const migratedTasks: ScheduledTask[] = []; - - for (const id of ids) { - const rawTask = await args.state.get(taskKey(id)); - const task = - parseStoredTask(rawTask) ?? parseLegacyStoredTaskForMigration(rawTask); - if (!task) { - missing += 1; - continue; - } - migratedTasks.push(task); - if (await store.getTask(task.id)) { - existing += 1; - continue; - } - await store.saveTask(task); - migrated += 1; - } - - const runs = await listIncompleteRunsForTasksFromState( - args.state, - migratedTasks, - ); - for (const run of runs) { - if (await store.getRun(run.id)) { - existing += 1; - continue; - } - await upsertSqlRun(args.db, run); - migrated += 1; - } - - return { - existing, - migrated, - missing, - scanned: ids.length + runs.length, - }; -} diff --git a/packages/junior/src/cli/upgrade/migrations/agent-turn-session-actor.ts b/packages/junior/migrations/0005_agent_turn_session_actor.ts similarity index 84% rename from packages/junior/src/cli/upgrade/migrations/agent-turn-session-actor.ts rename to packages/junior/migrations/0005_agent_turn_session_actor.ts index 116598fd4..7018c1653 100644 --- a/packages/junior/src/cli/upgrade/migrations/agent-turn-session-actor.ts +++ b/packages/junior/migrations/0005_agent_turn_session_actor.ts @@ -1,12 +1,28 @@ -import type { RedisStateAdapter } from "@chat-adapter/state-redis"; -import { THREAD_STATE_TTL_MS, type StateAdapter } from "chat"; -import { isRecord, toOptionalString } from "@/chat/coerce"; -import { getChatConfig } from "@/chat/config"; +import { THREAD_STATE_TTL_MS } from "chat"; import type { - MigrationContext, - MigrationResult, - UpgradeMigration, -} from "../types"; + MigrationRedisV1, + MigrationStateV1, + MigrationV1, +} from "@sentry/junior-migrations"; +import { + isRecord, + migrationRedisKey, + toOptionalString, +} from "@sentry/junior/migration-helpers/v1"; + +type StateAdapter = MigrationStateV1; +type RedisStateAdapter = { getClient(): MigrationRedisV1 }; +type MigrationContext = { + io?: { info(message: string): void }; + redisStateAdapter?: RedisStateAdapter; + stateAdapter: StateAdapter; +}; +type MigrationResult = { + existing: number; + migrated: number; + missing: number; + scanned: number; +}; const AGENT_TURN_SESSION_PREFIX = "junior:agent_turn_session"; const AGENT_TURN_SESSION_INDEX_KEY = `${AGENT_TURN_SESSION_PREFIX}:index`; @@ -31,11 +47,7 @@ function sessionRecordKey(conversationId: string, sessionId: string): string { } function logicalConversationIndexPrefix(): string { - const prefix = getChatConfig().state.keyPrefix; - return [ - ...(prefix ? [prefix] : []), - `${AGENT_TURN_SESSION_PREFIX}:conversation:`, - ].join(":"); + return migrationRedisKey(`${AGENT_TURN_SESSION_PREFIX}:conversation:`); } function conversationIdFromRedisListKey(key: string): string | undefined { @@ -151,7 +163,8 @@ async function migrateSessionRecord(args: { return "migrated"; } -async function migrateAgentTurnSessionActor( +/** Rewrite retained turn-session requester fields to actor fields. */ +export async function migrateAgentTurnSessionActor( context: MigrationContext, ): Promise { const result: MigrationResult = { @@ -230,7 +243,16 @@ async function migrateAgentTurnSessionActor( return result; } -export const agentTurnSessionActorMigration: UpgradeMigration = { - name: "migrate-agent-turn-session-requester-to-actor", - run: migrateAgentTurnSessionActor, -}; +const migration = { + apiVersion: 1, + async up(context) { + return await migrateAgentTurnSessionActor({ + stateAdapter: context.state, + ...(context.redis + ? { redisStateAdapter: { getClient: () => context.redis! } } + : {}), + }); + }, +} satisfies MigrationV1; + +export default migration; diff --git a/packages/junior/src/cli/upgrade/migrations/redis-conversation-state.ts b/packages/junior/migrations/0006_redis_conversation_state.ts similarity index 93% rename from packages/junior/src/cli/upgrade/migrations/redis-conversation-state.ts rename to packages/junior/migrations/0006_redis_conversation_state.ts index d2b903865..0463efb51 100644 --- a/packages/junior/src/cli/upgrade/migrations/redis-conversation-state.ts +++ b/packages/junior/migrations/0006_redis_conversation_state.ts @@ -1,25 +1,40 @@ -import type { RedisStateAdapter } from "@chat-adapter/state-redis"; -import type { StateAdapter } from "chat"; import type { Destination } from "@sentry/junior-plugin-api"; -import { isRecord, toOptionalNumber, toOptionalString } from "@/chat/coerce"; -import { getChatConfig } from "@/chat/config"; -import { parseDestination, sameDestination } from "@/chat/destination"; -import { coerceThreadConversationState } from "@/chat/state/conversation"; -import { JUNIOR_THREAD_STATE_TTL_MS } from "@/chat/state/ttl"; -import { - getConversation, - requestConversationWork, - type Conversation, - type ExecutionStatus, - type InboundMessage, - type Lease, - type Source, -} from "@/chat/task-execution/state"; import type { - MigrationContext, - MigrationResult, - UpgradeMigration, -} from "../types"; + MigrationRedisV1, + MigrationStateV1, + MigrationV1, +} from "@sentry/junior-migrations"; +import { + coerceThreadConversationState, + getMigrationConversation as getConversation, + isRecord, + JUNIOR_THREAD_STATE_TTL_MS, + migrationRedisKey, + parseDestination, + requestMigrationConversationWork as requestConversationWork, + sameDestination, + toOptionalNumber, + toOptionalString, + type MigrationRetainedConversationV1 as Conversation, + type MigrationExecutionStatusV1 as ExecutionStatus, + type MigrationInboundMessageV1 as InboundMessage, + type MigrationLeaseV1 as Lease, + type MigrationSourceV1 as Source, +} from "@sentry/junior/migration-helpers/v1"; + +type StateAdapter = MigrationStateV1; +type RedisStateAdapter = { getClient(): MigrationRedisV1 }; +type MigrationContext = { + io?: { info(message: string): void }; + redisStateAdapter?: RedisStateAdapter; + stateAdapter: StateAdapter; +}; +type MigrationResult = { + existing: number; + migrated: number; + missing: number; + scanned: number; +}; const CONVERSATION_PREFIX = "junior:conversation"; const CONVERSATION_SCHEMA_VERSION = 1; @@ -451,8 +466,7 @@ function uniqueAwaitingContinuationSummaries( } function redisIndexKey(indexKey: string): string { - const prefix = getChatConfig().state.keyPrefix; - return [...(prefix ? [prefix] : []), indexKey].join(":"); + return migrationRedisKey(indexKey); } async function upsertEmulatedIndexEntry(args: { @@ -771,7 +785,8 @@ async function seedAwaitingContinuationConversationWork( } } -async function migrateRedisConversationState( +/** Move legacy Redis conversation work into the retained conversation model. */ +export async function migrateRedisConversationState( context: MigrationContext, ): Promise { const result = await migrateLegacyConversationWorkRedisState(context); @@ -779,9 +794,16 @@ async function migrateRedisConversationState( return result; } -export const redisConversationStateMigration: UpgradeMigration = { - // TODO(after 2026-07-01): remove after deployed installs have had a release - // window to move legacy conversation-work Redis state forward. - name: "migrate-redis-conversation-state", - run: migrateRedisConversationState, -}; +const migration = { + apiVersion: 1, + async up(context) { + return await migrateRedisConversationState({ + stateAdapter: context.state, + ...(context.redis + ? { redisStateAdapter: { getClient: () => context.redis! } } + : {}), + }); + }, +} satisfies MigrationV1; + +export default migration; diff --git a/packages/junior/migrations/0007_conversations_to_sql.ts b/packages/junior/migrations/0007_conversations_to_sql.ts new file mode 100644 index 000000000..4285900ba --- /dev/null +++ b/packages/junior/migrations/0007_conversations_to_sql.ts @@ -0,0 +1,111 @@ +import type { MigrationStateV1, MigrationV1 } from "@sentry/junior-migrations"; +import { + addAgentTurnUsage, + createMigrationSqlStore, + createMigrationStateConversationStore, + listMigrationTurnSessionSummaries, +} from "@sentry/junior/migration-helpers/v1"; + +const CONVERSATION_BACKFILL_LIMIT = 10_000; + +type MigrationResult = { + existing: number; + migrated: number; + missing: number; + scanned: number; +}; + +type ConversationTarget = Pick< + ReturnType, + "backfillConversation" | "listByActivity" +>; + +/** Backfill retained state conversations and turn metrics into SQL. */ +export async function migrateConversationsToSql( + context: { + io?: { info(message: string): void }; + stateAdapter: MigrationStateV1; + }, + options: { batchSize?: number; target: ConversationTarget }, +): Promise { + const source = createMigrationStateConversationStore(context.stateAdapter); + const target = options.target; + const limit = Math.max(1, options.batchSize ?? CONVERSATION_BACKFILL_LIMIT); + const [stateConversations, sqlConversations] = await Promise.all([ + source.listByActivity({ limit }), + target.listByActivity({ limit }), + ]); + const byId = new Map( + sqlConversations.map((conversation) => [ + conversation.conversationId, + conversation, + ]), + ); + for (const conversation of stateConversations) { + const existing = byId.get(conversation.conversationId); + const existingExecutionAt = + existing?.execution.updatedAtMs ?? existing?.updatedAtMs ?? 0; + const stateExecutionAt = + conversation.execution.updatedAtMs ?? conversation.updatedAtMs; + if (!existing || stateExecutionAt >= existingExecutionAt) { + byId.set(conversation.conversationId, conversation); + } + } + const conversations = [...byId.values()] + .sort( + (left, right) => + right.lastActivityAtMs - left.lastActivityAtMs || + left.conversationId.localeCompare(right.conversationId), + ) + .slice(0, limit); + const summaries = await listMigrationTurnSessionSummaries( + context.stateAdapter, + conversations.map((conversation) => conversation.conversationId), + ); + for (const conversation of conversations) { + const conversationSummaries = + summaries.get(conversation.conversationId) ?? []; + const executionSummary = conversation.execution.runId + ? conversationSummaries.find( + (summary) => summary.sessionId === conversation.execution.runId, + ) + : undefined; + await target.backfillConversation( + conversation, + conversationSummaries.length > 0 + ? { + durationMs: conversationSummaries.reduce( + (total, summary) => total + summary.cumulativeDurationMs, + 0, + ), + usage: addAgentTurnUsage( + ...conversationSummaries.map( + (summary) => summary.cumulativeUsage, + ), + ), + executionDurationMs: executionSummary?.cumulativeDurationMs ?? 0, + executionUsage: executionSummary?.cumulativeUsage, + } + : undefined, + ); + } + + return { + existing: 0, + migrated: conversations.length, + missing: 0, + scanned: conversations.length, + }; +} + +const migration = { + apiVersion: 1, + async up(context) { + return await migrateConversationsToSql( + { stateAdapter: context.state }, + { target: createMigrationSqlStore(context.database) }, + ); + }, +} satisfies MigrationV1; + +export default migration; diff --git a/packages/junior/migrations/0008_conversation_history_to_sql.ts b/packages/junior/migrations/0008_conversation_history_to_sql.ts new file mode 100644 index 000000000..0f1828640 --- /dev/null +++ b/packages/junior/migrations/0008_conversation_history_to_sql.ts @@ -0,0 +1,1128 @@ +// @ts-nocheck -- frozen migration capsule; do not refactor against current Junior internals. +/* eslint-disable no-unused-vars */ + +// packages/junior/src/chat/conversations/legacy-import.ts +import { isDeepStrictEqual } from "node:util"; +import { eq as eq3 } from "drizzle-orm"; +import { z as z4 } from "zod"; + +// migration:visible +import { toMigrationStoredConversationMessage } from "@sentry/junior/migration-helpers/v1"; + +// migration:state +function getStateAdapter() { + throw new Error("state adapter required"); +} + +// migration:session-log +import { + migrationContextProvenance, + migrationLegacyActorProvenance, + decodeMigrationSessionLogEntry, + migrationPiMessageProvenanceSchema, +} from "@sentry/junior/migration-helpers/v1"; + +// migration:advisor +import { createMigrationLegacyAdvisorSessionReader } from "@sentry/junior/migration-helpers/v1"; + +// migration:db-schema +import { + migrationJuniorAgentSteps, + migrationJuniorConversationMessages, + migrationJuniorConversations, +} from "@sentry/junior/migration-helpers/v1"; + +// packages/junior/src/chat/conversations/sql/history.ts +import { and, asc, eq, isNotNull, sql as sql2 } from "drizzle-orm"; + +// packages/junior/src/chat/conversations/history.ts +import { z as z3 } from "zod"; + +// packages/junior/src/chat/pi/messages.ts +import { z } from "zod"; +var piMessageSchema = z + .object({ + role: z.string(), + }) + .passthrough() + .transform((value) => value); +var piContentMessageSchema = z + .object({ + content: z.array(z.unknown()), + role: z.string().min(1), + }) + .passthrough() + .transform((value) => value); + +// packages/junior/src/chat/model-profile.ts +import { z as z2 } from "zod"; +var modelProfileSchema = z2.string().regex(/^[a-z][a-z0-9_-]*$/); + +// packages/junior/src/chat/conversations/history.ts +var handoffModelProfileSchema = modelProfileSchema.refine( + (profile) => profile !== "standard", + "handoff profile must not be standard", +); +var piMessageStepEntrySchema = z3.object({ + type: z3.literal("pi_message"), + schemaVersion: z3.number().int().optional(), + message: piMessageSchema, + provenance: migrationPiMessageProvenanceSchema.optional(), +}); +var contextEpochStartedEntrySchema = z3.union([ + z3 + .object({ + type: z3.literal("context_epoch_started"), + reason: z3.literal("initial"), + modelProfile: z3.literal("standard"), + modelId: z3.string().min(1), + }) + .strict(), + z3 + .object({ + type: z3.literal("context_epoch_started"), + reason: z3.literal("handoff"), + modelProfile: handoffModelProfileSchema, + modelId: z3.string().min(1), + }) + .strict(), + z3 + .object({ + type: z3.literal("context_epoch_started"), + reason: z3.union([z3.literal("compaction"), z3.literal("rollback")]), + // This historical migration permanently accepts deployed markers that + // predate model bindings. + modelProfile: z3.undefined().optional(), + modelId: z3.undefined().optional(), + }) + .strict(), + z3 + .object({ + type: z3.literal("context_epoch_started"), + reason: z3.union([z3.literal("compaction"), z3.literal("rollback")]), + modelProfile: modelProfileSchema, + modelId: z3.string().min(1), + }) + .strict(), +]); +var piMessageStepSchema = z3 + .object({ + message: piMessageSchema, + createdAtMs: z3.number().finite(), + provenance: migrationPiMessageProvenanceSchema.optional(), + }) + .strict(); +var contextEpochStartSchema = z3.discriminatedUnion("reason", [ + z3 + .object({ + reason: z3.literal("initial"), + modelProfile: z3.literal("standard"), + modelId: z3.string().min(1), + messages: z3.array(piMessageStepSchema), + }) + .strict(), + z3 + .object({ + reason: z3.literal("handoff"), + modelProfile: handoffModelProfileSchema, + modelId: z3.string().min(1), + messages: z3.array(piMessageStepSchema), + }) + .strict(), + z3 + .object({ + reason: z3.union([z3.literal("compaction"), z3.literal("rollback")]), + modelProfile: modelProfileSchema, + modelId: z3.string().min(1), + messages: z3.array(piMessageStepSchema), + }) + .strict(), +]); +var mcpProviderConnectedEntrySchema = z3.object({ + type: z3.literal("mcp_provider_connected"), + provider: z3.string().min(1), +}); +var authorizationKindSchema = z3.union([ + z3.literal("plugin"), + z3.literal("mcp"), +]); +var authorizationRequestedEntrySchema = z3.object({ + type: z3.literal("authorization_requested"), + kind: authorizationKindSchema, + provider: z3.string().min(1), + actorId: z3.string().min(1), + authorizationId: z3.string().min(1), + delivery: z3.union([ + z3.literal("private_link_sent"), + z3.literal("private_link_reused"), + ]), +}); +var authorizationCompletedEntrySchema = z3.object({ + type: z3.literal("authorization_completed"), + kind: authorizationKindSchema, + provider: z3.string().min(1), + actorId: z3.string().min(1), + authorizationId: z3.string().min(1), +}); +var toolExecutionStartedEntrySchema = z3.object({ + type: z3.literal("tool_execution_started"), + toolCallId: z3.string().min(1), + toolName: z3.string().min(1), + args: z3.unknown().optional(), +}); +var visibleContextCompactedEntrySchema = z3.object({ + type: z3.literal("visible_context_compacted"), + compactions: z3.array( + z3.object({ + coveredMessageIds: z3.array(z3.string()), + createdAtMs: z3.number(), + id: z3.string().min(1), + summary: z3.string(), + }), + ), +}); +var subagentStartedEntrySchema = z3 + .object({ + type: z3.literal("subagent_started"), + subagentInvocationId: z3.string().min(1), + subagentKind: z3.string().min(1), + modelId: z3.string().min(1).optional(), + parentToolCallId: z3.string().min(1).optional(), + reasoningLevel: z3.string().min(1).optional(), + childConversationId: z3.string().min(1), + historyMode: z3.union([z3.literal("isolated"), z3.literal("shared")]), + }) + .strict(); +var subagentEndedEntrySchema = z3.object({ + type: z3.literal("subagent_ended"), + subagentInvocationId: z3.string().min(1), + outcome: z3.union([ + z3.literal("success"), + z3.literal("error"), + z3.literal("aborted"), + ]), + errorCode: z3.string().min(1).optional(), +}); +var appendableAgentStepEntrySchema = z3.union([ + piMessageStepEntrySchema, + mcpProviderConnectedEntrySchema, + authorizationRequestedEntrySchema, + authorizationCompletedEntrySchema, + toolExecutionStartedEntrySchema, + visibleContextCompactedEntrySchema, + subagentStartedEntrySchema, + subagentEndedEntrySchema, +]); +var agentStepEntrySchema = z3.union([ + appendableAgentStepEntrySchema, + contextEpochStartedEntrySchema, +]); +var newAgentStepSchema = z3 + .object({ + entry: appendableAgentStepEntrySchema, + createdAtMs: z3.number().finite(), + }) + .strict(); + +// packages/junior/src/chat/conversations/sql/conversation-row.ts +import { sql } from "drizzle-orm"; +async function ensureConversationRow(executor, conversationId, atMs) { + const at = new Date(atMs); + await executor + .db() + .insert(migrationJuniorConversations) + .values({ + conversationId, + createdAt: at, + lastActivityAt: at, + updatedAt: at, + executionStatus: "idle", + }) + .onConflictDoUpdate({ + target: migrationJuniorConversations.conversationId, + set: { + lastActivityAt: sql`greatest(${migrationJuniorConversations.lastActivityAt}, excluded.last_activity_at)`, + updatedAt: sql`greatest(${migrationJuniorConversations.updatedAt}, excluded.updated_at)`, + transcriptPurgedAt: null, + }, + }); +} + +// packages/junior/src/db/postgres-json.ts +function sanitizePostgresJson(value) { + if (typeof value === "string") { + return value.replaceAll("\0", " "); + } + if (Array.isArray(value)) { + return value.map((item) => sanitizePostgresJson(item)); + } + if (value === null || typeof value !== "object") { + return value; + } + const prototype = Object.getPrototypeOf(value); + if (prototype !== Object.prototype && prototype !== null) { + return value; + } + return Object.fromEntries( + Object.entries(value).map(([key, item]) => [ + key, + sanitizePostgresJson(item), + ]), + ); +} + +// packages/junior/src/chat/conversations/sql/history.ts +function messageRole(entry) { + if (entry.type !== "pi_message") { + return null; + } + const role = entry.message.role; + return typeof role === "string" ? role : null; +} +function insertFromStep(conversationId, seq, contextEpoch, step) { + const { type, ...payload } = agentStepEntrySchema.parse(step.entry); + return { + conversationId, + seq, + contextEpoch, + type, + role: messageRole(step.entry), + payload: sanitizePostgresJson(payload), + createdAt: new Date(step.createdAtMs), + }; +} +function stepFromRow(row) { + const entry = agentStepEntrySchema.parse({ type: row.type, ...row.payload }); + return { + seq: row.seq, + contextEpoch: row.contextEpoch, + createdAtMs: row.createdAt.getTime(), + entry, + }; +} +function piMessageStep(step) { + return { + entry: { + type: "pi_message", + message: step.message, + ...(step.provenance ? { provenance: step.provenance } : {}), + }, + createdAtMs: step.createdAtMs, + }; +} +var SqlAgentStepStore = class { + constructor(executor) { + this.executor = executor; + } + executor; + async append(conversationId, steps) { + const parsed = steps.map((step) => newAgentStepSchema.parse(step)); + if (parsed.length === 0) { + return; + } + const newestCreatedAtMs = Math.max( + ...parsed.map((step) => step.createdAtMs), + ); + await this.executor.transaction(async () => { + await ensureConversationRow( + this.executor, + conversationId, + newestCreatedAtMs, + ); + await this.executor + .db() + .update(migrationJuniorConversations) + .set({ archivedAt: null }) + .where( + and( + eq(migrationJuniorConversations.conversationId, conversationId), + isNotNull(migrationJuniorConversations.archivedAt), + ), + ); + const cursor = await this.readCursor(conversationId); + const contextEpoch = cursor.maxEpoch ?? 0; + let seq = cursor.nextSeq; + const rows = parsed.map((step) => + insertFromStep(conversationId, seq++, contextEpoch, step), + ); + await this.executor.db().insert(migrationJuniorAgentSteps).values(rows); + }); + } + async startEpoch(conversationId, opts) { + const parsed = contextEpochStartSchema.parse(opts); + await this.executor.transaction(async () => { + await ensureConversationRow(this.executor, conversationId, Date.now()); + await this.executor + .db() + .update(migrationJuniorConversations) + .set({ archivedAt: null }) + .where( + and( + eq(migrationJuniorConversations.conversationId, conversationId), + isNotNull(migrationJuniorConversations.archivedAt), + ), + ); + const cursor = await this.readCursor(conversationId); + const contextEpoch = + parsed.reason === "initial" + ? (cursor.maxEpoch ?? 0) + : (cursor.maxEpoch ?? -1) + 1; + let seq = cursor.nextSeq; + const { messages, ...binding } = parsed; + const marker = { + entry: { type: "context_epoch_started", ...binding }, + createdAtMs: Date.now(), + }; + const rows = [marker, ...messages.map(piMessageStep)].map((step) => + insertFromStep(conversationId, seq++, contextEpoch, step), + ); + await this.executor.db().insert(migrationJuniorAgentSteps).values(rows); + }); + } + async loadCurrentEpoch(conversationId) { + const cursor = await this.readCursor(conversationId); + if (cursor.maxEpoch === null) { + return []; + } + const rows = await this.executor + .db() + .select() + .from(migrationJuniorAgentSteps) + .where( + and( + eq(migrationJuniorAgentSteps.conversationId, conversationId), + eq(migrationJuniorAgentSteps.contextEpoch, cursor.maxEpoch), + ), + ) + .orderBy(asc(migrationJuniorAgentSteps.seq)); + return rows.map(stepFromRow); + } + async loadHistory(conversationId) { + const rows = await this.executor + .db() + .select() + .from(migrationJuniorAgentSteps) + .where(eq(migrationJuniorAgentSteps.conversationId, conversationId)) + .orderBy(asc(migrationJuniorAgentSteps.seq)); + return rows.map(stepFromRow); + } + /** Read the next `seq` and current highest epoch for one conversation. */ + async readCursor(conversationId) { + const rows = await this.executor + .db() + .select({ + maxSeq: sql2`max(${migrationJuniorAgentSteps.seq})`, + maxEpoch: sql2`max(${migrationJuniorAgentSteps.contextEpoch})`, + }) + .from(migrationJuniorAgentSteps) + .where(eq(migrationJuniorAgentSteps.conversationId, conversationId)); + const maxSeq = rows[0]?.maxSeq; + const maxEpoch = rows[0]?.maxEpoch; + return { + maxEpoch: + maxEpoch === null || maxEpoch === void 0 ? null : Number(maxEpoch), + nextSeq: maxSeq === null || maxSeq === void 0 ? 0 : Number(maxSeq) + 1, + }; + } +}; +function createSqlAgentStepStore(executor) { + return new SqlAgentStepStore(executor); +} + +// packages/junior/src/chat/conversations/sql/legacy-history-import.ts +import { eq as eq2, sql as sql3 } from "drizzle-orm"; + +// migration:xml +import { migrationUnescapeXml } from "@sentry/junior/migration-helpers/v1"; + +// packages/junior/src/chat/conversations/sql/legacy-history-import.ts +var INITIAL_SESSION_ID = "session_0"; +var ADVISOR_TASK_OPEN = "\n"; +var ADVISOR_TASK_CLOSE = "\n"; +var ADVISOR_CONTEXT_OPEN = "\n"; +var ADVISOR_CONTEXT_CLOSE = "\n"; +function importedAdvisorChildConversationId(parentConversationId) { + return `advisor:${parentConversationId}`; +} +function epochFromSessionId(sessionId) { + const match = /^session_(\d+)$/.exec(sessionId); + return match ? Number(match[1]) : 0; +} +function messageTimestampMs(message) { + const timestamp = message.timestamp; + return typeof timestamp === "number" ? timestamp : void 0; +} +function readAdvisorRequest(text) { + if ( + !text.startsWith(ADVISOR_TASK_OPEN) || + !text.endsWith(ADVISOR_CONTEXT_CLOSE) + ) { + return void 0; + } + const taskEnd = text.indexOf(ADVISOR_TASK_CLOSE, ADVISOR_TASK_OPEN.length); + if (taskEnd < 0) { + return void 0; + } + const contextStart = taskEnd + ADVISOR_TASK_CLOSE.length + 2; + if (!text.startsWith(ADVISOR_CONTEXT_OPEN, contextStart)) { + return void 0; + } + const task = text.slice(ADVISOR_TASK_OPEN.length, taskEnd); + const context = text.slice( + contextStart + ADVISOR_CONTEXT_OPEN.length, + -ADVISOR_CONTEXT_CLOSE.length, + ); + return `${migrationUnescapeXml(task)} + +Executor context: +${migrationUnescapeXml(context)}`; +} +function normalizeAdvisorMessage(message) { + const record = message; + if (record.role !== "user" || !Array.isArray(record.content)) { + return message; + } + let changed = false; + const content = record.content.map((part) => { + if ( + !part || + typeof part !== "object" || + part.type !== "text" || + typeof part.text !== "string" + ) { + return part; + } + const text = readAdvisorRequest(part.text); + if (text === void 0) { + return part; + } + changed = true; + return { ...part, text }; + }); + return changed ? { ...record, content } : message; +} +function piEntryProvenance(entry) { + if (entry.provenance) { + return entry.provenance; + } + if (entry.actor) { + return migrationLegacyActorProvenance(entry.actor); + } + return migrationContextProvenance; +} +function convertLegacySessionLog(args) { + const steps = []; + const fallback = args.fallbackCreatedAtMs; + let advisorChildConversationId; + let seq = 0; + const push = (contextEpoch, entry, createdAtMs) => { + steps.push({ seq, contextEpoch, entry, createdAtMs }); + seq += 1; + }; + for (const entry of args.entries) { + const epoch = epochFromSessionId(entry.sessionId ?? INITIAL_SESSION_ID); + switch (entry.type) { + case "pi_message": { + push( + epoch, + { + type: "pi_message", + message: entry.message, + provenance: piEntryProvenance(entry), + }, + messageTimestampMs(entry.message) ?? fallback, + ); + break; + } + case "projection_reset": { + const provenance = + entry.provenance ?? + entry.messages.map(() => migrationContextProvenance); + if (provenance.length !== entry.messages.length) { + throw new Error( + "projection_reset provenance must align one-to-one with messages", + ); + } + push( + epoch, + { type: "context_epoch_started", reason: "compaction" }, + fallback, + ); + entry.messages.forEach((message, index) => { + push( + epoch, + { + type: "pi_message", + message, + provenance: provenance[index], + }, + messageTimestampMs(message) ?? fallback, + ); + }); + break; + } + case "mcp_provider_connected": { + push( + epoch, + { type: "mcp_provider_connected", provider: entry.provider }, + fallback, + ); + break; + } + case "authorization_requested": { + push( + epoch, + { + type: "authorization_requested", + kind: entry.kind, + provider: entry.provider, + actorId: entry.actorId, + authorizationId: entry.authorizationId, + delivery: entry.delivery, + }, + entry.createdAtMs, + ); + break; + } + case "authorization_completed": { + push( + epoch, + { + type: "authorization_completed", + kind: entry.kind, + provider: entry.provider, + actorId: entry.actorId, + authorizationId: entry.authorizationId, + }, + entry.createdAtMs, + ); + break; + } + case "tool_execution_started": { + push( + epoch, + { + type: "tool_execution_started", + toolCallId: entry.toolCallId, + toolName: entry.toolName, + ...(entry.args !== void 0 ? { args: entry.args } : {}), + }, + entry.createdAtMs, + ); + break; + } + case "subagent_started": { + const childConversationId = importedAdvisorChildConversationId( + args.conversationId, + ); + advisorChildConversationId = childConversationId; + push( + epoch, + { + type: "subagent_started", + subagentInvocationId: entry.subagentInvocationId, + subagentKind: entry.subagentKind, + ...(entry.parentToolCallId + ? { parentToolCallId: entry.parentToolCallId } + : {}), + childConversationId, + historyMode: "shared", + }, + entry.createdAtMs, + ); + break; + } + case "subagent_ended": { + push( + epoch, + { + type: "subagent_ended", + subagentInvocationId: entry.subagentInvocationId, + outcome: entry.outcome, + ...(entry.errorCode ? { errorCode: entry.errorCode } : {}), + }, + entry.createdAtMs, + ); + break; + } + case "actor_recorded": { + break; + } + } + } + return { + steps, + ...(advisorChildConversationId ? { advisorChildConversationId } : {}), + }; +} +function convertAdvisorMessages(messages, fallbackCreatedAtMs) { + return messages.map((sourceMessage, seq) => { + const message = normalizeAdvisorMessage(sourceMessage); + return { + seq, + contextEpoch: 0, + entry: { + type: "pi_message", + message, + provenance: migrationContextProvenance, + }, + createdAtMs: messageTimestampMs(message) ?? fallbackCreatedAtMs, + }; + }); +} +function messageRole2(entry) { + if (entry.type !== "pi_message") { + return null; + } + const role = entry.message.role; + return typeof role === "string" ? role : null; +} +function insertRow(conversationId, step) { + const { type, ...payload } = agentStepEntrySchema.parse(step.entry); + return { + conversationId, + seq: step.seq, + contextEpoch: step.contextEpoch, + type, + role: messageRole2(step.entry), + payload: sanitizePostgresJson(payload), + createdAt: new Date(step.createdAtMs), + }; +} +async function writeLegacyImport(executor, args) { + return executor.withLock( + `junior_conversation:legacy-import:${args.conversationId}`, + () => + executor.transaction(async () => { + const db = executor.db(); + const conversations = await db + .select({ + transcriptPurgedAt: migrationJuniorConversations.transcriptPurgedAt, + }) + .from(migrationJuniorConversations) + .where( + eq2( + migrationJuniorConversations.conversationId, + args.conversationId, + ), + ) + .for("update"); + if (conversations[0]?.transcriptPurgedAt) { + return false; + } + const existing = await db + .select({ seq: migrationJuniorAgentSteps.seq }) + .from(migrationJuniorAgentSteps) + .where( + eq2(migrationJuniorAgentSteps.conversationId, args.conversationId), + ) + .limit(1); + if (existing.length > 0) { + return false; + } + const createdAt = new Date(args.fallbackCreatedAtMs); + await ensureConversationRow2( + executor, + args.conversationId, + createdAt, + new Date(args.lastActivityAtMs), + ); + if (args.messages && args.messages.length > 0) { + await db + .insert(migrationJuniorConversationMessages) + .values( + args.messages.map((message) => ({ + conversationId: args.conversationId, + messageId: message.messageId, + role: message.role, + authorIdentityId: message.authorIdentityId ?? null, + text: message.text, + meta: message.meta ?? null, + repliedAt: + message.repliedAtMs === void 0 + ? null + : new Date(message.repliedAtMs), + createdAt: new Date(message.createdAtMs), + })), + ) + .onConflictDoUpdate({ + target: [ + migrationJuniorConversationMessages.conversationId, + migrationJuniorConversationMessages.messageId, + ], + set: { + meta: sql3`nullif(coalesce(${migrationJuniorConversationMessages.meta}, '{}'::jsonb) || coalesce(excluded.meta, '{}'::jsonb), '{}'::jsonb)`, + repliedAt: sql3`coalesce(${migrationJuniorConversationMessages.repliedAt}, excluded.replied_at)`, + }, + }); + } + if (args.steps.length > 0) { + await db + .insert(migrationJuniorAgentSteps) + .values( + args.steps.map((step) => insertRow(args.conversationId, step)), + ); + } + if (args.child) { + const childCreatedAtMs = + args.child.steps.length > 0 + ? Math.min(...args.child.steps.map((step) => step.createdAtMs)) + : args.fallbackCreatedAtMs; + const childLastActivityAtMs = + args.child.steps.length > 0 + ? Math.max(...args.child.steps.map((step) => step.createdAtMs)) + : childCreatedAtMs; + await ensureChildConversationRow( + executor, + args.child.conversationId, + args.conversationId, + new Date(childCreatedAtMs), + new Date(childLastActivityAtMs), + ); + if (args.child.steps.length > 0) { + await db + .insert(migrationJuniorAgentSteps) + .values( + args.child.steps.map((step) => + insertRow(args.child.conversationId, step), + ), + ); + } + } + return true; + }), + ); +} +async function ensureConversationRow2( + executor, + conversationId, + createdAt, + lastActivityAt, +) { + await executor + .db() + .insert(migrationJuniorConversations) + .values({ + conversationId, + schemaVersion: 1, + createdAt, + lastActivityAt, + updatedAt: lastActivityAt, + executionStatus: "idle", + }) + .onConflictDoUpdate({ + target: migrationJuniorConversations.conversationId, + set: { + createdAt: sql3`least(${migrationJuniorConversations.createdAt}, excluded.created_at)`, + lastActivityAt: sql3`greatest(${migrationJuniorConversations.lastActivityAt}, excluded.last_activity_at)`, + updatedAt: sql3`greatest(${migrationJuniorConversations.updatedAt}, excluded.updated_at)`, + }, + }); +} +async function ensureChildConversationRow( + executor, + childConversationId, + parentConversationId, + createdAt, + lastActivityAt, +) { + await ensureConversationRow2( + executor, + parentConversationId, + createdAt, + lastActivityAt, + ); + await executor + .db() + .insert(migrationJuniorConversations) + .values({ + conversationId: childConversationId, + schemaVersion: 1, + parentConversationId, + createdAt, + lastActivityAt, + updatedAt: lastActivityAt, + executionStatus: "idle", + }) + .onConflictDoUpdate({ + target: migrationJuniorConversations.conversationId, + set: { + parentConversationId: sql3`coalesce(${migrationJuniorConversations.parentConversationId}, excluded.parent_conversation_id)`, + createdAt: sql3`least(${migrationJuniorConversations.createdAt}, excluded.created_at)`, + lastActivityAt: sql3`greatest(${migrationJuniorConversations.lastActivityAt}, excluded.last_activity_at)`, + updatedAt: sql3`greatest(${migrationJuniorConversations.updatedAt}, excluded.updated_at)`, + }, + }); +} + +// packages/junior/src/chat/conversations/legacy-import.ts +var legacyVisibleMessageSchema = z4.object({ + id: z4.string(), + role: z4.enum(["user", "assistant", "system"]), + text: z4.string(), + createdAtMs: z4.number().finite(), + author: z4.object({}).passthrough().optional(), + meta: z4.object({}).passthrough().optional(), +}); +var legacyCompactionSchema = z4.object({ + coveredMessageIds: z4.array(z4.string()), + createdAtMs: z4.number().finite(), + id: z4.string(), + summary: z4.string(), +}); +var legacyThreadStateSnapshotSchema = z4.object({ + conversation: z4 + .object({ + compactions: z4.array(legacyCompactionSchema).optional(), + messages: z4.array(legacyVisibleMessageSchema).optional(), + stats: z4 + .object({ updatedAtMs: z4.number().finite().optional() }) + .passthrough() + .optional(), + }) + .passthrough() + .optional(), +}); +async function loadThreadStateSnapshot(conversationId) { + const stateAdapter = getStateAdapter(); + await stateAdapter.connect(); + const raw = await stateAdapter.get(`thread-state:${conversationId}`); + if (!raw) { + return { compactions: [], messages: [] }; + } + const conversation = legacyThreadStateSnapshotSchema.parse(raw).conversation; + return { + compactions: conversation?.compactions ?? [], + messages: conversation?.messages ?? [], + ...(conversation?.stats?.updatedAtMs !== void 0 + ? { lastActivityAtMs: conversation.stats.updatedAtMs } + : {}), + }; +} +function intrinsicTimestamps(entries, visible, compactions) { + const candidates = []; + const pushMessageTs = (message) => { + const timestamp = message.timestamp; + if (typeof timestamp === "number") { + candidates.push(timestamp); + } + }; + for (const entry of entries) { + if (entry.type === "pi_message") { + pushMessageTs(entry.message); + } else if (entry.type === "projection_reset") { + entry.messages.forEach(pushMessageTs); + } else if ("createdAtMs" in entry) { + candidates.push(entry.createdAtMs); + } + } + for (const message of visible) { + candidates.push(message.createdAtMs); + } + for (const compaction of compactions) { + candidates.push(compaction.createdAtMs); + } + return candidates; +} +async function importConversationFromLegacy(conversationId, deps) { + const stepStore = createSqlAgentStepStore(deps.executor); + const existing = await stepStore.loadCurrentEpoch(conversationId); + if (existing.length > 0) { + return { imported: false }; + } + const entries = deps.sessionLogStore + ? await deps.sessionLogStore.read({ conversationId }) + : await decodeMigrationSessionLogEntry({ conversationId }); + const snapshot = deps.loadVisibleMessages + ? { + compactions: deps.legacyCompactions ?? [], + messages: await deps.loadVisibleMessages(conversationId), + } + : await loadThreadStateSnapshot(conversationId); + const { compactions, messages: visible } = snapshot; + if ( + entries.length === 0 && + visible.length === 0 && + compactions.length === 0 + ) { + return { imported: false }; + } + const hasAdvisor = entries.some((entry) => entry.type === "subagent_started"); + const advisorMessages = hasAdvisor + ? await ( + deps.advisorSessionStore ?? createMigrationLegacyAdvisorSessionReader() + ).load(conversationId) + : []; + const intrinsic = intrinsicTimestamps(entries, visible, compactions); + for (const message of advisorMessages) { + const timestamp = message.timestamp; + if (typeof timestamp === "number") { + intrinsic.push(timestamp); + } + } + const fallbackCreatedAtMs = + deps.conversationRecord?.createdAtMs ?? + (intrinsic.length > 0 ? Math.min(...intrinsic) : void 0) ?? + 0; + const lastActivityAtMs = Math.max( + fallbackCreatedAtMs, + deps.conversationRecord?.lastActivityAtMs ?? 0, + deps.legacyLastActivityAtMs ?? 0, + intrinsic.length > 0 ? Math.max(...intrinsic) : 0, + ); + const converted = convertLegacySessionLog({ + conversationId, + entries, + fallbackCreatedAtMs, + }); + if (compactions.length > 0) { + converted.steps.push({ + seq: converted.steps.length, + contextEpoch: converted.steps.at(-1)?.contextEpoch ?? 0, + entry: { type: "visible_context_compacted", compactions }, + createdAtMs: compactions.at(-1)?.createdAtMs ?? fallbackCreatedAtMs, + }); + } + if (converted.steps.length === 0 && visible.length > 0) { + const existingMessages = new Map( + (await deps.messageStore.list(conversationId)).map((message) => [ + message.messageId, + message, + ]), + ); + const fullyImported = visible.every((message) => { + const existingMessage = existingMessages.get(message.id); + const projected = toMigrationStoredConversationMessage(message); + return ( + existingMessage !== void 0 && + Object.entries(projected.meta ?? {}).every(([key, value]) => + isDeepStrictEqual(existingMessage.meta?.[key], value), + ) && + (message.meta?.replied !== true || + existingMessage.repliedAtMs !== void 0) + ); + }); + if (fullyImported) { + await writeLegacyImport(deps.executor, { + conversationId, + fallbackCreatedAtMs, + lastActivityAtMs, + steps: [], + }); + return { imported: false }; + } + } + let child; + if (converted.advisorChildConversationId) { + child = { + conversationId: converted.advisorChildConversationId, + steps: convertAdvisorMessages(advisorMessages, fallbackCreatedAtMs), + }; + } + const messages = visible.map((message) => ({ + ...toMigrationStoredConversationMessage(message), + ...(message.meta?.replied ? { repliedAtMs: message.createdAtMs } : {}), + })); + const imported = await writeLegacyImport(deps.executor, { + conversationId, + fallbackCreatedAtMs, + lastActivityAtMs, + ...(messages.length > 0 ? { messages } : {}), + steps: converted.steps, + ...(child ? { child } : {}), + }); + return { imported }; +} + +import { + createMigrationSqlConversationMessageStore, + createMigrationStateConversationStore, + decodeMigrationSessionLogEntry as decodeMigrationSessionLogEntry2, +} from "@sentry/junior/migration-helpers/v1"; +import { z as z5 } from "zod"; +var AGENT_SESSION_LOG_PREFIX = "junior:agent-session-log"; +var HISTORY_BACKFILL_LIMIT = 1e4; +var legacyVisibleMessageSchema2 = z5.object({ + id: z5.string(), + role: z5.enum(["user", "assistant", "system"]), + text: z5.string(), + createdAtMs: z5.number().finite(), + author: z5.object({}).passthrough().optional(), + meta: z5.object({}).passthrough().optional(), +}); +var legacyCompactionSchema2 = z5.object({ + coveredMessageIds: z5.array(z5.string()), + createdAtMs: z5.number().finite(), + id: z5.string(), + summary: z5.string(), +}); +var legacyThreadStateSnapshotSchema2 = z5.object({ + conversation: z5 + .object({ + compactions: z5.array(legacyCompactionSchema2).optional(), + messages: z5.array(legacyVisibleMessageSchema2).optional(), + }) + .passthrough() + .optional(), +}); +/** Import retained legacy session history and visible messages into SQL. */ +async function migrateConversationHistoryToSql(context, options = {}) { + const source = createMigrationStateConversationStore(context.stateAdapter); + const executor = options.executor; + const sessionLogStore = options.sessionLogStore ?? { + read: async ({ conversationId }) => { + const entries = await context.stateAdapter.get( + `${AGENT_SESSION_LOG_PREFIX}:${conversationId}`, + ); + return Array.isArray(entries) + ? entries.map(decodeMigrationSessionLogEntry2) + : []; + }, + append: async () => {}, + }; + const limit = Math.max(1, options.batchSize ?? HISTORY_BACKFILL_LIMIT); + const messageStore = createMigrationSqlConversationMessageStore(executor); + const conversations = await source.listByActivity({ limit }); + let migrated = 0; + let existing = 0; + for (const conversation of conversations) { + const raw = options.loadVisibleMessages + ? void 0 + : await context.stateAdapter.get( + `thread-state:${conversation.conversationId}`, + ); + const stored = raw + ? legacyThreadStateSnapshotSchema2.parse(raw).conversation + : void 0; + const result = await importConversationFromLegacy( + conversation.conversationId, + { + executor, + messageStore, + conversationRecord: conversation, + sessionLogStore, + ...(options.advisorSessionStore + ? { advisorSessionStore: options.advisorSessionStore } + : {}), + loadVisibleMessages: + options.loadVisibleMessages ?? (async () => stored?.messages ?? []), + legacyCompactions: + options.legacyCompactions ?? stored?.compactions ?? [], + }, + ); + if (result.imported) migrated += 1; + else existing += 1; + } + return { existing, migrated, missing: 0, scanned: conversations.length }; +} +var migration = { + apiVersion: 1, + async up(context) { + return await migrateConversationHistoryToSql( + { stateAdapter: context.state }, + { executor: context.database }, + ); + }, +}; +var entry_default = migration; +export { entry_default as default, migrateConversationHistoryToSql }; diff --git a/packages/junior/src/cli/upgrade/migrations/conversation-usage.ts b/packages/junior/migrations/0009_conversation_usage.ts similarity index 85% rename from packages/junior/src/cli/upgrade/migrations/conversation-usage.ts rename to packages/junior/migrations/0009_conversation_usage.ts index f3fb5a7ce..79378a234 100644 --- a/packages/junior/src/cli/upgrade/migrations/conversation-usage.ts +++ b/packages/junior/migrations/0009_conversation_usage.ts @@ -2,10 +2,18 @@ * Repair legacy usage from durable SQL steps in bounded, retry-safe batches. * Ephemeral run summaries are not an authority because TTL can erase evidence. */ -import { getChatConfig } from "@/chat/config"; -import { createJuniorSqlExecutor } from "@/db/executor"; -import type { JuniorSqlExecutor } from "@/db/db"; -import type { MigrationContext, MigrationResult } from "../types"; +import type { + MigrationDatabaseAdapter, + MigrationV1, +} from "@sentry/junior-migrations"; + +type MigrationResult = { + existing: number; + migrated: number; + missing: number; + scanned: number; + skipped?: number; +}; const CONVERSATION_USAGE_REPAIR_BATCH_SIZE = 500; const MAX_SAFE_INTEGER = 9_007_199_254_740_991; @@ -241,72 +249,68 @@ ORDER BY candidate.conversation_id /** Rebuild legacy conversation usage from canonical durable assistant messages. */ export async function repairConversationUsage( - _context: MigrationContext, + _context: { + io?: { info(message: string): void }; + stateAdapter?: unknown; + }, options: { batchSize?: number; - executor?: JuniorSqlExecutor; - } = {}, + executor: MigrationDatabaseAdapter; + }, ): Promise { - let executor = options.executor; - let closeExecutor: (() => Promise) | undefined; - if (!executor) { - const { sql } = getChatConfig(); - executor = createJuniorSqlExecutor({ - connectionString: sql.databaseUrl, - driver: sql.driver, - }); - closeExecutor = () => executor!.close(); - } - + const executor = options.executor; const batchSize = Math.max( 1, Math.floor(options.batchSize ?? CONVERSATION_USAGE_REPAIR_BATCH_SIZE), ); - try { - let cursor = ""; - let existing = 0; - let migrated = 0; - let missing = 0; - let scanned = 0; - let skipped = 0; + let cursor = ""; + let existing = 0; + let migrated = 0; + let missing = 0; + let scanned = 0; + let skipped = 0; - while (true) { - const sources = await executor.withLock(REPAIR_LOCK, () => - executor.query(USAGE_REPAIR_BATCH_SQL, [ - cursor, - batchSize, - ]), - ); - if (sources.length === 0) break; + while (true) { + const sources = await executor.withLock(REPAIR_LOCK, () => + executor.query(USAGE_REPAIR_BATCH_SQL, [ + cursor, + batchSize, + ]), + ); + if (sources.length === 0) break; - scanned += sources.length; - for (const source of sources) { - if (!source.repairable) { - missing += 1; - } else if (source.changed) { - migrated += 1; - } else if (source.matched) { - existing += 1; - } else { - skipped += 1; - } + scanned += sources.length; + for (const source of sources) { + if (!source.repairable) { + missing += 1; + } else if (source.changed) { + migrated += 1; + } else if (source.matched) { + existing += 1; + } else { + skipped += 1; } - cursor = sources.at(-1)!.conversationId; } - - return { - existing, - migrated, - missing, - scanned, - ...(skipped > 0 ? { skipped } : {}), - }; - } finally { - await closeExecutor?.(); + cursor = sources.at(-1)!.conversationId; } + + return { + existing, + migrated, + missing, + scanned, + ...(skipped > 0 ? { skipped } : {}), + }; } -export const conversationUsageRepairMigration = { - name: "repair-conversation-usage", - run: repairConversationUsage, -}; +const migration = { + apiVersion: 1, + async up(context) { + return await repairConversationUsage( + { io: { info: context.log } }, + { executor: context.database }, + ); + }, +} satisfies MigrationV1; + +export default migration; diff --git a/packages/junior/migrations/README.md b/packages/junior/migrations/README.md index a0b5c2aea..11604fd46 100644 --- a/packages/junior/migrations/README.md +++ b/packages/junior/migrations/README.md @@ -1,14 +1,32 @@ # SQL migrations for `@sentry/junior` -This is a standard Drizzle migration folder. `drizzle-kit generate` owns each -SQL file, snapshot, and journal entry; `junior upgrade` applies the folder with -Drizzle ORM's migrator before any data backfills run. +This migration folder extends Drizzle Kit's journal with self-contained +TypeScript data migrations. Drizzle Kit owns numbering, schema snapshots, and +journal entries; `junior upgrade` executes each entry as either `.sql` or +`.ts` through `@sentry/junior-migrations`. Data entries do not retain an +unchanged snapshot. 1. Edit the schema under `src/db/schema/`. 2. Run `pnpm --filter @sentry/junior db:generate --name `. 3. Commit the generated SQL file and `meta/` changes together. +For a data migration, run +`pnpm --filter @sentry/junior db:generate:data --name `. +TypeScript migrations target the versioned migration capability API and must +not import Junior runtime internals or current feature schemas. Their complete +implementation remains in the migration file and uses only the stable +database, state, Redis, and progress capabilities supplied by the runner. The +host database adapter owns connections, drivers, transactions, and locks so +those infrastructure modules are not frozen into every migration. + +Reusable infrastructure belongs in the append-only +`@sentry/junior/migration-helpers/v1` export. It may provide stable parsers, +stores, adapter projections, and key resolution, but must not implement a +specific data migration. The journal entry remains the only owner of one-off +record transformations. + The `0000_initial.sql` baseline represents the schema already deployed by the pre-Drizzle Junior migration runner. During upgrade, existing installations adopt that baseline once; new installations execute it normally. All later -migrations are applied by Drizzle in journal order. +migrations are applied by Junior in journal order. Drizzle ORM's stock +`migrate()` function is not compatible with TypeScript entries in this folder. diff --git a/packages/junior/migrations/meta/_journal.json b/packages/junior/migrations/meta/_journal.json index e97022daa..2bb84aa68 100644 --- a/packages/junior/migrations/meta/_journal.json +++ b/packages/junior/migrations/meta/_journal.json @@ -36,6 +36,41 @@ "when": 1784137322420, "tag": "0004_useful_magus", "breakpoints": true + }, + { + "idx": 5, + "version": "7", + "when": 1784137322421, + "tag": "0005_agent_turn_session_actor", + "breakpoints": true + }, + { + "idx": 6, + "version": "7", + "when": 1784137322422, + "tag": "0006_redis_conversation_state", + "breakpoints": true + }, + { + "idx": 7, + "version": "7", + "when": 1784137322423, + "tag": "0007_conversations_to_sql", + "breakpoints": true + }, + { + "idx": 8, + "version": "7", + "when": 1784137322424, + "tag": "0008_conversation_history_to_sql", + "breakpoints": true + }, + { + "idx": 9, + "version": "7", + "when": 1784137322425, + "tag": "0009_conversation_usage", + "breakpoints": true } ] } diff --git a/packages/junior/package.json b/packages/junior/package.json index d6bea10e8..4e192c0aa 100644 --- a/packages/junior/package.json +++ b/packages/junior/package.json @@ -24,6 +24,10 @@ "types": "./dist/instrumentation.d.ts", "default": "./dist/instrumentation.js" }, + "./migration-helpers/v1": { + "types": "./dist/migration-helpers/v1.d.ts", + "default": "./dist/migration-helpers/v1.js" + }, "./nitro": { "types": "./dist/nitro.d.ts", "default": "./dist/nitro.js" @@ -51,6 +55,7 @@ "prepack": "pnpm run build", "build": "tsup && tsc -p tsconfig.build.json --emitDeclarationOnly", "db:generate": "pnpm exec drizzle-kit generate --config drizzle.config.ts", + "db:generate:data": "pnpm exec junior-migrations generate --config drizzle.config.ts --out migrations", "lint": "oxlint --config .oxlintrc.json --deny-warnings src tests scripts bin tsup.config.ts && depcruise --config .dependency-cruiser.mjs src/chat", "lint:fix": "oxlint --config .oxlintrc.json --deny-warnings --fix src tests scripts bin tsup.config.ts", "test": "vitest run --maxWorkers=4", @@ -70,6 +75,7 @@ "@modelcontextprotocol/sdk": "1.29.0", "@neondatabase/serverless": "^1.1.0", "@sentry/junior-plugin-api": "workspace:*", + "@sentry/junior-migrations": "workspace:*", "@sentry/node": "catalog:", "@sinclair/typebox": "^0.34.49", "@slack/web-api": "^7.16.0", diff --git a/packages/junior/src/chat/conversations/sql/migrations.ts b/packages/junior/src/chat/conversations/sql/migrations.ts index 29c897b46..aa8637dcc 100644 --- a/packages/junior/src/chat/conversations/sql/migrations.ts +++ b/packages/junior/src/chat/conversations/sql/migrations.ts @@ -1,7 +1,15 @@ /** SQL schema migrations for durable Junior records. */ import { basename, dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; -import { readMigrationFiles } from "drizzle-orm/migrator"; +import { + resolveMigrations, + runMigrationJournal, + type MigrationContextV1, + type MigrationRunResult, + type TypeScriptMigrationLoader, +} from "@sentry/junior-migrations"; +import type { StateAdapter } from "chat"; +import type { RedisStateAdapter } from "@chat-adapter/state-redis"; import type { JuniorSqlMigrationExecutor } from "@/db/db"; import { juniorSqlSchema as schema } from "@/db/schema"; @@ -43,7 +51,7 @@ SELECT return; } - const migrations = readMigrationFiles({ migrationsFolder }); + const migrations = await resolveMigrations(migrationsFolder); const [metrics] = await executor.query<{ complete: boolean }>(` SELECT count(*) = 4 AS complete FROM information_schema.columns @@ -92,23 +100,63 @@ CREATE TABLE IF NOT EXISTS drizzle.__drizzle_junior_core ( await executor.execute( `INSERT INTO drizzle.__drizzle_junior_core (hash, created_at) VALUES ($1, $2)`, - [migration.hash, migration.folderMillis], + [migration.hash, migration.when], ); }); } +export type MigrateSchemaOptions = + | { mode: "schema-bootstrap" } + | { + loadTypeScript: TypeScriptMigrationLoader; + log?: MigrationContextV1["log"]; + mode: "all"; + redisStateAdapter?: RedisStateAdapter; + stateAdapter: StateAdapter; + }; + export { schema }; -/** Apply the packaged Drizzle migrations during `junior upgrade`. */ +/** Apply all migrations, or bootstrap an empty test database to the latest schema. */ export async function migrateSchema( executor: JuniorSqlMigrationExecutor, -): Promise { + options: MigrateSchemaOptions, +): Promise { const migrationsFolder = migrationFolder(); - await executor.withMigrationLock(MIGRATIONS_TABLE, async () => { - await adoptLegacyMigrationState(executor, migrationsFolder); - await executor.migrate({ - migrationsFolder, - migrationsTable: MIGRATIONS_TABLE, + const runAll = options.mode === "all"; + const baseOptions = { + beforeRun: async () => { + await adoptLegacyMigrationState(executor, migrationsFolder); + }, + executor, + migrationsFolder, + migrationsTable: MIGRATIONS_TABLE, + }; + if (!runAll) { + return await runMigrationJournal({ + ...baseOptions, + mode: "schema-bootstrap", }); + } + return await runMigrationJournal({ + ...baseOptions, + createContext: ({ progress }): MigrationContextV1 => ({ + database: executor, + log: options.log ?? (() => {}), + progress, + ...(options.redisStateAdapter + ? { + redis: { + sendCommand: async (args: readonly string[]) => + await options + .redisStateAdapter!.getClient() + .sendCommand([...args]), + }, + } + : {}), + state: options.stateAdapter, + }), + loadTypeScript: options.loadTypeScript, + mode: "all", }); } diff --git a/packages/junior/src/chat/plugins/migration-state.ts b/packages/junior/src/chat/plugins/migration-state.ts new file mode 100644 index 000000000..cea06e997 --- /dev/null +++ b/packages/junior/src/chat/plugins/migration-state.ts @@ -0,0 +1,49 @@ +import type { MigrationStateV1 } from "@sentry/junior-migrations"; +import type { StateAdapter } from "chat"; +import { + createPluginState, + pluginStateKey, + validatePluginStateKey, +} from "@/chat/plugins/state"; + +/** Project host state onto the permanent v1 migration API for one plugin. */ +export function createPluginMigrationStateV1( + plugin: string, + stateAdapter: StateAdapter, +): MigrationStateV1 { + const pluginState = createPluginState(plugin, stateAdapter); + const stateKey = (key: string): string => { + validatePluginStateKey(key); + return pluginStateKey(plugin, key); + }; + + return { + acquireLock: async (key, ttlMs) => { + await stateAdapter.connect(); + return await stateAdapter.acquireLock(stateKey(key), ttlMs); + }, + appendToList: async (key, value, options) => { + await stateAdapter.connect(); + await stateAdapter.appendToList(stateKey(key), value, options); + }, + connect: async () => { + await stateAdapter.connect(); + }, + delete: async (key) => { + await pluginState.delete(key); + }, + get: async (key: string) => await pluginState.get(key), + getList: async (key: string) => { + await stateAdapter.connect(); + return await stateAdapter.getList(stateKey(key)); + }, + releaseLock: async (lock) => { + await stateAdapter.releaseLock(lock); + }, + set: async (key, value, ttlMs) => { + await pluginState.set(key, value, ttlMs); + }, + setIfNotExists: async (key, value, ttlMs) => + await pluginState.setIfNotExists(key, value, ttlMs), + }; +} diff --git a/packages/junior/src/chat/plugins/migrations.ts b/packages/junior/src/chat/plugins/migrations.ts index d054f6e11..e3c620489 100644 --- a/packages/junior/src/chat/plugins/migrations.ts +++ b/packages/junior/src/chat/plugins/migrations.ts @@ -1,5 +1,13 @@ import { createHash } from "node:crypto"; -import { readMigrationFiles, type MigrationMeta } from "drizzle-orm/migrator"; +import { + resolveMigrations, + runMigrationJournal, + type MigrationContextV1, + type ResolvedMigration, + type TypeScriptMigrationLoader, +} from "@sentry/junior-migrations"; +import type { StateAdapter } from "chat"; +import { createPluginMigrationStateV1 } from "@/chat/plugins/migration-state"; import type { JuniorSqlMigrationExecutor } from "@/db/db"; interface PluginMigrationRoot { @@ -8,11 +16,21 @@ interface PluginMigrationRoot { pluginName: string; } -interface PluginMigrationResult { +type PluginMigrationResult = { existing: number; migrated: number; scanned: number; -} + skipped?: number; +}; + +type PluginMigrationOptions = + | { mode: "schema-bootstrap" } + | { + loadTypeScript: TypeScriptMigrationLoader; + log?: MigrationContextV1["log"]; + mode: "all"; + stateAdapter: StateAdapter; + }; const LEGACY_SCHEDULER_BASELINE_HASH = "d1d2f712181dd3a0557808f0fc67fd0722691d25f4c8cfb816b77c71d19e1e42"; @@ -30,28 +48,6 @@ function migrationTable(pluginName: string): string { return `__drizzle_${label}_${hash}`; } -async function appliedMigrationTime( - executor: JuniorSqlMigrationExecutor, - table: string, -): Promise { - const [exists] = await executor.query<{ tableName: string | null }>( - `SELECT to_regclass($1)::text AS "tableName"`, - [`drizzle.${table}`], - ); - if (!exists?.tableName) { - return undefined; - } - const [row] = await executor.query<{ createdAt: string | null }>( - `SELECT created_at::text AS "createdAt" - FROM drizzle.${table} - ORDER BY created_at DESC - LIMIT 1`, - ); - return row?.createdAt === null || row?.createdAt === undefined - ? undefined - : Number(row.createdAt); -} - async function legacyMigrationHashes( executor: JuniorSqlMigrationExecutor, pluginName: string, @@ -73,13 +69,13 @@ async function legacyMigrationHashes( } function adoptedMigration( - migrations: readonly MigrationMeta[], + migrations: readonly ResolvedMigration[], legacyHashes: ReadonlySet, pluginName: string, -): MigrationMeta | undefined { - let adopted: MigrationMeta | undefined; +): ResolvedMigration | undefined { + let adopted: ResolvedMigration | undefined; for (const migration of migrations) { - if (!legacyHashes.has(migration.hash)) { + if (migration.kind !== "sql" || !legacyHashes.has(migration.hash)) { break; } adopted = migration; @@ -96,10 +92,17 @@ function adoptedMigration( async function adoptLegacyMigrationState(args: { executor: JuniorSqlMigrationExecutor; - migrations: readonly MigrationMeta[]; + migrations: readonly ResolvedMigration[]; pluginName: string; table: string; -}): Promise { +}): Promise { + const [exists] = await args.executor.query<{ tableName: string | null }>( + `SELECT to_regclass($1)::text AS "tableName"`, + [`drizzle.${args.table}`], + ); + if (exists?.tableName) { + return; + } const legacyHashes = await legacyMigrationHashes( args.executor, args.pluginName, @@ -110,7 +113,7 @@ async function adoptLegacyMigrationState(args: { args.pluginName, ); if (!migration) { - return undefined; + return; } await args.executor.transaction(async () => { await args.executor.execute("CREATE SCHEMA IF NOT EXISTS drizzle"); @@ -123,26 +126,16 @@ CREATE TABLE IF NOT EXISTS drizzle.${args.table} ( `); await args.executor.execute( `INSERT INTO drizzle.${args.table} (hash, created_at) VALUES ($1, $2)`, - [migration.hash, migration.folderMillis], + [migration.hash, migration.when], ); }); - return migration.folderMillis; -} - -function appliedCount( - migrations: readonly MigrationMeta[], - createdAt: number | undefined, -): number { - return createdAt === undefined - ? 0 - : migrations.filter((migration) => migration.folderMillis <= createdAt) - .length; } -/** Apply enabled plugins' packaged Drizzle migrations in plugin-name order. */ +/** Apply enabled plugins' mixed migrations in plugin-name and journal order. */ export async function migratePluginSchemas( executor: JuniorSqlMigrationExecutor, roots: readonly PluginMigrationRoot[], + options: PluginMigrationOptions, ): Promise { const result: PluginMigrationResult = { existing: 0, @@ -153,26 +146,54 @@ export async function migratePluginSchemas( left.pluginName.localeCompare(right.pluginName), ); for (const root of orderedRoots) { - const migrations = readMigrationFiles({ migrationsFolder: root.dir }); + const migrations = await resolveMigrations(root.dir); const table = migrationTable(root.pluginName); - await executor.withMigrationLock(table, async () => { - const currentTime = - (await appliedMigrationTime(executor, table)) ?? - (await adoptLegacyMigrationState({ + const runAll = options.mode === "all"; + const baseOptions = { + beforeRun: async () => { + await adoptLegacyMigrationState({ executor, migrations, pluginName: root.pluginName, table, - })); - const existing = appliedCount(migrations, currentTime); - await executor.migrate({ - migrationsFolder: root.dir, - migrationsTable: table, - }); - result.scanned += migrations.length; - result.existing += existing; - result.migrated += migrations.length - existing; - }); + }); + }, + executor, + migrationsFolder: root.dir, + migrationsTable: table, + }; + const pluginResult = runAll + ? await runMigrationJournal({ + ...baseOptions, + createContext: ({ progress }): MigrationContextV1 => ({ + database: executor, + log: options.log ?? (() => {}), + progress, + state: createPluginMigrationStateV1( + root.pluginName, + options.stateAdapter, + ), + }), + loadTypeScript: options.loadTypeScript, + mode: "all", + }) + : await runMigrationJournal({ ...baseOptions, mode: "schema-bootstrap" }); + result.scanned += pluginResult.scanned; + result.existing += pluginResult.existing; + result.migrated += pluginResult.migrated; + if (pluginResult.skipped > 0) { + result.skipped = (result.skipped ?? 0) + pluginResult.skipped; + } } return result; } + +/** Construct enabled plugins' latest SQL schemas without running data entries. */ +export async function bootstrapPluginSchemas( + executor: JuniorSqlMigrationExecutor, + roots: readonly PluginMigrationRoot[], +): Promise { + return await migratePluginSchemas(executor, roots, { + mode: "schema-bootstrap", + }); +} diff --git a/packages/junior/src/chat/plugins/state.ts b/packages/junior/src/chat/plugins/state.ts index 4812176f3..ccd090b2f 100644 --- a/packages/junior/src/chat/plugins/state.ts +++ b/packages/junior/src/chat/plugins/state.ts @@ -9,7 +9,8 @@ function hashKeyPart(value: string): string { return createHash("sha256").update(value).digest("hex").slice(0, 32); } -function pluginStateKey(plugin: string, key: string): string { +/** Resolve one logical plugin key to its isolated host-state key. */ +export function pluginStateKey(plugin: string, key: string): string { const pluginPrefix = `junior:${plugin}`; if (key === pluginPrefix || key.startsWith(`${pluginPrefix}:`)) { return key; @@ -17,7 +18,8 @@ function pluginStateKey(plugin: string, key: string): string { return `junior:plugin_state:${hashKeyPart(plugin)}:${hashKeyPart(key)}`; } -function validatePluginStateKey(key: string): void { +/** Validate one logical plugin state key before namespacing it. */ +export function validatePluginStateKey(key: string): void { if (!key.trim()) { throw new Error("Plugin state key is required"); } diff --git a/packages/junior/src/chat/state/session-log.ts b/packages/junior/src/chat/state/session-log.ts index cb1be1908..559c07bac 100644 --- a/packages/junior/src/chat/state/session-log.ts +++ b/packages/junior/src/chat/state/session-log.ts @@ -554,9 +554,10 @@ function subagentEndedEntry(args: { }; } -function decode(value: unknown): SessionLogEntry { +/** Decode one persisted session-log entry for versioned migration helpers. */ +export function decodeSessionLogEntry(value: unknown): SessionLogEntry { if (typeof value === "string") { - return decode(JSON.parse(value) as unknown); + return decodeSessionLogEntry(JSON.parse(value) as unknown); } const parsed = sessionLogEntrySchema.safeParse(migrateStoredEntry(value)); @@ -812,7 +813,7 @@ function redisStore(redisStateAdapter: RedisStateAdapter): SessionLogStore { }, async read(scope) { const values = await client.lRange(key(scope), 0, -1); - return values.map(decode); + return values.map(decodeSessionLogEntry); }, }; } @@ -833,8 +834,8 @@ function stateStore(): SessionLogStore { try { const existingValue = await stateAdapter.get(listKey); const existingEntries = Array.isArray(existingValue) - ? existingValue.map(decode) - : (await stateAdapter.getList(listKey)).map(decode); + ? existingValue.map(decodeSessionLogEntry) + : (await stateAdapter.getList(listKey)).map(decodeSessionLogEntry); await stateAdapter.set( listKey, [...existingEntries, ...entries], @@ -848,10 +849,10 @@ function stateStore(): SessionLogStore { const listKey = rawKey(scope); const value = await stateAdapter.get(listKey); if (Array.isArray(value)) { - return value.map(decode); + return value.map(decodeSessionLogEntry); } const values = await stateAdapter.getList(listKey); - return values.map(decode); + return values.map(decodeSessionLogEntry); }, }; } @@ -872,7 +873,7 @@ async function loadEntries( }, ): Promise { const store = args.store ?? (await defaultStore()); - return (await store.read(args)).map(decode); + return (await store.read(args)).map(decodeSessionLogEntry); } /** diff --git a/packages/junior/src/cli/upgrade.ts b/packages/junior/src/cli/upgrade.ts index f5688ee2d..b5ff82c17 100644 --- a/packages/junior/src/cli/upgrade.ts +++ b/packages/junior/src/cli/upgrade.ts @@ -4,20 +4,13 @@ import { } from "@/chat/state/adapter"; import { createJiti } from "jiti"; import { loadAppPluginSet } from "@/plugin-module"; -import { sqlConversationMigration } from "./upgrade/migrations/conversations-sql"; -import { coreSqlSchemaMigration } from "./upgrade/migrations/core-sql"; -import { sqlConversationHistoryMigration } from "./upgrade/migrations/conversations-history-sql"; -import { pluginStorageMigration } from "./upgrade/migrations/plugin-storage"; -import { sqlPluginMigration } from "./upgrade/migrations/plugin-sql"; +import { migrateCoreJournal } from "./upgrade/migrations/core-journal"; +import { migratePluginJournals } from "./upgrade/migrations/plugin-journal"; import { resolveUpgradePlugins } from "./upgrade/migrations/upgrade-plugins"; -import { redisConversationStateMigration } from "./upgrade/migrations/redis-conversation-state"; -import { agentTurnSessionActorMigration } from "./upgrade/migrations/agent-turn-session-actor"; -import { conversationUsageRepairMigration } from "./upgrade/migrations/conversation-usage"; import type { MigrationContext, MigrationResult, UpgradeIo, - UpgradeMigration, } from "./upgrade/types"; import { type JuniorPluginSet } from "@/plugins"; @@ -26,17 +19,6 @@ const DEFAULT_IO: UpgradeIo = { }; const localPluginLoader = createJiti(import.meta.url, { moduleCache: false }); -const MIGRATIONS: UpgradeMigration[] = [ - agentTurnSessionActorMigration, - redisConversationStateMigration, - coreSqlSchemaMigration, - sqlConversationMigration, - sqlConversationHistoryMigration, - conversationUsageRepairMigration, - sqlPluginMigration, - pluginStorageMigration, -]; - function isMissingVirtualConfig(error: unknown): boolean { if (!(error instanceof Error)) { return false; @@ -90,14 +72,19 @@ export async function runUpgradeMigrations( const plugins = await resolveUpgradePlugins(context); const migrationContext = { ...context, ...plugins }; const results: MigrationResult[] = []; - for (const migration of MIGRATIONS) { - migrationContext.io.info(`Running migration ${migration.name}...`); - const result = await migration.run(migrationContext); + const run = async ( + name: string, + migrate: (context: MigrationContext) => Promise, + ): Promise => { + migrationContext.io.info(`Running migration ${name}...`); + const result = await migrate(migrationContext); migrationContext.io.info( - `Finished migration ${migration.name}: ${formatMigrationResult(result)}`, + `Finished migration ${name}: ${formatMigrationResult(result)}`, ); results.push(result); - } + }; + await run("core-migrations", migrateCoreJournal); + await run("plugin-migrations", migratePluginJournals); return results; } diff --git a/packages/junior/src/cli/upgrade/migrations/conversations-history-sql.ts b/packages/junior/src/cli/upgrade/migrations/conversations-history-sql.ts deleted file mode 100644 index 66e3e39a0..000000000 --- a/packages/junior/src/cli/upgrade/migrations/conversations-history-sql.ts +++ /dev/null @@ -1,87 +0,0 @@ -import { getChatConfig } from "@/chat/config"; -import { importConversationFromLegacy } from "@/chat/conversations/legacy-import"; -import { createSqlConversationMessageStore } from "@/chat/conversations/sql/messages"; -import { createStateConversationStore } from "@/chat/conversations/state"; -import type { LegacyAdvisorSessionReader } from "@/chat/conversations/legacy-advisor-session"; -import type { ConversationMessage as ThreadConversationMessage } from "@/chat/state/conversation"; -import type { SessionLogStore } from "@/chat/state/session-log"; -import { createJuniorSqlExecutor } from "@/db/executor"; -import type { JuniorSqlExecutor } from "@/db/db"; -import type { MigrationContext, MigrationResult } from "../types"; - -const HISTORY_BACKFILL_LIMIT = 10_000; - -/** - * Bulk-import legacy Redis conversation history (session logs, advisor blobs, - * and visible messages) into SQL, bounded newest-first over the same activity - * scan as the metadata backfill. Idempotent per conversation: it skips any - * conversation that already has step rows. - */ -export async function migrateConversationHistoryToSql( - context: MigrationContext, - options: { - batchSize?: number; - executor?: JuniorSqlExecutor; - sessionLogStore?: SessionLogStore; - advisorSessionStore?: LegacyAdvisorSessionReader; - loadVisibleMessages?: ( - conversationId: string, - ) => Promise; - } = {}, -): Promise { - const source = createStateConversationStore(context.stateAdapter); - let executor = options.executor; - let closeExecutor: (() => Promise) | undefined; - if (!executor) { - const { sql } = getChatConfig(); - executor = createJuniorSqlExecutor({ - connectionString: sql.databaseUrl, - driver: sql.driver, - }); - closeExecutor = () => executor!.close(); - } - const limit = Math.max(1, options.batchSize ?? HISTORY_BACKFILL_LIMIT); - try { - const messageStore = createSqlConversationMessageStore(executor); - const conversations = await source.listByActivity({ limit }); - let migrated = 0; - let existing = 0; - for (const conversation of conversations) { - const result = await importConversationFromLegacy( - conversation.conversationId, - { - executor, - messageStore, - conversationRecord: conversation, - ...(options.sessionLogStore - ? { sessionLogStore: options.sessionLogStore } - : {}), - ...(options.advisorSessionStore - ? { advisorSessionStore: options.advisorSessionStore } - : {}), - ...(options.loadVisibleMessages - ? { loadVisibleMessages: options.loadVisibleMessages } - : {}), - }, - ); - if (result.imported) { - migrated += 1; - } else { - existing += 1; - } - } - return { - existing, - migrated, - missing: 0, - scanned: conversations.length, - }; - } finally { - await closeExecutor?.(); - } -} - -export const sqlConversationHistoryMigration = { - name: "backfill-agent-steps-sql", - run: migrateConversationHistoryToSql, -}; diff --git a/packages/junior/src/cli/upgrade/migrations/conversations-sql.ts b/packages/junior/src/cli/upgrade/migrations/conversations-sql.ts deleted file mode 100644 index bb24c4283..000000000 --- a/packages/junior/src/cli/upgrade/migrations/conversations-sql.ts +++ /dev/null @@ -1,106 +0,0 @@ -import { getChatConfig } from "@/chat/config"; -import { createSqlStore, type SqlStore } from "@/chat/conversations/sql/store"; -import { createStateConversationStore } from "@/chat/conversations/state"; -import { addAgentTurnUsage } from "@/chat/usage"; -import { listAgentTurnSessionSummariesForConversations } from "@/chat/state/turn-session"; -import { createJuniorSqlExecutor } from "@/db/executor"; -import type { MigrationContext, MigrationResult } from "../types"; - -const CONVERSATION_BACKFILL_LIMIT = 10_000; - -/** Copy retained conversation records into the configured SQL store. */ -export async function migrateConversationsToSql( - context: MigrationContext, - options: { - batchSize?: number; - target?: Pick; - } = {}, -): Promise { - const source = createStateConversationStore(context.stateAdapter); - let target = options.target; - let closeTarget: (() => Promise) | undefined; - if (!target) { - const { sql } = getChatConfig(); - const executor = createJuniorSqlExecutor({ - connectionString: sql.databaseUrl, - driver: sql.driver, - }); - target = createSqlStore(executor); - closeTarget = () => executor.close(); - } - const limit = Math.max(1, options.batchSize ?? CONVERSATION_BACKFILL_LIMIT); - try { - const [stateConversations, sqlConversations] = await Promise.all([ - source.listByActivity({ limit }), - target.listByActivity({ limit }), - ]); - const byId = new Map( - sqlConversations.map((conversation) => [ - conversation.conversationId, - conversation, - ]), - ); - for (const conversation of stateConversations) { - const existing = byId.get(conversation.conversationId); - const existingExecutionAt = - existing?.execution.updatedAtMs ?? existing?.updatedAtMs ?? 0; - const stateExecutionAt = - conversation.execution.updatedAtMs ?? conversation.updatedAtMs; - if (!existing || stateExecutionAt >= existingExecutionAt) { - byId.set(conversation.conversationId, conversation); - } - } - const conversations = [...byId.values()] - .sort( - (left, right) => - right.lastActivityAtMs - left.lastActivityAtMs || - left.conversationId.localeCompare(right.conversationId), - ) - .slice(0, limit); - const summaries = await listAgentTurnSessionSummariesForConversations( - context.stateAdapter, - conversations.map((conversation) => conversation.conversationId), - ); - for (const conversation of conversations) { - const conversationSummaries = - summaries.get(conversation.conversationId) ?? []; - const executionSummary = conversation.execution.runId - ? conversationSummaries.find( - (summary) => summary.sessionId === conversation.execution.runId, - ) - : undefined; - await target.backfillConversation( - conversation, - conversationSummaries.length > 0 - ? { - durationMs: conversationSummaries.reduce( - (total, summary) => total + summary.cumulativeDurationMs, - 0, - ), - usage: addAgentTurnUsage( - ...conversationSummaries.map( - (summary) => summary.cumulativeUsage, - ), - ), - executionDurationMs: executionSummary?.cumulativeDurationMs ?? 0, - executionUsage: executionSummary?.cumulativeUsage, - } - : undefined, - ); - } - - return { - existing: 0, - migrated: conversations.length, - missing: 0, - scanned: conversations.length, - }; - } finally { - await closeTarget?.(); - } -} - -export const sqlConversationMigration = { - name: "backfill-conversations-sql", - run: migrateConversationsToSql, -}; diff --git a/packages/junior/src/cli/upgrade/migrations/core-journal.ts b/packages/junior/src/cli/upgrade/migrations/core-journal.ts new file mode 100644 index 000000000..7951c77d8 --- /dev/null +++ b/packages/junior/src/cli/upgrade/migrations/core-journal.ts @@ -0,0 +1,37 @@ +import { getChatConfig } from "@/chat/config"; +import { migrateSchema } from "@/chat/conversations/sql/migrations"; +import { createJuniorSqlExecutor } from "@/db/executor"; +import { createJiti } from "jiti"; +import type { MigrationContext, MigrationResult } from "../types"; + +const migrationLoader = createJiti(import.meta.url, { moduleCache: false }); + +/** Apply core schema and self-contained data migrations in journal order. */ +export async function migrateCoreJournal( + context: MigrationContext, +): Promise { + const { sql } = getChatConfig(); + const executor = createJuniorSqlExecutor({ + connectionString: sql.databaseUrl, + driver: sql.driver, + }); + try { + const result = await migrateSchema(executor, { + loadTypeScript: async (path) => + await migrationLoader.import>(path), + log: context.io.info, + mode: "all", + redisStateAdapter: context.redisStateAdapter, + stateAdapter: context.stateAdapter, + }); + return { + existing: result.existing, + migrated: result.migrated, + missing: 0, + scanned: result.scanned, + skipped: result.skipped, + }; + } finally { + await executor.close(); + } +} diff --git a/packages/junior/src/cli/upgrade/migrations/core-sql.ts b/packages/junior/src/cli/upgrade/migrations/core-sql.ts deleted file mode 100644 index a8b07ef98..000000000 --- a/packages/junior/src/cli/upgrade/migrations/core-sql.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { getChatConfig } from "@/chat/config"; -import { migrateSchema } from "@/chat/conversations/sql/migrations"; -import { createJuniorSqlExecutor } from "@/db/executor"; -import type { MigrationContext, MigrationResult } from "../types"; - -/** Apply core SQL schema migrations before upgrade backfills run. */ -export async function migrateCoreSqlSchema( - _context: MigrationContext, -): Promise { - const { sql } = getChatConfig(); - const executor = createJuniorSqlExecutor({ - connectionString: sql.databaseUrl, - driver: sql.driver, - }); - try { - await migrateSchema(executor); - return { existing: 0, migrated: 0, missing: 0, scanned: 0 }; - } finally { - await executor.close(); - } -} - -export const coreSqlSchemaMigration = { - name: "core-sql-schema", - run: migrateCoreSqlSchema, -}; diff --git a/packages/junior/src/cli/upgrade/migrations/plugin-sql.ts b/packages/junior/src/cli/upgrade/migrations/plugin-journal.ts similarity index 67% rename from packages/junior/src/cli/upgrade/migrations/plugin-sql.ts rename to packages/junior/src/cli/upgrade/migrations/plugin-journal.ts index c41ef3dc7..d03c89249 100644 --- a/packages/junior/src/cli/upgrade/migrations/plugin-sql.ts +++ b/packages/junior/src/cli/upgrade/migrations/plugin-journal.ts @@ -2,11 +2,14 @@ import { getChatConfig } from "@/chat/config"; import { migratePluginSchemas } from "@/chat/plugins/migrations"; import { pluginCatalogRuntime } from "@/chat/plugins/catalog-runtime"; import { createJuniorSqlExecutor } from "@/db/executor"; +import { createJiti } from "jiti"; import { resolveUpgradePlugins } from "./upgrade-plugins"; import type { MigrationContext, MigrationResult } from "../types"; -/** Apply SQL schema migrations owned by explicitly enabled plugins. */ -export async function migratePluginsToSql( +const migrationLoader = createJiti(import.meta.url, { moduleCache: false }); + +/** Apply mixed migration journals owned by explicitly enabled plugins. */ +export async function migratePluginJournals( context: MigrationContext, ): Promise { const { sql } = getChatConfig(); @@ -20,20 +23,23 @@ export async function migratePluginsToSql( const result = await migratePluginSchemas( executor, pluginCatalogRuntime.getMigrationRoots(), + { + loadTypeScript: async (path) => + await migrationLoader.import>(path), + log: context.io.info, + mode: "all", + stateAdapter: context.stateAdapter, + }, ); return { existing: result.existing, migrated: result.migrated, missing: 0, scanned: result.scanned, + ...(result.skipped === undefined ? {} : { skipped: result.skipped }), }; } finally { pluginCatalogRuntime.setConfig(previousConfig); await executor.close(); } } - -export const sqlPluginMigration = { - name: "migrate-plugin-sql", - run: migratePluginsToSql, -}; diff --git a/packages/junior/src/cli/upgrade/migrations/plugin-storage.ts b/packages/junior/src/cli/upgrade/migrations/plugin-storage.ts deleted file mode 100644 index bc340d266..000000000 --- a/packages/junior/src/cli/upgrade/migrations/plugin-storage.ts +++ /dev/null @@ -1,93 +0,0 @@ -import type { StorageMigrationResult } from "@sentry/junior-plugin-api"; -import { pluginRuntimeRegistrationsFromPluginSet } from "@/plugins"; -import { getDb } from "@/chat/db"; -import { createPluginLogger } from "@/chat/plugins/logging"; -import { createPluginState } from "@/chat/plugins/state"; -import { pluginCatalogRuntime } from "@/chat/plugins/catalog-runtime"; -import { getChatConfig } from "@/chat/config"; -import { createJuniorSqlExecutor } from "@/db/executor"; -import { resolveUpgradePlugins } from "./upgrade-plugins"; -import type { MigrationContext, MigrationResult } from "../types"; - -function emptyResult(): MigrationResult { - return { - existing: 0, - migrated: 0, - missing: 0, - scanned: 0, - }; -} - -function addResult( - left: MigrationResult, - right: StorageMigrationResult, -): MigrationResult { - return { - existing: left.existing + right.existing, - migrated: left.migrated + right.migrated, - missing: left.missing + right.missing, - scanned: left.scanned + right.scanned, - ...(left.skipped !== undefined || right.skipped !== undefined - ? { skipped: (left.skipped ?? 0) + (right.skipped ?? 0) } - : {}), - }; -} - -function dbForPlugin(context: MigrationContext, sqlUrlDb: unknown): unknown { - return context.db ?? sqlUrlDb ?? getDb(); -} - -/** Run plugin-owned storage migrations after plugin SQL schemas are available. */ -export async function runPluginStorageMigrations( - context: MigrationContext, -): Promise { - const { pluginCatalogConfig, pluginSet } = - await resolveUpgradePlugins(context); - if (!pluginSet) { - return emptyResult(); - } - - const previousConfig = pluginCatalogRuntime.setConfig(pluginCatalogConfig); - const sql = getChatConfig().sql; - const ownedExecutor = context.db - ? undefined - : createJuniorSqlExecutor({ - connectionString: sql.databaseUrl, - driver: sql.driver, - }); - const sqlUrlDb = ownedExecutor ? ownedExecutor.db() : undefined; - try { - let result = emptyResult(); - const plugins = pluginRuntimeRegistrationsFromPluginSet(pluginSet) - .filter((plugin) => plugin.hooks?.migrateStorage) - .sort((left, right) => - left.manifest.name.localeCompare(right.manifest.name), - ); - for (const plugin of plugins) { - const pluginName = plugin.manifest.name; - const hook = plugin.hooks?.migrateStorage; - if (!hook) { - continue; - } - const db = dbForPlugin(context, sqlUrlDb); - const pluginResult = await hook({ - db, - log: createPluginLogger(pluginName), - plugin: { name: pluginName }, - state: createPluginState(pluginName, context.stateAdapter), - }); - if (pluginResult) { - result = addResult(result, pluginResult); - } - } - return result; - } finally { - pluginCatalogRuntime.setConfig(previousConfig); - await ownedExecutor?.close(); - } -} - -export const pluginStorageMigration = { - name: "run-plugin-storage-migrations", - run: runPluginStorageMigrations, -}; diff --git a/packages/junior/src/cli/upgrade/types.ts b/packages/junior/src/cli/upgrade/types.ts index 0168d025d..913af8b3b 100644 --- a/packages/junior/src/cli/upgrade/types.ts +++ b/packages/junior/src/cli/upgrade/types.ts @@ -8,7 +8,6 @@ export interface UpgradeIo { } export interface MigrationContext { - db?: unknown; io: UpgradeIo; pluginCatalogConfig?: PluginCatalogConfig; pluginSet?: JuniorPluginSet; @@ -16,15 +15,10 @@ export interface MigrationContext { stateAdapter: StateAdapter; } -export interface MigrationResult { +export type MigrationResult = { existing: number; migrated: number; missing: number; scanned: number; skipped?: number; -} - -export interface UpgradeMigration { - name: string; - run(context: MigrationContext): Promise; -} +}; diff --git a/packages/junior/src/db/db.ts b/packages/junior/src/db/db.ts index 35c037e43..321184858 100644 --- a/packages/junior/src/db/db.ts +++ b/packages/junior/src/db/db.ts @@ -7,7 +7,6 @@ */ import type { PgDatabase } from "drizzle-orm/pg-core"; import type { PgQueryResultHKT } from "drizzle-orm/pg-core/session"; -import type { MigrationConfig } from "drizzle-orm/migrator"; import type { juniorSqlSchema } from "./schema"; export type JuniorDatabase = PgDatabase< @@ -23,7 +22,6 @@ export interface JuniorSqlDatabase { export interface JuniorSqlMigrationExecutor extends JuniorSqlDatabase { execute(statement: string, params?: readonly unknown[]): Promise; - migrate(config: MigrationConfig): Promise; query( statement: string, params?: readonly unknown[], diff --git a/packages/junior/src/db/neon.ts b/packages/junior/src/db/neon.ts index 62e7aa041..0ce11d06a 100644 --- a/packages/junior/src/db/neon.ts +++ b/packages/junior/src/db/neon.ts @@ -6,8 +6,6 @@ import { type QueryResultRow, } from "@neondatabase/serverless"; import { drizzle } from "drizzle-orm/neon-serverless"; -import { migrate } from "drizzle-orm/neon-serverless/migrator"; -import type { MigrationConfig } from "drizzle-orm/migrator"; import type { JuniorDatabase, JuniorSqlExecutor } from "./db"; import { juniorSqlSchema } from "./schema"; @@ -45,13 +43,6 @@ class NeonExecutor implements NeonJuniorSqlExecutor { return result.rows as T[]; } - async migrate(config: MigrationConfig): Promise { - await migrate( - drizzle(this.queryClient(), { schema: juniorSqlSchema }), - config, - ); - } - async transaction(callback: () => Promise): Promise { const existingClient = this.transactionClient.getStore(); if (existingClient) { diff --git a/packages/junior/src/db/postgres.ts b/packages/junior/src/db/postgres.ts index 4124bd9ff..53020ce44 100644 --- a/packages/junior/src/db/postgres.ts +++ b/packages/junior/src/db/postgres.ts @@ -5,8 +5,6 @@ import pg, { type QueryResultRow, } from "pg"; import { drizzle } from "drizzle-orm/node-postgres"; -import { migrate } from "drizzle-orm/node-postgres/migrator"; -import type { MigrationConfig } from "drizzle-orm/migrator"; import type { JuniorDatabase, JuniorSqlExecutor } from "./db"; import { juniorSqlSchema } from "./schema"; @@ -43,13 +41,6 @@ class PostgresExecutor implements JuniorSqlExecutor { return result.rows as T[]; } - async migrate(config: MigrationConfig): Promise { - await migrate( - drizzle(this.queryClient(), { schema: juniorSqlSchema }), - config, - ); - } - async transaction(callback: () => Promise): Promise { const existingClient = this.transactionClient.getStore(); if (existingClient) { diff --git a/packages/junior/src/migration-helpers/README.md b/packages/junior/src/migration-helpers/README.md new file mode 100644 index 000000000..f2550f515 --- /dev/null +++ b/packages/junior/src/migration-helpers/README.md @@ -0,0 +1,14 @@ +# Migration Helpers + +This directory owns versioned, append-only infrastructure helpers available to +packaged TypeScript migrations through `@sentry/junior/migration-helpers/v1`. + +Helpers may expose stable parsing primitives, state/database adapters, stores, +and key resolution. They must not contain one-off migration decisions or data +transforms. Logic such as mapping one retired record shape into another belongs +only in the journal entry that performs that migration. + +The behavior and signature of an existing version are permanent compatibility +contracts. Add a new versioned entrypoint when that contract must change. +Migration files may be updated while still unreleased, but shipped migration +sources are hash-verified ledger entries and must remain immutable. diff --git a/packages/junior/src/migration-helpers/v1.ts b/packages/junior/src/migration-helpers/v1.ts new file mode 100644 index 000000000..15665d5fd --- /dev/null +++ b/packages/junior/src/migration-helpers/v1.ts @@ -0,0 +1,298 @@ +import type { Destination } from "@sentry/junior-plugin-api"; +import type { + MigrationDatabaseAdapter, + MigrationStateV1, +} from "@sentry/junior-migrations"; +import type { StateAdapter } from "chat"; +import { + isRecord as runtimeIsRecord, + toOptionalNumber as runtimeToOptionalNumber, + toOptionalString as runtimeToOptionalString, +} from "@/chat/coerce"; +import { getChatConfig } from "@/chat/config"; +import { createLegacyAdvisorSessionReader } from "@/chat/conversations/legacy-advisor-session"; +import { createSqlConversationMessageStore } from "@/chat/conversations/sql/messages"; +import { createStateConversationStore } from "@/chat/conversations/state"; +import { createSqlStore } from "@/chat/conversations/sql/store"; +import { toStoredConversationMessage } from "@/chat/conversations/visible-messages"; +import { + parseDestination as runtimeParseDestination, + sameDestination as runtimeSameDestination, +} from "@/chat/destination"; +import { coerceThreadConversationState as runtimeCoerceThreadConversationState } from "@/chat/state/conversation"; +import { listAgentTurnSessionSummariesForConversations } from "@/chat/state/turn-session"; +import { + contextProvenance, + decodeSessionLogEntry, + legacyActorProvenance, + piMessageProvenanceSchema, +} from "@/chat/state/session-log"; +import { + getConversation, + requestConversationWork, +} from "@/chat/task-execution/state"; +import { addAgentTurnUsage as runtimeAddAgentTurnUsage } from "@/chat/usage"; +import { unescapeXml } from "@/chat/xml"; +import type { JuniorSqlDatabase } from "@/db/db"; +import { + juniorAgentSteps, + juniorConversationMessages, + juniorConversations, +} from "@/db/schema"; + +export const JUNIOR_THREAD_STATE_TTL_MS = 7 * 24 * 60 * 60 * 1_000; +export const migrationContextProvenance: unknown = contextProvenance; +export const migrationJuniorAgentSteps: unknown = juniorAgentSteps; +export const migrationJuniorConversationMessages: unknown = + juniorConversationMessages; +export const migrationJuniorConversations: unknown = juniorConversations; +export const migrationPiMessageProvenanceSchema: unknown = + piMessageProvenanceSchema; + +export type MigrationSourceV1 = + | "api" + | "internal" + | "local" + | "plugin" + | "resource_event" + | "scheduler" + | "slack"; + +export type MigrationExecutionStatusV1 = + | "awaiting_resume" + | "failed" + | "idle" + | "pending" + | "running"; + +/** Frozen retained inbound-message shape used by v1 data migrations. */ +export interface MigrationInboundMessageV1 { + attemptCount?: number; + conversationId: string; + createdAtMs: number; + destination: Destination; + inboundMessageId: string; + injectedAtMs?: number; + input: { + attachments?: unknown[]; + authorId?: string; + metadata?: Record; + text: string; + }; + receivedAtMs: number; + source: MigrationSourceV1; +} + +/** Frozen retained execution-lease shape used by v1 data migrations. */ +export interface MigrationLeaseV1 { + acquiredAtMs: number; + expiresAtMs: number; + lastCheckInAtMs: number; + token: string; +} + +/** Frozen conversation projection shared by v1 state and SQL helpers. */ +export interface MigrationConversationV1 { + actor?: unknown; + channelName?: string; + conversationId: string; + createdAtMs: number; + destination?: Destination; + execution: { + inboundMessageIds?: string[]; + lastCheckpointAtMs?: number; + lastEnqueuedAtMs?: number; + lease?: MigrationLeaseV1; + pendingCount?: number; + pendingMessages?: MigrationInboundMessageV1[]; + runId?: string; + status: MigrationExecutionStatusV1; + updatedAtMs?: number; + }; + lastActivityAtMs: number; + schemaVersion: 1; + source?: MigrationSourceV1; + title?: string; + updatedAtMs: number; +} + +/** State-backed conversation projection with retained mailbox fields. */ +export interface MigrationRetainedConversationV1 extends MigrationConversationV1 { + execution: MigrationConversationV1["execution"] & { + inboundMessageIds: string[]; + pendingCount: number; + pendingMessages: MigrationInboundMessageV1[]; + }; +} + +/** Legacy thread-state projection required by the v1 continuation migration. */ +export interface MigrationThreadConversationStateV1 { + processing: { activeTurnId?: string }; +} + +/** Frozen turn-summary projection used by the conversation SQL backfill. */ +export interface MigrationTurnSessionSummaryV1 { + cumulativeDurationMs: number; + cumulativeUsage?: Record; + sessionId: string; +} + +/** Narrow state conversation store exposed to v1 migrations. */ +export interface MigrationStateConversationStoreV1 { + listByActivity(args: { limit: number }): Promise; +} + +/** Narrow SQL conversation store exposed to v1 migrations. */ +export interface MigrationSqlConversationStoreV1 { + backfillConversation( + conversation: MigrationConversationV1, + metrics?: { + durationMs: number; + executionDurationMs: number; + executionUsage?: Record; + usage?: Record; + }, + ): Promise; + listByActivity(args: { limit: number }): Promise; +} + +/** Return whether a value is a non-null object record. */ +export function isRecord(value: unknown): value is Record { + return runtimeIsRecord(value); +} + +/** Return a finite number or undefined. */ +export function toOptionalNumber(value: unknown): number | undefined { + return runtimeToOptionalNumber(value); +} + +/** Return a non-empty string or undefined. */ +export function toOptionalString(value: unknown): string | undefined { + return runtimeToOptionalString(value); +} + +/** Parse one persisted destination through the stable destination schema. */ +export function parseDestination(value: unknown): Destination | undefined { + return runtimeParseDestination(value); +} + +/** Compare two persisted destinations by routing identity. */ +export function sameDestination( + left: Destination, + right: Destination, +): boolean { + return runtimeSameDestination(left, right); +} + +/** Coerce one legacy thread-state value into the retained conversation shape. */ +export function coerceThreadConversationState( + value: unknown, +): MigrationThreadConversationStateV1 { + return runtimeCoerceThreadConversationState( + value, + ) as unknown as MigrationThreadConversationStateV1; +} + +/** Merge retained turn usage records. */ +export function addAgentTurnUsage( + ...values: Array | undefined> +): Record | undefined { + return runtimeAddAgentTurnUsage(...values) as + | Record + | undefined; +} + +/** Resolve a logical state key to the configured physical Redis key. */ +export function migrationRedisKey(key: string): string { + const prefix = getChatConfig().state.keyPrefix; + return [...(prefix ? [prefix] : []), key].join(":"); +} + +/** Create the retained state-backed conversation store for a v1 migration. */ +export function createMigrationStateConversationStore( + state: MigrationStateV1, +): MigrationStateConversationStoreV1 { + return createStateConversationStore( + state as StateAdapter, + ) as unknown as MigrationStateConversationStoreV1; +} + +/** Create the SQL conversation store for a v1 migration database adapter. */ +export function createMigrationSqlStore( + database: MigrationDatabaseAdapter, +): MigrationSqlConversationStoreV1 { + return createSqlStore( + database as unknown as JuniorSqlDatabase, + ) as unknown as MigrationSqlConversationStoreV1; +} + +/** Create the SQL message store used by legacy history migrations. */ +export function createMigrationSqlConversationMessageStore( + database: MigrationDatabaseAdapter, +): unknown { + return createSqlConversationMessageStore( + database as unknown as JuniorSqlDatabase, + ); +} + +/** Create the legacy advisor reader used by history migrations. */ +export function createMigrationLegacyAdvisorSessionReader(): unknown { + return createLegacyAdvisorSessionReader(); +} + +/** Decode one retained session-log value. */ +export function decodeMigrationSessionLogEntry(value: unknown): unknown { + return decodeSessionLogEntry(value); +} + +/** Recover provenance from one legacy actor value. */ +export function migrationLegacyActorProvenance(value: unknown): unknown { + return legacyActorProvenance(value as never); +} + +/** Convert one legacy visible message into its SQL projection. */ +export function toMigrationStoredConversationMessage(value: unknown): unknown { + return toStoredConversationMessage(value as never); +} + +/** Unescape one legacy XML text fragment. */ +export function migrationUnescapeXml(value: string): string { + return unescapeXml(value); +} + +/** Read one retained conversation through the v1 migration state adapter. */ +export async function getMigrationConversation(args: { + conversationId: string; + state: MigrationStateV1; +}): Promise { + return (await getConversation({ + conversationId: args.conversationId, + state: args.state as StateAdapter, + })) as unknown as MigrationRetainedConversationV1 | undefined; +} + +/** Mark one retained conversation runnable through the v1 migration state adapter. */ +export async function requestMigrationConversationWork(args: { + conversationId: string; + destination: Destination; + nowMs?: number; + state: MigrationStateV1; +}): Promise { + await requestConversationWork({ + conversationId: args.conversationId, + destination: args.destination, + ...(args.nowMs === undefined ? {} : { nowMs: args.nowMs }), + state: args.state as StateAdapter, + }); +} + +/** Read retained turn summaries through the v1 migration state adapter. */ +export async function listMigrationTurnSessionSummaries( + state: MigrationStateV1, + conversationIds: string[], +): Promise> { + return (await listAgentTurnSessionSummariesForConversations( + state as StateAdapter, + conversationIds, + )) as Map; +} diff --git a/packages/junior/tests/component/cli/upgrade-cli.test.ts b/packages/junior/tests/component/cli/upgrade-cli.test.ts index 0a3cbbd3b..64f69e347 100644 --- a/packages/junior/tests/component/cli/upgrade-cli.test.ts +++ b/packages/junior/tests/component/cli/upgrade-cli.test.ts @@ -2,6 +2,8 @@ import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; import type { RedisStateAdapter } from "@chat-adapter/state-redis"; +import type { MigrationContextV1 } from "@sentry/junior-migrations"; +import { createJiti } from "jiti"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { getChatConfig } from "@/chat/config"; import { disconnectStateAdapter, getStateAdapter } from "@/chat/state/adapter"; @@ -19,10 +21,10 @@ import type { PiMessage } from "@/chat/pi/messages"; import { persistThreadStateById } from "@/chat/runtime/thread-state"; import { recordAgentTurnSessionSummary } from "@/chat/state/turn-session"; import { resolveUpgradePluginSet } from "@/cli/upgrade"; -import { agentTurnSessionActorMigration } from "@/cli/upgrade/migrations/agent-turn-session-actor"; -import { repairConversationUsage } from "@/cli/upgrade/migrations/conversation-usage"; -import { migrateConversationsToSql } from "@/cli/upgrade/migrations/conversations-sql"; -import { redisConversationStateMigration } from "@/cli/upgrade/migrations/redis-conversation-state"; +import { migrateAgentTurnSessionActor } from "../../../migrations/0005_agent_turn_session_actor"; +import { migrateRedisConversationState } from "../../../migrations/0006_redis_conversation_state"; +import { migrateConversationsToSql } from "../../../migrations/0007_conversations_to_sql"; +import { repairConversationUsage } from "../../../migrations/0009_conversation_usage"; import { CONVERSATION_ID, SLACK_DESTINATION, @@ -44,6 +46,7 @@ const OTHER_SLACK_DESTINATION = { ...SLACK_DESTINATION, channelId: "C999", } as const; +const migrationLoader = createJiti(import.meta.url, { moduleCache: false }); const stateOnlyConversationStore: ConversationStore = { get: async () => undefined, @@ -169,7 +172,7 @@ export const plugins = { ); await expect( - agentTurnSessionActorMigration.run({ + migrateAgentTurnSessionActor({ io: { info: () => {} }, stateAdapter, }), @@ -196,7 +199,7 @@ export const plugins = { expect(migratedRecord).not.toHaveProperty("requester"); await expect( - agentTurnSessionActorMigration.run({ + migrateAgentTurnSessionActor({ io: { info: () => {} }, stateAdapter, }), @@ -265,7 +268,7 @@ export const plugins = { } as unknown as RedisStateAdapter; await expect( - agentTurnSessionActorMigration.run({ + migrateAgentTurnSessionActor({ io: { info: () => {} }, redisStateAdapter, stateAdapter, @@ -290,6 +293,121 @@ export const plugins = { ), ).resolves.toEqual(expect.objectContaining({ actor: requester })); }); + it("runs journaled TypeScript state migrations exactly once", async () => { + const stateAdapter = getStateAdapter(); + await stateAdapter.connect(); + const fixture = await createLocalJuniorSqlFixture(); + const actor = { + platform: "slack", + teamId: "T123", + userId: "migration-user", + } as const; + const summary = { + version: 1, + conversationId: CONVERSATION_ID, + cumulativeDurationMs: 0, + lastProgressAtMs: 2, + sessionId: "session-one", + sliceId: 1, + startedAtMs: 1, + state: "completed", + updatedAtMs: 3, + requester: actor, + }; + await stateAdapter.appendToList("junior:agent_turn_session:index", summary); + await stateAdapter.appendToList( + `junior:agent_turn_session:conversation:${CONVERSATION_ID}:index`, + summary, + ); + await stateAdapter.set( + `junior:agent_turn_session:${CONVERSATION_ID}:session-one`, + summary, + ); + + try { + await expect( + migrateSchema(fixture.sql, { + loadTypeScript: async (migrationPath) => + await migrationLoader.import>( + migrationPath, + ), + mode: "all", + stateAdapter, + }), + ).resolves.toEqual({ + existing: 0, + migrated: 10, + scanned: 10, + skipped: 0, + }); + await expect( + stateAdapter.getList("junior:agent_turn_session:index"), + ).resolves.toEqual([ + expect.objectContaining({ + actor, + conversationId: CONVERSATION_ID, + sessionId: "session-one", + }), + ]); + await expect( + stateAdapter.get( + `junior:agent_turn_session:${CONVERSATION_ID}:session-one`, + ), + ).resolves.toEqual( + expect.objectContaining({ + actor, + conversationId: CONVERSATION_ID, + sessionId: "session-one", + }), + ); + await expect( + migrateSchema(fixture.sql, { + loadTypeScript: async (migrationPath) => + await migrationLoader.import>( + migrationPath, + ), + mode: "all", + stateAdapter, + }), + ).resolves.toEqual({ + existing: 10, + migrated: 0, + scanned: 10, + skipped: 0, + }); + } finally { + await fixture.close(); + } + }, 15_000); + + it("passes the host state adapter through without wrapping its identity", async () => { + const stateAdapter = getStateAdapter(); + await stateAdapter.connect(); + const fixture = await createLocalJuniorSqlFixture(); + const observedStates: unknown[] = []; + + try { + await migrateSchema(fixture.sql, { + loadTypeScript: async () => ({ + default: { + apiVersion: 1, + async up(context: MigrationContextV1) { + observedStates.push(context.state); + }, + }, + }), + mode: "all", + stateAdapter, + }); + + expect(observedStates).toHaveLength(5); + expect(observedStates.every((state) => state === stateAdapter)).toBe( + true, + ); + } finally { + await fixture.close(); + } + }); it("migrates legacy conversation work before SQL conversation backfill", async () => { const stateAdapter = getStateAdapter(); @@ -311,13 +429,13 @@ export const plugins = { const sqlStore = createSqlStore(fixture.sql); try { - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); const context = { io: { info: () => {} }, stateAdapter, }; const results = [ - await redisConversationStateMigration.run(context), + await migrateRedisConversationState(context), await migrateConversationsToSql(context, { target: sqlStore }), ]; @@ -368,7 +486,7 @@ export const plugins = { const sqlStore = createSqlStore(fixture.sql); try { - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); for (let index = 0; index < 3; index++) { const conversationId = `slack:C123:page-${index}`; await appendInboundMessage({ @@ -409,7 +527,7 @@ export const plugins = { const sqlStore = createSqlStore(fixture.sql); try { - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); const seedMs = Date.now() - 1_000; await sqlStore.recordExecution({ conversationId: CONVERSATION_ID, @@ -604,7 +722,7 @@ WHERE conversation_id = $1 } as PiMessage; try { - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); await sqlStore.recordExecution({ conversationId: CONVERSATION_ID, createdAtMs: 1_000, @@ -769,7 +887,7 @@ WHERE conversation_id = $1 const conversationIds = ["local:usage-batch-a", "local:usage-batch-b"]; try { - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); for (const [index, conversationId] of conversationIds.entries()) { await sqlStore.recordExecution({ conversationId, @@ -904,7 +1022,7 @@ WHERE conversation_id = $1 const stepStore = createSqlAgentStepStore(fixture.sql); try { - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); await sqlStore.recordExecution({ conversationId: CONVERSATION_ID, createdAtMs: 1_000, @@ -986,7 +1104,7 @@ WHERE conversation_id = $1 await persistActiveTurn(CONVERSATION_ID, "turn-timeout"); await expect( - redisConversationStateMigration.run({ + migrateRedisConversationState({ io: { info: () => {} }, stateAdapter, }), @@ -1043,7 +1161,7 @@ WHERE conversation_id = $1 await stateAdapter.set("junior:conversation-work:index", [CONVERSATION_ID]); await expect( - redisConversationStateMigration.run({ + migrateRedisConversationState({ io: { info: () => {} }, stateAdapter, }), @@ -1120,7 +1238,7 @@ WHERE conversation_id = $1 await stateAdapter.set("junior:conversation-work:index", [CONVERSATION_ID]); await expect( - redisConversationStateMigration.run({ + migrateRedisConversationState({ io: { info: () => {} }, stateAdapter, }), @@ -1154,7 +1272,7 @@ WHERE conversation_id = $1 await stateAdapter.set("junior:conversation-work:index", [CONVERSATION_ID]); await expect( - redisConversationStateMigration.run({ + migrateRedisConversationState({ io: { info: () => {} }, stateAdapter, }), @@ -1174,7 +1292,7 @@ WHERE conversation_id = $1 }); await expect( - redisConversationStateMigration.run({ + migrateRedisConversationState({ io: { info: () => {} }, stateAdapter, }), @@ -1200,13 +1318,13 @@ WHERE conversation_id = $1 const sqlStore = createSqlStore(fixture.sql); try { - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); const context = { io: { info: () => {} }, stateAdapter, }; const results = [ - await redisConversationStateMigration.run(context), + await migrateRedisConversationState(context), await migrateConversationsToSql(context, { target: sqlStore }), ]; diff --git a/packages/junior/tests/component/conversation-search.test.ts b/packages/junior/tests/component/conversation-search.test.ts index ffb7a47dd..4f587d4e8 100644 --- a/packages/junior/tests/component/conversation-search.test.ts +++ b/packages/junior/tests/component/conversation-search.test.ts @@ -12,7 +12,7 @@ describe("conversation search", () => { const fixture = await createLocalJuniorSqlFixture(); try { - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); const conversations = createSqlStore(fixture.sql); const messages = createSqlConversationMessageStore(fixture.sql); const search = createSqlConversationSearchStore(fixture.sql); diff --git a/packages/junior/tests/component/conversation-sql-store.test.ts b/packages/junior/tests/component/conversation-sql-store.test.ts index 164c0da33..afcf61cc2 100644 --- a/packages/junior/tests/component/conversation-sql-store.test.ts +++ b/packages/junior/tests/component/conversation-sql-store.test.ts @@ -36,7 +36,7 @@ describe("conversation SQL store", () => { try { const store = createSqlStore(fixture.sql); - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); await store.recordActivity({ conversationId: CONVERSATION_ID, @@ -169,7 +169,7 @@ describe("conversation SQL store", () => { try { const store = createSqlStore(fixture.sql); - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); const identity = await upsertIdentity( fixture.sql, @@ -265,7 +265,7 @@ describe("conversation SQL store", () => { const fixture = await createLocalJuniorSqlFixture(); try { - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); await fixture.sql .db() @@ -338,7 +338,7 @@ describe("conversation SQL store", () => { try { const store = createSqlStore(fixture.sql); - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); await store.recordActivity({ conversationId: CONVERSATION_ID, @@ -385,7 +385,7 @@ describe("conversation SQL store", () => { try { const store = createSqlStore(fixture.sql); - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); const destination = inboundMessage("visibility").destination; // Slack reports this C-prefixed channel private (channel_type: group). @@ -452,7 +452,7 @@ describe("conversation SQL store", () => { try { const store = createSqlStore(fixture.sql); - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); // A write without a live source signal fails closed to private even // though the channel id is C-prefixed. @@ -482,7 +482,7 @@ describe("conversation SQL store", () => { try { const store = createSqlStore(fixture.sql); - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); await fixture.sql.execute( ` INSERT INTO junior_conversations ( @@ -542,7 +542,7 @@ INSERT INTO junior_conversations ( try { const store = createSqlStore(fixture.sql); - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); await store.recordExecution({ conversationId: CONVERSATION_ID, @@ -595,7 +595,7 @@ INSERT INTO junior_conversations ( try { const store = createSqlStore(fixture.sql); - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); await store.recordExecution({ conversationId: CONVERSATION_ID, createdAtMs: 1_000, @@ -672,7 +672,7 @@ WHERE conversation_id = $1 try { const store = createSqlStore(fixture.sql); - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); await store.recordExecution({ conversationId: CONVERSATION_ID, createdAtMs: 1_000, @@ -755,7 +755,7 @@ WHERE conversation_id = $1 try { const store = createSqlStore(fixture.sql); - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); await store.recordExecution({ conversationId: CONVERSATION_ID, @@ -805,7 +805,7 @@ WHERE conversation_id = $1 try { const store = createSqlStore(fixture.sql); - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); await store.recordActivity({ conversationId: CONVERSATION_ID, @@ -841,7 +841,7 @@ WHERE conversation_id = $1 try { await disconnectStateAdapter(); const store = createSqlStore(fixture.sql); - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); await store.recordExecution({ conversationId: CONVERSATION_ID, createdAtMs: 1_000, @@ -875,7 +875,7 @@ WHERE conversation_id = $1 vi.useFakeTimers({ now: 302_000 }); await disconnectStateAdapter(); const store = createSqlStore(fixture.sql); - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); await store.recordExecution({ conversationId: CONVERSATION_ID, createdAtMs: 1_000, @@ -910,7 +910,7 @@ WHERE conversation_id = $1 vi.useFakeTimers({ now: 2_000 }); await disconnectStateAdapter(); const store = createSqlStore(fixture.sql); - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); await store.recordExecution({ conversationId: CONVERSATION_ID, createdAtMs: 1_000, @@ -948,7 +948,7 @@ WHERE conversation_id = $1 vi.useFakeTimers({ now: 600_000 }); await disconnectStateAdapter(); const store = createSqlStore(fixture.sql); - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); await store.recordExecution({ conversationId: CONVERSATION_ID, createdAtMs: 1_000, @@ -1002,7 +1002,7 @@ WHERE conversation_id = $1 await disconnectStateAdapter(); const state = getStateAdapter(); const store = createSqlStore(fixture.sql); - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); await appendInboundMessage({ message: inboundMessage("check-in"), conversationStore: store, @@ -1055,7 +1055,7 @@ WHERE conversation_id = $1 await disconnectStateAdapter(); const state = getStateAdapter(); const store = createSqlStore(fixture.sql); - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); await appendInboundMessage({ message: inboundMessage("drain-sql"), conversationStore: store, diff --git a/packages/junior/tests/component/conversation-transcripts-sql.test.ts b/packages/junior/tests/component/conversation-transcripts-sql.test.ts index 89d47b208..00081acc0 100644 --- a/packages/junior/tests/component/conversation-transcripts-sql.test.ts +++ b/packages/junior/tests/component/conversation-transcripts-sql.test.ts @@ -223,8 +223,8 @@ describe("conversation transcript SQL stores", () => { const fixture = await createLocalJuniorSqlFixture(); try { - await migrateSchema(fixture.sql); - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); const [applied] = await fixture.sql.query<{ count: number }>( "SELECT count(*)::integer AS count FROM drizzle.__drizzle_junior_core", @@ -239,7 +239,7 @@ describe("conversation transcript SQL stores", () => { const fixture = await createLocalJuniorSqlFixture(); try { - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); await seedConversation(fixture, CONVERSATION_ID); const store = createSqlAgentStepStore(fixture.sql); @@ -292,7 +292,7 @@ describe("conversation transcript SQL stores", () => { const fixture = await createLocalJuniorSqlFixture(); try { - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); await seedConversation(fixture, CONVERSATION_ID); const store = createSqlAgentStepStore(fixture.sql); @@ -323,7 +323,7 @@ describe("conversation transcript SQL stores", () => { const fixture = await createLocalJuniorSqlFixture(); try { - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); await seedConversation(fixture, CONVERSATION_ID); const store = createSqlAgentStepStore(fixture.sql); @@ -365,7 +365,7 @@ describe("conversation transcript SQL stores", () => { const fixture = await createLocalJuniorSqlFixture(); try { - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); await seedConversation(fixture, CONVERSATION_ID); const store = createSqlAgentStepStore(fixture.sql); const entry = { @@ -390,7 +390,7 @@ describe("conversation transcript SQL stores", () => { const fixture = await createLocalJuniorSqlFixture(); try { - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); await seedConversation(fixture, CONVERSATION_ID); const store = createSqlAgentStepStore(fixture.sql); await store.append(CONVERSATION_ID, [ @@ -435,7 +435,7 @@ describe("conversation transcript SQL stores", () => { const fixture = await createLocalJuniorSqlFixture(); try { - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); await seedConversation(fixture, CONVERSATION_ID); const store = createSqlAgentStepStore(fixture.sql); @@ -467,7 +467,7 @@ INSERT INTO junior_agent_steps ( const fixture = await createLocalJuniorSqlFixture(); try { - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); await seedConversation(fixture, CONVERSATION_ID); const store = createSqlConversationMessageStore(fixture.sql); @@ -524,7 +524,7 @@ INSERT INTO junior_agent_steps ( } try { - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); // Seed an old activity clock; content writes must refresh the window. await seedConversation(fixture, CONVERSATION_ID); await fixture.sql @@ -586,7 +586,7 @@ INSERT INTO junior_agent_steps ( const fixture = await createLocalJuniorSqlFixture(); try { - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); await seedConversation(fixture, CONVERSATION_ID); await seedConversation(fixture, CHILD_CONVERSATION_ID, CONVERSATION_ID); const steps = createSqlAgentStepStore(fixture.sql); diff --git a/packages/junior/tests/component/conversations/legacy-import.test.ts b/packages/junior/tests/component/conversations/legacy-import.test.ts index fb14774db..ee3842f64 100644 --- a/packages/junior/tests/component/conversations/legacy-import.test.ts +++ b/packages/junior/tests/component/conversations/legacy-import.test.ts @@ -18,7 +18,7 @@ import { createSqlConversationMessageStore } from "@/chat/conversations/sql/mess import { migrateSchema } from "@/chat/conversations/sql/migrations"; import { hydrateConversationMessages } from "@/chat/conversations/visible-messages"; import { coerceThreadConversationState } from "@/chat/state/conversation"; -import { migrateConversationHistoryToSql } from "@/cli/upgrade/migrations/conversations-history-sql"; +import { migrateConversationHistoryToSql } from "../../../migrations/0008_conversation_history_to_sql"; import type { LegacyAdvisorSessionReader } from "@/chat/conversations/legacy-advisor-session"; import type { Conversation } from "@/chat/conversations/store"; import type { PiMessage } from "@/chat/pi/messages"; @@ -40,7 +40,7 @@ const ORIGINAL_ENV = vi.hoisted(() => ({ async function processSqlStores() { const executor = getSqlExecutor(); - await migrateSchema(executor); + await migrateSchema(executor, { mode: "schema-bootstrap" }); return { executor, stepStore: getAgentStepStore(), @@ -112,7 +112,7 @@ describe("legacy conversation import", () => { it("imports steps, advisor child, and visible messages once, idempotently", async () => { const fixture = await createLocalJuniorSqlFixture(); - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); const stepStore = createSqlAgentStepStore(fixture.sql); const messageStore = createSqlConversationMessageStore(fixture.sql); const childId = `advisor:${CONVERSATION_ID}`; @@ -255,7 +255,7 @@ describe("legacy conversation import", () => { it("rolls back steps when the transactional message import fails", async () => { const fixture = await createLocalJuniorSqlFixture(); - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); const stepStore = createSqlAgentStepStore(fixture.sql); const messageStore = createSqlConversationMessageStore(fixture.sql); @@ -316,7 +316,7 @@ describe("legacy conversation import", () => { it("seals a completed message-only import without step rows", async () => { const fixture = await createLocalJuniorSqlFixture(); - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); const stepStore = createSqlAgentStepStore(fixture.sql); const messageStore = createSqlConversationMessageStore(fixture.sql); const loadVisibleMessages = vi.fn(async () => [ @@ -381,7 +381,7 @@ describe("legacy conversation import", () => { it("never fabricates import-time timestamps for timestamp-less rows", async () => { const fixture = await createLocalJuniorSqlFixture(); - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); const stepStore = createSqlAgentStepStore(fixture.sql); const messageStore = createSqlConversationMessageStore(fixture.sql); const before = Date.now(); @@ -563,7 +563,7 @@ describe("legacy conversation import", () => { it("rejects a legacy import when the SQL transcript was already purged", async () => { const fixture = await createLocalJuniorSqlFixture(); - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); const stepStore = createSqlAgentStepStore(fixture.sql); const messageStore = createSqlConversationMessageStore(fixture.sql); @@ -648,7 +648,7 @@ describe("legacy conversation import", () => { ]); try { - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); const context = { io: { info: () => {} }, stateAdapter }; await expect( migrateConversationHistoryToSql(context, { executor: fixture.sql }), @@ -682,7 +682,7 @@ describe("legacy conversation import", () => { it("replaces NUL characters in imported agent steps", async () => { const fixture = await createLocalJuniorSqlFixture(); - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); const stepStore = createSqlAgentStepStore(fixture.sql); const messageStore = createSqlConversationMessageStore(fixture.sql); @@ -720,7 +720,7 @@ describe("legacy conversation import", () => { it("reads legacy visible messages from a real thread-state payload", async () => { const fixture = await createLocalJuniorSqlFixture(); - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); const messageStore = createSqlConversationMessageStore(fixture.sql); // Persisted pre-cutover shape: the visible transcript nested under @@ -783,7 +783,7 @@ describe("legacy conversation import", () => { it("rejects malformed legacy visible messages", async () => { const fixture = await createLocalJuniorSqlFixture(); - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); const messageStore = createSqlConversationMessageStore(fixture.sql); const stateAdapter = getStateAdapter(); await stateAdapter.connect(); diff --git a/packages/junior/tests/component/conversations/retention.test.ts b/packages/junior/tests/component/conversations/retention.test.ts index 752b11b52..36ce61d0a 100644 --- a/packages/junior/tests/component/conversations/retention.test.ts +++ b/packages/junior/tests/component/conversations/retention.test.ts @@ -152,7 +152,7 @@ describe("retention purge job", () => { beforeEach(async () => { fixture = await createLocalJuniorSqlFixture(); - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); }); afterEach(async () => { diff --git a/packages/junior/tests/component/memory-plugin-storage.test.ts b/packages/junior/tests/component/memory-plugin-storage.test.ts index c6cbdbe29..02d219d6d 100644 --- a/packages/junior/tests/component/memory-plugin-storage.test.ts +++ b/packages/junior/tests/component/memory-plugin-storage.test.ts @@ -1,6 +1,7 @@ import path from "node:path"; import { readdirSync } from "node:fs"; import { createMemoryState } from "@chat-adapter/state-memory"; +import { resolveMigrations } from "@sentry/junior-migrations"; import { afterAll, describe, expect, it, vi } from "vitest"; import { createMemoryPlugin, @@ -13,10 +14,9 @@ import { } from "@sentry/junior-plugin-api"; import { defineJuniorPlugins } from "@/plugins"; import { getPluginTools, setPlugins } from "@/chat/plugins/agent-hooks"; -import { migratePluginSchemas } from "@/chat/plugins/migrations"; -import { readMigrationFiles } from "drizzle-orm/migrator"; +import { bootstrapPluginSchemas } from "@/chat/plugins/migrations"; import { closeDb } from "@/chat/db"; -import { migratePluginsToSql } from "@/cli/upgrade/migrations/plugin-sql"; +import { migratePluginJournals } from "@/cli/upgrade/migrations/plugin-journal"; import { createLocalJuniorSqlFixture } from "../fixtures/sql"; const NEON = vi.hoisted(() => ({ @@ -39,7 +39,6 @@ vi.mock("@/db/executor", () => ({ db: NEON.sql.db.bind(NEON.sql), execute: NEON.sql.execute.bind(NEON.sql), query: NEON.sql.query.bind(NEON.sql), - migrate: NEON.sql.migrate.bind(NEON.sql), transaction: NEON.sql.transaction.bind(NEON.sql), withLock: NEON.sql.withLock.bind(NEON.sql), withMigrationLock: NEON.sql.withMigrationLock.bind(NEON.sql), @@ -87,7 +86,7 @@ function memoryMigrationFiles(): string[] { async function migrateMemorySchema( fixture: Awaited>, ) { - await migratePluginSchemas(fixture.sql, [ + await bootstrapPluginSchemas(fixture.sql, [ { dir: memoryMigrationsDir(), pluginName: "memory", @@ -98,9 +97,7 @@ async function migrateMemorySchema( describe("memory plugin host wiring", () => { it("adopts exact legacy migration hashes without replaying them", async () => { const fixture = await createLocalJuniorSqlFixture(); - const migrations = readMigrationFiles({ - migrationsFolder: memoryMigrationsDir(), - }); + const migrations = await resolveMigrations(memoryMigrationsDir()); const migrationFiles = memoryMigrationFiles(); expect(migrationFiles).toHaveLength(migrations.length); @@ -131,7 +128,7 @@ CREATE TABLE junior_schema_migrations ( } await expect( - migratePluginSchemas(fixture.sql, [ + bootstrapPluginSchemas(fixture.sql, [ { dir: memoryMigrationsDir(), pluginName: "memory", @@ -149,9 +146,8 @@ CREATE TABLE junior_schema_migrations ( it("does not adopt an unknown memory legacy checksum", async () => { const fixture = await createLocalJuniorSqlFixture(); - const migrationCount = readMigrationFiles({ - migrationsFolder: memoryMigrationsDir(), - }).length; + const migrationCount = (await resolveMigrations(memoryMigrationsDir())) + .length; const [baselineFile] = memoryMigrationFiles(); expect(baselineFile).toBeDefined(); @@ -169,7 +165,7 @@ CREATE TABLE junior_schema_migrations ( ); await expect( - migratePluginSchemas(fixture.sql, [ + bootstrapPluginSchemas(fixture.sql, [ { dir: memoryMigrationsDir(), pluginName: "memory", @@ -192,11 +188,10 @@ CREATE TABLE junior_schema_migrations ( NEON.sql = fixture.sql; try { - const migrationCount = readMigrationFiles({ - migrationsFolder: memoryMigrationsDir(), - }).length; + const migrationCount = (await resolveMigrations(memoryMigrationsDir())) + .length; await expect( - migratePluginsToSql({ + migratePluginJournals({ io: { info: () => {} }, pluginSet: defineJuniorPlugins([createMemoryPlugin()]), stateAdapter, diff --git a/packages/junior/tests/component/migrations/mixed-runner-database.test.ts b/packages/junior/tests/component/migrations/mixed-runner-database.test.ts new file mode 100644 index 000000000..a972b989d --- /dev/null +++ b/packages/junior/tests/component/migrations/mixed-runner-database.test.ts @@ -0,0 +1,307 @@ +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + runMigrationJournal, + type MigrationContextV1, + type MigrationStateV1, +} from "@sentry/junior-migrations"; +import { createJiti } from "jiti"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + createLocalJuniorSqlFixture, + type LocalJuniorSqlFixture, +} from "../../fixtures/sql"; +import { hasJuniorPostgresTestDatabase } from "../../fixtures/postgres/fixture"; + +const migrationLoader = createJiti(import.meta.url, { moduleCache: false }); +const temporaryDirectories: string[] = []; + +const unusedState: MigrationStateV1 = { + acquireLock: async () => null, + appendToList: async () => {}, + connect: async () => {}, + delete: async () => {}, + get: async () => undefined, + getList: async () => [], + releaseLock: async () => {}, + set: async () => {}, + setIfNotExists: async () => false, +}; + +async function createMigrationFolder(args: { + source: string; + tag: string; + when: number; +}): Promise { + const folder = await mkdtemp(join(tmpdir(), "junior-mixed-runner-")); + temporaryDirectories.push(folder); + await mkdir(join(folder, "meta")); + await writeFile( + join(folder, "meta", "_journal.json"), + JSON.stringify({ + version: "7", + dialect: "postgresql", + entries: [ + { + idx: 0, + version: "7", + when: args.when, + tag: args.tag, + breakpoints: true, + }, + ], + }), + ); + await writeFile(join(folder, `${args.tag}.ts`), args.source); + return folder; +} + +function migrationOptions(args: { + executor?: LocalJuniorSqlFixture["sql"]; + fixture: LocalJuniorSqlFixture; + folder: string; + table: string; +}) { + const executor = args.executor ?? args.fixture.sql; + return { + executor, + migrationsFolder: args.folder, + migrationsTable: args.table, + loadTypeScript: async (migrationPath: string) => + await migrationLoader.import>(migrationPath), + createContext: ({ + progress, + }: { + progress: MigrationContextV1["progress"]; + }): MigrationContextV1 => ({ + database: executor, + log: () => {}, + progress, + state: unusedState, + }), + }; +} + +afterEach(async () => { + await Promise.all( + temporaryDirectories + .splice(0) + .map((directory) => rm(directory, { recursive: true, force: true })), + ); +}); + +describe("mixed migration runner database contract", () => { + it("persists failed progress and resumes a TypeScript migration", async () => { + const fixture = await createLocalJuniorSqlFixture(); + const folder = await createMigrationFolder({ + source: ` +export default { + apiVersion: 1, + async up(context) { + const progress = await context.progress.load(); + if (progress === undefined) { + await context.database.execute( + "INSERT INTO mixed_runner_events (stage) VALUES ($1)", + ["before-failure"], + ); + await context.progress.save({ cursor: 1 }); + throw new Error("intentional migration interruption"); + } + await context.database.execute( + "INSERT INTO mixed_runner_events (stage) VALUES ($1)", + ["after-resume"], + ); + return { resumedFrom: progress.cursor }; + }, +}; +`, + tag: "0000_resumable", + when: 2_026_071_600_001, + }); + const table = "__junior_mixed_runner_resume"; + const options = migrationOptions({ fixture, folder, table }); + + try { + await fixture.sql.execute(` +CREATE TABLE mixed_runner_events ( + id SERIAL PRIMARY KEY, + stage TEXT NOT NULL +) +`); + + await expect(runMigrationJournal(options)).rejects.toThrow( + "intentional migration interruption", + ); + + await expect( + fixture.sql.query( + `SELECT stage FROM mixed_runner_events ORDER BY id ASC`, + ), + ).resolves.toEqual([{ stage: "before-failure" }]); + await expect( + fixture.sql.query(` +SELECT + name, + kind, + status, + progress, + result, + completed_at IS NOT NULL AS "completed" +FROM drizzle.${table} +`), + ).resolves.toEqual([ + { + completed: false, + kind: "typescript", + name: "0000_resumable", + progress: { cursor: 1 }, + result: null, + status: "failed", + }, + ]); + + await expect(runMigrationJournal(options)).resolves.toEqual({ + existing: 0, + migrated: 1, + scanned: 1, + skipped: 0, + }); + + await expect( + fixture.sql.query( + `SELECT stage FROM mixed_runner_events ORDER BY id ASC`, + ), + ).resolves.toEqual([ + { stage: "before-failure" }, + { stage: "after-resume" }, + ]); + await expect( + fixture.sql.query(` +SELECT + status, + progress, + result, + completed_at IS NOT NULL AS "completed" +FROM drizzle.${table} +`), + ).resolves.toEqual([ + { + completed: true, + progress: { cursor: 1 }, + result: { resumedFrom: 1 }, + status: "completed", + }, + ]); + } finally { + await fixture.close(); + } + }); + + it.skipIf(!hasJuniorPostgresTestDatabase())( + "serializes concurrent runs and executes a TypeScript migration once", + async () => { + const fixture = await createLocalJuniorSqlFixture(); + const folder = await createMigrationFolder({ + source: ` +export default { + apiVersion: 1, + async up(context) { + await context.database.execute( + "INSERT INTO mixed_runner_lock_events (stage) VALUES ($1)", + ["body"], + ); + while (true) { + const [barrier] = await context.database.query( + "SELECT released FROM mixed_runner_lock_barrier WHERE id = 1", + ); + if (barrier?.released === true) break; + await context.database.query("SELECT pg_sleep(0.01)"); + } + return { executed: true }; + }, +}; +`, + tag: "0000_locked", + when: 2_026_071_600_002, + }); + const table = "__junior_mixed_runner_lock"; + let lockAttempts = 0; + const observedExecutor = new Proxy(fixture.sql, { + get(target, key, receiver) { + if (key === "withMigrationLock") { + return async ( + migrationTable: string, + callback: () => Promise, + ) => { + lockAttempts += 1; + return await target.withMigrationLock(migrationTable, callback); + }; + } + const value = Reflect.get(target, key, receiver) as unknown; + return typeof value === "function" ? value.bind(target) : value; + }, + }); + const options = migrationOptions({ + executor: observedExecutor, + fixture, + folder, + table, + }); + + try { + await fixture.sql.execute(` +CREATE TABLE mixed_runner_lock_events ( + id SERIAL PRIMARY KEY, + stage TEXT NOT NULL +); +CREATE TABLE mixed_runner_lock_barrier ( + id INTEGER PRIMARY KEY, + released BOOLEAN NOT NULL +); +INSERT INTO mixed_runner_lock_barrier (id, released) VALUES (1, false) +`); + + const first = runMigrationJournal(options); + await vi.waitFor(async () => { + await expect( + fixture.sql.query( + "SELECT count(*)::integer AS count FROM mixed_runner_lock_events", + ), + ).resolves.toEqual([{ count: 1 }]); + }); + const second = runMigrationJournal(options); + await vi.waitFor(() => expect(lockAttempts).toBe(2)); + await expect( + fixture.sql.query( + "SELECT count(*)::integer AS count FROM mixed_runner_lock_events", + ), + ).resolves.toEqual([{ count: 1 }]); + await fixture.sql.execute( + "UPDATE mixed_runner_lock_barrier SET released = true WHERE id = 1", + ); + const results = await Promise.all([first, second]); + + expect(results).toEqual( + expect.arrayContaining([ + { existing: 0, migrated: 1, scanned: 1, skipped: 0 }, + { existing: 1, migrated: 0, scanned: 1, skipped: 0 }, + ]), + ); + await expect( + fixture.sql.query( + `SELECT stage FROM mixed_runner_lock_events ORDER BY id ASC`, + ), + ).resolves.toEqual([{ stage: "body" }]); + await expect( + fixture.sql.query(`SELECT status, result FROM drizzle.${table}`), + ).resolves.toEqual([ + { result: { executed: true }, status: "completed" }, + ]); + } finally { + await fixture.close(); + } + }, + 15_000, + ); +}); diff --git a/packages/junior/tests/component/scheduler-sql-plugin.test.ts b/packages/junior/tests/component/scheduler-sql-plugin.test.ts index 5304eb74a..14ef04386 100644 --- a/packages/junior/tests/component/scheduler-sql-plugin.test.ts +++ b/packages/junior/tests/component/scheduler-sql-plugin.test.ts @@ -2,22 +2,29 @@ import path from "node:path"; import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { createMemoryState } from "@chat-adapter/state-memory"; -import { readMigrationFiles } from "drizzle-orm/migrator"; -import { afterAll, afterEach, describe, expect, it, vi } from "vitest"; +import { + resolveMigrations, + type MigrationContextV1, +} from "@sentry/junior-migrations"; import { defineJuniorPlugin } from "@sentry/junior-plugin-api"; +import { afterAll, afterEach, describe, expect, it, vi } from "vitest"; import { createSchedulerSqlStore, schedulerPlugin, type SchedulerDb, type ScheduledTask, } from "@sentry/junior-scheduler"; +import schedulerStateToSqlMigration from "../../../junior-scheduler/migrations/0002_scheduler_state_to_sql"; import { createSchedulerStore } from "../../../junior-scheduler/src/store"; import { defineJuniorPlugins } from "@/plugins"; -import { migratePluginSchemas } from "@/chat/plugins/migrations"; +import { createPluginMigrationStateV1 } from "@/chat/plugins/migration-state"; +import { + bootstrapPluginSchemas, + migratePluginSchemas, +} from "@/chat/plugins/migrations"; import { createPluginState } from "@/chat/plugins/state"; -import { disconnectStateAdapter, getStateAdapter } from "@/chat/state/adapter"; -import { runPluginStorageMigrations } from "@/cli/upgrade/migrations/plugin-storage"; -import { migratePluginsToSql } from "@/cli/upgrade/migrations/plugin-sql"; +import { disconnectStateAdapter } from "@/chat/state/adapter"; +import { migratePluginJournals } from "@/cli/upgrade/migrations/plugin-journal"; import { createLocalJuniorSqlFixture } from "../fixtures/sql"; const NEON = vi.hoisted(() => ({ @@ -41,7 +48,6 @@ vi.mock("@/db/executor", () => ({ db: NEON.sql.db.bind(NEON.sql), execute: NEON.sql.execute.bind(NEON.sql), query: NEON.sql.query.bind(NEON.sql), - migrate: NEON.sql.migrate.bind(NEON.sql), transaction: NEON.sql.transaction.bind(NEON.sql), withLock: NEON.sql.withLock.bind(NEON.sql), withMigrationLock: NEON.sql.withMigrationLock.bind(NEON.sql), @@ -66,7 +72,7 @@ function memoryMigrationsDir(): string { async function migrateSchedulerSchema( fixture: Awaited>, ) { - await migratePluginSchemas(fixture.sql, [ + await bootstrapPluginSchemas(fixture.sql, [ { dir: schedulerMigrationsDir(), pluginName: "scheduler", @@ -74,6 +80,21 @@ async function migrateSchedulerSchema( ]); } +async function runSchedulerStateMigration(args: { + fixture: Awaited>; + stateAdapter: ReturnType; +}) { + return await schedulerStateToSqlMigration.up({ + database: args.fixture.sql, + log: () => {}, + progress: { + load: async () => undefined, + save: async () => {}, + }, + state: createPluginMigrationStateV1("scheduler", args.stateAdapter), + }); +} + function createTask(overrides: Partial = {}): ScheduledTask { return { id: "sched_sql_1", @@ -114,6 +135,127 @@ describe("scheduler SQL plugin storage", () => { await disconnectStateAdapter(); }); + it("scopes migration state while preserving plugin legacy keys", async () => { + const stateAdapter = createMemoryState(); + await stateAdapter.connect(); + const state = createPluginMigrationStateV1("scheduler", stateAdapter); + + try { + await stateAdapter.set("global-key", "global-value"); + await stateAdapter.set("junior:memory:secret", "memory-value"); + await state.set("global-key", "scheduler-value"); + await state.set("junior:scheduler:legacy", "legacy-value"); + + await expect(state.get("global-key")).resolves.toBe("scheduler-value"); + await expect(stateAdapter.get("global-key")).resolves.toBe( + "global-value", + ); + await expect(state.get("junior:memory:secret")).resolves.toBeUndefined(); + await expect(state.get("junior:scheduler:legacy")).resolves.toBe( + "legacy-value", + ); + await expect(stateAdapter.get("junior:scheduler:legacy")).resolves.toBe( + "legacy-value", + ); + + await stateAdapter.appendToList("global-list", "global-item"); + await state.appendToList("global-list", "scheduler-item"); + await expect(state.getList("global-list")).resolves.toEqual([ + "scheduler-item", + ]); + await expect(stateAdapter.getList("global-list")).resolves.toEqual([ + "global-item", + ]); + + const scopedLock = await state.acquireLock("migration-lock", 1_000); + const globalLock = await stateAdapter.acquireLock( + "migration-lock", + 1_000, + ); + expect(scopedLock).not.toBeNull(); + expect(globalLock).not.toBeNull(); + if (scopedLock) { + await state.releaseLock(scopedLock); + } + if (globalLock) { + await stateAdapter.releaseLock(globalLock); + } + } finally { + await stateAdapter.disconnect(); + } + }); + + it("passes scoped state through the plugin migration host", async () => { + const stateAdapter = createMemoryState(); + await stateAdapter.connect(); + const fixture = await createLocalJuniorSqlFixture(); + const migrationsDir = mkdtempSync( + path.join(tmpdir(), "junior-plugin-state-migration-"), + ); + mkdirSync(path.join(migrationsDir, "meta")); + writeFileSync( + path.join(migrationsDir, "meta", "_journal.json"), + JSON.stringify({ + version: "7", + dialect: "postgresql", + entries: [ + { + idx: 0, + version: "7", + when: 2_026_071_600_003, + tag: "0000_state_scope", + breakpoints: true, + }, + ], + }), + ); + writeFileSync( + path.join(migrationsDir, "0000_state_scope.ts"), + "export default { apiVersion: 1, async up() {} };\n", + ); + const observed: unknown[] = []; + + try { + await stateAdapter.set("global-key", "global-value"); + await stateAdapter.set("junior:other:secret", "other-value"); + await migratePluginSchemas( + fixture.sql, + [{ dir: migrationsDir, pluginName: "isolation" }], + { + mode: "all", + stateAdapter, + loadTypeScript: async () => ({ + default: { + apiVersion: 1, + async up(context: MigrationContextV1) { + observed.push( + await context.state.get("global-key"), + await context.state.get("junior:other:secret"), + ); + await context.state.set("global-key", "scoped-value"); + return { scoped: true }; + }, + }, + }), + }, + ); + + expect(observed).toEqual([undefined, undefined]); + await expect(stateAdapter.get("global-key")).resolves.toBe( + "global-value", + ); + await expect( + createPluginMigrationStateV1("isolation", stateAdapter).get( + "global-key", + ), + ).resolves.toBe("scoped-value"); + } finally { + rmSync(migrationsDir, { force: true, recursive: true }); + await stateAdapter.disconnect(); + await fixture.close(); + } + }); + it("adopts deployed scheduler schema state into its Drizzle journal", async () => { const fixture = await createLocalJuniorSqlFixture(); @@ -174,16 +316,19 @@ INSERT INTO junior_scheduler_tasks ( ); await expect( - migratePluginSchemas(fixture.sql, [ + bootstrapPluginSchemas(fixture.sql, [ { dir: schedulerMigrationsDir(), pluginName: "scheduler", }, ]), - ).resolves.toEqual({ existing: 1, migrated: 1, scanned: 2 }); - const migrations = readMigrationFiles({ - migrationsFolder: schedulerMigrationsDir(), + ).resolves.toEqual({ + existing: 1, + migrated: 1, + scanned: 3, + skipped: 1, }); + const migrations = await resolveMigrations(schedulerMigrationsDir()); const migrationRows = await fixture.sql.query<{ createdAt: string; hash: string; @@ -193,10 +338,12 @@ FROM drizzle.${migrationTable!.tablename} ORDER BY created_at `); expect(migrationRows).toEqual( - migrations.map((migration) => ({ - createdAt: String(migration.folderMillis), - hash: migration.hash, - })), + migrations + .filter((migration) => migration.kind === "sql") + .map((migration) => ({ + createdAt: String(migration.when), + hash: migration.hash, + })), ); const [migratedTask] = await fixture.sql.query<{ record: unknown }>( `SELECT record FROM junior_scheduler_tasks WHERE id = $1`, @@ -207,13 +354,18 @@ ORDER BY created_at }); expect(migratedTask?.record).not.toHaveProperty("credentialSubject"); await expect( - migratePluginSchemas(fixture.sql, [ + bootstrapPluginSchemas(fixture.sql, [ { dir: schedulerMigrationsDir(), pluginName: "scheduler", }, ]), - ).resolves.toEqual({ existing: 2, migrated: 0, scanned: 2 }); + ).resolves.toEqual({ + existing: 2, + migrated: 0, + scanned: 3, + skipped: 1, + }); await expect( fixture.sql.query<{ createdAt: string; @@ -235,18 +387,24 @@ ORDER BY created_at { dir: schedulerMigrationsDir(), pluginName: "scheduler" }, { dir: memoryMigrationsDir(), pluginName: "memory" }, ]; - const migrationCount = roots.reduce( - (count, root) => - count + readMigrationFiles({ migrationsFolder: root.dir }).length, - 0, - ); - + const migrations = ( + await Promise.all( + roots.map(async (root) => await resolveMigrations(root.dir)), + ) + ).flat(); + const migrationCount = migrations.length; + const sqlMigrationCount = migrations.filter( + (migration) => migration.kind === "sql", + ).length; try { - await expect(migratePluginSchemas(fixture.sql, roots)).resolves.toEqual({ - existing: 0, - migrated: migrationCount, - scanned: migrationCount, - }); + await expect(bootstrapPluginSchemas(fixture.sql, roots)).resolves.toEqual( + { + existing: 0, + migrated: sqlMigrationCount, + scanned: migrationCount, + skipped: 1, + }, + ); const migrationTables = await fixture.sql.query<{ tablename: string }>(` SELECT tablename FROM pg_tables @@ -256,11 +414,12 @@ ORDER BY tablename `); expect(migrationTables).toHaveLength(2); await expect( - migratePluginSchemas(fixture.sql, [...roots].reverse()), + bootstrapPluginSchemas(fixture.sql, [...roots].reverse()), ).resolves.toEqual({ - existing: migrationCount, + existing: sqlMigrationCount, migrated: 0, scanned: migrationCount, + skipped: 1, }); } finally { await fixture.close(); @@ -280,12 +439,12 @@ ORDER BY tablename try { await expect( - migratePluginSchemas(fixture.sql, [ + bootstrapPluginSchemas(fixture.sql, [ { dir: missingJournal, pluginName: "missing" }, ]), ).rejects.toThrow("Can't find meta/_journal.json file"); await expect( - migratePluginSchemas(fixture.sql, [ + bootstrapPluginSchemas(fixture.sql, [ { dir: invalidJournal, pluginName: "invalid" }, ]), ).rejects.toThrow("Expected property name"); @@ -515,26 +674,47 @@ ORDER BY tablename const run = await stateStore.claimDueRun({ nowMs: TEST_NOW_MS }); expect(run).toBeDefined(); - const context = { - db, - io: { info: () => {} }, - pluginSet: defineJuniorPlugins([schedulerPlugin()]), - stateAdapter, - }; - - await expect(runPluginStorageMigrations(context)).resolves.toEqual({ + await expect( + runSchedulerStateMigration({ fixture, stateAdapter }), + ).resolves.toEqual({ existing: 0, migrated: 2, missing: 0, scanned: 2, }); - await expect(runPluginStorageMigrations(context)).resolves.toEqual({ + await expect( + runSchedulerStateMigration({ fixture, stateAdapter }), + ).resolves.toEqual({ existing: 2, migrated: 0, missing: 0, scanned: 2, }); + const { credentialMode: _credentialMode, ...legacySqlTask } = task; + await fixture.sql.execute( + "UPDATE junior_scheduler_tasks SET record = $2::jsonb WHERE id = $1", + [ + task.id, + JSON.stringify({ + ...legacySqlTask, + credentialSubject: { + allowedWhen: "private-direct-conversation", + type: "user", + userId: "U123", + }, + }), + ], + ); + await expect( + runSchedulerStateMigration({ fixture, stateAdapter }), + ).resolves.toEqual({ + existing: 1, + migrated: 1, + missing: 0, + scanned: 2, + }); + const sqlStore = createSchedulerSqlStore(db); await expect(sqlStore.getTask(task.id)).resolves.toMatchObject({ id: task.id, @@ -563,7 +743,7 @@ ORDER BY tablename const badRunId = `${task.id}:${TEST_RUN_AT_MS}`; await state.set( "junior:scheduler:tasks", - ["sched_state_sql_bad", task.id], + ["sched_state_sql_bad", "sched_state_sql_missing_required", task.id], 5 * 60 * 1000, ); await state.set( @@ -587,6 +767,12 @@ ORDER BY tablename }, 5 * 60 * 1000, ); + const { schedule: _schedule, ...missingSchedule } = task; + await state.set( + "junior:scheduler:task:sched_state_sql_missing_required", + { ...missingSchedule, id: "sched_state_sql_missing_required" }, + 5 * 60 * 1000, + ); await state.set( `junior:scheduler:active:${task.id}`, { @@ -598,22 +784,22 @@ ORDER BY tablename ); await state.set( `junior:scheduler:run:${badRunId}`, - { id: badRunId }, + { + id: badRunId, + scheduledForMs: TEST_RUN_AT_MS, + status: "pending", + taskId: task.id, + }, 5 * 60 * 1000, ); await expect( - runPluginStorageMigrations({ - db, - io: { info: () => {} }, - pluginSet: defineJuniorPlugins([schedulerPlugin()]), - stateAdapter, - }), + runSchedulerStateMigration({ fixture, stateAdapter }), ).resolves.toEqual({ existing: 0, migrated: 1, - missing: 1, - scanned: 2, + missing: 2, + scanned: 3, }); const sqlStore = createSchedulerSqlStore(db); @@ -624,6 +810,9 @@ ORDER BY tablename await expect(sqlStore.getTask("sched_state_sql_bad")).resolves.toBe( undefined, ); + await expect( + sqlStore.getTask("sched_state_sql_missing_required"), + ).resolves.toBe(undefined); await expect(sqlStore.getRun(badRunId)).resolves.toBe(undefined); } finally { await stateAdapter.disconnect(); @@ -631,27 +820,17 @@ ORDER BY tablename } }, 15_000); - it("does not load scheduler storage migration from package-only plugin set", async () => { + it("does not apply scheduler SQL migrations from package-only config", async () => { const stateAdapter = createMemoryState(); await stateAdapter.connect(); const fixture = await createLocalJuniorSqlFixture(); + NEON.sql = fixture.sql; try { - await migrateSchedulerSchema(fixture); - const db = fixture.sql.db() as unknown as SchedulerDb; - const stateStore = createSchedulerStore( - createPluginState("scheduler", stateAdapter), - ); - const task = createTask({ id: "sched_package_config" }); - await stateStore.saveTask(task); - const run = await stateStore.claimDueRun({ nowMs: TEST_NOW_MS }); - expect(run).toBeDefined(); - await expect( - runPluginStorageMigrations({ - db, + migratePluginJournals({ io: { info: () => {} }, - pluginSet: defineJuniorPlugins(["@sentry/junior-scheduler"]), + pluginCatalogConfig: { packages: ["@sentry/junior-scheduler"] }, stateAdapter, }), ).resolves.toEqual({ @@ -660,17 +839,13 @@ ORDER BY tablename missing: 0, scanned: 0, }); - - const sqlStore = createSchedulerSqlStore(db); - await expect(sqlStore.getTask(task.id)).resolves.toBe(undefined); - await expect(sqlStore.getRun(run!.id)).resolves.toBe(undefined); } finally { await stateAdapter.disconnect(); await fixture.close(); } - }, 15_000); + }); - it("does not apply scheduler SQL migrations from package-only config", async () => { + it("applies scheduler SQL migrations from registration-only config", async () => { const stateAdapter = createMemoryState(); await stateAdapter.connect(); const fixture = await createLocalJuniorSqlFixture(); @@ -678,16 +853,24 @@ ORDER BY tablename try { await expect( - migratePluginsToSql({ + migratePluginJournals({ io: { info: () => {} }, - pluginCatalogConfig: { packages: ["@sentry/junior-scheduler"] }, + pluginSet: defineJuniorPlugins([schedulerPlugin()]), stateAdapter, }), ).resolves.toEqual({ existing: 0, - migrated: 0, + migrated: 3, missing: 0, - scanned: 0, + scanned: 3, + }); + + const db = fixture.sql.db() as unknown as SchedulerDb; + const store = createSchedulerSqlStore(db); + const task = createTask({ id: "sched_schema_registration_config" }); + await store.saveTask(task); + await expect(store.getTask(task.id)).resolves.toMatchObject({ + id: task.id, }); } finally { await stateAdapter.disconnect(); @@ -695,32 +878,29 @@ ORDER BY tablename } }); - it("applies scheduler SQL migrations from registration-only config", async () => { + it("runs a migration-only plugin registration", async () => { const stateAdapter = createMemoryState(); await stateAdapter.connect(); const fixture = await createLocalJuniorSqlFixture(); NEON.sql = fixture.sql; + const scheduler = schedulerPlugin(); + const migrationOnly = defineJuniorPlugin({ + manifest: scheduler.manifest, + packageName: scheduler.packageName, + }); try { await expect( - migratePluginsToSql({ + migratePluginJournals({ io: { info: () => {} }, - pluginSet: defineJuniorPlugins([schedulerPlugin()]), + pluginSet: defineJuniorPlugins([migrationOnly]), stateAdapter, }), ).resolves.toEqual({ existing: 0, - migrated: 2, + migrated: 3, missing: 0, - scanned: 2, - }); - - const db = fixture.sql.db() as unknown as SchedulerDb; - const store = createSchedulerSqlStore(db); - const task = createTask({ id: "sched_schema_registration_config" }); - await store.saveTask(task); - await expect(store.getTask(task.id)).resolves.toMatchObject({ - id: task.id, + scanned: 3, }); } finally { await stateAdapter.disconnect(); @@ -736,7 +916,7 @@ ORDER BY tablename try { await expect( - migratePluginsToSql({ + migratePluginJournals({ io: { info: () => {} }, pluginSet: defineJuniorPlugins([ "@sentry/junior-scheduler", @@ -746,9 +926,9 @@ ORDER BY tablename }), ).resolves.toEqual({ existing: 0, - migrated: 2, + migrated: 3, missing: 0, - scanned: 2, + scanned: 3, }); } finally { await stateAdapter.disconnect(); @@ -861,50 +1041,4 @@ INSERT INTO junior_scheduler_runs ( await fixture.close(); } }, 15_000); - - it("passes database access to plugin storage migrations", async () => { - const stateAdapter = getStateAdapter(); - await stateAdapter.connect(); - const fixture = await createLocalJuniorSqlFixture(); - - try { - const db = fixture.sql.db() as unknown as SchedulerDb; - let receivedDb: unknown; - const plugin = defineJuniorPlugin({ - manifest: { - name: "stateless", - displayName: "Stateless", - description: "Storage migration with database access", - }, - hooks: { - migrateStorage(ctx) { - receivedDb = ctx.db; - return { - existing: 0, - migrated: 0, - missing: 0, - scanned: 1, - }; - }, - }, - }); - - await expect( - runPluginStorageMigrations({ - db, - io: { info: () => {} }, - pluginSet: defineJuniorPlugins([plugin]), - stateAdapter, - }), - ).resolves.toEqual({ - existing: 0, - migrated: 0, - missing: 0, - scanned: 1, - }); - expect(receivedDb).toBe(db); - } finally { - await fixture.close(); - } - }, 15_000); }); diff --git a/packages/junior/tests/fixtures/postgres/executor.ts b/packages/junior/tests/fixtures/postgres/executor.ts index 2e90d9ad0..420d0f3f4 100644 --- a/packages/junior/tests/fixtures/postgres/executor.ts +++ b/packages/junior/tests/fixtures/postgres/executor.ts @@ -1,7 +1,5 @@ import { type PoolClient, type QueryResultRow } from "pg"; import { drizzle } from "drizzle-orm/node-postgres"; -import { migrate } from "drizzle-orm/node-postgres/migrator"; -import type { MigrationConfig } from "drizzle-orm/migrator"; import type { JuniorDatabase, JuniorSqlExecutor } from "@/db/db"; import { createPostgresJuniorSqlExecutor } from "@/db/postgres"; import { juniorSqlSchema } from "@/db/schema"; @@ -37,10 +35,6 @@ class ClientJuniorSqlExecutor implements JuniorSqlExecutor { return result.rows as T[]; } - async migrate(config: MigrationConfig): Promise { - await migrate(drizzle(this.client, { schema: juniorSqlSchema }), config); - } - async transaction(callback: () => Promise): Promise { const savepoint = `junior_test_savepoint_${++this.savepointId}`; await this.client.query(`SAVEPOINT ${savepoint}`); @@ -66,10 +60,10 @@ class ClientJuniorSqlExecutor implements JuniorSqlExecutor { } async withMigrationLock( - _migrationTable: string, + migrationTable: string, callback: () => Promise, ): Promise { - return await callback(); + return await this.withLock(`junior:migrate:${migrationTable}`, callback); } async close(): Promise { diff --git a/packages/junior/tests/fixtures/postgres/global-setup.ts b/packages/junior/tests/fixtures/postgres/global-setup.ts index 8fa39271b..59a007636 100644 --- a/packages/junior/tests/fixtures/postgres/global-setup.ts +++ b/packages/junior/tests/fixtures/postgres/global-setup.ts @@ -6,7 +6,7 @@ import { type PostgresHarnessConfig, } from "@sentry/junior-testing/postgres"; import { migrateSchema } from "@/chat/conversations/sql/migrations"; -import { migratePluginSchemas } from "@/chat/plugins/migrations"; +import { bootstrapPluginSchemas } from "@/chat/plugins/migrations"; import type { JuniorSqlMigrationExecutor } from "@/db/db"; import { createPostgresJuniorSqlExecutor } from "@/db/postgres"; @@ -47,8 +47,8 @@ export async function setupJuniorPostgresHarness( migrateTemplate: async (connectionString) => { const executor = createPostgresJuniorSqlExecutor({ connectionString }); try { - await migrateSchema(executor); - await migratePluginSchemas(executor, [ + await migrateSchema(executor, { mode: "schema-bootstrap" }); + await bootstrapPluginSchemas(executor, [ { dir: path.resolve(process.cwd(), "../junior-scheduler/migrations"), pluginName: "scheduler", diff --git a/packages/junior/tests/fixtures/sql.ts b/packages/junior/tests/fixtures/sql.ts index 2d88accac..6b86e3524 100644 --- a/packages/junior/tests/fixtures/sql.ts +++ b/packages/junior/tests/fixtures/sql.ts @@ -5,7 +5,6 @@ import { createLocalPgliteFixture, type LocalPgliteFixture, } from "@sentry/junior-testing/pglite"; -import { migrate } from "drizzle-orm/pglite/migrator"; import type { PgliteDatabase } from "drizzle-orm/pglite"; import { createEmptyJuniorSqlFixture, @@ -43,12 +42,12 @@ export async function createLocalJuniorSqlFixture(): Promise fixture.close(), db: () => fixture.db() as unknown as JuniorDatabase, execute: (statement, params) => fixture.execute(statement, params), - migrate: (config) => migrate(fixture.db(), config), query: (statement: string, params?: readonly unknown[]) => fixture.query(statement, params), transaction: (callback) => fixture.transaction(callback), withLock: (lockName, callback) => fixture.withLock(lockName, callback), - withMigrationLock: (_migrationTable, callback) => callback(), + withMigrationLock: (migrationTable, callback) => + fixture.withLock(`junior:migrate:${migrationTable}`, callback), }; return { diff --git a/packages/junior/tests/integration/api/conversations/list.test.ts b/packages/junior/tests/integration/api/conversations/list.test.ts index 611f9d7af..d0c5ec0fa 100644 --- a/packages/junior/tests/integration/api/conversations/list.test.ts +++ b/packages/junior/tests/integration/api/conversations/list.test.ts @@ -11,7 +11,7 @@ describe("conversation list API", () => { const fixture = createConfiguredJuniorSqlFixture(); const store = createSqlStore(fixture.sql); try { - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); await store.recordActivity({ actor: { email: "other@example.com", diff --git a/packages/junior/tests/integration/api/conversations/stats.test.ts b/packages/junior/tests/integration/api/conversations/stats.test.ts index 30880c5e1..f63339739 100644 --- a/packages/junior/tests/integration/api/conversations/stats.test.ts +++ b/packages/junior/tests/integration/api/conversations/stats.test.ts @@ -15,7 +15,7 @@ describe("conversation stats API", () => { const fixture = createConfiguredJuniorSqlFixture(); const store = createSqlStore(fixture.sql); try { - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); await store.recordActivity({ conversationId: "slack:C1:recent", channelName: "proj-alpha", @@ -179,7 +179,7 @@ describe("conversation stats API", () => { const fixture = createConfiguredJuniorSqlFixture(); try { - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); const now = new Date("2026-06-15T11:00:00.000Z"); await fixture.sql .db() diff --git a/packages/junior/tests/integration/api/locations.test.ts b/packages/junior/tests/integration/api/locations.test.ts index 9420114df..c17dd0ae9 100644 --- a/packages/junior/tests/integration/api/locations.test.ts +++ b/packages/junior/tests/integration/api/locations.test.ts @@ -19,7 +19,7 @@ describe("locations API", () => { const store = createSqlStore(fixture.sql); try { - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); const nowMs = Date.parse("2026-06-15T11:00:00.000Z"); await store.recordActivity({ conversationId: "slack:C1:seed", diff --git a/packages/junior/tests/integration/api/people/fixture.ts b/packages/junior/tests/integration/api/people/fixture.ts index c35dc0ee1..99356d1ad 100644 --- a/packages/junior/tests/integration/api/people/fixture.ts +++ b/packages/junior/tests/integration/api/people/fixture.ts @@ -6,7 +6,7 @@ import type { LocalJuniorSqlFixture } from "../../../fixtures/sql"; /** Seed representative verified and untrusted people rows for people API tests. */ export async function seedPeople(fixture: LocalJuniorSqlFixture) { const store = createSqlStore(fixture.sql); - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); await store.recordActivity({ conversationId: "slack:C1:123", diff --git a/packages/junior/tests/integration/api/people/profile.test.ts b/packages/junior/tests/integration/api/people/profile.test.ts index 99e172041..e259bcf56 100644 --- a/packages/junior/tests/integration/api/people/profile.test.ts +++ b/packages/junior/tests/integration/api/people/profile.test.ts @@ -120,7 +120,7 @@ describe("people profile API", () => { const store = createSqlStore(fixture.sql); try { - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); const nowMs = Date.parse("2026-06-15T11:00:00.000Z"); await store.recordActivity({ conversationId: "slack:C1:seed", diff --git a/packages/junior/tests/integration/conversation-sql.test.ts b/packages/junior/tests/integration/conversation-sql.test.ts index aee638578..889d40dfe 100644 --- a/packages/junior/tests/integration/conversation-sql.test.ts +++ b/packages/junior/tests/integration/conversation-sql.test.ts @@ -20,7 +20,7 @@ describe("conversation SQL local mode", () => { const fixture = await createLocalJuniorSqlFixture(); try { - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); const rows = await fixture.sql.query<{ column_name: string; @@ -132,7 +132,7 @@ INSERT INTO drizzle.__drizzle_migrations (hash, created_at) VALUES ('host-migration', 9999999999999) `); - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); const [host] = await fixture.sql.query<{ count: number }>( "SELECT count(*)::integer AS count FROM drizzle.__drizzle_migrations", @@ -156,7 +156,7 @@ VALUES ('host-migration', 9999999999999) }); try { - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); await fixture.sql.execute(` CREATE TABLE junior_schema_migrations ( id TEXT PRIMARY KEY, @@ -191,7 +191,10 @@ ALTER TABLE junior_conversations second.query("SELECT 1"), ]); - await Promise.all([migrateSchema(fixture.sql), migrateSchema(second)]); + await Promise.all([ + migrateSchema(fixture.sql, { mode: "schema-bootstrap" }), + migrateSchema(second, { mode: "schema-bootstrap" }), + ]); const [journal] = await fixture.sql.query<{ count: number }>( "SELECT count(*)::integer AS count FROM drizzle.__drizzle_junior_core", ); @@ -207,8 +210,8 @@ ALTER TABLE junior_conversations const fixture = await createLocalJuniorSqlFixture(); try { - await migrateSchema(fixture.sql); - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); const conversation = buildJuniorSqlConversation({ conversationId: "slack:C123:1718123456.000000", @@ -291,7 +294,7 @@ WHERE conversation_id = $1 const fixture = await createLocalJuniorSqlFixture(); try { - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); await fixture.sql.execute(` CREATE TABLE junior_schema_migrations ( id TEXT PRIMARY KEY, @@ -322,7 +325,7 @@ ALTER TABLE junior_conversations DROP COLUMN metric_run_id `); - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); const metricColumns = await fixture.sql.query<{ column_name: string }>(` SELECT column_name @@ -357,7 +360,7 @@ ORDER BY column_name const fixture = await createLocalJuniorSqlFixture(); try { - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); await fixture.sql.execute(` CREATE TABLE junior_schema_migrations ( id TEXT PRIMARY KEY, @@ -383,7 +386,7 @@ VALUES "ALTER TABLE junior_conversations DROP COLUMN archived_at", ); - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); const [migrationRows] = await fixture.sql.query<{ count: number }>( "SELECT count(*)::integer AS count FROM drizzle.__drizzle_junior_core", @@ -404,7 +407,7 @@ WHERE table_schema = 'public' ) ORDER BY column_name `); - expect(migrationRows?.count).toBe(4); + expect(migrationRows?.count).toBe(5); expect(searchIndex?.exists).toBe(true); expect(metricColumns.map((row) => row.column_name)).toEqual([ "duration_ms", @@ -433,9 +436,9 @@ INSERT INTO junior_schema_migrations (id, checksum) VALUES ('0001_conversation_core', 'legacy-checksum-1') `); - await expect(migrateSchema(fixture.sql)).rejects.toThrow( - "Cannot adopt partial legacy core migration state", - ); + await expect( + migrateSchema(fixture.sql, { mode: "schema-bootstrap" }), + ).rejects.toThrow("Cannot adopt partial legacy core migration state"); } finally { await fixture.close(); } @@ -445,7 +448,7 @@ VALUES ('0001_conversation_core', 'legacy-checksum-1') const fixture = await createLocalJuniorSqlFixture(); try { - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); const store = createSqlStore(fixture.sql); await recordAgentTurnSessionSummary({ diff --git a/packages/junior/tests/integration/slack-conversation-search.test.ts b/packages/junior/tests/integration/slack-conversation-search.test.ts index 61a8f50bf..67add69bf 100644 --- a/packages/junior/tests/integration/slack-conversation-search.test.ts +++ b/packages/junior/tests/integration/slack-conversation-search.test.ts @@ -25,7 +25,7 @@ describe("searchConversationHistory", () => { const fixture = await createLocalJuniorSqlFixture(); try { - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); const conversations = createSqlStore(fixture.sql); const messages = createSqlConversationMessageStore(fixture.sql); const search = createSqlConversationSearchStore(fixture.sql); diff --git a/packages/junior/tests/integration/slack-schedule-tools.test.ts b/packages/junior/tests/integration/slack-schedule-tools.test.ts index d68de51f1..93a4b62eb 100644 --- a/packages/junior/tests/integration/slack-schedule-tools.test.ts +++ b/packages/junior/tests/integration/slack-schedule-tools.test.ts @@ -17,7 +17,7 @@ import { type SchedulerDb, type SchedulerToolContext, } from "@sentry/junior-scheduler"; -import { migratePluginSchemas } from "@/chat/plugins/migrations"; +import { bootstrapPluginSchemas } from "@/chat/plugins/migrations"; import * as dbModule from "@/chat/db"; import { getPluginTools, setPlugins } from "@/chat/plugins/agent-hooks"; import { disconnectStateAdapter } from "@/chat/state/adapter"; @@ -41,7 +41,7 @@ function schedulerMigrationsDir(): string { async function useSchedulerSqlPlugin() { const fixture = await createLocalJuniorSqlFixture(); - await migratePluginSchemas(fixture.sql, [ + await bootstrapPluginSchemas(fixture.sql, [ { dir: schedulerMigrationsDir(), pluginName: "scheduler", diff --git a/packages/junior/tests/unit/sql/executor.test.ts b/packages/junior/tests/unit/sql/executor.test.ts index 4950889e4..329ca8e0f 100644 --- a/packages/junior/tests/unit/sql/executor.test.ts +++ b/packages/junior/tests/unit/sql/executor.test.ts @@ -13,7 +13,6 @@ function executor(name: string): JuniorSqlExecutor { throw new Error(`${name} test executor does not expose Drizzle`); }), execute: vi.fn(), - migrate: vi.fn(), query: vi.fn(), transaction: vi.fn(async (callback) => await callback()), withLock: vi.fn(async (_lockName, callback) => await callback()), diff --git a/packages/junior/tsconfig.build.json b/packages/junior/tsconfig.build.json index ca568009e..3edb702bb 100644 --- a/packages/junior/tsconfig.build.json +++ b/packages/junior/tsconfig.build.json @@ -13,6 +13,7 @@ "src/app.ts", "src/handlers/**/*.ts", "src/instrumentation.ts", + "src/migration-helpers/**/*.ts", "src/nitro.ts", "src/reporting.ts", "src/vercel.ts", diff --git a/packages/junior/tsup.config.ts b/packages/junior/tsup.config.ts index a1791cd3b..38fe661b6 100644 --- a/packages/junior/tsup.config.ts +++ b/packages/junior/tsup.config.ts @@ -15,6 +15,7 @@ export default defineConfig({ api: "src/api.ts", "api/schema": "src/api/schema.ts", instrumentation: "src/instrumentation.ts", + "migration-helpers/v1": "src/migration-helpers/v1.ts", nitro: "src/nitro.ts", vercel: "src/vercel.ts", }, @@ -35,6 +36,7 @@ export default defineConfig({ "@chat-adapter/state-redis", "@earendil-works/pi-agent-core", "@earendil-works/pi-ai", + "@sentry/junior-migrations", "@sinclair/typebox", "@slack/web-api", "@vercel/functions", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5628a4e19..2621bc436 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -169,6 +169,9 @@ importers: '@neondatabase/serverless': specifier: ^1.1.0 version: 1.1.0 + '@sentry/junior-migrations': + specifier: workspace:* + version: link:../junior-migrations '@sentry/junior-plugin-api': specifier: workspace:* version: link:../junior-plugin-api @@ -466,6 +469,9 @@ importers: specifier: 'catalog:' version: 4.4.3 devDependencies: + '@sentry/junior-migrations': + specifier: workspace:* + version: link:../junior-migrations '@sentry/junior-testing': specifier: workspace:* version: file:packages/junior-testing @@ -488,6 +494,30 @@ importers: specifier: ^4.1.7 version: 4.1.7(@types/node@25.9.1)(tsx@4.22.3) + packages/junior-migrations: + devDependencies: + '@types/node': + specifier: ^25.9.1 + version: 25.9.1 + drizzle-kit: + specifier: 'catalog:' + version: 0.31.10 + drizzle-orm: + specifier: 'catalog:' + version: 0.45.2 + oxlint: + specifier: ^1.66.0 + version: 1.66.0 + tsup: + specifier: ^8.5.1 + version: 8.5.1(tsx@4.22.3)(typescript@6.0.3) + typescript: + specifier: ^6.0.3 + version: 6.0.3 + vitest: + specifier: ^4.1.7 + version: 4.1.7(@types/node@25.9.1)(tsx@4.22.3) + packages/junior-notion: {} packages/junior-plugin-api: @@ -521,6 +551,9 @@ importers: specifier: 'catalog:' version: 4.4.3 devDependencies: + '@sentry/junior-migrations': + specifier: workspace:* + version: link:../junior-migrations '@types/node': specifier: ^25.9.1 version: 25.9.1 @@ -3164,6 +3197,10 @@ packages: '@sentry/junior-memory@file:packages/junior-memory': resolution: {directory: packages/junior-memory, type: directory} + '@sentry/junior-migrations@file:packages/junior-migrations': + resolution: {directory: packages/junior-migrations, type: directory} + hasBin: true + '@sentry/junior-plugin-api@file:packages/junior-plugin-api': resolution: {directory: packages/junior-plugin-api, type: directory} @@ -6879,8 +6916,8 @@ packages: resolution: {integrity: sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==} engines: {node: '>=18'} - string-width@8.2.1: - resolution: {integrity: sha512-IIaP0g3iy9Cyy18w3M9YcaDudujEAVHKt3a3QJg1+sr/oX96TbaGUubG0hJyCjCBThFH+tFpcIyoUHUn1ogaLA==} + string-width@8.2.2: + resolution: {integrity: sha512-GaPUh5gfdrYzqeVNZvUfT23vYYxXzKYidUcnMtJg/3rxRV63EFZy3k6xfKlmfeJD0176lnUV/Usr3XcwSvFzpg==} engines: {node: '>=20'} string_decoder@1.3.0: @@ -10102,6 +10139,8 @@ snapshots: - sql.js - sqlite3 + '@sentry/junior-migrations@file:packages/junior-migrations': {} + '@sentry/junior-plugin-api@file:packages/junior-plugin-api': dependencies: commander: 14.0.3 @@ -10262,6 +10301,7 @@ snapshots: '@logtape/logtape': 2.1.1 '@modelcontextprotocol/sdk': 1.29.0(zod@4.4.3) '@neondatabase/serverless': 1.1.0 + '@sentry/junior-migrations': file:packages/junior-migrations '@sentry/junior-plugin-api': file:packages/junior-plugin-api '@sentry/node': 10.53.1 '@sinclair/typebox': 0.34.49 @@ -11713,7 +11753,7 @@ snapshots: cli-truncate@5.2.0: dependencies: slice-ansi: 8.0.0 - string-width: 8.2.1 + string-width: 8.2.2 cli-width@4.1.0: {} @@ -14362,6 +14402,12 @@ snapshots: postcss: 8.5.15 tsx: 4.22.3 + postcss-load-config@6.0.1(tsx@4.22.3): + dependencies: + lilconfig: 3.1.3 + optionalDependencies: + tsx: 4.22.3 + postcss-nested@6.2.0(postcss@8.5.15): dependencies: postcss: 8.5.15 @@ -15145,7 +15191,7 @@ snapshots: get-east-asian-width: 1.6.0 strip-ansi: 7.2.0 - string-width@8.2.1: + string-width@8.2.2: dependencies: get-east-asian-width: 1.6.0 strip-ansi: 7.2.0 @@ -15412,6 +15458,33 @@ snapshots: - tsx - yaml + tsup@8.5.1(tsx@4.22.3)(typescript@6.0.3): + dependencies: + bundle-require: 5.1.0(esbuild@0.27.7) + cac: 6.7.14 + chokidar: 4.0.3 + consola: 3.4.2 + debug: 4.4.3 + esbuild: 0.27.7 + fix-dts-default-cjs-exports: 1.0.1 + joycon: 3.1.1 + picocolors: 1.1.1 + postcss-load-config: 6.0.1(tsx@4.22.3) + resolve-from: 5.0.0 + rollup: 4.60.4 + source-map: 0.7.6 + sucrase: 3.35.1 + tinyexec: 1.1.2 + tinyglobby: 0.2.16 + tree-kill: 1.2.2 + optionalDependencies: + typescript: 6.0.3 + transitivePeerDependencies: + - jiti + - supports-color + - tsx + - yaml + tsx@4.21.0: dependencies: esbuild: 0.27.7 @@ -15979,7 +16052,7 @@ snapshots: wrap-ansi@10.0.0: dependencies: ansi-styles: 6.2.3 - string-width: 8.2.1 + string-width: 8.2.2 strip-ansi: 7.2.0 wrap-ansi@7.0.0: diff --git a/scripts/bump-release-versions.mjs b/scripts/bump-release-versions.mjs index 07646a8d5..d37091931 100644 --- a/scripts/bump-release-versions.mjs +++ b/scripts/bump-release-versions.mjs @@ -10,6 +10,7 @@ if (!newVersion) { const files = [ "packages/junior/package.json", + "packages/junior-migrations/package.json", "packages/junior-plugin-api/package.json", "packages/junior-agent-browser/package.json", "packages/junior-amplitude/package.json",