diff --git a/.changeset/audit-fixes-and-tests.md b/.changeset/audit-fixes-and-tests.md new file mode 100644 index 00000000..c467d8c0 --- /dev/null +++ b/.changeset/audit-fixes-and-tests.md @@ -0,0 +1,17 @@ +--- +"api": patch +"everything-dev": patch +--- + +Fix bugs and add test coverage for Phase 3+4 infra/alchemy features + +- **B1**: Fix typo in `alchemy.ts` generated comment (`[d eploy]` → `[deploy]`) +- **B2**: Fix fire-and-forget schema creation in `db/index.ts` — `pool.on("connect")` handlers now `await` both `CREATE SCHEMA` and `SET search_path` queries in sequence (both Neon and pg paths) +- **B3**: Fix Railway redeploy in `plugin.ts` to read `deploy.service` first, falling back to `ci.railway.service` for backward compatibility +- **S3+S6**: Extract shared pool helpers (`buildPoolConfig`, `attachPoolSchemaHandlers`, `createCloseHandler`) in `db/index.ts` to eliminate ~70% code duplication between Neon WebSocket and pg connection paths +- **S5**: Replace non-idiomatic `throw` with `yield* Effect.fail(...)` in `db/layer.ts` drift handlers + +**New tests (15 total)**: +- `merge.test.ts`: 4 tests for `ci.railway` → `deploy` backward compat mapping, plus updated `BOS_CONFIG_ORDER` field assertions for `ci`/`infra`/`deploy` +- `infra.test.ts`: 3 tests for `buildDatabaseConfigs` with `infraConfig` (shared DATABASE_URL, convention fallback, port preservation) +- `tests/e2e/toml-config-pipeline.test.ts`: 8 e2e tests covering the full TOML config pipeline — read full config with `[infra]`/`[deploy]`, path resolution, extends merge, `ci.railway` mapping, shared database config, `generateAlchemyRun` output correctness, and TOML round-trip diff --git a/.changeset/infra-alchemy-phases.md b/.changeset/infra-alchemy-phases.md new file mode 100644 index 00000000..eb5525a2 --- /dev/null +++ b/.changeset/infra-alchemy-phases.md @@ -0,0 +1,23 @@ +--- +"api": minor +"everything-dev": minor +--- + +Phase 3: Add `[infra]` and `[deploy]` config sections for explicit infrastructure declarations + +- **`InfraConfigSchema`** / **`DeployConfigSchema`** in `types.ts` — new Zod schemas for `[infra.database]` (type, schemaMode, dedicated, secret), `[infra.redis]` (enabled, slug), and `[deploy]` (provider, service, redeploy). Added to both `BosConfigInputSchema` and `BosConfigSchema`. +- **Backward compat**: `ci.railway` → `deploy` mapping during extends resolution in `mergeBosConfigWithExtends`. When `[deploy]` is absent but `ci.railway.service` is present, deplo y is derived automatically. +- **Config ordering**: `"infra"` and `"deploy"` added to `BOS_CONFIG_ORDER` (before `"app"`). +- **Infra planner**: `InfraInput` gains optional `infraConfig` field. `allocateDatabases` and `buildDatabaseConfigs` check `infraConfig` first, falling back to convention-based `*_DATABASE_URL` scanning when absent. +- **Shared database**: When `[infra.database]` is declared, a single shared `DATABASE_URL` is provisioned (per-plugin schema isolation via search_path). + +Phase 4: Split DatabaseLive into DriverLive + MigrationLive, add Neon WebSocket Pool + +- **Layer split**: `DatabaseLive` decomposed into: + - `DriverLive` — manages driver acquire/release only + - `MigrationLive` — handles migration apply + drift detection (depends on `DatabaseTag`) + - `DatabaseLive = Layer.provideMerge(DriverLive(url), MigrationLive)` — composed, backward-compatible + - Enables skipping migrations when Alchemy manages prod migrations. +- **Neon WebSocket Pool**: `createDatabaseDriver` now checks for `neon.tech` in the URL and uses `@neondatabase/serverless` Pool + `drizzle-orm/neon-serverless` with same `search_path` + `CREATE SCHEMA` support. Gracefully falls back to `pg` Pool if `@neondatabase/serverless` is not installed. +- **Deploy script generation**: `alchemy.ts` with `generateAlchemyRun()` that writes `alchemy.run.ts` from `[deploy]` config, supporting both Railway and Alchemy providers. +- **Dependency**: `@neondatabase/serverless: "^1.0.0"` added to api/package.json. diff --git a/.changeset/schema-cleanup-and-secret.md b/.changeset/schema-cleanup-and-secret.md new file mode 100644 index 00000000..8956c6f4 --- /dev/null +++ b/.changeset/schema-cleanup-and-secret.md @@ -0,0 +1,13 @@ +--- +"api": patch +"everything-dev": patch +--- + +Clean up InfraDatabaseSchema, implement `secret` field, deduplicate `isPlainObject` + +- **Remove `schemaMode` from `InfraDatabaseSchema`** — schema isolation (`search_path`) is handled at the driver layer in `api/src/db/layer.ts`, not in the Docker compose infra layer. Having it in both places was misleading +- **Implement `secret` field** — when a per-plugin record specifies `secret = "CUSTOM_DB_URL"`, that env var name is used instead of the conventional `{PLUGIN}_DATABASE_URL`. Falls back to conventional naming when absent +- **Import `isPlainObject` from `../merge`** in `cli/infra.ts` instead of redefining it locally +- **Update `alchemy.ts`** to no longer reference `schemaMode` — replaces the broken inline ternary with a driver-layer isolation comment +- **Update all test fixtures** to remove `schemaMode` from test data across `infra.test.ts` and `toml-config-pipeline.test.ts` +- **Add 2 tests** for `secret` field: custom secret name resolution and conventional fallback diff --git a/.changeset/toml-config-format.md b/.changeset/toml-config-format.md new file mode 100644 index 00000000..8ab34d43 --- /dev/null +++ b/.changeset/toml-config-format.md @@ -0,0 +1,14 @@ +--- +"everything-dev": minor +--- + +Add TOML config format support (`bos.config.toml`) as a dual-format alternative to `bos.config.json` + +- **`bos.config.toml`**: `findBosConfigPath` probes `bos.config.toml` first, then `bos.config.json` in directory tree walks. Both can't coexist — throws on duplicate. +- **Format-preserving writes**: `saveBosConfig` and `reportDeployResult` detect the original config format (TOML or JSON) and write back in the same format. +- **`disabled = true` marker**: Replaces the `null` sentinel pattern for TOML (which has no null). Schema field `disabled` on `BosPluginRefSchema` ensures Zod doesn't strip it. `cleanNullSentinels` strips `disabled: true` entries during merge. +- **Effect-free sync helpers**: `readBosConfigSource`, `findBosConfigPath`, and `findBosConfigPathInDir` are plain sync functions that throw native errors (not `FiberFailure`), preserving readable error messages. +- **Directory-only lookup**: Added `findBosConfigPathInDir` for cases where walking up the tree is incorrect (plugin local path resolution in `resolveComposableReference`). +- **Published form stays JSON**: FastKV registry keys remain `apps/{account}/{gateway}/bos.config.json`; TOML is local authoring only. +- **Bootstrapping**: `bos init` now copies `bos.config.toml` alongside `bos.config.json` in init patterns, and reads/writes configs via format-aware helpers. +- **Migration**: 12 raw `JSON.parse(readFileSync(...))` sites migrated to `readBosConfigSource` across build, plugin, upgrade, status, and init modules. diff --git a/.changeset/toml-gap-fixes.md b/.changeset/toml-gap-fixes.md new file mode 100644 index 00000000..f2d77b92 --- /dev/null +++ b/.changeset/toml-gap-fixes.md @@ -0,0 +1,12 @@ +--- +"api": patch +"everything-dev": patch +--- + +Fix remaining TOML config implementation gaps + +- **G2**: Add missing `title` and `description` fields to `BosConfigInputSchema` so they are not stripped during Zod validation (previously existed on the TypeScript interface but not the Zod schema) +- **G3**: Handle per-plugin record variant of `[infra.database]` in `buildDatabaseConfigs`. When `database` is a `Record` (e.g. `[infra.database.auth]`, `[infra.database.api]`), the function now iterates entries and matches them to `*_DATABASE_URL` secrets instead of treating all truthy database values as a single shared config. Extracted `resolveDatabasePort` and `buildDatabaseConfigFromSecret` helpers to share logic with the convention-based scanning path. +- **G1**: Wire `generateAlchemyRun` into `publish.ts` — after a successful publish, if `bosConfig.deploy` is present, `generateAlchemyRun` is called to write `alchemy.run.ts` with the deploy configuration +- **G4**: Make 22 user-facing error messages format-agnostic (no longer hardcode `"bos.config.json"`). Affected files: `config.ts`, `plugin.ts`, `cli.ts`, `sync.ts`, `shared-deps.ts`, `upgrade.ts`, `dag.ts` +- **G5**: Fix `upgrade.ts` plugin config scanning glob/regex to match both `bos.config.json` and `bos.config.toml` diff --git a/Dockerfile b/Dockerfile index bc3732ae..990ee860 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,7 +30,7 @@ COPY --from=builder --chown=appuser:appgroup /app/node_modules ./node_modules COPY --from=builder --chown=appuser:appgroup /app/package.json . COPY --from=builder --chown=appuser:appgroup /app/bun.lock . COPY --from=builder --chown=appuser:appgroup /app/bunfig.toml . -COPY --from=builder --chown=appuser:appgroup /app/bos.config.json ./ +COPY --from=builder --chown=appuser:appgroup /app/bos.config.* ./ COPY --from=builder --chown=appuser:appgroup /app/packages/everything-dev ./packages/everything-dev COPY --from=builder --chown=appuser:appgroup /app/packages/every-plugin ./packages/every-plugin diff --git a/api/package.json b/api/package.json index d665ad33..d55c1553 100644 --- a/api/package.json +++ b/api/package.json @@ -26,6 +26,7 @@ "effect": "catalog:", "every-plugin": "catalog:", "pg": "catalog:", + "@neondatabase/serverless": "^1.0.0", "@electric-sql/pglite": "catalog:", "zod": "catalog:" }, diff --git a/api/src/db/index.ts b/api/src/db/index.ts index 67d3fa7e..c071d5cc 100644 --- a/api/src/db/index.ts +++ b/api/src/db/index.ts @@ -2,6 +2,7 @@ import { mkdirSync } from "node:fs"; import { dirname } from "node:path"; import type { PgDatabase, PgQueryResultHKT } from "drizzle-orm/pg-core"; import { Data } from "every-plugin/effect"; +import type { PoolConfig } from "pg"; import * as schema from "./schema"; export type Database = PgDatabase; @@ -11,6 +12,16 @@ export interface DatabaseDriver { close(): Promise; } +interface PoolLike { + on(event: "error", listener: (err: Error) => void): this; + on( + event: "connect", + listener: (client: { query: (sql: string) => Promise }) => void, + ): this; + removeAllListeners(event?: string | symbol): this; + end(): Promise; +} + export class DatabaseError extends Data.TaggedError("DatabaseError")<{ stage: "driver" | "migration" | "load" | "close"; migrationTag?: string; @@ -37,7 +48,46 @@ export function unwrapDatabaseError(error: unknown): string { return parts.join(": "); } -export async function createDatabaseDriver(url: string): Promise { +function buildPoolConfig(url: string): PoolConfig { + const isLocal = url.includes("localhost") || url.includes("127.0.0.1"); + return { + connectionString: url, + ssl: isLocal + ? false + : { rejectUnauthorized: process.env.DB_SSL_REJECT_UNAUTHORIZED === "true" }, + max: Number(process.env.DB_POOL_MAX) || 10, + connectionTimeoutMillis: Number(process.env.DB_CONNECTION_TIMEOUT_MS) || 30_000, + idleTimeoutMillis: Number(process.env.DB_IDLE_TIMEOUT_MS) || 30_000, + }; +} + +function attachPoolSchemaHandlers(pool: PoolLike, schemaName: string | undefined): void { + pool.on("error", (err: Error) => { + console.error("[Database] Unexpected pool error:", err.message); + }); + if (schemaName) { + pool.on("connect", async (client) => { + await client.query(`CREATE SCHEMA IF NOT EXISTS "${schemaName}"`); + await client.query(`SET search_path TO "${schemaName}", public`); + }); + } +} + +function createCloseHandler(pool: PoolLike): () => Promise { + let closed = false; + return async () => { + if (closed) return; + closed = true; + pool.removeAllListeners("error"); + pool.removeAllListeners("connect"); + await pool.end(); + }; +} + +export async function createDatabaseDriver( + url: string, + schemaName?: string, +): Promise { if (url.startsWith("pglite:") || url === ":memory:") { const { drizzle } = await import("drizzle-orm/pglite"); const { PGlite } = await import("@electric-sql/pglite"); @@ -47,6 +97,10 @@ export async function createDatabaseDriver(url: string): Promise mkdirSync(dirname(dataDir), { recursive: true }); } const pglite = new PGlite(dataDir); + if (schemaName) { + await pglite.exec(`CREATE SCHEMA IF NOT EXISTS "${schemaName}"`); + await pglite.exec(`SET search_path TO "${schemaName}", public`); + } const db = drizzle(pglite, { schema }); return { db, @@ -56,33 +110,28 @@ export async function createDatabaseDriver(url: string): Promise }; } + // Neon WebSocket Pool (pooled connection, works with PgBouncer) + if (url.includes("neon.tech")) { + try { + const { Pool } = await import("@neondatabase/serverless"); + const { drizzle } = await import("drizzle-orm/neon-serverless"); + const pool = new Pool(buildPoolConfig(url)); + attachPoolSchemaHandlers(pool, schemaName); + return { + db: drizzle(pool, { schema }), + close: createCloseHandler(pool), + }; + } catch { + // @neondatabase/serverless not installed, fall through to pg + } + } + const { Pool } = await import("pg"); const { drizzle } = await import("drizzle-orm/node-postgres"); - const isLocal = url.includes("localhost") || url.includes("127.0.0.1"); - const pool = new Pool({ - connectionString: url, - ssl: isLocal - ? false - : { rejectUnauthorized: process.env.DB_SSL_REJECT_UNAUTHORIZED === "true" }, - max: Number(process.env.DB_POOL_MAX) || 10, - connectionTimeoutMillis: Number(process.env.DB_CONNECTION_TIMEOUT_MS) || 30_000, - idleTimeoutMillis: Number(process.env.DB_IDLE_TIMEOUT_MS) || 30_000, - }); - pool.on("error", (err: Error) => { - console.error("[Database] Unexpected pool error:", err.message); - }); - let closed = false; + const pool = new Pool(buildPoolConfig(url)); + attachPoolSchemaHandlers(pool, schemaName); return { db: drizzle(pool, { schema }), - close: async () => { - if (closed) return; - closed = true; - pool.removeAllListeners("error"); - console.error( - "[Database] pool.end() called from:", - new Error("pool.end() stack trace").stack, - ); - await pool.end(); - }, + close: createCloseHandler(pool), }; } diff --git a/api/src/db/layer.ts b/api/src/db/layer.ts index 5db1b1d1..5a0802c1 100644 --- a/api/src/db/layer.ts +++ b/api/src/db/layer.ts @@ -6,13 +6,17 @@ import { detectDrift, loadMigrations, migrate } from "./migrate"; export class DatabaseTag extends Context.Tag("Database")() {} -export const DatabaseLive = (url: string) => +export const DriverLive = (url: string) => Layer.scoped( DatabaseTag, Effect.gen(function* () { + const pluginId = yield* PluginIdTag; + const slug = pluginMigrationSlug(pluginId); + const schemaName = `plugin_${slug}`; + const driver = yield* Effect.acquireRelease( Effect.tryPromise({ - try: () => createDatabaseDriver(url), + try: () => createDatabaseDriver(url, schemaName), catch: (cause) => new DatabaseError({ stage: "driver", cause }), }), (driver) => @@ -22,38 +26,50 @@ export const DatabaseLive = (url: string) => }).pipe(Effect.ignore), ); - const pluginId = yield* PluginIdTag; - const storage = getMigrationStorage(pluginMigrationSlug(pluginId)); - const { migrations, source } = yield* loadMigrations(); + return driver.db; + }), + ); - if (migrations.length === 0) { - yield* Effect.logWarning( - `[Database] No migrations found (source: ${source}) — schema may be missing`, +export const MigrationLive = Layer.effect( + DatabaseTag, + Effect.gen(function* () { + const db = yield* DatabaseTag; + const pluginId = yield* PluginIdTag; + const slug = pluginMigrationSlug(pluginId); + const schemaName = `plugin_${slug}`; + const storage = getMigrationStorage(slug); + + const { migrations, source } = yield* loadMigrations(); + + if (migrations.length === 0) { + yield* Effect.logWarning( + `[Database] No migrations found (source: ${source}) — schema may be missing`, + ); + } else { + const applied = yield* migrate(db, migrations, storage, schemaName); + + if (applied === 0) { + yield* Effect.logInfo( + `[Database] Schema up to date (0 migrations needed, source: ${source})`, ); } else { - const applied = yield* migrate(driver.db, migrations, storage); - - if (applied === 0) { - yield* Effect.logInfo( - `[Database] Schema up to date (0 migrations needed, source: ${source})`, - ); - } else { - yield* Effect.logInfo( - `[Database] Applied ${applied}/${migrations.length} migration(s) (source: ${source}, journal: ${storage.schema}.${storage.table})`, - ); - } + yield* Effect.logInfo( + `[Database] Applied ${applied}/${migrations.length} migration(s) (source: ${source}, journal: ${storage.schema}.${storage.table}, schema: ${schemaName})`, + ); + } - const drift = yield* detectDrift(driver.db, migrations, storage); - if (drift.status === "healthy" || drift.status === "untracked-existing-schema") { - yield* Effect.logInfo(`[Database] Ready`); - } else if (drift.status === "drift-safe-repair") { - yield* Effect.logWarning( - `[Database] ⚠️ Migration drift detected: ${drift.missingTables.length} expected table(s) missing: ${drift.missingTables.join(", ")}`, - ); - yield* Effect.logWarning( - `[Database] Run \`bos db doctor ${storage.slug}\` to diagnose and \`bos db repair ${storage.slug}\` to fix.`, - ); - throw new DatabaseError({ + const drift = yield* detectDrift(db, migrations, storage, schemaName); + if (drift.status === "healthy" || drift.status === "untracked-existing-schema") { + yield* Effect.logInfo(`[Database] Ready`); + } else if (drift.status === "drift-safe-repair") { + yield* Effect.logWarning( + `[Database] ⚠️ Migration drift detected: ${drift.missingTables.length} expected table(s) missing: ${drift.missingTables.join(", ")}`, + ); + yield* Effect.logWarning( + `[Database] Run \`bos db doctor ${storage.slug}\` to diagnose and \`bos db repair ${storage.slug}\` to fix.`, + ); + yield* Effect.fail( + new DatabaseError({ stage: "migration", migrationTag: "drift-safe-repair", cause: new Error( @@ -61,13 +77,15 @@ export const DatabaseLive = (url: string) => `Run \`bos db repair ${storage.slug}\` to reset the migration history and reapply migrations. ` + `Missing tables: ${drift.missingTables.join(", ")}`, ), - }); - } - if (drift.status === "drift-manual") { - yield* Effect.logWarning( - `[Database] ⚠️ Partial migration drift detected: ${drift.missingTables.length}/${drift.expectedTables.length} expected table(s) missing.`, - ); - throw new DatabaseError({ + }), + ); + } + if (drift.status === "drift-manual") { + yield* Effect.logWarning( + `[Database] ⚠️ Partial migration drift detected: ${drift.missingTables.length}/${drift.expectedTables.length} expected table(s) missing.`, + ); + yield* Effect.fail( + new DatabaseError({ stage: "migration", migrationTag: "drift-manual", cause: new Error( @@ -75,10 +93,13 @@ export const DatabaseLive = (url: string) => `Run \`bos db doctor ${storage.slug}\` for details. Manual intervention required. ` + `Missing tables: ${drift.missingTables.join(", ")}`, ), - }); - } + }), + ); } + } - return driver.db; - }), - ); + return db; + }), +); + +export const DatabaseLive = (url: string) => Layer.provideMerge(DriverLive(url), MigrationLive); diff --git a/api/src/db/migrate.ts b/api/src/db/migrate.ts index 4eb75abd..a065b87b 100644 --- a/api/src/db/migrate.ts +++ b/api/src/db/migrate.ts @@ -41,13 +41,15 @@ export interface DriftReport { function getExistingTables( db: Database, tables: string[], + schemaName?: string, ): Effect.Effect, DatabaseError> { if (tables.length === 0) return Effect.succeed(new Set()); + const schema = schemaName ?? "public"; return Effect.tryPromise({ try: () => db.execute(sql` SELECT table_name FROM information_schema.tables - WHERE table_schema = 'public' + WHERE table_schema = ${sql.raw(`'${schema}'`)} AND table_name = ANY(${sql.raw(toSqlArray(tables))}) `), catch: (cause) => @@ -180,6 +182,7 @@ export function migrate( db: Database, migrations: Migration[], storage?: MigrationStorage, + schemaName?: string, ): Effect.Effect { return Effect.gen(function* () { const sorted = [...migrations].sort((a, b) => a.idx - b.idx); @@ -187,6 +190,14 @@ export function migrate( yield* ensureMigrationTable(db, journal); + if (schemaName) { + yield* Effect.tryPromise({ + try: () => db.execute(sql`CREATE SCHEMA IF NOT EXISTS ${sql.raw(`"${schemaName}"`)}`), + catch: (cause) => + new DatabaseError({ stage: "migration", migrationTag: "init-data-schema", cause }), + }); + } + const ref = journalRef(journal); const appliedHashes = yield* readAppliedHashes(db, ref); @@ -200,7 +211,7 @@ export function migrate( // as applied rather than crashing on a duplicate DDL error. const expectedTables = extractExpectedTables([migration]); if (expectedTables.length > 0) { - const existing = yield* getExistingTables(db, expectedTables); + const existing = yield* getExistingTables(db, expectedTables, schemaName); const missingTables = expectedTables.filter((t) => !existing.has(t)); if (missingTables.length === 0) { yield* Effect.logWarning( @@ -302,6 +313,7 @@ export function detectDrift( db: Database, migrations: Migration[], storage?: MigrationStorage, + schemaName?: string, ): Effect.Effect { return Effect.gen(function* () { const journal = storage ?? getMigrationStorage(); @@ -322,7 +334,7 @@ export function detectDrift( }; } - const existing = yield* getExistingTables(db, expectedTables); + const existing = yield* getExistingTables(db, expectedTables, schemaName); const missingTables = expectedTables.filter((t) => !existing.has(t)); if (appliedCount === 0 && missingTables.length === 0) { diff --git a/api/tests/unit/schema-isolation.test.ts b/api/tests/unit/schema-isolation.test.ts new file mode 100644 index 00000000..01bc4571 --- /dev/null +++ b/api/tests/unit/schema-isolation.test.ts @@ -0,0 +1,135 @@ +import type { Migration } from "virtual:drizzle-migrations.sql"; +import { sql } from "drizzle-orm"; +import { Effect } from "every-plugin/effect"; +import { getMigrationStorage } from "everything-dev/db"; +import { describe, expect, it } from "vitest"; +import { createDatabaseDriver } from "../../src/db/index"; +import { detectDrift, migrate } from "../../src/db/migrate"; + +const TEST_MIGRATIONS: Migration[] = [ + { + idx: 0, + when: 1778470014668, + tag: "0000_test_init", + hash: "a".repeat(64), + sql: [ + 'CREATE TABLE "things" ("thing_id" text PRIMARY KEY NOT NULL, "plugin_id" text NOT NULL, "created_at" timestamp with time zone DEFAULT now() NOT NULL, "updated_at" timestamp with time zone DEFAULT now() NOT NULL)', + ], + }, +]; + +async function withDriver( + url: string, + schemaName: string | undefined, + fn: (db: Awaited>) => Promise, +) { + const driver = await createDatabaseDriver(url, schemaName); + try { + await fn(driver); + } finally { + await driver.close(); + } +} + +describe("schema isolation", () => { + describe("PGlite with schemaName", () => { + it("creates the plugin schema and sets search_path", async () => { + await withDriver(":memory:", "plugin_test", async (driver) => { + const result = await driver.db.execute(sql`SELECT current_schema() as schema`); + const rows = result as unknown as { rows: { schema: string }[] }; + expect(rows.rows[0]?.schema).toBe("plugin_test"); + }); + }); + + it("creates tables in the plugin schema, not public", async () => { + await withDriver(":memory:", "plugin_test", async (driver) => { + const storage = getMigrationStorage("test"); + await Effect.runPromise(migrate(driver.db, TEST_MIGRATIONS, storage, "plugin_test")); + + const tablesInPlugin = await driver.db.execute(sql` + SELECT table_name FROM information_schema.tables + WHERE table_schema = 'plugin_test' + `); + const pluginRows = tablesInPlugin as unknown as { rows: { table_name: string }[] }; + const pluginTableNames = pluginRows.rows.map((r) => r.table_name); + expect(pluginTableNames).toContain("things"); + + const tablesInPublic = await driver.db.execute(sql` + SELECT table_name FROM information_schema.tables + WHERE table_schema = 'public' + `); + const publicRows = tablesInPublic as unknown as { rows: { table_name: string }[] }; + const publicTableNames = publicRows.rows.map((r) => r.table_name); + expect(publicTableNames).not.toContain("things"); + }); + }); + + it("detectDrift finds tables in the plugin schema", async () => { + await withDriver(":memory:", "plugin_test", async (driver) => { + const storage = getMigrationStorage("test"); + await (await import("effect")).Effect.runPromise( + migrate(driver.db, TEST_MIGRATIONS, storage, "plugin_test"), + ); + + const drift = await Effect.runPromise( + detectDrift(driver.db, TEST_MIGRATIONS, storage, "plugin_test"), + ); + expect(drift.status).toBe("healthy"); + expect(drift.missingTables).toHaveLength(0); + }); + }); + + it("isolates two plugins with same table name", async () => { + const driverA = await createDatabaseDriver(":memory:", "plugin_a"); + const driverB = await createDatabaseDriver(":memory:", "plugin_b"); + try { + const storage = getMigrationStorage("test"); + + await Effect.runPromise(migrate(driverA.db, TEST_MIGRATIONS, storage, "plugin_a")); + await Effect.runPromise(migrate(driverB.db, TEST_MIGRATIONS, storage, "plugin_b")); + + await driverA.db.execute( + sql`INSERT INTO "things" ("thing_id", "plugin_id") VALUES ('thing-a', 'a')`, + ); + await driverB.db.execute( + sql`INSERT INTO "things" ("thing_id", "plugin_id") VALUES ('thing-b', 'b')`, + ); + + const rowsA = await driverA.db.execute(sql`SELECT thing_id FROM "things"`); + const resultA = rowsA as unknown as { rows: { thing_id: string }[] }; + expect(resultA.rows.map((r) => r.thing_id)).toEqual(["thing-a"]); + + const rowsB = await driverB.db.execute(sql`SELECT thing_id FROM "things"`); + const resultB = rowsB as unknown as { rows: { thing_id: string }[] }; + expect(resultB.rows.map((r) => r.thing_id)).toEqual(["thing-b"]); + } finally { + await driverA.close(); + await driverB.close(); + } + }); + }); + + describe("PGlite without schemaName (backward compat)", () => { + it("defaults to public schema", async () => { + await withDriver(":memory:", undefined, async (driver) => { + const result = await driver.db.execute(sql`SELECT current_schema() as schema`); + const rows = result as unknown as { rows: { schema: string }[] }; + expect(rows.rows[0]?.schema).toBe("public"); + }); + }); + + it("migrations land in public", async () => { + await withDriver(":memory:", undefined, async (driver) => { + const storage = getMigrationStorage("test"); + await Effect.runPromise(migrate(driver.db, TEST_MIGRATIONS, storage)); + + const tablesInPublic = await driver.db.execute(sql` + SELECT table_name FROM information_schema.tables + WHERE table_schema = 'public' + `); + const publicRows = tablesInPublic as unknown as { rows: { table_name: string }[] }; + expect(publicRows.rows.map((r) => r.table_name)).toContain("things"); + }); + }); + }); +}); diff --git a/bun.lock b/bun.lock index 54ac9683..cd8d3312 100644 --- a/bun.lock +++ b/bun.lock @@ -42,6 +42,7 @@ "version": "2.7.7", "dependencies": { "@electric-sql/pglite": "catalog:", + "@neondatabase/serverless": "^1.0.0", "@orpc/contract": "catalog:", "@orpc/server": "catalog:", "drizzle-kit": "catalog:", @@ -71,7 +72,7 @@ }, "host": { "name": "host", - "version": "1.14.0", + "version": "1.15.0", "dependencies": { "@electric-sql/pglite": "catalog:", "@hono/node-server": "^2.0.1", @@ -153,7 +154,7 @@ }, "packages/everything-dev": { "name": "everything-dev", - "version": "1.51.7", + "version": "1.52.0", "bin": { "everything-dev": "./dist/cli.mjs", "bos": "./dist/cli.mjs", @@ -175,6 +176,7 @@ "gradient-string": "^3.0.0", "ink": "^6.8.0", "pg": "catalog:", + "smol-toml": "^1.7.1", "tar": "^7.4.3", }, "devDependencies": { @@ -389,7 +391,7 @@ "drizzle-orm": "^0.45.1", "effect": "3.21.2", "every-plugin": "^2.10.0", - "everything-dev": "^1.51.7", + "everything-dev": "^1.52.0", "near-kit": "^0.14.0", "pg": "^8.12.0", "react": "19.2.4", @@ -837,6 +839,8 @@ "@neon-rs/load": ["@neon-rs/load@0.0.4", "", {}, "sha512-kTPhdZyTQxB+2wpiRcFWrDcejc4JI6tkPuS7UZCG4l6Zvc5kU/gGQ/ozvHTh1XR5tS+UlfAfGuPajjzQjCiHCw=="], + "@neondatabase/serverless": ["@neondatabase/serverless@1.1.0", "", {}, "sha512-r3ZZhRjEcfEdKIZnoB1RusNgvHuaBRqfCzV4Gi+5A9yUX0S4HTws/ASWqt13wL4y4I+0rqsWGdA2w7EQXHi3+Q=="], + "@noble/ciphers": ["@noble/ciphers@2.2.0", "", {}, "sha512-Z6pjIZ/8IJcCGzb2S/0Px5J81yij85xASuk1teLNeg75bfT07MV3a/O2Mtn1I2se43k3lkVEcFaR10N4cgQcZA=="], "@noble/curves": ["@noble/curves@2.2.0", "", { "dependencies": { "@noble/hashes": "2.2.0" } }, "sha512-T/BoHgFXirb0ENSPBquzX0rcjXeM6Lo892a2jlYJkqk83LqZx0l1Of7DzlKJ6jkpvMrkHSnAcgb5JegL8SeIkQ=="], @@ -2485,6 +2489,8 @@ "slice-ansi": ["slice-ansi@8.0.0", "", { "dependencies": { "ansi-styles": "^6.2.3", "is-fullwidth-code-point": "^5.1.0" } }, "sha512-stxByr12oeeOyY2BlviTNQlYV5xOj47GirPr4yA1hE9JCtxfQN0+tVbkxwCtYDQWhEKWFHsEK48ORg5jrouCAg=="], + "smol-toml": ["smol-toml@1.7.1", "", {}, "sha512-PPlsspAZ4jbMBu5DMFhfUGDQLu/vrL4SyBROVS37x8ynnVmFIs1VPBz1Co8Xks3TvpIaZXmU85y4DrQ+UyVFoQ=="], + "sockjs": ["sockjs@0.3.24", "", { "dependencies": { "faye-websocket": "^0.11.3", "uuid": "^8.3.2", "websocket-driver": "^0.7.4" } }, "sha512-GJgLTZ7vYb/JtPSSZ10hsOYIvEYsjbNU+zPdIHcUaWVNUEPivzxku31865sSSud0Da0W4lEeOPlmw93zLQchuQ=="], "solid-js": ["solid-js@1.9.14", "", { "dependencies": { "csstype": "^3.1.0", "seroval": "~1.5.4", "seroval-plugins": "~1.5.4" } }, "sha512-sAEXC0Kk0S1EDg+8ysEWJDbYhA3RRoEjwuySUGlKIemeo0I5YZfOyumNjNs9Sv3y2nmhD+0rW66ag2HsMuQiGQ=="], diff --git a/packages/everything-dev/package.json b/packages/everything-dev/package.json index 2605efa1..b0f76110 100644 --- a/packages/everything-dev/package.json +++ b/packages/everything-dev/package.json @@ -220,12 +220,13 @@ "defu": "^6.1.7", "dotenv": "^17.3.1", "effect": "^3.21.0", - "execa": "^9.5.2", "every-plugin": "catalog:", + "execa": "^9.5.2", "glob": "^13.0.6", "gradient-string": "^3.0.0", "ink": "^6.8.0", "pg": "catalog:", + "smol-toml": "^1.7.1", "tar": "^7.4.3" }, "peerDependencies": { diff --git a/packages/everything-dev/src/alchemy.ts b/packages/everything-dev/src/alchemy.ts new file mode 100644 index 00000000..ef3d7281 --- /dev/null +++ b/packages/everything-dev/src/alchemy.ts @@ -0,0 +1,73 @@ +import { writeFileSync } from "node:fs"; +import { join, resolve } from "node:path"; +import type { DeployConfig, InfraConfig } from "./types"; + +export interface AlchemyRunSpec { + provider: "railway" | "alchemy"; + service?: string; + neonProject?: string; + stage: "production" | "staging" | "preview"; +} + +export function generateAlchemyRun( + deployConfig: DeployConfig, + infraConfig: InfraConfig | undefined, + projectDir: string, + stage: AlchemyRunSpec["stage"] = "production", +): void { + const spec: AlchemyRunSpec = { + provider: deployConfig.provider, + service: deployConfig.service, + neonProject: process.env.NEON_PROJECT ?? "everything-dev", + stage, + }; + + const script = generateScript(spec, infraConfig); + const outPath = resolve(join(projectDir, "alchemy.run.ts")); + writeFileSync(outPath, script); +} + +function generateScript(spec: AlchemyRunSpec, infraConfig: InfraConfig | undefined): string { + const lines: string[] = [ + "// Generated by `bos deploy` — do not edit manually", + "// To regenerate: update [deploy] in bos.config.toml and run `bos deploy`", + "", + `const provider = "${spec.provider}";`, + `const stage = "${spec.stage}";`, + ]; + + if (spec.service) { + lines.push(`const service = "${spec.service}";`); + } + + if (spec.provider === "alchemy") { + lines.push(`const neonProject = "${spec.neonProject}";`); + lines.push(""); + lines.push("// Steps:"); + lines.push("// 1. Drizzle.Schema: generate migrations from schema files"); + lines.push("// 2. Neon.Branch: create/get branch for stage, apply migrations"); + lines.push("// 3. Set DATABASE_URL = branch pooled connection URI"); + lines.push("// 4. Deploy MF remotes to Zephyr CDN"); + lines.push("// 5. Redeploy host container with new DATABASE_URL"); + } + + if (infraConfig?.database) { + lines.push(""); + lines.push("// Database isolation handled via search_path at the driver layer"); + } + + if (spec.provider === "railway") { + lines.push(""); + lines.push("// Railway deployment:"); + if (spec.service) { + lines.push(`// Redeploy service: ${spec.service}`); + } else { + lines.push("// No service configured — set [deploy].service in bos.config.toml"); + } + } + + lines.push(""); + lines.push(`export default { provider: "${spec.provider}", stage: "${spec.stage}" };`); + + return `${lines.join("\n")}\n`; +} diff --git a/packages/everything-dev/src/build.ts b/packages/everything-dev/src/build.ts index e8d436b2..7f6a4c1b 100644 --- a/packages/everything-dev/src/build.ts +++ b/packages/everything-dev/src/build.ts @@ -4,6 +4,7 @@ import { join } from "node:path"; import process from "node:process"; import { formatDuration } from "./cli/timing"; import { resolveLocalDevelopmentPath } from "./config"; +import { findBosConfigPathInDir } from "./config-source"; import type { WorkspaceDeployResult } from "./contract"; import { applyDeployResults, type DeployResultEntry, parseDeployLines } from "./integrity"; import { syncResolvedSharedDeps } from "./shared-deps"; @@ -346,7 +347,8 @@ export async function buildWorkspaceTargets(opts: { delete env.DEPLOY; } - const bosConfigPath = join(opts.configDir, "bos.config.json"); + const bosConfigPath = + findBosConfigPathInDir(opts.configDir) ?? join(opts.configDir, "bos.config.json"); let configSnapshot: string | undefined; if (opts.deploy && existsSync(bosConfigPath)) { configSnapshot = readFileSync(bosConfigPath, "utf-8"); diff --git a/packages/everything-dev/src/cli.ts b/packages/everything-dev/src/cli.ts index c8941bc6..010fa16b 100644 --- a/packages/everything-dev/src/cli.ts +++ b/packages/everything-dev/src/cli.ts @@ -561,7 +561,7 @@ async function main() { if (descriptor.key === "config") { if (!result.config) { - console.error("No bos.config.json found"); + console.error("No bos.config file found"); process.exit(1); } diff --git a/packages/everything-dev/src/cli/infra.ts b/packages/everything-dev/src/cli/infra.ts index 92a0731f..4ecc62ec 100644 --- a/packages/everything-dev/src/cli/infra.ts +++ b/packages/everything-dev/src/cli/infra.ts @@ -3,7 +3,9 @@ import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { dirname, join } from "node:path"; import * as p from "@clack/prompts"; import { config as loadDotenv } from "dotenv"; -import type { RuntimeConfig } from "../types"; +import { findBosConfigPath, readBosConfigSource } from "../config-source"; +import { isPlainObject } from "../merge"; +import type { InfraConfig, InfraDatabase, RuntimeConfig } from "../types"; const POSTGRES_USER = "everythingdev"; const POSTGRES_PASSWORD = "everythingdev"; @@ -175,7 +177,7 @@ export function buildOriginMap( configDir: string, runtimeConfig: RuntimeConfig, ): Map { - const configPath = join(configDir, "bos.config.json"); + const configPath = findBosConfigPath(configDir); const originMap = new Map(); const account = runtimeConfig.account; @@ -188,8 +190,8 @@ export function buildOriginMap( return null; }; - const rawConfig = existsSync(configPath) - ? (JSON.parse(readFileSync(configPath, "utf-8")) as Record) + const rawConfig = configPath + ? (readBosConfigSource(configPath) as Record) : null; const rawPlugins = rawConfig?.plugins as Record | undefined; @@ -226,11 +228,96 @@ export function buildOriginMap( return originMap; } +function isSingleDatabaseConfig(value: InfraConfig["database"]): value is InfraDatabase { + return isPlainObject(value) && "type" in value; +} + +function resolveDatabasePort( + secret: string, + slug: string, + portMap: Record, +): number { + if (portMap[slug] !== undefined) return portMap[slug]; + if (secret === API_DATABASE_SECRET) { + portMap[slug] = 5432; + } else if (secret === AUTH_DATABASE_SECRET) { + portMap[slug] = 5433; + } else { + resolvePort(slug, portMap, BASE_POSTGRES_PORT); + } + return portMap[slug]; +} + +function buildDatabaseConfigFromSecret( + secret: string, + originMap: Map, + portMap: Record, +): DatabaseSecretConfig { + const slug = normalizeDatabaseSlug(secret); + const fromKey = originMap.get(secret) ?? ""; + const port = resolveDatabasePort(secret, slug, portMap); + + const volumeName = fromKey + ? `${fromKey.replace(/\./g, "_")}_postgres_${slug}_data` + : `postgres_${slug}_data`; + const containerName = fromKey ? `${fromKey}-postgres-${slug}` : `postgres-${slug}`; + + return { + secret, + slug, + fromKey, + port, + serviceName: `postgres-${slug.replace(/_/g, "-")}`, + containerName, + databaseName: `${slug}_db`, + volumeName, + url: `postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@localhost:${port}/${slug}_db`, + }; +} + export function buildDatabaseConfigs( secrets: string[], originMap: Map, portMap: Record, + infraConfig?: InfraConfig, ): DatabaseSecretConfig[] { + // When infra.database is explicitly declared + if (infraConfig?.database) { + if (isSingleDatabaseConfig(infraConfig.database)) { + // Single shared database config + portMap.shared = portMap.shared ?? 5432; + return [ + { + secret: "DATABASE_URL", + slug: "shared", + fromKey: "", + port: portMap.shared, + serviceName: "postgres-shared", + containerName: "everythingdev-postgres-shared", + databaseName: "shared_db", + volumeName: "everythingdev_postgres_shared_data", + url: `postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@localhost:${portMap.shared}/shared_db`, + }, + ]; + } + + // Per-plugin database record config + // NOTE: per-plugin InfraDatabase.dedicated is aspirational (separate containers + // per plugin) — currently all plugins share one Docker compose postgres instance + // via search_path isolation. The .secret field overrides the conventional + // {PLUGIN}_DATABASE_URL naming when present. + const databaseRecord = infraConfig.database as Record; + const results: DatabaseSecretConfig[] = []; + for (const [pluginId, pluginDb] of Object.entries(databaseRecord)) { + const expectedSecret = pluginDb.secret ?? `${pluginId.toUpperCase()}_DATABASE_URL`; + const matchingSecret = secrets.find((s) => s.toUpperCase() === expectedSecret.toUpperCase()); + if (matchingSecret) { + results.push(buildDatabaseConfigFromSecret(matchingSecret, originMap, portMap)); + } + } + return results; + } + const databaseSecrets = uniqueSecrets( secrets.filter((secret) => secret.endsWith("_DATABASE_URL")), ); diff --git a/packages/everything-dev/src/cli/init.ts b/packages/everything-dev/src/cli/init.ts index 1fd382c8..c31b9140 100644 --- a/packages/everything-dev/src/cli/init.ts +++ b/packages/everything-dev/src/cli/init.ts @@ -15,6 +15,7 @@ import { dirname, join, relative, resolve } from "node:path"; import { pipeline } from "node:stream/promises"; import { execa } from "execa"; import { glob } from "glob"; +import { findBosConfigPathInDir, readBosConfigSource } from "../config-source"; import type { OverrideSection } from "../contract"; import { fetchBosConfigFromFastKv } from "../fastkv"; import { fetchResponse } from "../http-client"; @@ -30,6 +31,7 @@ import { getExtendsRef, parseBosRef, readJsonFile } from "./utils/helpers"; const require = createRequire(import.meta.url); export const INIT_ROOT_PATTERNS = [ + "bos.config.toml", "bos.config.json", "package.json", ".env.example", @@ -88,7 +90,7 @@ export async function resolveCatalogChainSource(opts: { let repository: string | undefined; let currentRef = `bos://${opts.extendsAccount}/${opts.extendsGateway}`; let sourceDir = opts.sourceDir ? resolve(opts.sourceDir) : undefined; - let configPath = sourceDir ? join(sourceDir, "bos.config.json") : undefined; + let configPath = sourceDir ? findBosConfigPathInDir(sourceDir) : undefined; try { while (true) { @@ -172,12 +174,11 @@ export async function resolveSourceDir(opts: { }): Promise { if (opts.source) { const sourceDir = resolve(opts.source); - if (!existsSync(join(sourceDir, "bos.config.json"))) { - throw new Error(`No bos.config.json found in source directory: ${sourceDir}`); + const configPath = findBosConfigPathInDir(sourceDir); + if (!configPath) { + throw new Error(`No bos.config file found in source directory: ${sourceDir}`); } - const parentConfig = JSON.parse( - readFileSync(join(sourceDir, "bos.config.json"), "utf-8"), - ) as BosConfig; + const parentConfig = readBosConfigSource(configPath) as BosConfig; return { sourceDir, parentConfig, cleanup: async () => {} }; } @@ -630,9 +631,9 @@ export async function personalizeConfig( .map(([key]) => key), ); - const configPath = join(destination, "bos.config.json"); - if (existsSync(configPath)) { - const config = JSON.parse(readFileSync(configPath, "utf-8")) as Record; + const configPath = findBosConfigPathInDir(destination); + if (configPath) { + const config = readBosConfigSource(configPath) as Record; config.extends = `bos://${opts.extendsAccount}/${opts.extendsGateway}`; @@ -743,7 +744,11 @@ export async function personalizeConfig( } } - await saveBosConfig(destination, config); + await saveBosConfig(destination, config, opts.mode === "init" ? "toml" : undefined); + + if (opts.mode === "init" && configPath?.endsWith(".json")) { + rmSync(configPath, { force: true }); + } } for (const relPath of ["ui/src/lib/api-types.gen.ts", "api/src/lib/plugins-types.gen.ts"]) { @@ -1228,7 +1233,7 @@ export async function scaffoldMinimalProject( config.plugins = {}; } - await saveBosConfig(destination, config); + await saveBosConfig(destination, config, "toml"); const workspacePackages: string[] = []; for (const section of opts.overrides) { @@ -1494,9 +1499,9 @@ The API plugin receives typed client factories for all other plugins via \`creat ### Generated Types -\`api/src/lib/plugins-types.gen.ts\`, \`api/src/lib/auth-types.gen.ts\`, \`ui/src/lib/api-types.gen.ts\`, and \`ui/src/lib/auth-types.gen.ts\` are generated by \`bos types gen\` from \`bos.config.json\`. These files are gitignored and auto-regenerated on \`bun install\`, \`typecheck\`, \`bos dev\`, \`bos build\`, and bos plugin management commands. +\`api/src/lib/plugins-types.gen.ts\`, \`api/src/lib/auth-types.gen.ts\`, \`ui/src/lib/api-types.gen.ts\`, and \`ui/src/lib/auth-types.gen.ts\` are generated by \`bos types gen\` from \`bos.config.toml\` or \`bos.config.json\`. These files are gitignored and auto-regenerated on \`bun install\`, \`typecheck\`, \`bos dev\`, \`bos build\`, and bos plugin management commands. -If you hand-edit \`bos.config.json\`, run \`bos types gen\` or restart \`bos dev\` to regenerate.`); +If you hand-edit \`bos.config.toml\` or \`bos.config.json\`, run \`bos types gen\` or restart \`bos dev\` to regenerate.`); } const testCommands: string[] = []; @@ -1552,7 +1557,7 @@ bun run dev # Restart \`\`\` **Module Federation errors:** -- Check \`bos.config.json\` URLs are accessible +- Check \`bos.config.toml\` or \`bos.config.json\` URLs are accessible - Verify shared dependency versions match in package.json - Clear browser cache @@ -1566,7 +1571,7 @@ bun run db:studio # Open Drizzle Studio **Required files:** - \`.env\` — Secrets (see \`.env.example\`) -- \`bos.config.json\` — Runtime configuration (committed)`); +- \`bos.config.toml\` or \`bos.config.json\` — Runtime configuration (committed)`); return `${parts.join("\n\n")}\n`; } diff --git a/packages/everything-dev/src/cli/status.ts b/packages/everything-dev/src/cli/status.ts index 0ff3b1c3..5e288928 100644 --- a/packages/everything-dev/src/cli/status.ts +++ b/packages/everything-dev/src/cli/status.ts @@ -1,5 +1,6 @@ -import { existsSync, readFileSync } from "node:fs"; +import { existsSync } from "node:fs"; import { join } from "node:path"; +import { findBosConfigPathInDir, readBosConfigSource } from "../config-source"; import type { StatusResult } from "../contract"; import { fetchBosConfigFromFastKv } from "../fastkv"; import { fetchJsonOrNull } from "../http-client"; @@ -49,17 +50,17 @@ async function checkParentReachable(extendsRef: string | undefined): Promise { - const configPath = join(projectDir, "bos.config.json"); - if (!existsSync(configPath)) { + const configPath = findBosConfigPathInDir(projectDir); + if (!configPath) { return { status: "error", - error: "No bos.config.json found in current directory", + error: "No bos config file found in current directory", packages: [], envFile: "missing", }; } - const config = JSON.parse(readFileSync(configPath, "utf-8")) as Record; + const config = readBosConfigSource(configPath) as Record; const packageNames = [...FRAMEWORK_PACKAGES]; for (const name of CATALOG_TOOL_PACKAGES) { diff --git a/packages/everything-dev/src/cli/sync.ts b/packages/everything-dev/src/cli/sync.ts index 079fd12c..220aef33 100644 --- a/packages/everything-dev/src/cli/sync.ts +++ b/packages/everything-dev/src/cli/sync.ts @@ -3,6 +3,7 @@ import { copyFileSync, existsSync, mkdirSync, readFileSync, writeFileSync } from import { dirname, join } from "node:path"; import { glob } from "glob"; import { loadResolvedConfig } from "../config"; +import { findBosConfigPath, readBosConfigSource, stringifyBosConfig } from "../config-source"; import type { SyncOptions, SyncResult } from "../contract"; import { isPlainObject as isPlainObjectFromMerge, @@ -25,6 +26,7 @@ const FRAMEWORK_OWNED_SYNC_FILES = new Set([ ".env.example", ".gitignore", "biome.json", + "bos.config.toml", "bos.config.json", "bunfig.toml", "package.json", @@ -273,15 +275,23 @@ function buildSyncedFileContent( : filePath); const dest = join(projectDir, destPath); - if (filePath.endsWith("bos.config.json")) { - const localContent = existsSync(dest) ? readFileSync(dest, "utf-8") : null; + if (filePath.endsWith("bos.config.json") || filePath.endsWith("bos.config.toml")) { + const localTomlDest = join(projectDir, "bos.config.toml"); + const _localJsonDest = join(projectDir, "bos.config.json"); + const actualDest = existsSync(localTomlDest) ? localTomlDest : dest; + const isLocalToml = actualDest.endsWith(".toml"); + const localContent = existsSync(actualDest) ? readFileSync(actualDest, "utf-8") : null; const templateContent = readFileSync(src, "utf-8"); if (localContent) { - const local = JSON.parse(localContent) as Record; + const local = isLocalToml + ? (readBosConfigSource(actualDest) as Record) + : (JSON.parse(localContent) as Record); const template = JSON.parse(templateContent) as Record; const merged = mergeBosConfigWithTemplate(local, template); - return `${JSON.stringify(merged, null, 2)}\n`; + return isLocalToml + ? `${stringifyBosConfig(merged as Record)}\n` + : `${JSON.stringify(merged, null, 2)}\n`; } } @@ -396,13 +406,12 @@ function hasPluginsWorkspace(projectDir: string): boolean { } export async function syncTemplate(projectDir: string, options: SyncOptions): Promise { - // Sync reads the raw bos.config.json (not the resolved config) because it needs + // Sync reads the raw config (not the resolved config) because it needs // the user's explicit local settings: their extends ref, selected plugins, etc. // The resolved config is the merged result and would include inherited parent // values that the user didn't explicitly choose, which would break sync filtering. - const localConfig = JSON.parse( - readFileSync(join(projectDir, "bos.config.json"), "utf-8"), - ) as Record; + const configPath = findBosConfigPath(projectDir) ?? join(projectDir, "bos.config.json"); + const localConfig = readBosConfigSource(configPath) as Record; let extendsRef: string | undefined; if (typeof localConfig.extends === "string") { @@ -417,7 +426,7 @@ export async function syncTemplate(projectDir: string, options: SyncOptions): Pr skipped: [], added: [], conflicted: [], - error: "No extends field found in bos.config.json — cannot determine parent", + error: "No extends field found in bos.config — cannot determine parent", }; } diff --git a/packages/everything-dev/src/cli/upgrade.ts b/packages/everything-dev/src/cli/upgrade.ts index 11d1eb8a..cc3d7a14 100644 --- a/packages/everything-dev/src/cli/upgrade.ts +++ b/packages/everything-dev/src/cli/upgrade.ts @@ -5,6 +5,7 @@ import process from "node:process"; import * as p from "@clack/prompts"; import { glob } from "glob"; import { loadResolvedConfig } from "../config"; +import { findBosConfigPathInDir, readBosConfigSource } from "../config-source"; import type { PhaseTiming, UpgradeOptions, UpgradeResult } from "../contract"; import { syncResolvedSharedDeps } from "../shared-deps"; import { saveBosConfig } from "../utils/save-config"; @@ -271,12 +272,12 @@ function syncRootCatalogWithParent( } async function readExtendedRootSource(projectDir: string): Promise { - const configPath = join(projectDir, "bos.config.json"); - if (!existsSync(configPath)) { + const configPath = findBosConfigPathInDir(projectDir); + if (!configPath) { return { catalog: {}, extendsChain: [] }; } - const localConfig = JSON.parse(readFileSync(configPath, "utf-8")) as Record; + const localConfig = readBosConfigSource(configPath) as Record; let extendsRef = getExtendsRef(localConfig); if (!extendsRef?.startsWith("bos://")) { return { @@ -587,13 +588,13 @@ function getApiEntry(pluginConfig: Record): Record { const migrated: string[] = []; - const rootConfigPath = join(projectDir, "bos.config.json"); + const rootConfigPath = findBosConfigPathInDir(projectDir) ?? join(projectDir, "bos.config.json"); if (existsSync(rootConfigPath)) { - const rootConfig = JSON.parse(readFileSync(rootConfigPath, "utf-8")) as Record; + const rootConfig = readBosConfigSource(rootConfigPath) as Record; let rootChanged = migrateRootConfigTargets(rootConfig); - const pluginConfigPaths = await glob("plugins/*/bos.config.json", { + const pluginConfigPaths = await glob("plugins/*/bos.config.{json,toml}", { cwd: projectDir, nodir: true, dot: false, @@ -601,13 +602,13 @@ export async function migrateBosConfigFiles(projectDir: string): Promise; + const pluginConfig = readBosConfigSource(filePath) as Record; rootChanged = mergePluginConfigIntoRoot(rootConfig, pluginKey, pluginConfig) || rootChanged; } catch (e) { console.warn(`[Upgrade] Failed to parse plugin config at ${filePath}: ${e}`); @@ -643,12 +644,12 @@ async function loadParentPluginOptions(projectDir: string): Promise<{ parentPlugins: Record; newPluginKeys: string[]; } | null> { - const configPath = join(projectDir, "bos.config.json"); - if (!existsSync(configPath)) { + const configPath = findBosConfigPathInDir(projectDir); + if (!configPath) { return null; } - const localConfig = JSON.parse(readFileSync(configPath, "utf-8")) as Record; + const localConfig = readBosConfigSource(configPath) as Record; const extendsRef = getExtendsRef(localConfig); if (!extendsRef?.startsWith("bos://")) { return null; @@ -770,13 +771,13 @@ async function findWorkspacePackageJsons(projectDir: string): Promise } export async function migrateChildRootPackageJson(projectDir: string): Promise { - const configPath = join(projectDir, "bos.config.json"); + const configPath = findBosConfigPathInDir(projectDir); const pkgPath = join(projectDir, "package.json"); - if (!existsSync(configPath) || !existsSync(pkgPath)) { + if (!configPath || !existsSync(pkgPath)) { return false; } - const config = readJsonFile>(configPath); + const config = readBosConfigSource(configPath) as Record; const extendsRef = getExtendsRef(config); if (!extendsRef?.startsWith("bos://")) { return false; @@ -1166,7 +1167,7 @@ async function runMigrationPhase( await timePhase(timings, "sync shared deps", async () => { const configResult = await loadResolvedConfig({ cwd: projectDir }); if (!configResult) { - throw new Error("No bos.config.json found in current directory"); + throw new Error("No bos.config file found in current directory"); } return syncResolvedSharedDeps({ diff --git a/packages/everything-dev/src/config-source.ts b/packages/everything-dev/src/config-source.ts new file mode 100644 index 00000000..70a8dd5c --- /dev/null +++ b/packages/everything-dev/src/config-source.ts @@ -0,0 +1,220 @@ +import { existsSync, readFileSync } from "node:fs"; +import { dirname, join, resolve } from "node:path"; +import { Data, Effect } from "effect"; +import { parse as parseToml, stringify as stringifyToml } from "smol-toml"; +import { isPlainObject } from "./merge"; +import type { BosConfigInput } from "./types"; + +export const CONFIG_FILENAMES = ["bos.config.toml", "bos.config.json"] as const; + +export class ConfigReadError extends Data.TaggedError("ConfigReadError")<{ + path: string; + cause: unknown; +}> {} + +export class ConfigParseError extends Data.TaggedError("ConfigParseError")<{ + path: string; + format: "toml" | "json"; + cause: unknown; +}> {} + +export class ConfigBothExistError extends Data.TaggedError("ConfigBothExistError")<{ + dir: string; +}> {} + +export type ConfigSourceError = ConfigReadError | ConfigParseError | ConfigBothExistError; + +function detectFormat(path: string, content: string): "toml" | "json" { + if (path.endsWith(".toml")) return "toml"; + if (path.endsWith(".json")) return "json"; + const trimmed = content.trim(); + return trimmed.length > 0 && trimmed[0] === "{" ? "json" : "toml"; +} + +export const readBosConfigSourceEff = ( + configPath: string, +): Effect.Effect => + Effect.gen(function* () { + const content = yield* Effect.try({ + try: () => readFileSync(configPath, "utf-8"), + catch: (error) => new ConfigReadError({ path: configPath, cause: error }), + }); + + const format = detectFormat(configPath, content); + + return yield* Effect.try({ + try: () => + format === "toml" + ? (parseToml(content) as unknown as BosConfigInput) + : (JSON.parse(content) as BosConfigInput), + catch: (error) => new ConfigParseError({ path: configPath, format, cause: error }), + }); + }); + +export const findBosConfigPathEff = ( + dir: string, +): Effect.Effect => + Effect.gen(function* () { + let current = resolve(dir); + while (current !== "/") { + const tomlPath = join(current, "bos.config.toml"); + const jsonPath = join(current, "bos.config.json"); + const hasToml = existsSync(tomlPath); + const hasJson = existsSync(jsonPath); + + if (hasToml && hasJson) { + return yield* Effect.fail(new ConfigBothExistError({ dir: current })); + } + if (hasToml) return tomlPath; + if (hasJson) return jsonPath; + + const parent = dirname(current); + if (parent === current) break; + current = parent; + } + return null; + }); + +const RESOLVED_CONFIG_FILENAME = "bos.resolved-config.json"; + +export const readBosConfigWithResolvedFallbackEff = ( + configDir: string, +): Effect.Effect, ConfigSourceError, never> => + Effect.gen(function* () { + const resolvedPath = join(configDir, ".bos", RESOLVED_CONFIG_FILENAME); + if (existsSync(resolvedPath)) { + const raw = yield* Effect.try({ + try: () => JSON.parse(readFileSync(resolvedPath, "utf-8")), + catch: (error) => + new ConfigParseError({ path: resolvedPath, format: "json", cause: error }), + }).pipe( + Effect.tapError((error) => + Effect.logWarning( + `[Config] Failed to parse ${resolvedPath}, falling back to source: ${error}`, + ), + ), + Effect.catchAll(() => Effect.succeed(null)), + ); + + if (isPlainObject(raw)) { + const { _resolved: _ignored, ...configData } = raw as Record; + if (Object.keys(configData).length > 0) { + return configData; + } + } + } + + const sourcePath = yield* findBosConfigPathEff(configDir); + if (!sourcePath) { + return yield* Effect.fail( + new ConfigReadError({ + path: join(configDir, "bos.config.json"), + cause: new Error("No bos.config.toml or bos.config.json found"), + }), + ); + } + const config = yield* readBosConfigSourceEff(sourcePath); + return config as unknown as Record; + }); + +function stripNulls(value: unknown): unknown { + if (value === null || value === undefined) return undefined; + if (isPlainObject(value)) { + const out: Record = {}; + for (const [key, val] of Object.entries(value as Record)) { + const cleaned = stripNulls(val); + if (cleaned !== undefined) { + out[key] = cleaned; + } + } + return Object.keys(out).length > 0 ? out : undefined; + } + if (Array.isArray(value)) { + const arr = value.map((item) => stripNulls(item)).filter((item) => item !== undefined); + return arr.length > 0 ? arr : undefined; + } + return value; +} + +export function stringifyBosConfig(value: Record): string { + const cleaned = stripNulls(value); + if ( + !cleaned || + (typeof cleaned === "object" && !Array.isArray(cleaned) && Object.keys(cleaned).length === 0) + ) { + return ""; + } + return stringifyToml(cleaned); +} + +export function findFormat(configPath: string): "json" | "toml" { + return detectFormat(configPath, ""); +} + +export function readBosConfigSource(configPath: string): BosConfigInput { + const content = readFileSync(configPath, "utf-8"); + const format = detectFormat(configPath, content); + if (format === "toml") { + return parseToml(content) as unknown as BosConfigInput; + } + return JSON.parse(content) as BosConfigInput; +} + +export function findBosConfigPathInDir(dir: string): string | null { + const tomlPath = join(dir, "bos.config.toml"); + const jsonPath = join(dir, "bos.config.json"); + if (existsSync(tomlPath) && existsSync(jsonPath)) { + throw new Error( + `Both bos.config.toml and bos.config.json exist in ${dir}. Remove one of them.`, + ); + } + if (existsSync(tomlPath)) return tomlPath; + if (existsSync(jsonPath)) return jsonPath; + return null; +} + +export function findBosConfigPath(dir: string): string | null { + let current = resolve(dir); + while (current !== "/") { + const tomlPath = join(current, "bos.config.toml"); + const jsonPath = join(current, "bos.config.json"); + const hasToml = existsSync(tomlPath); + const hasJson = existsSync(jsonPath); + + if (hasToml && hasJson) { + throw new Error( + `Both bos.config.toml and bos.config.json exist in ${current}. Remove one of them.`, + ); + } + if (hasToml) return tomlPath; + if (hasJson) return jsonPath; + + const parent = dirname(current); + if (parent === current) break; + current = parent; + } + return null; +} + +export function readBosConfigWithResolvedFallback(configDir: string): Record { + const resolvedPath = join(configDir, ".bos", RESOLVED_CONFIG_FILENAME); + if (existsSync(resolvedPath)) { + try { + const raw = JSON.parse(readFileSync(resolvedPath, "utf-8")); + if (isPlainObject(raw)) { + const { _resolved: _ignored, ...configData } = raw as Record; + if (Object.keys(configData).length > 0) { + return configData; + } + } + } catch (e) { + console.warn(`[Config] Failed to parse ${resolvedPath}, falling back to source: ${e}`); + } + } + + const sourcePath = findBosConfigPathInDir(configDir); + if (!sourcePath) { + throw new Error(`No bos.config.toml or bos.config.json found in ${configDir}`); + } + return readBosConfigSource(sourcePath) as unknown as Record; +} diff --git a/packages/everything-dev/src/config.ts b/packages/everything-dev/src/config.ts index f9ac3a56..95491291 100644 --- a/packages/everything-dev/src/config.ts +++ b/packages/everything-dev/src/config.ts @@ -1,6 +1,12 @@ import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { dirname, isAbsolute, join, resolve } from "node:path"; import { fetchApiPluginManifest } from "./api-contract"; +import { + findBosConfigPath, + findBosConfigPathInDir, + readBosConfigSource, + readBosConfigWithResolvedFallback, +} from "./config-source"; import { manifestPluginsToNodes } from "./dag"; import { fetchBosConfigFromFastKv } from "./fastkv"; import { fetchJsonOrNull } from "./http-client"; @@ -81,17 +87,9 @@ export function findConfigPath(cwd?: string): string | null { const cached = configPathCache.get(cacheKey); if (cached !== undefined) return cached; - let dir = cacheKey; - while (dir !== "/") { - const configPath = join(dir, "bos.config.json"); - if (existsSync(configPath)) { - configPathCache.set(cacheKey, configPath); - return configPath; - } - dir = dirname(dir); - } - configPathCache.set(cacheKey, null); - return null; + const configPath = findBosConfigPath(cacheKey); + configPathCache.set(cacheKey, configPath); + return configPath; } export function getConfig(): BosConfig | null { @@ -240,7 +238,7 @@ export async function loadBosConfig(options?: { }): Promise { const result = await loadResolvedConfig(options); if (!result) { - throw new Error("No bos.config.json found"); + throw new Error("No bos.config file found"); } return result.runtime; @@ -473,26 +471,11 @@ export function writeResolvedConfig( export function resolveBosConfigPath(configDir: string): string { const resolvedPath = getResolvedConfigPath(configDir); if (existsSync(resolvedPath)) return resolvedPath; - return join(configDir, "bos.config.json"); + return findBosConfigPath(configDir) ?? join(configDir, "bos.config.json"); } export function readBosConfigForBuild(configDir: string): Record { - const resolvedPath = getResolvedConfigPath(configDir); - if (existsSync(resolvedPath)) { - try { - const raw = JSON.parse(readFileSync(resolvedPath, "utf-8")); - if (isPlainObject(raw)) { - const { _resolved, ...configData } = raw; - return configData as Record; - } - } catch (e) { - console.warn( - `[Config] Failed to parse _resolved.json, falling back to bos.config.json: ${e}`, - ); - } - } - const bosConfigPath = join(configDir, "bos.config.json"); - return JSON.parse(readFileSync(bosConfigPath, "utf-8")) as Record; + return readBosConfigWithResolvedFallback(configDir); } function parseExtendsTarget(ref: string): ParsedExtendsTarget { @@ -635,8 +618,8 @@ export async function resolveComposableReference( if (localDevelopmentPath) { const localPath = localDevelopmentPath; - const localConfigPath = join(localPath, "bos.config.json"); - if (existsSync(localConfigPath)) { + const localConfigPath = findBosConfigPathInDir(localPath); + if (localConfigPath) { const localConfig = await resolveConfigWithExtends( localConfigPath, localPath, @@ -912,7 +895,7 @@ async function loadConfigFile(configPath: string, baseDir: string): Promise): stri if (result.length !== nodes.size) { const cyclic = [...nodes.keys()].filter((k) => !result.includes(k)); throw new Error( - `Circular dependency detected among: ${cyclic.join(" -> ")}. Check dependsOn declarations in bos.config.json.`, + `Circular dependency detected among: ${cyclic.join(" -> ")}. Check dependsOn declarations in bos.config.`, ); } diff --git a/packages/everything-dev/src/infra/planner.ts b/packages/everything-dev/src/infra/planner.ts index 2efb3939..1acde1b2 100644 --- a/packages/everything-dev/src/infra/planner.ts +++ b/packages/everything-dev/src/infra/planner.ts @@ -13,7 +13,7 @@ import { savePortState, } from "../cli/infra"; import { buildDescription } from "../service-descriptor"; -import type { RuntimeConfig } from "../types"; +import type { InfraConfig, RuntimeConfig } from "../types"; import type { ClaimRecord, CliPorts, @@ -173,6 +173,7 @@ function allocateServices( function allocateDatabases( runtimeConfig: RuntimeConfig, configDir: string, + infraConfig?: InfraConfig, ): Effect.Effect< { postgres: Record; @@ -191,7 +192,7 @@ function allocateDatabases( const allocator = yield* PortAllocator; const infraDatabases: DatabaseSecretConfig[] = yield* Effect.sync(() => - buildDatabaseConfigs(allSecrets, originMap, { ...persisted.postgresPorts }), + buildDatabaseConfigs(allSecrets, originMap, { ...persisted.postgresPorts }, infraConfig), ); const infraRedis: RedisSecretConfig[] = yield* Effect.sync(() => buildRedisConfigs(allSecrets, originMap, { ...persisted.redisPorts }), @@ -405,7 +406,7 @@ export function planInfra(input: InfraInput): Effect.Effect; + const isToml = opts.bosConfigPath.endsWith(".toml"); + const config = readBosConfigSource(opts.bosConfigPath) as Record; setNestedPath(config, opts.urlField, opts.url); if (opts.integrityField) { if (opts.integrity) { @@ -257,13 +259,18 @@ export function reportDeployResult(opts: { deleteNestedPath(config, opts.integrityField); } } - writeFileSync(opts.bosConfigPath, `${JSON.stringify(config, null, 2)}\n`); - console.log(` ✅ Updated bos.config.json: ${opts.urlField}`); + const content = isToml + ? `${stringifyBosConfig(config)}\n` + : `${JSON.stringify(config, null, 2)}\n`; + writeFileSync(opts.bosConfigPath, content); + const label = isToml ? "bos.config.toml" : "bos.config.json"; + console.log(` ✅ Updated ${label}: ${opts.urlField}`); if (opts.integrityField && opts.integrity) { - console.log(` ✅ Updated bos.config.json: ${opts.integrityField}`); + console.log(` ✅ Updated ${label}: ${opts.integrityField}`); } } catch (err) { - console.error(" ❌ Failed to update bos.config.json:", (err as Error).message); + const label = opts.bosConfigPath.endsWith(".toml") ? "bos.config.toml" : "bos.config.json"; + console.error(` ❌ Failed to update ${label}:`, (err as Error).message); } } @@ -305,7 +312,7 @@ export function applyDeployResults( } export function findPluginKey(bosConfigPath: string, pluginDir: string): string | null { - const config = JSON.parse(readFileSync(bosConfigPath, "utf8")) as Record; + const config = readBosConfigSource(bosConfigPath) as Record; const plugins = config.plugins as Record> | undefined; if (!plugins) return null; const configRoot = join(bosConfigPath, ".."); diff --git a/packages/everything-dev/src/merge.ts b/packages/everything-dev/src/merge.ts index d420b3c1..2211a35c 100644 --- a/packages/everything-dev/src/merge.ts +++ b/packages/everything-dev/src/merge.ts @@ -11,6 +11,8 @@ export const BOS_CONFIG_ORDER = [ "staging", "repository", "ci", + "infra", + "deploy", "app", "plugins", ] as const; @@ -48,10 +50,14 @@ function unionArrays(a: unknown, b: unknown): unknown[] | undefined { return result; } -function cleanNullSentinels(obj: Record): Record { +function cleanNullSentinels( + obj: Record, + stripDisabled = false, +): Record { const out: Record = {}; for (const [key, value] of Object.entries(obj)) { if (value === null || value === undefined) continue; + if (stripDisabled && isPlainObject(value) && value.disabled === true) continue; if (isPlainObject(value)) { const cleaned = cleanNullSentinels(value); if (Object.keys(cleaned).length > 0) { @@ -100,6 +106,7 @@ export function mergeBosConfigWithExtends( if (child.plugins !== undefined && isPlainObject(child.plugins)) { (merged as Record).plugins = cleanNullSentinels( child.plugins as Record, + true, ); } @@ -133,6 +140,19 @@ export function mergeBosConfigWithExtends( } } + // Backward compat: map ci.railway → deploy if deploy section is absent + if ( + !mergedRecord.deploy && + isPlainObject(mergedRecord.ci) && + isPlainObject((mergedRecord.ci as Record).railway) + ) { + const railway = (mergedRecord.ci as Record).railway as Record; + mergedRecord.deploy = { + provider: "railway", + ...(railway.service ? { service: railway.service } : {}), + }; + } + return rebuildOrderedConfig(mergedRecord) as BosConfigInput; } diff --git a/packages/everything-dev/src/plugin.ts b/packages/everything-dev/src/plugin.ts index aa2a623d..3f78d431 100644 --- a/packages/everything-dev/src/plugin.ts +++ b/packages/everything-dev/src/plugin.ts @@ -1,5 +1,5 @@ import { EventEmitter } from "node:events"; -import { existsSync, readFileSync, writeFileSync } from "node:fs"; +import { existsSync, writeFileSync } from "node:fs"; import { basename, dirname, join, relative, resolve } from "node:path"; import process from "node:process"; import { createInterface } from "node:readline/promises"; @@ -54,6 +54,7 @@ import { resumeWarnings, suppressWarnings, } from "./config"; +import { findBosConfigPathInDir, readBosConfigSource } from "./config-source"; import { type BosConfigResult, bosContract, @@ -343,7 +344,7 @@ export default createPlugin({ return { status: "error" as const, key: "", - error: "No bos.config.json found", + error: "No bos.config file found", }; } @@ -401,7 +402,7 @@ export default createPlugin({ return { status: "error" as const, key: input.key, - error: "No bos.config.json found", + error: "No bos.config file found", }; } @@ -442,7 +443,7 @@ export default createPlugin({ return { status: "error" as const, key: input.key, - error: "No bos.config.json found", + error: "No bos.config file found", }; } @@ -531,12 +532,10 @@ export default createPlugin({ const version = manifest?.plugin.version ?? pkgJson.version; if (publishedUrl) { - const rootConfigPath = join(deps.configDir, "bos.config.json"); + const rootConfigPath = + findBosConfigPathInDir(deps.configDir) ?? join(deps.configDir, "bos.config.json"); try { - const rootConfig = JSON.parse(readFileSync(rootConfigPath, "utf-8")) as Record< - string, - unknown - >; + const rootConfig = readBosConfigSource(rootConfigPath) as Record; if (!rootConfig.plugins || typeof rootConfig.plugins !== "object") { rootConfig.plugins = {}; } @@ -552,12 +551,9 @@ export default createPlugin({ delete entry.integrity; } writeFileSync(rootConfigPath, `${JSON.stringify(rootConfig, null, 2)}\n`); - console.log(` ✅ Updated bos.config.json: plugins.${input.key}.production`); + console.log(` ✅ Updated config: plugins.${input.key}.production`); } catch (err) { - console.error( - ` ❌ Failed to update bos.config.json:`, - err instanceof Error ? err.message : err, - ); + console.error(` ❌ Failed to update config:`, err instanceof Error ? err.message : err); } await generateCodeArtifacts(deps.configDir, deps.bosConfig); @@ -642,7 +638,7 @@ export default createPlugin({ if (!deps.bosConfig) { return { status: "error" as const, - description: "No bos.config.json found", + description: "No bos.config file found", processes: [], timings: devTimings, }; @@ -651,7 +647,7 @@ export default createPlugin({ if (proxy && !resolveProxyUrl(deps.bosConfig)) { return { status: "error" as const, - description: "No valid proxy URL configured in bos.config.json", + description: "No valid proxy URL configured in bos.config", processes: [], timings: devTimings, }; @@ -807,7 +803,7 @@ export default createPlugin({ status: "error" as const, url: "", error: - "No configuration found. Provide --account and --gateway flags, or create a local bos.config.json.", + "No configuration found. Provide --account and --gateway flags, or create a local bos.config.", }; } @@ -1042,7 +1038,7 @@ export default createPlugin({ return { status: "error" as const, registryUrl: "", - error: "No bos.config.json found", + error: "No bos.config file found", }; } @@ -1084,7 +1080,7 @@ export default createPlugin({ status: "error" as const, registryUrl: "", redeployed: false, - error: "No bos.config.json found", + error: "No bos.config file found", }; } @@ -1136,12 +1132,13 @@ export default createPlugin({ let service: string | undefined; if (process.env.RAILWAY_TOKEN) { - const railwayService = input.service ?? deps.bosConfig.ci?.railway?.service; + const railwayService = + input.service ?? deps.bosConfig.deploy?.service ?? deps.bosConfig.ci?.railway?.service; if (!railwayService) { console.log(); console.log( colors.yellow( - " Railway redeploy skipped: ci.railway.service is not configured in bos.config.json", + " Railway redeploy skipped: no service configured — set [deploy].service in bos.config.toml", ), ); return { @@ -1153,7 +1150,7 @@ export default createPlugin({ redeployed: false, deployResults: result.deployResults, error: - "Config published but Railway redeploy failed: ci.railway.service is not configured in bos.config.json", + "Config published but Railway redeploy failed: no service configured — set [deploy].service in bos.config.toml", }; } @@ -1220,7 +1217,7 @@ export default createPlugin({ contract: "", allowance: input.allowance, functionNames: PUBLISH_FUNCTION_NAMES, - error: "No bos.config.json found", + error: "No bos.config file found", }; } @@ -1451,6 +1448,7 @@ export default createPlugin({ await timePhase(timings, "personalize config", () => personalizeConfig(targetDir, { + mode: "init", extendsAccount, extendsGateway, account: account || extendsAccount, @@ -1564,7 +1562,7 @@ export default createPlugin({ updated: [], skipped: [], added: [], - error: "No bos.config.json found in current directory", + error: "No bos.config file found in current directory", }; } @@ -1597,7 +1595,7 @@ export default createPlugin({ return { status: "error" as const, packages: [], - error: "No bos.config.json found in current directory", + error: "No bos.config file found in current directory", }; } @@ -1622,7 +1620,7 @@ export default createPlugin({ fetched: [], skipped: [], failed: [], - error: "No bos.config.json found in current directory", + error: "No bos.config file found in current directory", }; } @@ -1642,7 +1640,7 @@ export default createPlugin({ fetched: [], skipped: [], failed: [], - error: "Failed to load bos.config.json", + error: "Failed to load bos.config", }; } @@ -1783,7 +1781,7 @@ export default createPlugin({ plugin: input.plugin, source: "remote" as const, section: "", - error: "No bos.config.json found in current directory", + error: "No bos.config file found in current directory", }; } @@ -1796,7 +1794,7 @@ export default createPlugin({ plugin: input.plugin, source: "remote" as const, section: "", - error: "Failed to load bos.config.json", + error: "Failed to load bos.config", }; } @@ -1838,7 +1836,7 @@ export default createPlugin({ appliedHashCount: 0, expectedTables: [], missingTables: [], - error: "No bos.config.json", + error: "No bos.config file", }; } @@ -1894,7 +1892,7 @@ export default createPlugin({ if (!configPath) { return { status: "error" as const, - message: "No bos.config.json found", + message: "No bos.config file found", diagnosis: null, error: "No config", }; @@ -1940,7 +1938,7 @@ export default createPlugin({ status: "error" as const, packages: [], envFile: "missing" as const, - error: "No bos.config.json found in current directory", + error: "No bos.config file found in current directory", }; } diff --git a/packages/everything-dev/src/publish.ts b/packages/everything-dev/src/publish.ts index 4f721102..68535d72 100644 --- a/packages/everything-dev/src/publish.ts +++ b/packages/everything-dev/src/publish.ts @@ -1,10 +1,10 @@ -import { readFileSync } from "node:fs"; -import { join } from "node:path"; import process from "node:process"; import { Effect } from "effect"; +import { generateAlchemyRun } from "./alchemy"; import { buildWorkspaceTargets, selectWorkspaceTargets } from "./build"; import { generateCodeArtifacts } from "./code-artifacts"; import { loadResolvedConfig } from "./config"; +import { findBosConfigPath, readBosConfigSource } from "./config-source"; import type { WorkspaceDeployResult } from "./contract"; import { buildRegistryConfigUrlForNetwork, @@ -103,7 +103,7 @@ export async function publishToFastKv(input: PublishToFastKvInput): Promise = { @@ -264,6 +274,18 @@ export async function publishToFastKv(input: PublishToFastKvInput): Promise { - const bosConfigPath = join(opts.configDir, "bos.config.json"); const resolvedConfigPath = join(opts.configDir, ".bos", "bos.resolved-config.json"); const packageJsonPath = join(opts.configDir, "package.json"); const generatedPath = join(opts.configDir, ".bos", "generated", "shared-deps.json"); - const bosConfig: unknown = opts.bosConfig ?? JSON.parse(readFileSync(bosConfigPath, "utf-8")); + const bosConfig: unknown = + opts.bosConfig ?? + (() => { + const configPath = findBosConfigPath(opts.configDir); + if (!configPath) throw new Error("No bos.config.toml or bos.config.json found"); + return readBosConfigSource(configPath); + })(); if (!isPlainObject(bosConfig)) { - throw new Error("bos.config.json must be an object"); + throw new Error("bos.config must be an object"); } const pkgJson = existsSync(packageJsonPath) diff --git a/packages/everything-dev/src/types.ts b/packages/everything-dev/src/types.ts index 8a3ae256..ff2411ce 100644 --- a/packages/everything-dev/src/types.ts +++ b/packages/everything-dev/src/types.ts @@ -84,6 +84,7 @@ export const BosPluginRefSchema = ComposableAppEntrySchema.extend({ app: z.record(z.string(), z.unknown()).optional(), plugins: z.record(z.string(), z.unknown()).optional(), dependsOn: z.array(z.string()).optional(), + disabled: z.boolean().optional(), }); export type BosPluginRef = z.infer; export type PluginEntryValue = string | BosPluginRef; @@ -196,6 +197,8 @@ export const BosConfigInputSchema: z.ZodType = z.lazy(() => extends: ExtendsSchema.optional(), account: z.string().optional(), domain: z.string().optional(), + title: z.string().optional(), + description: z.string().optional(), testnet: z.string().optional(), template: z.string().optional(), gateway: z @@ -217,6 +220,8 @@ export const BosConfigInputSchema: z.ZodType = z.lazy(() => app: z.record(z.string(), BosConfigInputAppEntrySchema).optional(), plugins: z.record(z.string(), z.union([z.string(), BosConfigInputSchema])).optional(), ci: CiConfigSchema.optional(), + infra: InfraConfigSchema.optional(), + deploy: DeployConfigSchema.optional(), }), ); @@ -245,6 +250,8 @@ export interface BosConfigInput { app?: Record; plugins?: Record; ci?: CiConfig; + infra?: InfraConfig; + deploy?: DeployConfig; } export const RailwayCiSchema = z.object({ @@ -257,6 +264,31 @@ export const CiConfigSchema = z.object({ }); export type CiConfig = z.infer; +export const InfraDatabaseSchema = z.object({ + type: z.enum(["postgres"]).default("postgres"), + dedicated: z.boolean().optional(), + secret: z.string().optional(), +}); +export type InfraDatabase = z.infer; + +export const InfraConfigSchema = z.object({ + database: z.union([InfraDatabaseSchema, z.record(z.string(), InfraDatabaseSchema)]).optional(), + redis: z + .object({ + enabled: z.boolean().optional(), + slug: z.string().optional(), + }) + .optional(), +}); +export type InfraConfig = z.infer; + +export const DeployConfigSchema = z.object({ + provider: z.enum(["railway", "alchemy"]).default("railway"), + service: z.string().optional(), + redeploy: z.boolean().optional(), +}); +export type DeployConfig = z.infer; + export const BosConfigSchema = z.object({ account: z.string(), extends: ExtendsSchema.optional(), @@ -267,6 +299,8 @@ export const BosConfigSchema = z.object({ staging: BosStagingSchema.optional(), repository: z.string().optional(), ci: CiConfigSchema.optional(), + infra: InfraConfigSchema.optional(), + deploy: DeployConfigSchema.optional(), plugins: z.record(z.string(), z.union([z.string(), BosPluginRefSchema])).optional(), app: z.object({ host: HostConfigSchema, diff --git a/packages/everything-dev/src/utils/save-config.ts b/packages/everything-dev/src/utils/save-config.ts index 7ae22d02..e70bae22 100644 --- a/packages/everything-dev/src/utils/save-config.ts +++ b/packages/everything-dev/src/utils/save-config.ts @@ -1,15 +1,29 @@ import { readFileSync, writeFileSync } from "node:fs"; import { join } from "node:path"; +import { findBosConfigPath, stringifyBosConfig } from "../config-source"; import { rebuildOrderedConfig } from "../merge"; import type { BosConfig } from "../types"; export async function saveBosConfig( configDir: string, config: BosConfig | Record, + format?: "json" | "toml", ): Promise { - const filePath = join(configDir, "bos.config.json"); + const existingPath = findBosConfigPath(configDir); + const forceToml = format === "toml"; + const forceJson = format === "json"; + const isToml = forceToml ? true : forceJson ? false : (existingPath?.endsWith(".toml") ?? true); + const filePath = forceToml + ? join(configDir, "bos.config.toml") + : forceJson + ? join(configDir, "bos.config.json") + : (existingPath ?? join(configDir, "bos.config.toml")); + const ordered = rebuildOrderedConfig(config as Record); - const next = `${JSON.stringify(ordered, null, 2)}\n`; + const next = isToml + ? `${stringifyBosConfig(ordered)}\n` + : `${JSON.stringify(ordered, null, 2)}\n`; + try { if (readFileSync(filePath, "utf8") === next) return; } catch { diff --git a/packages/everything-dev/tests/e2e/toml-config-pipeline.test.ts b/packages/everything-dev/tests/e2e/toml-config-pipeline.test.ts new file mode 100644 index 00000000..cf72bba2 --- /dev/null +++ b/packages/everything-dev/tests/e2e/toml-config-pipeline.test.ts @@ -0,0 +1,180 @@ +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { generateAlchemyRun } from "../../src/alchemy"; +import { buildDatabaseConfigs } from "../../src/cli/infra"; +import { + findBosConfigPathInDir, + readBosConfigSource, + stringifyBosConfig, +} from "../../src/config-source"; +import { mergeBosConfigWithExtends } from "../../src/merge"; + +describe("TOML config e2e pipeline", () => { + let tmpDir: string; + + beforeAll(() => { + tmpDir = mkdtempSync(join(tmpdir(), "bos-e2e-toml-")); + }); + + afterAll(() => { + rmSync(tmpDir, { recursive: true, force: true }); + }); + + it("reads a full bos.config.toml with infra + deploy sections", () => { + writeFileSync( + join(tmpDir, "bos.config.toml"), + [ + `account = "test.everything.near"`, + `domain = "test.everything.dev"`, + ``, + `[ci]`, + `[ci.railway]`, + `service = "test-api"`, + ``, + `[infra.database]`, + `type = "postgres"`, + ``, + `[deploy]`, + `provider = "railway"`, + `service = "test-api"`, + `redeploy = true`, + ``, + `[app.ui]`, + `development = "local:ui"`, + `production = "https://cdn.example.com/ui"`, + ``, + `[app.api]`, + `development = "local:api"`, + `production = "https://cdn.example.com/api"`, + ``, + `[app.host]`, + `development = "local:host"`, + `production = "https://cdn.example.com/host"`, + ``, + `[plugins.apps]`, + `development = "local:plugins/apps"`, + `production = "https://cdn.example.com/apps"`, + ].join("\n"), + ); + + const config = readBosConfigSource(join(tmpDir, "bos.config.toml")); + expect(config.account).toBe("test.everything.near"); + expect(config.domain).toBe("test.everything.dev"); + expect(config.ci?.railway?.service).toBe("test-api"); + expect(config.deploy?.provider).toBe("railway"); + expect(config.deploy?.service).toBe("test-api"); + expect(config.deploy?.redeploy).toBe(true); + + const infra = config.infra!; + expect(infra.database).toBeDefined(); + if (infra.database && !Array.isArray(infra.database) && typeof infra.database === "object") { + const db = infra.database as Record; + expect(db.type).toBe("postgres"); + } + + const appUi = config.app?.ui as Record | undefined; + expect(appUi?.development).toBe("local:ui"); + + const plugins = config.plugins as Record | undefined; + const appsPlugin = plugins?.apps as Record | undefined; + expect(appsPlugin?.development).toBe("local:plugins/apps"); + }); + + it("finds bos.config.toml in directory", () => { + const found = findBosConfigPathInDir(tmpDir); + expect(found).toBe(join(tmpDir, "bos.config.toml")); + }); + + it("merges infra + deploy through extends chain", () => { + const parent = { + account: "parent.near", + infra: { database: { type: "postgres" as const } }, + }; + const child = { + account: "child.near", + deploy: { provider: "railway" as const, service: "child-api" }, + }; + const merged = mergeBosConfigWithExtends(parent, child); + const m = merged as Record; + expect(m.account).toBe("child.near"); + expect(m.infra).toEqual(parent.infra); + expect(m.deploy).toEqual({ provider: "railway", service: "child-api" }); + }); + + it("ci.railway maps to deploy in merge", () => { + const merged = mergeBosConfigWithExtends( + {}, + { + ci: { railway: { service: "mapped-service" } }, + account: "test.near", + }, + ); + const m = merged as Record; + expect(m.deploy).toEqual({ provider: "railway", service: "mapped-service" }); + }); + + it("buildDatabaseConfigs uses shared DATABASE_URL with infraConfig", () => { + const configs = buildDatabaseConfigs( + ["API_DATABASE_URL", "AUTH_DATABASE_URL"], + new Map(), + {}, + { database: { type: "postgres" } }, + ); + expect(configs).toHaveLength(1); + expect(configs[0].secret).toBe("DATABASE_URL"); + }); + + it("generateAlchemyRun produces valid output", () => { + generateAlchemyRun( + { provider: "railway", service: "test-api", redeploy: true }, + { database: { type: "postgres" } }, + tmpDir, + ); + + const output = readFileSync(join(tmpDir, "alchemy.run.ts"), "utf-8"); + expect(output).toContain('provider = "railway"'); + expect(output).toContain('stage = "production"'); + expect(output).toContain('const service = "test-api"'); + expect(output).toContain("Redeploy service: test-api"); + expect(output).not.toContain("[d eploy]"); + }); + + it("stringifyBosConfig round-trips through readBosConfigSource", () => { + const original = { + account: "roundtrip.near", + domain: "roundtrip.dev", + deploy: { provider: "railway" as const, service: "rt-service", redeploy: true }, + }; + const toml = stringifyBosConfig(original); + writeFileSync(join(tmpDir, "roundtrip.toml"), toml); + const parsed = readBosConfigSource(join(tmpDir, "roundtrip.toml")); + expect(parsed.account).toBe("roundtrip.near"); + expect(parsed.deploy?.service).toBe("rt-service"); + }); + + it("TOML with infra section produces correct infra config", () => { + writeFileSync( + join(tmpDir, "infra-test.toml"), + [ + `[infra.database]`, + `type = "postgres"`, + ``, + `[infra.redis]`, + `enabled = true`, + ``, + `[deploy]`, + `provider = "railway"`, + `service = "my-app"`, + ].join("\n"), + ); + const config = readBosConfigSource(join(tmpDir, "infra-test.toml")); + expect(config.infra?.database).toBeDefined(); + const db = config.infra!.database as Record; + expect(db.type).toBe("postgres"); + expect(config.infra?.redis?.enabled).toBe(true); + expect(config.deploy?.provider).toBe("railway"); + expect(config.deploy?.service).toBe("my-app"); + }); +}); diff --git a/packages/everything-dev/tests/integration/personalize-config.test.ts b/packages/everything-dev/tests/integration/personalize-config.test.ts index 5305c3ac..0c8de9a5 100644 --- a/packages/everything-dev/tests/integration/personalize-config.test.ts +++ b/packages/everything-dev/tests/integration/personalize-config.test.ts @@ -3,6 +3,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import { buildInitPatterns, copyFilteredFiles, personalizeConfig } from "../../src/cli/init"; +import { findBosConfigPathInDir, readBosConfigSource } from "../../src/config-source"; import { loadManifestNormalizationSpec } from "../../src/internal/manifest-normalizer"; const REPO_ROOT = join(import.meta.dirname, "../../../../"); @@ -214,4 +215,32 @@ describe("personalizeConfig with real root config", () => { production: "https://auth.child.dev", }); }); + + it("mode=init writes bos.config.toml and cleans up stale json from template copy", async () => { + const testDir = mkdtempSync(join(tmpdir(), "bos-init-toml-")); + tempDirs.push(testDir); + const repoConfig = readFileSync(join(REPO_ROOT, "bos.config.json"), "utf-8"); + + writeFileSync(join(testDir, "bos.config.json"), repoConfig); + + await personalizeConfig(testDir, { + mode: "init", + extendsAccount: "dev.everything.near", + extendsGateway: "everything.dev", + account: "test.near", + domain: "test.dev", + overrides: ["ui", "api"], + workspaceOpts: { sourceDir: REPO_ROOT }, + }); + + const configPath = findBosConfigPathInDir(testDir); + expect(configPath).toBe(join(testDir, "bos.config.toml")); + expect(configPath).not.toBeNull(); + expect(existsSync(join(testDir, "bos.config.json"))).toBe(false); + + const config = readBosConfigSource(configPath!); + expect(config.account).toBe("test.near"); + expect(config.domain).toBe("test.dev"); + expect(config.extends).toBe("bos://dev.everything.near/everything.dev"); + }); }); diff --git a/packages/everything-dev/tests/unit/config-source.test.ts b/packages/everything-dev/tests/unit/config-source.test.ts new file mode 100644 index 00000000..c2a05bf8 --- /dev/null +++ b/packages/everything-dev/tests/unit/config-source.test.ts @@ -0,0 +1,217 @@ +import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { + CONFIG_FILENAMES, + findBosConfigPath, + readBosConfigSource, + readBosConfigWithResolvedFallback, + stringifyBosConfig, +} from "../../src/config-source"; + +describe("CONFIG_FILENAMES", () => { + it("has toml first, json second", () => { + expect(CONFIG_FILENAMES).toEqual(["bos.config.toml", "bos.config.json"]); + }); +}); + +describe("readBosConfigSource", () => { + let tmpDir: string; + + beforeAll(() => { + tmpDir = mkdtempSync(join(tmpdir(), "bos-config-source-")); + writeFileSync( + join(tmpDir, "bos.config.toml"), + `account = "dev.everything.near"\ndomain = "everything.dev"\n`, + ); + writeFileSync( + join(tmpDir, "bos.config.json"), + JSON.stringify({ account: "dev.everything.near", domain: "everything.dev" }), + ); + }); + + afterAll(() => { + rmSync(tmpDir, { recursive: true, force: true }); + }); + + it("parses TOML config", () => { + const config = readBosConfigSource(join(tmpDir, "bos.config.toml")); + expect(config.account).toBe("dev.everything.near"); + expect(config.domain).toBe("everything.dev"); + }); + + it("parses JSON config", () => { + const config = readBosConfigSource(join(tmpDir, "bos.config.json")); + expect(config.account).toBe("dev.everything.near"); + expect(config.domain).toBe("everything.dev"); + }); + + it("parses TOML with nested tables", () => { + writeFileSync( + join(tmpDir, "nested.toml"), + `[app.ui]\ndevelopment = "local:ui"\nproduction = "https://ui.example.com"\nintegrity = "sha384-abc123"\n`, + ); + const config = readBosConfigSource(join(tmpDir, "nested.toml")); + expect(config.app?.ui).toBeDefined(); + const ui = config.app?.ui as Record; + expect(ui.development).toBe("local:ui"); + expect(ui.production).toBe("https://ui.example.com"); + expect(ui.integrity).toBe("sha384-abc123"); + }); + + it("throws on missing file", () => { + expect(() => readBosConfigSource(join(tmpDir, "nonexistent.toml"))).toThrow(); + }); + + it("throws on invalid TOML", () => { + writeFileSync(join(tmpDir, "invalid.toml"), "<<< NOT TOML >>>"); + expect(() => readBosConfigSource(join(tmpDir, "invalid.toml"))).toThrow(); + }); + + it("throws on invalid JSON", () => { + writeFileSync(join(tmpDir, "invalid.json"), "{invalid}"); + expect(() => readBosConfigSource(join(tmpDir, "invalid.json"))).toThrow(); + }); +}); + +describe("findBosConfigPath", () => { + let tmpDir: string; + + beforeAll(() => { + tmpDir = mkdtempSync(join(tmpdir(), "bos-find-config-")); + }); + + afterAll(() => { + rmSync(tmpDir, { recursive: true, force: true }); + }); + + afterEach(() => { + // Clean up any config files created during test + for (const name of ["bos.config.toml", "bos.config.json"]) { + const p = join(tmpDir, name); + if (existsSync(p)) rmSync(p); + } + }); + + it("errors when both config files exist", () => { + writeFileSync(join(tmpDir, "bos.config.json"), "{}"); + writeFileSync(join(tmpDir, "bos.config.toml"), 'key = "value"'); + expect(() => findBosConfigPath(tmpDir)).toThrow(); + }); + + it("finds JSON when TOML absent", () => { + writeFileSync(join(tmpDir, "bos.config.json"), "{}"); + const found = findBosConfigPath(tmpDir); + expect(found).toBe(join(tmpDir, "bos.config.json")); + }); + + it("walks up directory tree", () => { + const subDir = join(tmpDir, "sub", "deep"); + writeFileSync(join(tmpDir, "bos.config.json"), "{}"); + const found = findBosConfigPath(subDir); + expect(found).toBe(join(tmpDir, "bos.config.json")); + }); + + it("returns null when no config found", () => { + const emptyDir = join(tmpDir, "empty"); + const found = findBosConfigPath(emptyDir); + expect(found).toBeNull(); + }); +}); + +describe("readBosConfigWithResolvedFallback", () => { + let tmpDir: string; + + beforeAll(() => { + tmpDir = mkdtempSync(join(tmpdir(), "bos-resolved-fallback-")); + }); + + afterAll(() => { + rmSync(tmpDir, { recursive: true, force: true }); + }); + + it("reads from source when no resolved config exists", () => { + writeFileSync(join(tmpDir, "bos.config.toml"), `account = "test.near"\ndomain = "test.dev"\n`); + const config = readBosConfigWithResolvedFallback(tmpDir); + expect(config.account).toBe("test.near"); + expect(config.domain).toBe("test.dev"); + rmSync(join(tmpDir, "bos.config.toml")); + }); + + it("prefers resolved config over source", () => { + writeFileSync(join(tmpDir, "bos.config.toml"), `account = "source.near"\n`); + mkdirSync(join(tmpDir, ".bos"), { recursive: true }); + writeFileSync( + join(tmpDir, ".bos/bos.resolved-config.json"), + JSON.stringify({ + _resolved: { env: "development", resolvedAt: "2024-01-01" }, + account: "resolved.near", + }), + ); + const config = readBosConfigWithResolvedFallback(tmpDir); + expect(config.account).toBe("resolved.near"); + // Clean up + rmSync(join(tmpDir, "bos.config.toml")); + rmSync(join(tmpDir, ".bos/bos.resolved-config.json"), { force: true }); + }); + + it("falls back to source when resolved config is empty", () => { + writeFileSync(join(tmpDir, "bos.config.toml"), `account = "source.near"\n`); + mkdirSync(join(tmpDir, ".bos"), { recursive: true }); + writeFileSync(join(tmpDir, ".bos/bos.resolved-config.json"), JSON.stringify({ _resolved: {} })); + const config = readBosConfigWithResolvedFallback(tmpDir); + expect(config.account).toBe("source.near"); + rmSync(join(tmpDir, "bos.config.toml")); + rmSync(join(tmpDir, ".bos/bos.resolved-config.json"), { force: true }); + }); +}); + +describe("stringifyBosConfig", () => { + it("produces valid TOML", () => { + const toml = stringifyBosConfig({ + account: "test.near", + domain: "test.dev", + app: { + ui: { + development: "local:ui", + production: "https://ui.example.com", + }, + }, + }); + expect(toml).toContain('account = "test.near"'); + expect(toml).toContain('domain = "test.dev"'); + expect(toml).toContain("[app.ui]"); + expect(toml).toContain('development = "local:ui"'); + expect(toml).toContain('production = "https://ui.example.com"'); + }); + + it("strips null and undefined values", () => { + const toml = stringifyBosConfig({ + account: "test.near", + integrity: null, + extra: undefined, + nested: { + value: "kept", + removed: null, + }, + }); + expect(toml).toContain('account = "test.near"'); + expect(toml).toContain('value = "kept"'); + expect(toml).not.toContain("integrity"); + expect(toml).not.toContain("removed"); + }); + + it("handles arrays", () => { + const toml = stringifyBosConfig({ + secrets: ["API_KEY", "DB_URL"], + }); + expect(toml).toContain("API_KEY"); + expect(toml).toContain("DB_URL"); + }); + + it("handles empty objects", () => { + const toml = stringifyBosConfig({}); + expect(toml).toBe(""); + }); +}); diff --git a/packages/everything-dev/tests/unit/infra.test.ts b/packages/everything-dev/tests/unit/infra.test.ts index 918f1275..366b1b10 100644 --- a/packages/everything-dev/tests/unit/infra.test.ts +++ b/packages/everything-dev/tests/unit/infra.test.ts @@ -3,6 +3,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import { + buildDatabaseConfigs, ensureEnvFile, loadPortState, loadProjectEnv, @@ -519,3 +520,127 @@ describe("generated infra", () => { expect(loaded.postgresPorts.api).toBe(5432); }); }); + +describe("buildDatabaseConfigs with infraConfig", () => { + it("returns shared DATABASE_URL when infraConfig.database present", () => { + const configs = buildDatabaseConfigs( + ["API_DATABASE_URL", "AUTH_DATABASE_URL"], + new Map(), + {}, + { database: { type: "postgres" } }, + ); + expect(configs).toHaveLength(1); + expect(configs[0].secret).toBe("DATABASE_URL"); + expect(configs[0].slug).toBe("shared"); + expect(configs[0].port).toBe(5432); + expect(configs[0].url).toContain("shared_db"); + }); + + it("falls back to convention scanning when infraConfig.database is absent", () => { + const originMap = new Map(); + originMap.set("API_DATABASE_URL", "everything.near"); + originMap.set("AUTH_DATABASE_URL", "everything.near"); + const configs = buildDatabaseConfigs(["API_DATABASE_URL", "AUTH_DATABASE_URL"], originMap, {}); + expect(configs.length).toBeGreaterThan(1); + expect(configs.map((c) => c.secret)).toContain("API_DATABASE_URL"); + expect(configs.map((c) => c.secret)).toContain("AUTH_DATABASE_URL"); + }); + + it("uses existing port when infraConfig.database present", () => { + const configs = buildDatabaseConfigs( + ["API_DATABASE_URL"], + new Map(), + { shared: 5433 }, + { database: { type: "postgres" } }, + ); + expect(configs[0].port).toBe(5433); + }); + + it("returns per-plugin configs from record variant", () => { + const originMap = new Map(); + originMap.set("AUTH_DATABASE_URL", "test.near"); + originMap.set("API_DATABASE_URL", "test.near"); + const configs = buildDatabaseConfigs( + ["API_DATABASE_URL", "EXAMPLE_DATABASE_URL", "AUTH_DATABASE_URL"], + originMap, + {}, + { + database: { + auth: { type: "postgres" }, + api: { type: "postgres" }, + }, + }, + ); + expect(configs).toHaveLength(2); + const secrets = configs.map((c) => c.secret).sort(); + expect(secrets).toEqual(["API_DATABASE_URL", "AUTH_DATABASE_URL"]); + }); + + it("skips per-plugin entries without matching secrets", () => { + const configs = buildDatabaseConfigs( + ["API_DATABASE_URL"], + new Map(), + {}, + { + database: { + auth: { type: "postgres" }, + nonexistent: { type: "postgres" }, + }, + }, + ); + expect(configs).toHaveLength(0); + }); + + it("assigns 5432 and 5433 for API and AUTH ports in per-plugin record", () => { + const originMap = new Map(); + originMap.set("API_DATABASE_URL", "test.near"); + originMap.set("AUTH_DATABASE_URL", "test.near"); + const configs = buildDatabaseConfigs( + ["API_DATABASE_URL", "AUTH_DATABASE_URL"], + originMap, + {}, + { + database: { + api: { type: "postgres" }, + auth: { type: "postgres" }, + }, + }, + ); + const apiConfig = configs.find((c) => c.secret === "API_DATABASE_URL"); + const authConfig = configs.find((c) => c.secret === "AUTH_DATABASE_URL"); + expect(apiConfig?.port).toBe(5432); + expect(authConfig?.port).toBe(5433); + }); + + it("uses custom secret name from per-plugin InfraDatabase.secret", () => { + const originMap = new Map(); + originMap.set("CUSTOM_DB_URL", "test.near"); + const configs = buildDatabaseConfigs( + ["CUSTOM_DB_URL", "AUTH_DATABASE_URL"], + originMap, + {}, + { + database: { + auth: { type: "postgres", secret: "CUSTOM_DB_URL" }, + }, + }, + ); + expect(configs).toHaveLength(1); + expect(configs[0].secret).toBe("CUSTOM_DB_URL"); + }); + + it("falls back to conventional secret when InfraDatabase.secret is absent", () => { + const configs = buildDatabaseConfigs( + ["AUTH_DATABASE_URL"], + new Map(), + {}, + { + database: { + auth: { type: "postgres" }, + }, + }, + ); + expect(configs).toHaveLength(1); + expect(configs[0].secret).toBe("AUTH_DATABASE_URL"); + }); +}); diff --git a/packages/everything-dev/tests/unit/merge.test.ts b/packages/everything-dev/tests/unit/merge.test.ts index b1d3b1b7..443e7ffd 100644 --- a/packages/everything-dev/tests/unit/merge.test.ts +++ b/packages/everything-dev/tests/unit/merge.test.ts @@ -6,6 +6,7 @@ import { rebuildOrderedConfig, resolveExtendsRef, } from "../../src/merge"; +import type { BosConfigInput } from "../../src/types"; describe("mergeBosConfigWithExtends", () => { it("child scalars override parent scalars", () => { @@ -491,7 +492,49 @@ describe("BOS_CONFIG_ORDER", () => { expect(BOS_CONFIG_ORDER).toContain("testnet"); expect(BOS_CONFIG_ORDER).toContain("staging"); expect(BOS_CONFIG_ORDER).toContain("repository"); + expect(BOS_CONFIG_ORDER).toContain("ci"); + expect(BOS_CONFIG_ORDER).toContain("infra"); + expect(BOS_CONFIG_ORDER).toContain("deploy"); expect(BOS_CONFIG_ORDER).toContain("app"); expect(BOS_CONFIG_ORDER).toContain("plugins"); }); }); + +describe("ci.railway → deploy backward compat", () => { + it("maps ci.railway to deploy when deploy section is absent", () => { + const child = { + ci: { railway: { service: "my-service" } }, + account: "test.near", + }; + const merged = mergeBosConfigWithExtends({}, child as BosConfigInput); + expect((merged as Record).deploy).toEqual({ + provider: "railway", + service: "my-service", + }); + }); + + it("does not overwrite deploy when it already exists", () => { + const child = { + ci: { railway: { service: "ci-service" } }, + deploy: { provider: "railway", service: "deploy-service" }, + account: "test.near", + }; + const merged = mergeBosConfigWithExtends({}, child as BosConfigInput); + expect((merged as Record).deploy).toEqual({ + provider: "railway", + service: "deploy-service", + }); + }); + + it("does not create deploy when ci.railway is absent", () => { + const child = { ci: {}, account: "test.near" }; + const merged = mergeBosConfigWithExtends({}, child as BosConfigInput); + expect((merged as Record).deploy).toBeUndefined(); + }); + + it("does not create deploy when ci is absent", () => { + const child = { account: "test.near" }; + const merged = mergeBosConfigWithExtends({}, child as BosConfigInput); + expect((merged as Record).deploy).toBeUndefined(); + }); +}); diff --git a/plans/README.md b/plans/README.md index fbb94a35..7f7e7e12 100644 --- a/plans/README.md +++ b/plans/README.md @@ -11,6 +11,7 @@ honor its STOP conditions, and update your row when done. | 003 | Require API_DATABASE_URL in production | P1 | S | — | DONE | | 006 | Add warnings to empty catch blocks | P1 | S | — | DONE | | 007 | Add CSRF protection to state-changing endpoints | P1 | M | — | DONE | +| toml-infra-alchemy | TOML config + shared databases + Alchemy integration | P2 | L | — | PLANNED | ## Dependency notes