diff --git a/.changeset/quiet-tables-load.md b/.changeset/quiet-tables-load.md new file mode 100644 index 0000000..05a9819 --- /dev/null +++ b/.changeset/quiet-tables-load.md @@ -0,0 +1,8 @@ +--- +"@will-be-done/hyperdb": patch +--- + +Speed up repeated SQLite `loadTables()` calls with persisted schema signatures +and make SQLite schema reconciliation handle identifiers case-insensitively. +Ensure rejected async driver commands run generator cleanup, including +transaction rollback and lock release. diff --git a/README.md b/README.md index a7fe9c7..70d2b2c 100644 --- a/README.md +++ b/README.md @@ -173,6 +173,9 @@ export async function createAppDB() { } ``` +`AsyncSqlDriver` is also exercised against Turso Database's browser WASM +engine in the shared driver conformance suite. + If your whole app state can be loaded into memory at startup, you may not need `HybridDB`. A plain `new SubscribableDB(new DB(new BptreeInmemDriver()))` keeps reads and writes synchronous, so you can use `useSyncSelector`, `useSyncDispatch`, diff --git a/packages/hyperdb-doc/src/content/docs/runtime/drivers.md b/packages/hyperdb-doc/src/content/docs/runtime/drivers.md index b4680c0..f1c3e39 100644 --- a/packages/hyperdb-doc/src/content/docs/runtime/drivers.md +++ b/packages/hyperdb-doc/src/content/docs/runtime/drivers.md @@ -13,12 +13,12 @@ the sync or async runtime helpers, which depends on the storage path. ## Choosing a driver -| Driver | Import | Mode | Environment | Use for | -| ------------------- | ---------------------- | ----- | ----------- | -------------------------------------------------------------- | -| `BptreeInmemDriver` | `.../drivers/inmemory` | sync | both | Tests, fully loaded app state, and the fast `HybridDB` cache | -| `IdbDriver` | `.../drivers/idb` | async | browser | Browser persistence, usually as a `HybridDB` primary store | -| `SqlDriver` | `.../drivers/sqlite` | sync | both | Any synchronous SQLite binding (native server SQLite, sql.js) | -| `AsyncSqlDriver` | `.../drivers/sqlite` | async | both | Async SQLite, including browser SQLite as a `HybridDB` primary | +| Driver | Import | Mode | Environment | Use for | +| ------------------- | ---------------------- | ----- | ----------- | ------------------------------------------------------------- | +| `BptreeInmemDriver` | `.../drivers/inmemory` | sync | both | Tests, fully loaded app state, and the fast `HybridDB` cache | +| `IdbDriver` | `.../drivers/idb` | async | browser | Browser persistence, usually as a `HybridDB` primary store | +| `SqlDriver` | `.../drivers/sqlite` | sync | both | Any synchronous SQLite binding (native server SQLite, sql.js) | +| `AsyncSqlDriver` | `.../drivers/sqlite` | async | both | Async SQLite, including Turso WASM as a `HybridDB` primary | Sync drivers work with `execSync` / `syncDispatch` / `selectSync`. Async drivers require `execAsync` / `asyncDispatch` / `selectAsync`. `HybridDB` also uses the @@ -107,12 +107,25 @@ not prepare or emit SQL diagnostic events. Use `formatAsyncSqlDriverDebugEvent(event)` when you want the old one-line message but need to send it to a custom logger. +HyperDB's shared driver conformance suite runs `AsyncSqlDriver` against the +browser build of Turso Database (`@tursodatabase/database-wasm`) in Chromium, +covering the same runtime behavior as the SQL.js, IndexedDB, and in-memory +drivers. + The SQLite storage codec encodes `bigint`, `ArrayBuffer`, and typed-array/data-view values around JSON storage so they round-trip exactly. `uniqhash` indexes are created as SQLite `UNIQUE` indexes. Upserts delete the same primary keys first and then insert the new rows, so a secondary unique conflict throws instead of replacing a different row. +The SQLite drivers persist each table's physical-layout signature in the +reserved `_hyperdb_schema_metadata` table. After the first successful +`loadTables()` reconciliation, loading unchanged definitions takes one metadata +read instead of inspecting and rebuilding every table and index. The stored +signature is tied to SQLite's `schema_version`, so external schema changes make +the next load run the full transactional reconciliation and refresh the +metadata only after it succeeds. + ## SQLite Recipes ### SQL.js sync diff --git a/packages/hyperdb-doc/src/content/docs/start/llm-cheat-sheet.md b/packages/hyperdb-doc/src/content/docs/start/llm-cheat-sheet.md index bdf3415..e3e4fc5 100644 --- a/packages/hyperdb-doc/src/content/docs/start/llm-cheat-sheet.md +++ b/packages/hyperdb-doc/src/content/docs/start/llm-cheat-sheet.md @@ -42,7 +42,8 @@ and views that should re-run only when the exact index ranges they read change. trace metadata helpers. - `@will-be-done/hyperdb/drivers/inmemory`: `BptreeInmemDriver`. - `@will-be-done/hyperdb/drivers/sqlite`: `SqlDriver`, `AsyncSqlDriver`, and - SQLite adapter types. + SQLite adapter types. `AsyncSqlDriver` is compatible with Turso Database's + browser WASM build. - `@will-be-done/hyperdb/drivers/idb`: `openIndexedDBDriver`, `IdbDriver`, and IndexedDB driver options. - `@will-be-done/hyperdb-devtool/react`: separate package with @@ -285,6 +286,11 @@ preparation. Use `logIdbDriverDebugEvent` from console style, and `logAsyncSqlDriverDebugEvent` from `@will-be-done/hyperdb/drivers/sqlite` for async SQLite. +SQLite `loadTables()` stores physical-layout signatures in the reserved +`_hyperdb_schema_metadata` table. Unchanged definitions use a one-query fast +path; a changed definition or SQLite `schema_version` triggers full +transactional schema reconciliation. + Use a B-tree full-scan index such as `byIds` for `preloadTables`; the built-in `byId` index is a `uniqhash` index for exact id lookups. `SubscribableDB` adds revisions, subscriptions, selector invalidation, and lifecycle hooks. Pure diff --git a/packages/hyperdb/package.json b/packages/hyperdb/package.json index a70bbd7..6dd243a 100644 --- a/packages/hyperdb/package.json +++ b/packages/hyperdb/package.json @@ -66,6 +66,7 @@ }, "devDependencies": { "@eslint/js": "^9.39.1", + "@tursodatabase/database-wasm": "0.7.1", "@types/react": "19.2.17", "@types/react-dom": "19.2.3", "@types/sql.js": "^1.4.9", diff --git a/packages/hyperdb/src/hyperdb/core/executor.test.ts b/packages/hyperdb/src/hyperdb/core/executor.test.ts index ff0b442..db93ef3 100644 --- a/packages/hyperdb/src/hyperdb/core/executor.test.ts +++ b/packages/hyperdb/src/hyperdb/core/executor.test.ts @@ -38,6 +38,36 @@ describe("executor", () => { await expect(execAsync(command())).rejects.toThrow("boom"); }); + it("throws rejected async commands back into generators", async () => { + let cleanedUp = false; + + function* command(): Generator { + try { + yield* unwrap(Promise.reject(new Error("boom"))); + } finally { + cleanedUp = true; + } + } + + await expect(execAsync(command())).rejects.toThrow("boom"); + expect(cleanedUp).toBe(true); + }); + + it("throws rejected execMaybeAsync commands back into generators", async () => { + let cleanedUp = false; + + function* command(): Generator { + try { + yield* unwrap(Promise.reject(new Error("boom"))); + } finally { + cleanedUp = true; + } + } + + await expect(execMaybeAsync(command())).rejects.toThrow("boom"); + expect(cleanedUp).toBe(true); + }); + it("continues through noop commands in execSync", () => { function* command(): Generator { yield* noop(); diff --git a/packages/hyperdb/src/hyperdb/core/executor.ts b/packages/hyperdb/src/hyperdb/core/executor.ts index 59c5c45..180e1b7 100644 --- a/packages/hyperdb/src/hyperdb/core/executor.ts +++ b/packages/hyperdb/src/hyperdb/core/executor.ts @@ -35,11 +35,11 @@ async function execAsyncFrom( cmd: Generator, value: unknown, ): Promise { - let result = cmd.next(await value); + let result = await resumeAfterAsyncCommand(cmd, value); while (!result.done) { if (isUnwrapCmd(result.value)) { - result = cmd.next(await result.value.data); + result = await resumeAfterAsyncCommand(cmd, result.value.data); } else if (isNoopCmd(result.value)) { result = cmd.next(); } else { @@ -50,12 +50,26 @@ async function execAsyncFrom( return result.value as T; } +async function resumeAfterAsyncCommand( + cmd: Generator, + data: unknown, +): Promise> { + let value: unknown; + try { + value = await data; + } catch (error) { + return cmd.throw(error); + } + + return cmd.next(value); +} + export async function execAsync(cmd: Generator): Promise { let result = cmd.next(); while (!result.done) { if (isUnwrapCmd(result.value)) { - result = cmd.next(await result.value.data); + result = await resumeAfterAsyncCommand(cmd, result.value.data); } else if (isNoopCmd(result.value)) { result = cmd.next(); } else { diff --git a/packages/hyperdb/src/hyperdb/drivers/sqlite/async-sql-driver.test.ts b/packages/hyperdb/src/hyperdb/drivers/sqlite/async-sql-driver.test.ts index 2bda9d8..910d665 100644 --- a/packages/hyperdb/src/hyperdb/drivers/sqlite/async-sql-driver.test.ts +++ b/packages/hyperdb/src/hyperdb/drivers/sqlite/async-sql-driver.test.ts @@ -3,11 +3,16 @@ import { defineTable } from "../../schema/table"; import { execAsync } from "../../core/executor"; import { DB } from "../../runtime/db"; import { v } from "../../schema/values"; -import { createSqlJsAsyncDriver } from "../../test-utils/sql-js-driver"; +import { + createInspectableSqlAsyncDriver, + createSqlJsAsyncDriver, +} from "../../test-utils/sql-js-driver"; +import { createTursoWasmDriver } from "../../test-utils/turso-wasm-driver"; import { AsyncSqlDriver, formatAsyncSqlDriverDebugEvent, } from "./async-sql-driver"; +import { SQLITE_SCHEMA_METADATA_TABLE } from "./sqlite-common"; type Task = { type: "task"; @@ -89,7 +94,94 @@ describe("db", async () => { ]); }); - for (const driver of [createSqlJsAsyncDriver]) { + it("uses one metadata read when async table schemas are unchanged", async () => { + const { driver, execLog } = await createInspectableSqlAsyncDriver(); + const db = new DB(driver); + + await execAsync(db.loadTables([tasksTable])); + execLog.length = 0; + await execAsync(db.loadTables([tasksTable])); + + expect(execLog).toHaveLength(1); + expect(execLog[0]).toContain(`LEFT JOIN ${SQLITE_SCHEMA_METADATA_TABLE}`); + expect(execLog[0]).not.toContain("BEGIN TRANSACTION"); + }); + + it("initializes async schema metadata when the adapter hides missing-table details", async () => { + let obscureMetadataReadError = true; + const { driver, sqldb, execLog } = await createInspectableSqlAsyncDriver( + {}, + { + beforePrepare(sql) { + if ( + obscureMetadataReadError && + sql.includes(`LEFT JOIN ${SQLITE_SCHEMA_METADATA_TABLE}`) + ) { + throw new Error("statement preparation failed"); + } + }, + }, + ); + const db = new DB(driver); + + await execAsync(db.loadTables([tasksTable])); + expect( + sqldb.exec( + `SELECT name FROM sqlite_schema WHERE name = '${SQLITE_SCHEMA_METADATA_TABLE}'`, + )[0]?.values, + ).toEqual([[SQLITE_SCHEMA_METADATA_TABLE]]); + + await expect(execAsync(db.loadTables([tasksTable]))).rejects.toThrow( + "statement preparation failed", + ); + + obscureMetadataReadError = false; + execLog.length = 0; + await execAsync(db.loadTables([tasksTable])); + expect(execLog).toHaveLength(1); + expect(execLog[0]).toContain(`LEFT JOIN ${SQLITE_SCHEMA_METADATA_TABLE}`); + }); + + it("treats async SQLite index identifiers as case-insensitive", async () => { + const { driver, sqldb, execLog } = await createInspectableSqlAsyncDriver(); + const db = new DB(driver); + + await execAsync(db.loadTables([tasksTable])); + sqldb.exec("DROP INDEX idx_tasks_byTitle_sort_key_v2"); + sqldb.exec( + "CREATE INDEX idx_tasks_bytitle_sort_key_v2 ON tasks(idx_byTitle_sort_key_v2, id) WHERE idx_byTitle_sort_key_v2 IS NOT NULL", + ); + execLog.length = 0; + + await execAsync(db.loadTables([tasksTable])); + + expect(execLog.some((sql) => sql.startsWith("DROP INDEX"))).toBe(false); + expect( + (sqldb.exec("PRAGMA index_list(tasks)")[0]?.values ?? []).some( + (row) => String(row[1]) === "idx_tasks_bytitle_sort_key_v2", + ), + ).toBe(true); + }); + + it("uses the schema metadata fast path with Turso WASM", async () => { + const debug = vi.fn(); + const db = new DB(await createTursoWasmDriver({ debug })); + + await execAsync(db.loadTables([tasksTable])); + debug.mockClear(); + await execAsync(db.loadTables([tasksTable])); + + expect(debug).toHaveBeenCalledTimes(1); + expect(debug.mock.calls[0]?.[0]).toMatchObject({ + operation: "scan", + status: "success", + }); + expect(debug.mock.calls[0]?.[0].sql).toContain( + `LEFT JOIN ${SQLITE_SCHEMA_METADATA_TABLE}`, + ); + }); + + for (const driver of [createSqlJsAsyncDriver, createTursoWasmDriver]) { it("preserves physical indexes whose logical names end in _sort_key", async () => { const debug = vi.fn(); const db = new DB(await driver({ debug })); diff --git a/packages/hyperdb/src/hyperdb/drivers/sqlite/async-sql-driver.ts b/packages/hyperdb/src/hyperdb/drivers/sqlite/async-sql-driver.ts index e40b7b1..d64636e 100644 --- a/packages/hyperdb/src/hyperdb/drivers/sqlite/async-sql-driver.ts +++ b/packages/hyperdb/src/hyperdb/drivers/sqlite/async-sql-driver.ts @@ -27,6 +27,13 @@ import { isSqliteSortKeyColumn, SQLITE_SORT_KEY_SUFFIX, LEGACY_SQLITE_SORT_KEY_SUFFIX, + createSqliteTableSchemaSignature, + createSqliteSchemaMetadataTableSQL, + selectSqliteSchemaMetadataSQL, + selectSqliteSchemaMetadataTableExistsSQL, + upsertSqliteSchemaMetadataSQL, + isMissingSqliteSchemaMetadataError, + sqliteIdentifierKey, assertSafeTableDefinition, buildRowInsertParams, parseSqliteStoredRow, @@ -674,46 +681,192 @@ export class AsyncSqlDriver implements DBDriver { *loadTables( tableDefinitions: TableDefinition[], ): Generator { - yield* unwrapCb(async () => { - await this.txAndQueryLock.acquireAsync(); - }); + for (const tableDef of tableDefinitions) { + assertSafeTableDefinition(tableDef); + } - try { - for (const tableDef of tableDefinitions) { - assertSafeTableDefinition(tableDef); - } + tableDefinitions = cloneDeep(tableDefinitions); + if (tableDefinitions.length === 0) return; - yield* unwrapCb(async () => { - await runAsyncSQL(this.db, "BEGIN TRANSACTION", undefined, this.debug); + yield* unwrapCb(async () => { + await this.txAndQueryLock.acquireAsync(); + try { + if (await this.schemaMetadataMatches(tableDefinitions)) { + this.installTableDefinitions(tableDefinitions); + return; + } - tableDefinitions = cloneDeep(tableDefinitions); - for (const tableDef of tableDefinitions) { - await this.createTable(tableDef); - const indexUniqueness = await this.getGeneratedIndexUniqueness( - tableDef.tableName, + let transactionStarted = false; + try { + await runAsyncSQL( + this.db, + "BEGIN TRANSACTION", + undefined, + this.debug, ); - const reencodedColumns = this.reencodedSortKeyColumns( - tableDef, - indexUniqueness, + transactionStarted = true; + await runAsyncSQL( + this.db, + createSqliteSchemaMetadataTableSQL(), + undefined, + this.debug, ); - await this.dropStaleSortKeyIndexes(tableDef, indexUniqueness); - await this.dropStaleSortKeyColumns(tableDef); - await this.addMissingSortKeyColumns(tableDef); - await this.resetSortKeyColumns(tableDef, reencodedColumns); - await this.backfillSortKeyColumns(tableDef); - await this.createIndexes(tableDef); - this.tableDefinitions.set(tableDef.tableName, tableDef); + + for (const tableDef of tableDefinitions) { + await this.createTable(tableDef); + const indexUniqueness = await this.getGeneratedIndexUniqueness( + tableDef.tableName, + ); + const reencodedColumns = this.reencodedSortKeyColumns( + tableDef, + indexUniqueness, + ); + const existingColumns = await this.getTableColumns( + tableDef.tableName, + ); + await this.dropStaleSortKeyIndexes(tableDef, indexUniqueness); + await this.dropStaleSortKeyColumns(tableDef, existingColumns); + await this.addMissingSortKeyColumns(tableDef, existingColumns); + await this.resetSortKeyColumns( + tableDef, + reencodedColumns, + existingColumns, + ); + await this.backfillSortKeyColumns(tableDef); + await this.createIndexes(tableDef); + } + + await this.writeSchemaMetadata(tableDefinitions); + await runAsyncSQL(this.db, "COMMIT", undefined, this.debug); + transactionStarted = false; + this.installTableDefinitions(tableDefinitions); + } catch (error) { + if (transactionStarted) { + await rollbackAsyncQuietly(this.db, error, this.debug); + } + throw error; } + } finally { + this.txAndQueryLock.release(); + } + }); + } - await runAsyncSQL(this.db, "COMMIT", undefined, this.debug); + private async schemaMetadataMatches( + tableDefinitions: TableDefinition[], + ): Promise { + const sql = selectSqliteSchemaMetadataSQL(tableDefinitions.length); + const startedAt = this.debug ? nowMs() : 0; + let statement: AsyncSQLStatement | undefined; + + try { + statement = await this.db.prepare(sql); + const rows = await statement.values( + tableDefinitions.map((tableDef) => tableDef.tableName), + ); + emitAsyncSqlDebug(this.debug, "scan", sql, startedAt, () => ({ + rowCount: rows.length, + })); + + const schemaVersion = Number(rows[0]?.[3]); + if (!Number.isInteger(schemaVersion)) return false; + const metadataByTable = new Map( + rows + .filter((row) => typeof row[0] === "string") + .map((row) => [ + String(row[0]), + { + signature: String(row[1]), + verifiedSchemaVersion: Number(row[2]), + }, + ]), + ); + + return tableDefinitions.every((tableDef) => { + const metadata = metadataByTable.get(tableDef.tableName); + return ( + metadata?.signature === createSqliteTableSchemaSignature(tableDef) && + metadata.verifiedSchemaVersion === schemaVersion + ); }); } catch (error) { - yield* unwrapCb(async () => { - await rollbackAsyncQuietly(this.db, error, this.debug); - }); + if (isMissingSqliteSchemaMetadataError(error)) return false; + await statement?.finalize(); + statement = undefined; + try { + if (!(await this.schemaMetadataTableExists())) return false; + } catch { + // Preserve the original metadata-read error when even the fallback + // schema inspection cannot run. + } + emitAsyncSqlDebug(this.debug, "scan", sql, startedAt, () => ({}), error); throw error; } finally { - this.txAndQueryLock.release(); + await statement?.finalize(); + } + } + + private async schemaMetadataTableExists(): Promise { + const sql = selectSqliteSchemaMetadataTableExistsSQL(); + const startedAt = this.debug ? nowMs() : 0; + const statement = await this.db.prepare(sql); + try { + const rows = await statement.values([]); + emitAsyncSqlDebug(this.debug, "scan", sql, startedAt, () => ({ + rowCount: rows.length, + })); + return rows.length > 0; + } catch (error) { + emitAsyncSqlDebug(this.debug, "scan", sql, startedAt, () => ({}), error); + throw error; + } finally { + await statement.finalize(); + } + } + + private async getSchemaVersion(): Promise { + const sql = "SELECT schema_version FROM pragma_schema_version"; + const startedAt = this.debug ? nowMs() : 0; + const statement = await this.db.prepare(sql); + try { + const rows = await statement.values([]); + emitAsyncSqlDebug(this.debug, "scan", sql, startedAt, () => ({ + rowCount: rows.length, + })); + const schemaVersion = Number(rows[0]?.[0]); + if (!Number.isInteger(schemaVersion)) { + throw new Error("SQLite did not return a valid schema version"); + } + return schemaVersion; + } catch (error) { + emitAsyncSqlDebug(this.debug, "scan", sql, startedAt, () => ({}), error); + throw error; + } finally { + await statement.finalize(); + } + } + + private async writeSchemaMetadata( + tableDefinitions: TableDefinition[], + ): Promise { + const schemaVersion = await this.getSchemaVersion(); + await runAsyncSQL( + this.db, + upsertSqliteSchemaMetadataSQL(tableDefinitions.length), + tableDefinitions.flatMap((tableDef) => [ + tableDef.tableName, + createSqliteTableSchemaSignature(tableDef), + schemaVersion, + ]), + this.debug, + ); + } + + private installTableDefinitions( + tableDefinitions: TableDefinition[], + ): void { + for (const tableDef of tableDefinitions) { + this.tableDefinitions.set(tableDef.tableName, tableDef); } } @@ -722,15 +875,18 @@ export class AsyncSqlDriver implements DBDriver { await runAsyncSQL(this.db, sql, undefined, this.debug); } - private async getTableColumns(tableName: string): Promise> { - const columns = new Set(); + private async getTableColumns( + tableName: string, + ): Promise> { + const columns = new Map(); const sql = `PRAGMA table_info(${tableName})`; const startedAt = this.debug ? nowMs() : 0; const stmt = await this.db.prepare(sql); try { for (const row of await stmt.values([])) { - columns.add(String(row[1])); + const columnName = String(row[1]); + columns.set(sqliteIdentifierKey(columnName), columnName); } emitAsyncSqlDebug(this.debug, "scan", sql, startedAt, () => ({ tableName, @@ -797,7 +953,7 @@ export class AsyncSqlDriver implements DBDriver { ): Set { return new Set( getPersistentIndexPlan(tableDef).physicalIndexes.map((physicalIndex) => - sqliteIndexSortKeyColumn(physicalIndex.name), + sqliteIdentifierKey(sqliteIndexSortKeyColumn(physicalIndex.name)), ), ); } @@ -805,16 +961,35 @@ export class AsyncSqlDriver implements DBDriver { private getExpectedIndexNames(tableDef: TableDefinition): Set { return new Set( getPersistentIndexPlan(tableDef).physicalIndexes.map((physicalIndex) => - sqliteIndexIdentifier(tableDef.tableName, physicalIndex.name), + sqliteIdentifierKey( + sqliteIndexIdentifier(tableDef.tableName, physicalIndex.name), + ), ), ); } private isGeneratedIndexName(tableName: string, indexName: string): boolean { + const normalizedIndexName = sqliteIdentifierKey(indexName); return ( - indexName.startsWith(`idx_${tableName}_`) && - (indexName.endsWith(SQLITE_SORT_KEY_SUFFIX) || - indexName.endsWith(LEGACY_SQLITE_SORT_KEY_SUFFIX)) + normalizedIndexName.startsWith( + sqliteIdentifierKey(`idx_${tableName}_`), + ) && + (normalizedIndexName.endsWith(SQLITE_SORT_KEY_SUFFIX) || + normalizedIndexName.endsWith(LEGACY_SQLITE_SORT_KEY_SUFFIX)) + ); + } + + private physicalIndexForGeneratedIdentifier( + tableDef: TableDefinition, + generatedIndexName: string, + ) { + const normalizedGeneratedIndexName = + sqliteIdentifierKey(generatedIndexName); + return getPersistentIndexPlan(tableDef).physicalIndexes.find( + (physicalIndex) => + sqliteIdentifierKey( + sqliteIndexIdentifier(tableDef.tableName, physicalIndex.name), + ) === normalizedGeneratedIndexName, ); } @@ -825,13 +1000,9 @@ export class AsyncSqlDriver implements DBDriver { const expectedIndexes = this.getExpectedIndexNames(tableDef); for (const [indexName, unique] of indexUniqueness) { if (!this.isGeneratedIndexName(tableDef.tableName, indexName)) continue; - if (expectedIndexes.has(indexName)) { - const tableIndexName = this.tableIndexNameFromGenerated( - tableDef.tableName, - indexName, - ); + if (expectedIndexes.has(sqliteIdentifierKey(indexName))) { const expectedUnique = - getPersistentIndexPlan(tableDef).byLogicalName.get(tableIndexName) + this.physicalIndexForGeneratedIdentifier(tableDef, indexName) ?.unique ?? false; if (unique === expectedUnique) continue; } @@ -845,17 +1016,6 @@ export class AsyncSqlDriver implements DBDriver { } } - private tableIndexNameFromGenerated( - tableName: string, - generatedIndexName: string, - ): string { - const indexName = generatedIndexName.slice(`idx_${tableName}_`.length); - const suffix = indexName.endsWith(SQLITE_SORT_KEY_SUFFIX) - ? SQLITE_SORT_KEY_SUFFIX - : LEGACY_SQLITE_SORT_KEY_SUFFIX; - return indexName.slice(0, -suffix.length); - } - // Sort-key columns whose encoding changed because the index flipped between // uniqhash (value only) and hash/btree (value + id). Their existing values // are encoded with the old shape, so they must be recomputed for every row @@ -867,16 +1027,14 @@ export class AsyncSqlDriver implements DBDriver { const columns: string[] = []; for (const [indexName, unique] of indexUniqueness) { if (!this.isGeneratedIndexName(tableDef.tableName, indexName)) continue; - const tableIndexName = this.tableIndexNameFromGenerated( - tableDef.tableName, + const physicalIndex = this.physicalIndexForGeneratedIdentifier( + tableDef, indexName, ); - const physicalIndex = - getPersistentIndexPlan(tableDef).byLogicalName.get(tableIndexName); if (!physicalIndex) continue; const expectedUnique = physicalIndex.unique; if (unique !== expectedUnique) { - columns.push(sqliteIndexSortKeyColumn(tableIndexName)); + columns.push(sqliteIndexSortKeyColumn(physicalIndex.name)); } } return columns; @@ -885,10 +1043,10 @@ export class AsyncSqlDriver implements DBDriver { private async resetSortKeyColumns( tableDef: TableDefinition, sortKeyColumns: string[], + existingColumns: Map, ): Promise { - const existingColumns = await this.getTableColumns(tableDef.tableName); for (const sortKeyColumn of sortKeyColumns) { - if (!existingColumns.has(sortKeyColumn)) continue; + if (!existingColumns.has(sqliteIdentifierKey(sortKeyColumn))) continue; await runAsyncSQL( this.db, `UPDATE ${tableDef.tableName} SET ${sortKeyColumn} = NULL`, @@ -900,11 +1058,12 @@ export class AsyncSqlDriver implements DBDriver { private async dropStaleSortKeyColumns( tableDef: TableDefinition, + existingColumns: Map, ): Promise { const expectedColumns = this.getExpectedSortKeyColumns(tableDef); - for (const columnName of await this.getTableColumns(tableDef.tableName)) { + for (const [normalizedColumnName, columnName] of existingColumns) { if (!isSqliteSortKeyColumn(columnName)) continue; - if (expectedColumns.has(columnName)) continue; + if (expectedColumns.has(normalizedColumnName)) continue; await runAsyncSQL( this.db, @@ -912,21 +1071,23 @@ export class AsyncSqlDriver implements DBDriver { undefined, this.debug, ); + existingColumns.delete(normalizedColumnName); } } private async addMissingSortKeyColumns( tableDef: TableDefinition, + existingColumns: Map, ): Promise { - const existingColumns = await this.getTableColumns(tableDef.tableName); for (const physicalIndex of getPersistentIndexPlan(tableDef) .physicalIndexes) { const sortKeyColumn = sqliteIndexSortKeyColumn(physicalIndex.name); - if (existingColumns.has(sortKeyColumn)) continue; + const normalizedSortKeyColumn = sqliteIdentifierKey(sortKeyColumn); + if (existingColumns.has(normalizedSortKeyColumn)) continue; const sql = addSortKeyColumnSQL(tableDef.tableName, sortKeyColumn); await runAsyncSQL(this.db, sql, undefined, this.debug); - existingColumns.add(sortKeyColumn); + existingColumns.set(normalizedSortKeyColumn, sortKeyColumn); } } diff --git a/packages/hyperdb/src/hyperdb/drivers/sqlite/driver-edge-cases.test.ts b/packages/hyperdb/src/hyperdb/drivers/sqlite/driver-edge-cases.test.ts index 89e603c..6f8f74d 100644 --- a/packages/hyperdb/src/hyperdb/drivers/sqlite/driver-edge-cases.test.ts +++ b/packages/hyperdb/src/hyperdb/drivers/sqlite/driver-edge-cases.test.ts @@ -9,10 +9,15 @@ import { import { createInspectableSqlDriver, createSqlJsDriver, + createSqlJsDriverFromDatabase, type InspectableSqlDatabase, } from "../../test-utils/sql-js-driver"; import { v } from "../../schema/values"; -import { buildSortKeyWhereClause, type SqlValue } from "./sqlite-common"; +import { + buildSortKeyWhereClause, + SQLITE_SCHEMA_METADATA_TABLE, + type SqlValue, +} from "./sqlite-common"; const noSideTablesTable = defineTable("driverEdgeNoSideTables", { id: v.string(), @@ -89,6 +94,158 @@ function sqliteRows(sqldb: InspectableSqlDatabase, sql: string): SqlValue[][] { } describe("SQLite driver edge case regressions", () => { + it("uses one metadata read when loaded table schemas are unchanged", async () => { + const { driver, execLog } = await createInspectableSqlDriver(); + const db = new SyncDB(new DB(driver)); + + db.loadTables([noSideTablesTable]); + execLog.length = 0; + db.loadTables([noSideTablesTable]); + + expect(execLog).toHaveLength(1); + expect(execLog[0]).toContain(`LEFT JOIN ${SQLITE_SCHEMA_METADATA_TABLE}`); + expect(execLog[0]).not.toContain("BEGIN TRANSACTION"); + }); + + it("initializes schema metadata when the adapter hides missing-table details", async () => { + const { sqldb } = await createInspectableSqlDriver(); + let obscureMetadataReadError = true; + const { driver, execLog } = createSqlJsDriverFromDatabase(sqldb, [], { + beforePrepare(sql) { + if ( + obscureMetadataReadError && + sql.includes(`LEFT JOIN ${SQLITE_SCHEMA_METADATA_TABLE}`) + ) { + throw new Error("statement preparation failed"); + } + }, + }); + const db = new SyncDB(new DB(driver)); + + db.loadTables([noSideTablesTable]); + expect( + sqliteRows( + sqldb, + `SELECT name FROM sqlite_schema WHERE name = '${SQLITE_SCHEMA_METADATA_TABLE}'`, + ), + ).toEqual([[SQLITE_SCHEMA_METADATA_TABLE]]); + + expect(() => db.loadTables([noSideTablesTable])).toThrow( + "statement preparation failed", + ); + + obscureMetadataReadError = false; + execLog.length = 0; + db.loadTables([noSideTablesTable]); + expect(execLog).toHaveLength(1); + expect(execLog[0]).toContain(`LEFT JOIN ${SQLITE_SCHEMA_METADATA_TABLE}`); + }); + + it("refreshes per-table metadata after another table changes the schema", async () => { + const { driver, execLog } = await createInspectableSqlDriver(); + const db = new SyncDB(new DB(driver)); + + db.loadTables([noSideTablesTable]); + db.loadTables([manyPrefixRangesTable]); + + execLog.length = 0; + db.loadTables([noSideTablesTable]); + expect(execLog).toContain("BEGIN TRANSACTION"); + + execLog.length = 0; + db.loadTables([noSideTablesTable]); + expect(execLog).toHaveLength(1); + expect(execLog[0]).toContain(`LEFT JOIN ${SQLITE_SCHEMA_METADATA_TABLE}`); + }); + + it("repairs an externally dropped index despite matching table signatures", async () => { + const { driver, sqldb, execLog } = await createInspectableSqlDriver(); + const db = new SyncDB(new DB(driver)); + db.loadTables([noSideTablesTable]); + + sqldb.run("DROP INDEX idx_driverEdgeNoSideTables_byTitle_sort_key_v2"); + execLog.length = 0; + db.loadTables([noSideTablesTable]); + + expect(execLog).toContain("BEGIN TRANSACTION"); + expect( + execLog.some((sql) => + sql.startsWith( + "CREATE INDEX IF NOT EXISTS idx_driverEdgeNoSideTables_byTitle_sort_key_v2", + ), + ), + ).toBe(true); + expect( + sqliteRows(sqldb, "PRAGMA index_list(driverEdgeNoSideTables)").some( + (row) => + String(row[1]) === "idx_driverEdgeNoSideTables_byTitle_sort_key_v2", + ), + ).toBe(true); + }); + + it("treats SQLite index identifiers as case-insensitive", async () => { + const { driver, sqldb, execLog } = await createInspectableSqlDriver(); + const db = new SyncDB(new DB(driver)); + db.loadTables([noSideTablesTable]); + + sqldb.run("DROP INDEX idx_driverEdgeNoSideTables_byTitle_sort_key_v2"); + sqldb.run( + "CREATE INDEX idx_driveredgenosidetables_bytitle_sort_key_v2 ON driverEdgeNoSideTables(idx_byTitle_sort_key_v2, id) WHERE idx_byTitle_sort_key_v2 IS NOT NULL", + ); + execLog.length = 0; + + db.loadTables([noSideTablesTable]); + + expect(execLog.some((sql) => sql.startsWith("DROP INDEX"))).toBe(false); + expect( + sqliteRows(sqldb, "PRAGMA index_list(driverEdgeNoSideTables)").some( + (row) => + String(row[1]) === "idx_driveredgenosidetables_bytitle_sort_key_v2", + ), + ).toBe(true); + }); + + it("commits schema metadata only after reconciliation succeeds", async () => { + const { sqldb } = await createInspectableSqlDriver(); + const initialDb = new SyncDB( + new DB(createSqlJsDriverFromDatabase(sqldb).driver), + ); + initialDb.loadTables([sortKeyBackfillTableV1]); + const metadataBefore = sqliteRows( + sqldb, + `SELECT table_name, signature, verified_schema_version FROM ${SQLITE_SCHEMA_METADATA_TABLE}`, + ); + + const failingDriver = createSqlJsDriverFromDatabase(sqldb, [], { + beforeExec(sql) { + if (sql.includes("ADD COLUMN idx_byTitle_sort_key_v2")) { + throw new Error("injected schema failure"); + } + }, + }).driver; + const failingDb = new SyncDB(new DB(failingDriver)); + + expect(() => failingDb.loadTables([sortKeyBackfillTableV2])).toThrow( + "injected schema failure", + ); + expect( + sqliteRows( + sqldb, + `SELECT table_name, signature, verified_schema_version FROM ${SQLITE_SCHEMA_METADATA_TABLE}`, + ), + ).toEqual(metadataBefore); + + const recoveredDb = new SyncDB( + new DB(createSqlJsDriverFromDatabase(sqldb).driver), + ); + recoveredDb.loadTables([sortKeyBackfillTableV2]); + expect( + sqliteRows(sqldb, "PRAGMA table_info(driverEdgeSortKeyBackfill)").some( + (row) => String(row[1]) === "idx_byTitle_sort_key_v2", + ), + ).toBe(true); + }); + it("preserves physical indexes whose logical names end in _sort_key", async () => { const { driver, execLog } = await createInspectableSqlDriver(); const db = new SyncDB(new DB(driver)); @@ -406,7 +563,10 @@ describe("SQLite driver edge case regressions", () => { sqldb, "SELECT name FROM sqlite_master WHERE type = 'table' ORDER BY name", ).map(([name]) => String(name)); - expect(tableNames).toEqual(["driverEdgeNoSideTables"]); + expect(tableNames).toEqual([ + SQLITE_SCHEMA_METADATA_TABLE, + "driverEdgeNoSideTables", + ]); expect(tableNames.some((name) => name.endsWith("__idx"))).toBe(false); const columns = sqliteRows( diff --git a/packages/hyperdb/src/hyperdb/drivers/sqlite/sql-driver.ts b/packages/hyperdb/src/hyperdb/drivers/sqlite/sql-driver.ts index 3e94221..e76df2f 100644 --- a/packages/hyperdb/src/hyperdb/drivers/sqlite/sql-driver.ts +++ b/packages/hyperdb/src/hyperdb/drivers/sqlite/sql-driver.ts @@ -25,6 +25,13 @@ import { isSqliteSortKeyColumn, SQLITE_SORT_KEY_SUFFIX, LEGACY_SQLITE_SORT_KEY_SUFFIX, + createSqliteTableSchemaSignature, + createSqliteSchemaMetadataTableSQL, + selectSqliteSchemaMetadataSQL, + selectSqliteSchemaMetadataTableExistsSQL, + upsertSqliteSchemaMetadataSQL, + isMissingSqliteSchemaMetadataError, + sqliteIdentifierKey, assertSafeTableDefinition, buildRowInsertParams, parseSqliteStoredRow, @@ -341,9 +348,17 @@ export class SqlDriver implements DBDriver { assertSafeTableDefinition(tableDef); } + tableDefinitions = cloneDeep(tableDefinitions); + if (tableDefinitions.length === 0) return; + + if (this.schemaMetadataMatches(tableDefinitions)) { + this.installTableDefinitions(tableDefinitions); + return; + } + this.db.exec("BEGIN TRANSACTION"); try { - tableDefinitions = cloneDeep(tableDefinitions); + this.db.exec(createSqliteSchemaMetadataTableSQL()); for (const tableDef of tableDefinitions) { this.createTable(tableDef); const indexUniqueness = this.getGeneratedIndexUniqueness( @@ -353,30 +368,132 @@ export class SqlDriver implements DBDriver { tableDef, indexUniqueness, ); + const existingColumns = this.getTableColumns(tableDef.tableName); this.dropStaleSortKeyIndexes(tableDef, indexUniqueness); - this.dropStaleSortKeyColumns(tableDef); - this.addMissingSortKeyColumns(tableDef); - this.resetSortKeyColumns(tableDef, reencodedColumns); + this.dropStaleSortKeyColumns(tableDef, existingColumns); + this.addMissingSortKeyColumns(tableDef, existingColumns); + this.resetSortKeyColumns(tableDef, reencodedColumns, existingColumns); this.backfillSortKeyColumns(tableDef); this.createIndexes(tableDef); - this.tableDefinitions.set(tableDef.tableName, tableDef); } + this.writeSchemaMetadata(tableDefinitions); this.db.exec("COMMIT"); + this.installTableDefinitions(tableDefinitions); } catch (error) { rollbackQuietly(this.db); throw error; } } + private schemaMetadataMatches( + tableDefinitions: TableDefinition[], + ): boolean { + const sql = selectSqliteSchemaMetadataSQL(tableDefinitions.length); + let statement: SQLStatement | undefined; + + try { + statement = this.db.prepare(sql); + const rows = statement.values( + tableDefinitions.map((tableDef) => tableDef.tableName), + ); + const schemaVersion = Number(rows[0]?.[3]); + if (!Number.isInteger(schemaVersion)) return false; + + const metadataByTable = new Map( + rows + .filter((row) => typeof row[0] === "string") + .map((row) => [ + String(row[0]), + { + signature: String(row[1]), + verifiedSchemaVersion: Number(row[2]), + }, + ]), + ); + + return tableDefinitions.every((tableDef) => { + const metadata = metadataByTable.get(tableDef.tableName); + return ( + metadata?.signature === createSqliteTableSchemaSignature(tableDef) && + metadata.verifiedSchemaVersion === schemaVersion + ); + }); + } catch (error) { + if (isMissingSqliteSchemaMetadataError(error)) return false; + statement?.finalize(); + statement = undefined; + try { + if (!this.schemaMetadataTableExists()) return false; + } catch { + // Preserve the original metadata-read error when even the fallback + // schema inspection cannot run. + } + throw error; + } finally { + statement?.finalize(); + } + } + + private schemaMetadataTableExists(): boolean { + const statement = this.db.prepare( + selectSqliteSchemaMetadataTableExistsSQL(), + ); + try { + return statement.values([]).length > 0; + } finally { + statement.finalize(); + } + } + + private getSchemaVersion(): number { + const statement = this.db.prepare( + "SELECT schema_version FROM pragma_schema_version", + ); + try { + const schemaVersion = Number(statement.values([])[0]?.[0]); + if (!Number.isInteger(schemaVersion)) { + throw new Error("SQLite did not return a valid schema version"); + } + return schemaVersion; + } finally { + statement.finalize(); + } + } + + private writeSchemaMetadata(tableDefinitions: TableDefinition[]): void { + const schemaVersion = this.getSchemaVersion(); + this.db.exec( + upsertSqliteSchemaMetadataSQL(tableDefinitions.length), + tableDefinitions.flatMap((tableDef) => [ + tableDef.tableName, + createSqliteTableSchemaSignature(tableDef), + schemaVersion, + ]), + ); + } + + private installTableDefinitions( + tableDefinitions: TableDefinition[], + ): void { + for (const tableDef of tableDefinitions) { + this.tableDefinitions.set(tableDef.tableName, tableDef); + } + } + private createTable(tableDef: TableDefinition): void { const sql = createTableSQL(tableDef); this.db.exec(sql); } - private getTableColumns(tableName: string): Set { + private getTableColumns(tableName: string): Map { const q = this.db.prepare(`PRAGMA table_info(${tableName})`); try { - return new Set(q.values([]).map((row) => String(row[1]))); + return new Map( + q.values([]).map((row) => { + const columnName = String(row[1]); + return [sqliteIdentifierKey(columnName), columnName]; + }), + ); } finally { q.finalize(); } @@ -398,7 +515,7 @@ export class SqlDriver implements DBDriver { ): Set { return new Set( getPersistentIndexPlan(tableDef).physicalIndexes.map((physicalIndex) => - sqliteIndexSortKeyColumn(physicalIndex.name), + sqliteIdentifierKey(sqliteIndexSortKeyColumn(physicalIndex.name)), ), ); } @@ -406,16 +523,35 @@ export class SqlDriver implements DBDriver { private getExpectedIndexNames(tableDef: TableDefinition): Set { return new Set( getPersistentIndexPlan(tableDef).physicalIndexes.map((physicalIndex) => - sqliteIndexIdentifier(tableDef.tableName, physicalIndex.name), + sqliteIdentifierKey( + sqliteIndexIdentifier(tableDef.tableName, physicalIndex.name), + ), ), ); } private isGeneratedIndexName(tableName: string, indexName: string): boolean { + const normalizedIndexName = sqliteIdentifierKey(indexName); return ( - indexName.startsWith(`idx_${tableName}_`) && - (indexName.endsWith(SQLITE_SORT_KEY_SUFFIX) || - indexName.endsWith(LEGACY_SQLITE_SORT_KEY_SUFFIX)) + normalizedIndexName.startsWith( + sqliteIdentifierKey(`idx_${tableName}_`), + ) && + (normalizedIndexName.endsWith(SQLITE_SORT_KEY_SUFFIX) || + normalizedIndexName.endsWith(LEGACY_SQLITE_SORT_KEY_SUFFIX)) + ); + } + + private physicalIndexForGeneratedIdentifier( + tableDef: TableDefinition, + generatedIndexName: string, + ) { + const normalizedGeneratedIndexName = + sqliteIdentifierKey(generatedIndexName); + return getPersistentIndexPlan(tableDef).physicalIndexes.find( + (physicalIndex) => + sqliteIdentifierKey( + sqliteIndexIdentifier(tableDef.tableName, physicalIndex.name), + ) === normalizedGeneratedIndexName, ); } @@ -426,13 +562,9 @@ export class SqlDriver implements DBDriver { const expectedIndexes = this.getExpectedIndexNames(tableDef); for (const [indexName, unique] of indexUniqueness) { if (!this.isGeneratedIndexName(tableDef.tableName, indexName)) continue; - if (expectedIndexes.has(indexName)) { - const tableIndexName = this.tableIndexNameFromGenerated( - tableDef.tableName, - indexName, - ); + if (expectedIndexes.has(sqliteIdentifierKey(indexName))) { const expectedUnique = - getPersistentIndexPlan(tableDef).byLogicalName.get(tableIndexName) + this.physicalIndexForGeneratedIdentifier(tableDef, indexName) ?.unique ?? false; if (unique === expectedUnique) continue; } @@ -441,17 +573,6 @@ export class SqlDriver implements DBDriver { } } - private tableIndexNameFromGenerated( - tableName: string, - generatedIndexName: string, - ): string { - const indexName = generatedIndexName.slice(`idx_${tableName}_`.length); - const suffix = indexName.endsWith(SQLITE_SORT_KEY_SUFFIX) - ? SQLITE_SORT_KEY_SUFFIX - : LEGACY_SQLITE_SORT_KEY_SUFFIX; - return indexName.slice(0, -suffix.length); - } - // Sort-key columns whose encoding changed because the index flipped between // uniqhash (value only) and hash/btree (value + id). Their existing values // are encoded with the old shape, so they must be recomputed for every row @@ -463,16 +584,14 @@ export class SqlDriver implements DBDriver { const columns: string[] = []; for (const [indexName, unique] of indexUniqueness) { if (!this.isGeneratedIndexName(tableDef.tableName, indexName)) continue; - const tableIndexName = this.tableIndexNameFromGenerated( - tableDef.tableName, + const physicalIndex = this.physicalIndexForGeneratedIdentifier( + tableDef, indexName, ); - const physicalIndex = - getPersistentIndexPlan(tableDef).byLogicalName.get(tableIndexName); if (!physicalIndex) continue; const expectedUnique = physicalIndex.unique; if (unique !== expectedUnique) { - columns.push(sqliteIndexSortKeyColumn(tableIndexName)); + columns.push(sqliteIndexSortKeyColumn(physicalIndex.name)); } } return columns; @@ -481,34 +600,41 @@ export class SqlDriver implements DBDriver { private resetSortKeyColumns( tableDef: TableDefinition, sortKeyColumns: string[], + existingColumns: Map, ): void { - const existingColumns = this.getTableColumns(tableDef.tableName); for (const sortKeyColumn of sortKeyColumns) { - if (!existingColumns.has(sortKeyColumn)) continue; + if (!existingColumns.has(sqliteIdentifierKey(sortKeyColumn))) continue; this.db.exec(`UPDATE ${tableDef.tableName} SET ${sortKeyColumn} = NULL`); } } - private dropStaleSortKeyColumns(tableDef: TableDefinition): void { + private dropStaleSortKeyColumns( + tableDef: TableDefinition, + existingColumns: Map, + ): void { const expectedColumns = this.getExpectedSortKeyColumns(tableDef); - for (const columnName of this.getTableColumns(tableDef.tableName)) { + for (const [normalizedColumnName, columnName] of existingColumns) { if (!isSqliteSortKeyColumn(columnName)) continue; - if (expectedColumns.has(columnName)) continue; + if (expectedColumns.has(normalizedColumnName)) continue; this.db.exec(dropSortKeyColumnSQL(tableDef.tableName, columnName)); + existingColumns.delete(normalizedColumnName); } } - private addMissingSortKeyColumns(tableDef: TableDefinition): void { - const existingColumns = this.getTableColumns(tableDef.tableName); + private addMissingSortKeyColumns( + tableDef: TableDefinition, + existingColumns: Map, + ): void { for (const physicalIndex of getPersistentIndexPlan(tableDef) .physicalIndexes) { const sortKeyColumn = sqliteIndexSortKeyColumn(physicalIndex.name); - if (existingColumns.has(sortKeyColumn)) continue; + const normalizedSortKeyColumn = sqliteIdentifierKey(sortKeyColumn); + if (existingColumns.has(normalizedSortKeyColumn)) continue; const sql = addSortKeyColumnSQL(tableDef.tableName, sortKeyColumn); this.db.exec(sql); - existingColumns.add(sortKeyColumn); + existingColumns.set(normalizedSortKeyColumn, sortKeyColumn); } } diff --git a/packages/hyperdb/src/hyperdb/drivers/sqlite/sqlite-common.ts b/packages/hyperdb/src/hyperdb/drivers/sqlite/sqlite-common.ts index b6f564d..928bba9 100644 --- a/packages/hyperdb/src/hyperdb/drivers/sqlite/sqlite-common.ts +++ b/packages/hyperdb/src/hyperdb/drivers/sqlite/sqlite-common.ts @@ -30,6 +30,8 @@ export const SQL_BIND_PARAM_LIMIT = 900; const SAFE_IDENTIFIER = /^[A-Za-z_][A-Za-z0-9_]*$/; export const SQLITE_SORT_KEY_SUFFIX = "_sort_key_v2"; export const LEGACY_SQLITE_SORT_KEY_SUFFIX = "_sort_key"; +export const SQLITE_SCHEMA_METADATA_TABLE = "_hyperdb_schema_metadata"; +const SQLITE_SCHEMA_SIGNATURE_VERSION = 1; export function chunkArray(array: T[], size: number): T[][] { const chunks: T[][] = []; @@ -61,12 +63,70 @@ export function assertSafeIdentifier(kind: string, value: string): void { export function assertSafeTableDefinition(tableDef: TableDefinition): void { assertSafeIdentifier("Table name", tableDef.tableName); + if ( + sqliteIdentifierKey(tableDef.tableName) === + sqliteIdentifierKey(SQLITE_SCHEMA_METADATA_TABLE) + ) { + throw new Error( + `Table name is reserved for HyperDB SQLite metadata: ${tableDef.tableName}`, + ); + } for (const indexName of Object.keys(tableDef.indexes)) { assertSafeIdentifier("Index name", indexName); } } +export function sqliteIdentifierKey(identifier: string): string { + return identifier.toLowerCase(); +} + +export function createSqliteTableSchemaSignature( + tableDef: TableDefinition, +): string { + return JSON.stringify({ + version: SQLITE_SCHEMA_SIGNATURE_VERSION, + layout: "inline-sort-keys", + tableName: tableDef.tableName, + physicalIndexes: getPersistentIndexPlan(tableDef).physicalIndexes.map( + ({ name, sortColumns, unique, mode }) => ({ + name, + sortColumns, + unique, + mode, + }), + ), + }); +} + +export function createSqliteSchemaMetadataTableSQL(): string { + return `CREATE TABLE IF NOT EXISTS ${SQLITE_SCHEMA_METADATA_TABLE} ( table_name TEXT PRIMARY KEY, signature TEXT NOT NULL, verified_schema_version INTEGER NOT NULL )`; +} + +export function selectSqliteSchemaMetadataSQL(tableCount: number): string { + const placeholders = Array(tableCount).fill("?").join(", "); + return `SELECT metadata.table_name, metadata.signature, metadata.verified_schema_version, schema_state.schema_version FROM pragma_schema_version AS schema_state LEFT JOIN ${SQLITE_SCHEMA_METADATA_TABLE} AS metadata ON metadata.table_name IN (${placeholders})`; +} + +export function selectSqliteSchemaMetadataTableExistsSQL(): string { + return `SELECT 1 FROM sqlite_schema WHERE type = 'table' AND name = '${SQLITE_SCHEMA_METADATA_TABLE}' COLLATE NOCASE LIMIT 1`; +} + +export function upsertSqliteSchemaMetadataSQL(tableCount: number): string { + const values = Array(tableCount).fill("(?, ?, ?)").join(", "); + return `INSERT INTO ${SQLITE_SCHEMA_METADATA_TABLE} (table_name, signature, verified_schema_version) VALUES ${values} ON CONFLICT(table_name) DO UPDATE SET signature = excluded.signature, verified_schema_version = excluded.verified_schema_version`; +} + +export function isMissingSqliteSchemaMetadataError(error: unknown): boolean { + const message = error instanceof Error ? error.message : String(error); + return ( + message.toLowerCase().includes("no such table") && + message + .toLowerCase() + .includes(sqliteIdentifierKey(SQLITE_SCHEMA_METADATA_TABLE)) + ); +} + export function sqliteIndexSortKeyColumn(indexName: string): string { assertSafeIdentifier("Index name", indexName); const columnName = `idx_${indexName}${SQLITE_SORT_KEY_SUFFIX}`; @@ -86,10 +146,11 @@ export function sqliteIndexIdentifier( } export function isSqliteSortKeyColumn(columnName: string): boolean { + const normalizedColumnName = sqliteIdentifierKey(columnName); return ( - columnName.startsWith("idx_") && - (columnName.endsWith(SQLITE_SORT_KEY_SUFFIX) || - columnName.endsWith(LEGACY_SQLITE_SORT_KEY_SUFFIX)) + normalizedColumnName.startsWith("idx_") && + (normalizedColumnName.endsWith(SQLITE_SORT_KEY_SUFFIX) || + normalizedColumnName.endsWith(LEGACY_SQLITE_SORT_KEY_SUFFIX)) ); } diff --git a/packages/hyperdb/src/hyperdb/runtime/edge-cases.test.ts b/packages/hyperdb/src/hyperdb/runtime/edge-cases.test.ts index 854d7e2..72e872c 100644 --- a/packages/hyperdb/src/hyperdb/runtime/edge-cases.test.ts +++ b/packages/hyperdb/src/hyperdb/runtime/edge-cases.test.ts @@ -366,11 +366,13 @@ describe("runtime edge case regressions", () => { expect(await scanOrderIds(mixedOrderTable, rows, "asc")).toEqual({ SqlDriver: asc, + "AsyncSqlDriver (Turso WASM)": asc, BptreeInmemDriver: asc, IdbDriver: asc, }); expect(await scanOrderIds(mixedOrderTable, rows, "desc")).toEqual({ SqlDriver: desc, + "AsyncSqlDriver (Turso WASM)": desc, BptreeInmemDriver: desc, IdbDriver: desc, }); @@ -414,11 +416,13 @@ describe("runtime edge case regressions", () => { expect(await scanOrderIds(fullValueOrderTable, rows, "asc")).toEqual({ SqlDriver: asc, + "AsyncSqlDriver (Turso WASM)": asc, BptreeInmemDriver: asc, IdbDriver: asc, }); expect(await scanOrderIds(fullValueOrderTable, rows, "desc")).toEqual({ SqlDriver: [...asc].reverse(), + "AsyncSqlDriver (Turso WASM)": [...asc].reverse(), BptreeInmemDriver: [...asc].reverse(), IdbDriver: [...asc].reverse(), }); @@ -518,6 +522,7 @@ describe("runtime edge case regressions", () => { await scanOrderIds(testCase.table, testCase.rows, "asc"), ).toEqual({ SqlDriver: testCase.asc, + "AsyncSqlDriver (Turso WASM)": testCase.asc, BptreeInmemDriver: testCase.asc, IdbDriver: testCase.asc, }); @@ -525,6 +530,7 @@ describe("runtime edge case regressions", () => { await scanOrderIds(testCase.table, testCase.rows, "desc"), ).toEqual({ SqlDriver: testCase.desc, + "AsyncSqlDriver (Turso WASM)": testCase.desc, BptreeInmemDriver: testCase.desc, IdbDriver: testCase.desc, }); diff --git a/packages/hyperdb/src/hyperdb/test-utils/driver-factories.ts b/packages/hyperdb/src/hyperdb/test-utils/driver-factories.ts index 0731bb4..966685b 100644 --- a/packages/hyperdb/src/hyperdb/test-utils/driver-factories.ts +++ b/packages/hyperdb/src/hyperdb/test-utils/driver-factories.ts @@ -3,6 +3,7 @@ import type { DBDriver } from "../core/driver"; import { BptreeInmemDriver } from "../drivers/inmemory/bptree-inmem-driver"; import { IdbDriver, openIndexedDBDriver } from "../drivers/idb/idb-driver"; import { createSqlJsDriver } from "./sql-js-driver"; +import { createTursoWasmDriver } from "./turso-wasm-driver"; export type DriverFactory = [string, () => Promise]; @@ -48,6 +49,7 @@ export function createDriverFactories(options?: { if (options?.includeSql !== false) { factories.push(["SqlDriver", () => createSqlJsDriver()]); + factories.push(["AsyncSqlDriver (Turso WASM)", createTursoWasmDriver]); } factories.push(["BptreeInmemDriver", async () => new BptreeInmemDriver()]); diff --git a/packages/hyperdb/src/hyperdb/test-utils/sql-js-driver.ts b/packages/hyperdb/src/hyperdb/test-utils/sql-js-driver.ts index f079f8c..e41d999 100644 --- a/packages/hyperdb/src/hyperdb/test-utils/sql-js-driver.ts +++ b/packages/hyperdb/src/hyperdb/test-utils/sql-js-driver.ts @@ -32,6 +32,11 @@ export type InspectableSqlDatabase = { }; }; +export type SqlJsDriverHooks = { + beforeExec?: (sql: string) => void; + beforePrepare?: (sql: string) => void; +}; + export async function createSqlJsDriver(): Promise { const SQL = await initSqlJs({ locateFile: () => normalizeWasmUrl(wasmUrl), @@ -44,6 +49,7 @@ export async function createSqlJsDriver(): Promise { export function createSqlJsDriverFromDatabase( sqldb: InspectableSqlDatabase, execLog?: string[], + hooks: SqlJsDriverHooks = {}, ): { driver: SqlDriver; sqldb: InspectableSqlDatabase; execLog: string[] } { const log = execLog ?? []; @@ -53,10 +59,12 @@ export function createSqlJsDriverFromDatabase( driver: new SqlDriver({ exec(sql: string, params?: SqlValue[]): void { log.push(sql); + hooks.beforeExec?.(sql); sqldb.exec(sql, params); }, prepare(sql: string): SQLStatement { log.push(sql); + hooks.beforePrepare?.(sql); const prepared = sqldb.prepare(sql); return { @@ -81,16 +89,28 @@ export function createSqlJsDriverFromDatabase( class SqlJsAsyncAdapter implements AsyncSQLiteDB { private sqldb: InspectableSqlDatabase; - - constructor(sqldb: InspectableSqlDatabase) { + private execLog?: string[]; + private hooks: SqlJsDriverHooks; + + constructor( + sqldb: InspectableSqlDatabase, + execLog?: string[], + hooks: SqlJsDriverHooks = {}, + ) { this.sqldb = sqldb; + this.execLog = execLog; + this.hooks = hooks; } async exec(sql: string, params?: SqlValue[] | null): Promise { + this.execLog?.push(sql); + this.hooks.beforeExec?.(sql); this.sqldb.exec(sql, params ?? undefined); } async prepare(sql: string) { + this.execLog?.push(sql); + this.hooks.beforePrepare?.(sql); const prepared = this.sqldb.prepare(sql); return { @@ -134,3 +154,27 @@ export async function createInspectableSqlDriver(): Promise<{ return createSqlJsDriverFromDatabase(sqldb); } + +export async function createInspectableSqlAsyncDriver( + options: AsyncSqlDriverOptions = {}, + hooks: SqlJsDriverHooks = {}, +): Promise<{ + driver: AsyncSqlDriver; + sqldb: InspectableSqlDatabase; + execLog: string[]; +}> { + const SQL = await initSqlJs({ + locateFile: () => normalizeWasmUrl(wasmUrl), + }); + const sqldb: InspectableSqlDatabase = new SQL.Database(); + const execLog: string[] = []; + + return { + driver: new AsyncSqlDriver( + new SqlJsAsyncAdapter(sqldb, execLog, hooks), + options, + ), + sqldb, + execLog, + }; +} diff --git a/packages/hyperdb/src/hyperdb/test-utils/turso-wasm-driver.ts b/packages/hyperdb/src/hyperdb/test-utils/turso-wasm-driver.ts new file mode 100644 index 0000000..0400da4 --- /dev/null +++ b/packages/hyperdb/src/hyperdb/test-utils/turso-wasm-driver.ts @@ -0,0 +1,66 @@ +import { afterEach } from "vitest"; +import { + connect, + type Database as TursoDatabase, +} from "@tursodatabase/database-wasm/bundle"; +import { + AsyncSqlDriver, + type AsyncSqlDriverOptions, + type AsyncSQLiteDB, + type SqlValue, +} from "../drivers/sqlite"; + +const openDatabases = new Set(); + +afterEach(async () => { + await Promise.all( + [...openDatabases].map(async (database) => { + await database.close(); + }), + ); + openDatabases.clear(); +}); + +class TursoWasmAsyncAdapter implements AsyncSQLiteDB { + private readonly database: TursoDatabase; + + constructor(database: TursoDatabase) { + this.database = database; + } + + async exec(sql: string, params?: SqlValue[]): Promise { + if (!params || params.length === 0) { + await this.database.exec(sql); + return; + } + + const statement = await this.database.prepare(sql); + try { + await statement.run(params); + } finally { + statement.close(); + } + } + + async prepare(sql: string) { + const statement = (await this.database.prepare(sql)).raw(true); + + return { + async values(values: SqlValue[]): Promise { + return (await statement.all(values)) as SqlValue[][]; + }, + finalize(): void { + statement.close(); + }, + }; + } +} + +export async function createTursoWasmDriver( + options: AsyncSqlDriverOptions = {}, +): Promise { + const database = await connect(":memory:"); + openDatabases.add(database); + + return new AsyncSqlDriver(new TursoWasmAsyncAdapter(database), options); +} diff --git a/packages/hyperdb/vite.config.ts b/packages/hyperdb/vite.config.ts index 688d413..1ef9ef1 100644 --- a/packages/hyperdb/vite.config.ts +++ b/packages/hyperdb/vite.config.ts @@ -6,6 +6,15 @@ import { visualizer } from "rollup-plugin-visualizer"; const shouldAnalyze = process.env.ANALYZE === "true"; export default defineConfig({ + define: { + "process.env.NODE_DEBUG_NATIVE": "false", + }, + server: { + headers: { + "Cross-Origin-Embedder-Policy": "require-corp", + "Cross-Origin-Opener-Policy": "same-origin", + }, + }, build: { lib: { entry: { @@ -46,6 +55,7 @@ export default defineConfig({ : [], test: { name: "browser", + testTimeout: 120_000, include: ["src/**/*.test.ts", "src/**/*.test.tsx"], exclude: [...configDefaults.exclude, "**/*.browser.test.ts", "e2e/**"], browser: { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index fd59c7d..1ee2cfb 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -27,6 +27,9 @@ importers: '@eslint/js': specifier: ^9.39.1 version: 9.39.4 + '@tursodatabase/database-wasm': + specifier: 0.7.1 + version: 0.7.1 '@types/react': specifier: 19.2.17 version: 19.2.17 @@ -417,12 +420,21 @@ packages: '@emnapi/core@1.10.0': resolution: {integrity: sha512-yq6OkJ4p82CAfPl0u9mQebQHKPJkY7WrIuk205cTYnYe+k2Z8YBh11FrbRG/H6ihirqcacOgl2BIO8oyMQLeXw==} + '@emnapi/core@1.8.1': + resolution: {integrity: sha512-AvT9QFpxK0Zd8J0jopedNm+w/2fIzvtPKPjqyw9jwvBaReTTqPBk9Hixaz7KbjimP+QNz605/XnjFcDAL2pqBg==} + '@emnapi/runtime@1.10.0': resolution: {integrity: sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA==} '@emnapi/runtime@1.11.1': resolution: {integrity: sha512-vgj7R3y3Wgx24IQaGPA/R6YFXLHVMOZ0uVEyIQPaWs+rd1AzfEMXlAC22FYwO1XkKR6NPsq7mUandH8oIRdZFw==} + '@emnapi/runtime@1.8.1': + resolution: {integrity: sha512-mehfKSMWjjNol8659Z8KxEMrdSJDDot5SXMq00dM8BN4o+CLNXQ0xH2V7EchNHV4RmbZLmmPdEaXZc5H2FXmDg==} + + '@emnapi/wasi-threads@1.1.0': + resolution: {integrity: sha512-WI0DdZ8xFSbgMjR1sFsKABJ/C5OnRrjT06JXbZKexJGrDuPTzZdDYfFlsgcCXCyf+suG5QU2e/y1Wo2V/OapLQ==} + '@emnapi/wasi-threads@1.2.1': resolution: {integrity: sha512-uTII7OYF+/Mes/MrcIOYp5yOtSMLBWSIoLPpcgwipoiKbli6k322tcoFsxoIIxPDqW01SQGAgko4EzZi2BNv2w==} @@ -868,6 +880,9 @@ packages: '@mdx-js/mdx@3.1.1': resolution: {integrity: sha512-f6ZO2ifpwAQIpzGWaBQT2TXxPv6z3RBzQKpVftEWN78Vl/YweF1uwussDx8ECAXVtr3Rs89fKyG9YlzUs9DyGQ==} + '@napi-rs/wasm-runtime@1.0.7': + resolution: {integrity: sha512-SeDnOO0Tk7Okiq6DbXmmBODgOAb9dp9gjlphokTUxmt8U3liIP1ZsozBahH69j/RJv+Rfs6IwUKHTgQYJ/HBAw==} + '@napi-rs/wasm-runtime@1.1.5': resolution: {integrity: sha512-AWPoBRJ9tsnVhor4sjO7rkni+7p+2IAEFj6cx06UgP10jkQHqay/36uRV/bFkgrh18D9vb4cr8Q0Pthskgzy+Q==} peerDependencies: @@ -1346,6 +1361,15 @@ packages: peerDependencies: '@tanstack/ai': ^0.32.0 + '@tursodatabase/database-common@0.7.2': + resolution: {integrity: sha512-c4uWaA5m7nyDemUrjvX6lQQ89cBxv1jCv7nmwq5JoagyXgbMxDeBfLcY7N5kn0oEcnD0Y+cf09S0wKEXKFKbfg==} + + '@tursodatabase/database-wasm-common@0.7.2': + resolution: {integrity: sha512-6i4zEHg3DoqUtI48sunSHKVg+TgkubQwrl3MQWjnVGCbidjCg/ULd+6n9qaebHYbTh/p3Sm9OssctbdW1wEigg==} + + '@tursodatabase/database-wasm@0.7.1': + resolution: {integrity: sha512-KyoHKDzOgo+xgvXLQyqxmuCHSY3C4UraycyVNodl+Qa0tl0TcF0/6z0FzkofAKI2Ja0fuJWX+dnDy0kBxdzvbA==} + '@tybys/wasm-util@0.10.2': resolution: {integrity: sha512-RoBvJ2X0wuKlWFIjrwffGw1IqZHKQqzIchKaadZZfnNpsAYp2mM0h36JtPCjNDAHGgYez/15uMBpfGwchhiMgg==} @@ -4602,7 +4626,11 @@ snapshots: dependencies: '@emnapi/wasi-threads': 1.2.1 tslib: 2.8.1 - optional: true + + '@emnapi/core@1.8.1': + dependencies: + '@emnapi/wasi-threads': 1.1.0 + tslib: 2.8.1 '@emnapi/runtime@1.10.0': dependencies: @@ -4612,12 +4640,18 @@ snapshots: '@emnapi/runtime@1.11.1': dependencies: tslib: 2.8.1 - optional: true + + '@emnapi/runtime@1.8.1': + dependencies: + tslib: 2.8.1 + + '@emnapi/wasi-threads@1.1.0': + dependencies: + tslib: 2.8.1 '@emnapi/wasi-threads@1.2.1': dependencies: tslib: 2.8.1 - optional: true '@esbuild/aix-ppc64@0.27.7': optional: true @@ -4984,6 +5018,12 @@ snapshots: transitivePeerDependencies: - supports-color + '@napi-rs/wasm-runtime@1.0.7': + dependencies: + '@emnapi/core': 1.10.0 + '@emnapi/runtime': 1.11.1 + '@tybys/wasm-util': 0.10.2 + '@napi-rs/wasm-runtime@1.1.5(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)': dependencies: '@emnapi/core': 1.10.0 @@ -5321,10 +5361,22 @@ snapshots: - ws - zod + '@tursodatabase/database-common@0.7.2': {} + + '@tursodatabase/database-wasm-common@0.7.2': + dependencies: + '@emnapi/core': 1.8.1 + '@emnapi/runtime': 1.8.1 + '@napi-rs/wasm-runtime': 1.0.7 + + '@tursodatabase/database-wasm@0.7.1': + dependencies: + '@tursodatabase/database-common': 0.7.2 + '@tursodatabase/database-wasm-common': 0.7.2 + '@tybys/wasm-util@0.10.2': dependencies: tslib: 2.8.1 - optional: true '@types/chai@5.2.3': dependencies: @@ -8685,8 +8737,7 @@ snapshots: minimist: 1.2.8 strip-bom: 3.0.0 - tslib@2.8.1: - optional: true + tslib@2.8.1: {} type-check@0.4.0: dependencies: