From 004cbe56975dda6db5ed4278727403fafd8f7298 Mon Sep 17 00:00:00 2001 From: Jack Zhuang <277994282+os-zhuang@users.noreply.github.com> Date: Tue, 23 Jun 2026 16:24:44 +0800 Subject: [PATCH] =?UTF-8?q?fix(cli):=20extend=20better-sqlite3=20=E2=86=92?= =?UTF-8?q?=20wasm=20SQLite=20auto-fallback=20to=20the=20persistent-file?= =?UTF-8?q?=20/=20--artifact=20dev=20path=20(#2229)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The native-better-sqlite3 → wasm SQLite → in-memory step-down previously only guarded the zero-config `:memory:` dev branch of `serve`. A normal `objectstack dev` run never reached it: `dev` injects a persistent `file:` DB so AI-authored data survives restarts, and `--artifact` boots resolve sqlite through the datasource factory — both constructed `better-sqlite3` directly with no probe and no fallback. better-sqlite3 loads its native addon lazily (first query), so an ABI mismatch was invisible at boot and surfaced later as a runtime `Find operation failed`. Hoist the probe-by-connect + step-down into a shared `resolveSqliteDriver` helper (@objectstack/service-datasource) and apply it to both previously unguarded sqlite construction sites: the explicit `sqlite`/`file:` branch in serve.ts and the sqlite branch of the default datasource driver factory. The helper forces the native load with `connect()` + `SELECT 1` and, in dev only, steps down to wasm SQLite (real SQL + on-disk persistence — the same `file:` keeps working) then to in-memory as a last resort, emitting the existing `⚠ native better-sqlite3 unavailable …` warning. In production the native driver is returned unprobed so a load failure surfaces loudly (fail-closed) instead of silently degrading. Dev is plumbed through standalone-stack so the --artifact boot enables the fallback explicitly (NODE_ENV fallback otherwise). Adds a co-located unit test simulating a NODE_MODULE_VERSION load failure (asserts wasm fallback + warning, in-memory last resort, and prod fail-closed). Co-Authored-By: Claude Opus 4.8 --- .../sqlite-wasm-fallback-artifact-path.md | 29 +++ packages/cli/src/commands/serve.ts | 133 ++++-------- packages/runtime/src/standalone-stack.ts | 13 +- .../services/service-datasource/package.json | 1 + .../src/default-datasource-driver-factory.ts | 35 +++- .../services/service-datasource/src/index.ts | 11 + .../src/sqlite-driver-fallback.test.ts | 184 +++++++++++++++++ .../src/sqlite-driver-fallback.ts | 195 ++++++++++++++++++ pnpm-lock.yaml | 3 + 9 files changed, 500 insertions(+), 104 deletions(-) create mode 100644 .changeset/sqlite-wasm-fallback-artifact-path.md create mode 100644 packages/services/service-datasource/src/sqlite-driver-fallback.test.ts create mode 100644 packages/services/service-datasource/src/sqlite-driver-fallback.ts diff --git a/.changeset/sqlite-wasm-fallback-artifact-path.md b/.changeset/sqlite-wasm-fallback-artifact-path.md new file mode 100644 index 0000000000..df774f6a77 --- /dev/null +++ b/.changeset/sqlite-wasm-fallback-artifact-path.md @@ -0,0 +1,29 @@ +--- +"@objectstack/service-datasource": patch +"@objectstack/cli": patch +"@objectstack/runtime": patch +--- + +fix(cli): extend native better-sqlite3 → wasm SQLite auto-fallback to the persistent-file / `--artifact` dev path (#2229) + +The native-`better-sqlite3` → wasm SQLite → in-memory step-down previously only +guarded the zero-config `:memory:` dev branch of `serve`. A normal +`objectstack dev` run never reaches it — `dev` injects a persistent `file:` DB +(so AI-authored data survives restarts) and `--artifact` boots resolve sqlite +through the datasource factory — both of which constructed +`better-sqlite3` directly with no probe and no fallback. An ABI mismatch (e.g. +a cached prebuilt binary built for a different Node version) was therefore not +caught at boot and surfaced later as a runtime `Find operation failed` on the +first query. + +The probe-by-connect + step-down is now hoisted into a shared +`resolveSqliteDriver` helper (`@objectstack/service-datasource`) and applied to +both previously-unguarded sqlite construction sites: the explicit `sqlite` / +`file:` branch in `serve.ts` and the sqlite branch of the default datasource +driver factory. better-sqlite3 loads its native addon lazily (first query), so +the helper forces the load with a `SELECT 1` and, **in dev only**, steps down to +wasm SQLite (real SQL + on-disk persistence — the same `file:` keeps working) +then to the in-memory driver as a last resort, emitting the existing +`⚠ native better-sqlite3 unavailable …` warning. In production the native driver +is returned unprobed so a load failure surfaces loudly (fail-closed) rather than +silently degrading to a different engine. diff --git a/packages/cli/src/commands/serve.ts b/packages/cli/src/commands/serve.ts index 6350fc090d..12811184e2 100644 --- a/packages/cli/src/commands/serve.ts +++ b/packages/cli/src/commands/serve.ts @@ -425,7 +425,7 @@ export default class Serve extends Command { // "missing artifact" error and assemble a bare kernel that // can later install marketplace apps at runtime. const { createDefaultHostConfig } = await import('@objectstack/runtime'); - const bootResult = await createDefaultHostConfig({ requireArtifact: !useEmptyBoot }); + const bootResult = await createDefaultHostConfig({ requireArtifact: !useEmptyBoot, dev: isDev }); config = { ...originalConfig, ...bootResult } as any; } else if (resolvedMode === 'standalone') { const { createStandaloneStack } = await import('@objectstack/runtime'); @@ -435,6 +435,9 @@ export default class Serve extends Command { const standaloneInput = { ...(config.standalone ?? {}), projectRoot: (config.standalone?.projectRoot ?? path.dirname(absolutePath)), + // #2229: dev enables the native-better-sqlite3 → wasm → in-memory + // step-down in the shared datasource factory; prod fails loudly. + dev: isDev, }; const bootResult = await createStandaloneStack(standaloneInput); config = { ...originalConfig, ...bootResult } as any; @@ -630,19 +633,25 @@ export default class Serve extends Command { resolvedDriverLabel = 'MongoDBDriver'; resolvedDatabaseUrl = databaseUrl ?? 'mongodb://localhost:27017/objectstack'; } else if (driverType === 'sqlite' || driverType === 'sql') { - const { SqlDriver } = await import('@objectstack/driver-sql'); const filePath = (databaseUrl ?? ':memory:').replace(/^file:/, '').replace(/^sqlite:/, '').replace(/^sql:\/\//, ''); - await kernel.use(new DriverPlugin(new SqlDriver({ - client: 'better-sqlite3', - connection: { filename: filePath }, - useNullAsDefault: true, + // Probe-by-connect with a dev-only native → wasm → in-memory + // step-down (#2229). better-sqlite3 loads its native addon lazily + // (first query), so an ABI mismatch is invisible here and would + // otherwise surface much later as a runtime crash. resolveSqliteDriver + // forces the load and degrades gracefully in dev / fails loudly in prod. + const { resolveSqliteDriver } = await import('@objectstack/service-datasource'); + const resolved = await resolveSqliteDriver({ + filename: filePath, + dev: isDev, // #2186: in dev, self-heal a persisted DB when a metadata change // relaxes a constraint (loosen-only; never destructive / never in prod). autoMigrate: isDev ? 'safe' : undefined, - }) as any)); - trackPlugin('SqlDriver'); - resolvedDriverLabel = 'SqlDriver(sqlite)'; - resolvedDatabaseUrl = databaseUrl ?? ':memory:'; + warn: (m) => console.warn(chalk.yellow(m)), + }); + await kernel.use(new DriverPlugin(resolved.driver)); + trackPlugin(resolved.engine === 'memory' ? 'MemoryDriver' : resolved.engine === 'sqlite-wasm' ? 'SqliteWasmDriver' : 'SqlDriver'); + resolvedDriverLabel = resolved.label; + resolvedDatabaseUrl = resolved.engine === 'memory' ? '(in-memory)' : (databaseUrl ?? ':memory:'); } else if (driverType === 'sqlite-wasm' || driverType === 'wasm-sqlite' || driverType === 'wasm') { const { SqliteWasmDriver } = await import('@objectstack/driver-sqlite-wasm'); const filePath = (databaseUrl ?? ':memory:').replace(/^file:/, '').replace(/^wasm-sqlite:\/\//, '').replace(/^sqlite:/, ''); @@ -676,90 +685,26 @@ export default class Serve extends Command { resolvedDriverLabel = 'SqlDriver(mysql2)'; resolvedDatabaseUrl = databaseUrl; } else if (isDev) { - // Default in dev: prefer native SQLite for production-like SQL - // semantics at native speed. When the native `better-sqlite3` - // binary is unavailable — not built, ABI mismatch after a Node - // upgrade (e.g. Node 25 → NODE_MODULE_VERSION mismatch), or a - // blocked prebuild download — fall back to the pure-JS wasm SQLite - // driver, which keeps *real* SQL semantics (and on-disk - // persistence) without any native build step. Only if wasm also - // fails to load do we drop to the in-memory driver (mingo), which - // is neither real SQL nor persistent. - // - // knex loads its client lazily (at first query, not at construction), - // so the only reliable signal inside this registration window is to - // actually open a connection: connect() runs `SELECT 1`, which forces - // better-sqlite3 to load. If that throws we step down the chain here - // instead of letting the failure surface much later — as a - // missing-module crash on the first real query — or be swallowed by - // the silent catch below, leaving the kernel with no driver at all. - let sqliteDriver: any; - let sqliteOk = false; - try { - const { SqlDriver } = await import('@objectstack/driver-sql'); - sqliteDriver = new SqlDriver({ - client: 'better-sqlite3', - connection: { filename: ':memory:' }, - useNullAsDefault: true, - autoMigrate: 'safe', // #2186 dev loosen-only self-heal - }); - await sqliteDriver.connect(); - sqliteOk = true; - } catch { - sqliteOk = false; - if (sqliteDriver?.disconnect) { - try { await sqliteDriver.disconnect(); } catch { /* ignore */ } - } - } - - if (sqliteOk) { - await kernel.use(new DriverPlugin(sqliteDriver)); - trackPlugin('SqlDriver'); - resolvedDriverLabel = 'SqlDriver(sqlite)'; - resolvedDatabaseUrl = ':memory:'; - } else { - // Native unavailable → try the pure-JS wasm SQLite driver before - // giving up on SQL fidelity entirely. Same probe-by-connect - // approach: actually open the connection so a load failure is - // caught here rather than on the first real query. - let wasmDriver: any; - let wasmOk = false; - try { - const { SqliteWasmDriver } = await import('@objectstack/driver-sqlite-wasm'); - wasmDriver = new SqliteWasmDriver({ - filename: ':memory:', - persist: 'on-disconnect', - }); - await wasmDriver.connect(); - wasmOk = true; - } catch { - wasmOk = false; - if (wasmDriver?.disconnect) { - try { await wasmDriver.disconnect(); } catch { /* ignore */ } - } - } - - if (wasmOk) { - await kernel.use(new DriverPlugin(wasmDriver)); - trackPlugin('SqliteWasmDriver'); - resolvedDriverLabel = 'SqliteWasmDriver'; - resolvedDatabaseUrl = ':memory:'; - console.warn(chalk.yellow( - ' ⚠ native better-sqlite3 unavailable (ABI mismatch or not built) — dev using wasm SQLite (real SQL, slower).\n' + - ' Rebuild better-sqlite3 for native speed, or set OS_DATABASE_DRIVER=sqlite-wasm to silence this.' - )); - } else { - const { InMemoryDriver } = await import('@objectstack/driver-memory'); - await kernel.use(new DriverPlugin(new InMemoryDriver())); - trackPlugin('MemoryDriver'); - resolvedDriverLabel = 'InMemoryDriver'; - resolvedDatabaseUrl = '(in-memory)'; - console.warn(chalk.yellow( - ' ⚠ neither native nor wasm SQLite available — dev falling back to InMemoryDriver (mingo, not real SQL).\n' + - ' Rebuild better-sqlite3, or set OS_DATABASE_URL / OS_DATABASE_DRIVER for SQL fidelity.' - )); - } - } + // Default in dev (no DB configured): prefer native SQLite for + // production-like SQL at native speed, with a graceful step-down to + // wasm SQLite (real SQL + on-disk persistence) then in-memory when the + // native better-sqlite3 binary is unavailable — not built, ABI mismatch + // after a Node upgrade (e.g. NODE_MODULE_VERSION change), or a blocked + // prebuild download. Shared with the explicit-file branch and the + // datasource factory via resolveSqliteDriver (#2229), which probes by + // actually opening a connection + running SELECT 1 (better-sqlite3 loads + // its native addon lazily at first query, not at construction). + const { resolveSqliteDriver } = await import('@objectstack/service-datasource'); + const resolved = await resolveSqliteDriver({ + filename: ':memory:', + dev: true, + autoMigrate: 'safe', // #2186 dev loosen-only self-heal + warn: (m) => console.warn(chalk.yellow(m)), + }); + await kernel.use(new DriverPlugin(resolved.driver)); + trackPlugin(resolved.engine === 'memory' ? 'MemoryDriver' : resolved.engine === 'sqlite-wasm' ? 'SqliteWasmDriver' : 'SqlDriver'); + resolvedDriverLabel = resolved.label; + resolvedDatabaseUrl = resolved.engine === 'memory' ? '(in-memory)' : ':memory:'; } } catch (e: any) { // silent diff --git a/packages/runtime/src/standalone-stack.ts b/packages/runtime/src/standalone-stack.ts index 91baf10ab6..9c41a72dac 100644 --- a/packages/runtime/src/standalone-stack.ts +++ b/packages/runtime/src/standalone-stack.ts @@ -71,6 +71,13 @@ export const StandaloneStackConfigSchema = z.object({ * precedence over this default. */ projectRoot: z.string().optional(), + /** + * Dev gate for the sqlite driver factory's native-better-sqlite3 → wasm → + * in-memory step-down (#2229). When omitted, defaults to + * `process.env.NODE_ENV === 'development'`. In production a native load + * failure is NOT silently swapped for wasm/mingo (fail-closed). + */ + dev: z.boolean().optional(), }); export type StandaloneStackConfig = z.input; @@ -183,6 +190,10 @@ export async function createStandaloneStack(config?: StandaloneStackConfig): Pro ); } else { const { createDefaultDatasourceDriverFactory } = await import('@objectstack/service-datasource'); + // #2229: in dev, a native better-sqlite3 ABI/load failure steps down to + // wasm SQLite (real SQL + on-disk persistence) then in-memory; in prod it + // fails loudly. Falls back to NODE_ENV when the caller did not pass `dev`. + const factoryDev = cfg.dev ?? process.env.NODE_ENV === 'development'; let driverId: string; let driverConfig: Record; if (dbDriver === 'memory') { @@ -211,7 +222,7 @@ export async function createStandaloneStack(config?: StandaloneStackConfig): Pro let driverHandle: { driver?: unknown } | unknown; try { - driverHandle = await createDefaultDatasourceDriverFactory().create({ driver: driverId, config: driverConfig }); + driverHandle = await createDefaultDatasourceDriverFactory({ dev: factoryDev }).create({ driver: driverId, config: driverConfig }); } catch (err: any) { // Preserve the actionable hint the bespoke path gave for the optional // mongo peer dep (the factory throws a generic "not installed" message). diff --git a/packages/services/service-datasource/package.json b/packages/services/service-datasource/package.json index a774e1bf39..7bb14f183f 100644 --- a/packages/services/service-datasource/package.json +++ b/packages/services/service-datasource/package.json @@ -32,6 +32,7 @@ "devDependencies": { "@objectstack/driver-memory": "workspace:*", "@objectstack/driver-sql": "workspace:*", + "@objectstack/driver-sqlite-wasm": "workspace:*", "@objectstack/plugin-hono-server": "workspace:*", "@types/node": "^26.0.0", "tsup": "^8.5.1", diff --git a/packages/services/service-datasource/src/default-datasource-driver-factory.ts b/packages/services/service-datasource/src/default-datasource-driver-factory.ts index 7d0ba35af0..99be7e45fb 100644 --- a/packages/services/service-datasource/src/default-datasource-driver-factory.ts +++ b/packages/services/service-datasource/src/default-datasource-driver-factory.ts @@ -113,7 +113,19 @@ function buildMongoUrl(spec: DatasourceConnectionSpec): string { * lazily so a host that never builds (e.g.) a mongo connection doesn't pay for * the mongo SDK. */ -export function createDefaultDatasourceDriverFactory(): IDatasourceDriverFactory { +export interface DefaultDatasourceDriverFactoryOptions { + /** + * Enables the dev-only native-`better-sqlite3` → wasm → in-memory step-down + * for sqlite construction (#2229). When omitted, defaults per call to + * `process.env.NODE_ENV === 'development'`. In production a native load + * failure is NOT silently swapped for a different engine (fail-closed). + */ + dev?: boolean; +} + +export function createDefaultDatasourceDriverFactory( + options: DefaultDatasourceDriverFactoryOptions = {}, +): IDatasourceDriverFactory { return { supports(driverId: string): boolean { return resolveKind(driverId) !== undefined; @@ -140,14 +152,19 @@ export function createDefaultDatasourceDriverFactory(): IDatasourceDriverFactory } if (kind === 'sqlite') { - const { SqlDriver } = await import('@objectstack/driver-sql'); - const driver = new SqlDriver({ - client: 'better-sqlite3', - connection: buildSqlConnection(spec, 'better-sqlite3') as any, - useNullAsDefault: true, - ...(schemaMode ? { schemaMode: schemaMode as any } : {}), - } as any); - return toHandle(driver, () => sqlServerVersion(driver, 'sqlite')); + // better-sqlite3 loads its native addon lazily (first query), so an ABI + // mismatch is invisible at construction and crashes later. resolveSqliteDriver + // probes up-front and, IN DEV ONLY, steps down to wasm SQLite (real SQL + + // on-disk persistence) then in-memory; in production it returns the native + // driver unprobed so a failure surfaces loudly (fail-closed). (#2229) + const conn = buildSqlConnection(spec, 'better-sqlite3') as { filename?: string }; + const { resolveSqliteDriver } = await import('./sqlite-driver-fallback.js'); + const resolved = await resolveSqliteDriver({ + filename: conn.filename ?? ':memory:', + dev: options.dev, + ...(schemaMode ? { schemaMode } : {}), + }); + return toHandle(resolved.driver, () => sqlServerVersion(resolved.driver, 'sqlite')); } if (kind === 'mongodb') { diff --git a/packages/services/service-datasource/src/index.ts b/packages/services/service-datasource/src/index.ts index 264291bbc2..6b918c7b4c 100644 --- a/packages/services/service-datasource/src/index.ts +++ b/packages/services/service-datasource/src/index.ts @@ -76,6 +76,17 @@ export type { // Host glue: dev driver factory + fail-closed secret binder. export { createDefaultDatasourceDriverFactory } from './default-datasource-driver-factory.js'; +// Shared native-better-sqlite3 → wasm → in-memory step-down (#2229). +export { + resolveSqliteDriver, + NATIVE_SQLITE_WASM_FALLBACK_WARNING, + NATIVE_SQLITE_MEMORY_FALLBACK_WARNING, +} from './sqlite-driver-fallback.js'; +export type { + ResolveSqliteDriverOptions, + ResolvedSqliteDriver, + SqliteFallbackEngine, +} from './sqlite-driver-fallback.js'; export { createDatasourceSecretBinder, toCredentialsRef, diff --git a/packages/services/service-datasource/src/sqlite-driver-fallback.test.ts b/packages/services/service-datasource/src/sqlite-driver-fallback.test.ts new file mode 100644 index 0000000000..8e83fb59d8 --- /dev/null +++ b/packages/services/service-datasource/src/sqlite-driver-fallback.test.ts @@ -0,0 +1,184 @@ +// Copyright (c) 2025 ObjectStack. Licensed under the Apache-2.0 license. + +import { describe, it, expect, beforeEach, afterAll, vi } from 'vitest'; +import { + resolveSqliteDriver, + NATIVE_SQLITE_WASM_FALLBACK_WARNING, + NATIVE_SQLITE_MEMORY_FALLBACK_WARNING, +} from './sqlite-driver-fallback.js'; + +// Shared, mutable test state read by the mocked driver constructors. `vi.hoisted` +// makes it available inside the hoisted `vi.mock` factories below. +const state = vi.hoisted(() => ({ + /** Make the native better-sqlite3 driver throw a NODE_MODULE_VERSION-style error. */ + nativeFails: false, + /** Make the wasm SQLite driver fail to connect (forces the in-memory last resort). */ + wasmFails: false, + nativeConfigs: [] as any[], + wasmConfigs: [] as any[], + memoryCount: 0, +})); + +const ABI_ERROR_MESSAGE = + "The module '/x/better_sqlite3.node' was compiled against a different Node.js version " + + 'using NODE_MODULE_VERSION 141. This version of Node.js requires NODE_MODULE_VERSION 127. ' + + 'Please try re-compiling or re-installing the module.'; + +vi.mock('@objectstack/driver-sql', () => { + class SqlDriver { + public readonly name = 'com.objectstack.driver.sql'; + constructor(public readonly config: any) { + state.nativeConfigs.push(config); + } + async connect(): Promise { + // Mirrors the real driver: connect() runs mkdir + a PRAGMA whose error it + // swallows — so it is NOT where the ABI failure surfaces. + } + async execute(_sql: string): Promise { + // better-sqlite3 loads its native addon lazily at the first query, so the + // ABI mismatch surfaces here (the SELECT 1 probe), not at construction. + if (state.nativeFails) throw new Error(ABI_ERROR_MESSAGE); + return [{ ok: 1 }]; + } + async disconnect(): Promise {} + } + return { SqlDriver }; +}); + +vi.mock('@objectstack/driver-sqlite-wasm', () => { + class SqliteWasmDriver { + public readonly name = 'com.objectstack.driver.sqlite-wasm'; + constructor(public readonly config: any) { + state.wasmConfigs.push(config); + } + async connect(): Promise { + if (state.wasmFails) throw new Error('wasm sqlite failed to initialise'); + } + async execute(): Promise { + return []; + } + async disconnect(): Promise {} + } + return { SqliteWasmDriver }; +}); + +vi.mock('@objectstack/driver-memory', () => { + class InMemoryDriver { + public readonly name = 'com.objectstack.driver.memory'; + constructor() { + state.memoryCount += 1; + } + } + return { InMemoryDriver }; +}); + +describe('resolveSqliteDriver — native better-sqlite3 → wasm → in-memory step-down (#2229)', () => { + beforeEach(() => { + state.nativeFails = false; + state.wasmFails = false; + state.nativeConfigs = []; + state.wasmConfigs = []; + state.memoryCount = 0; + }); + + it('uses native better-sqlite3 on the happy path (no fallback, no warning)', async () => { + const warn = vi.fn(); + const resolved = await resolveSqliteDriver({ filename: ':memory:', dev: true, warn }); + + expect(resolved.engine).toBe('better-sqlite3'); + expect(resolved.label).toBe('SqlDriver(sqlite)'); + expect(resolved.driver.name).toBe('com.objectstack.driver.sql'); + expect(warn).not.toHaveBeenCalled(); + expect(state.wasmConfigs).toHaveLength(0); + expect(state.memoryCount).toBe(0); + }); + + it('falls back to wasm SQLite when the native addon fails to load, emitting the warning', async () => { + state.nativeFails = true; + const warn = vi.fn(); + + const resolved = await resolveSqliteDriver({ + filename: '/tmp/proj/.objectstack/data/dev.db', + dev: true, + warn, + }); + + expect(resolved.engine).toBe('sqlite-wasm'); + expect(resolved.label).toBe('SqliteWasmDriver'); + expect(resolved.driver.name).toBe('com.objectstack.driver.sqlite-wasm'); + expect(warn).toHaveBeenCalledTimes(1); + expect(warn).toHaveBeenCalledWith(NATIVE_SQLITE_WASM_FALLBACK_WARNING); + // The persistent file path is preserved (real on-disk persistence via wasm). + expect(state.wasmConfigs[0].filename).toBe('/tmp/proj/.objectstack/data/dev.db'); + expect(state.wasmConfigs[0].persist).toBe('on-write'); + expect(state.memoryCount).toBe(0); + }); + + it('uses on-disconnect persistence for an ephemeral :memory: wasm fallback', async () => { + state.nativeFails = true; + const resolved = await resolveSqliteDriver({ filename: ':memory:', dev: true, warn: vi.fn() }); + + expect(resolved.engine).toBe('sqlite-wasm'); + expect(state.wasmConfigs[0].filename).toBe(':memory:'); + expect(state.wasmConfigs[0].persist).toBe('on-disconnect'); + }); + + it('drops to InMemoryDriver as a dev-only last resort when neither native nor wasm load', async () => { + state.nativeFails = true; + state.wasmFails = true; + const warn = vi.fn(); + + const resolved = await resolveSqliteDriver({ filename: ':memory:', dev: true, warn }); + + expect(resolved.engine).toBe('memory'); + expect(resolved.label).toBe('InMemoryDriver'); + expect(state.memoryCount).toBe(1); + expect(warn).toHaveBeenCalledWith(NATIVE_SQLITE_MEMORY_FALLBACK_WARNING); + }); + + it('forwards autoMigrate / schemaMode to the native driver', async () => { + await resolveSqliteDriver({ filename: ':memory:', dev: true, autoMigrate: 'safe', schemaMode: 'managed' }); + expect(state.nativeConfigs[0]).toMatchObject({ + client: 'better-sqlite3', + useNullAsDefault: true, + autoMigrate: 'safe', + schemaMode: 'managed', + }); + }); + + it('production (dev=false) is fail-closed — returns native unprobed, never degrades', async () => { + state.nativeFails = true; + const warn = vi.fn(); + + const resolved = await resolveSqliteDriver({ filename: '/tmp/prod.db', dev: false, warn }); + + // The native driver is handed back as-is so the ABI failure surfaces loudly + // at first use — we must NOT swap in wasm/mingo behind the operator's back. + expect(resolved.engine).toBe('better-sqlite3'); + expect(resolved.label).toBe('SqlDriver(sqlite)'); + expect(warn).not.toHaveBeenCalled(); + expect(state.wasmConfigs).toHaveLength(0); + expect(state.memoryCount).toBe(0); + }); + + describe('dev gate defaults to NODE_ENV when not passed explicitly', () => { + const originalNodeEnv = process.env.NODE_ENV; + afterAll(() => { + process.env.NODE_ENV = originalNodeEnv; + }); + + it('falls back to wasm when dev is omitted and NODE_ENV=development', async () => { + state.nativeFails = true; + process.env.NODE_ENV = 'development'; + const resolved = await resolveSqliteDriver({ filename: ':memory:', warn: vi.fn() }); + expect(resolved.engine).toBe('sqlite-wasm'); + }); + + it('is fail-closed when dev is omitted and NODE_ENV=production', async () => { + state.nativeFails = true; + process.env.NODE_ENV = 'production'; + const resolved = await resolveSqliteDriver({ filename: ':memory:', warn: vi.fn() }); + expect(resolved.engine).toBe('better-sqlite3'); + }); + }); +}); diff --git a/packages/services/service-datasource/src/sqlite-driver-fallback.ts b/packages/services/service-datasource/src/sqlite-driver-fallback.ts new file mode 100644 index 0000000000..aba044b993 --- /dev/null +++ b/packages/services/service-datasource/src/sqlite-driver-fallback.ts @@ -0,0 +1,195 @@ +// Copyright (c) 2025 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * Shared native-`better-sqlite3` → wasm SQLite → in-memory step-down for any + * sqlite-via-`better-sqlite3` construction (issue #2229). + * + * ## Why a probe is necessary + * + * `better-sqlite3` loads its native `.node` addon LAZILY — not at + * `require('better-sqlite3')`, and not even at knex construction, but at the + * first pool-connection acquire (`new Database(file)`), i.e. the first query. + * So an ABI mismatch (a cached prebuilt binary built for a different Node + * version — `NODE_MODULE_VERSION` mismatch) is invisible at boot and only + * surfaces much later as a runtime `Find operation failed` on the first read. + * + * This helper makes the failure observable up-front by actively probing: it + * opens a connection and runs a cheap `SELECT 1`, which forces the native addon + * to load. (`connect()` alone is NOT a reliable probe: for SQLite it only runs + * `mkdir` + a `PRAGMA` whose error is swallowed internally — so we additionally + * issue a raw `SELECT 1`, which propagates the load error.) On failure it steps + * down: + * + * 1. native `better-sqlite3` — fast, real SQL + * 2. wasm SQLite — pure-JS, real SQL + on-disk persistence, slower [dev only] + * 3. in-memory (mingo) — neither real SQL nor persistent [dev only, last resort] + * + * ## Dev vs production + * + * The wasm + in-memory step-down is GATED to dev. In production a native load + * failure is NOT silently swapped for a different engine: the error is re-thrown + * so it surfaces loudly (fail-closed) instead of an operator unknowingly running + * on wasm/mingo. This mirrors the existing `serve.ts` default-dev fallback and + * hoists it into one place shared by every sqlite construction site. + */ + +/** Which engine the resolver ultimately produced. */ +export type SqliteFallbackEngine = 'better-sqlite3' | 'sqlite-wasm' | 'memory'; + +export interface ResolveSqliteDriverOptions { + /** + * SQLite filename — `:memory:` for an ephemeral database, or an absolute / + * relative path for a persistent file. Preserved across the wasm fallback so + * a persistent `file:` database keeps its on-disk persistence through wasm. + * Pass the raw filename (callers strip any `file:` / `sqlite:` scheme first). + */ + filename: string; + /** + * Gates the wasm + in-memory step-down. When `true` (dev) a native ABI/load + * failure steps down the chain with a warning. When `false` (production) the + * native driver is returned unprobed so a failure surfaces loudly at first use + * (fail-closed) — we never silently degrade behind the operator's back. + * Defaults to `process.env.NODE_ENV === 'development'`. + */ + dev?: boolean; + /** Forwarded to the native SqlDriver (dev loosen-only self-heal, #2186). */ + autoMigrate?: 'off' | 'safe'; + /** Forwarded to the SQL drivers (external schema mode, ADR-0015). */ + schemaMode?: string; + /** + * Warning sink for the step-down messages. Defaults to `console.warn`. + * `serve.ts` passes a `chalk.yellow` wrapper so the banner stays consistent. + */ + warn?: (message: string) => void; +} + +export interface ResolvedSqliteDriver { + /** The concrete engine driver to register (e.g. via `DriverPlugin`). */ + driver: any; + /** Which engine actually resolved. */ + engine: SqliteFallbackEngine; + /** Banner label, matching `serve.ts`'s existing strings. */ + label: string; +} + +/** + * Warning emitted when native `better-sqlite3` is unavailable but wasm SQLite + * loads. Kept byte-for-byte identical to the original `serve.ts` text so the + * dev experience is the same regardless of which construction site triggers it. + */ +export const NATIVE_SQLITE_WASM_FALLBACK_WARNING = + ' ⚠ native better-sqlite3 unavailable (ABI mismatch or not built) — dev using wasm SQLite (real SQL, slower).\n' + + ' Rebuild better-sqlite3 for native speed, or set OS_DATABASE_DRIVER=sqlite-wasm to silence this.'; + +/** Warning emitted when neither native nor wasm SQLite loads (dev last resort). */ +export const NATIVE_SQLITE_MEMORY_FALLBACK_WARNING = + ' ⚠ neither native nor wasm SQLite available — dev falling back to InMemoryDriver (mingo, not real SQL).\n' + + ' Rebuild better-sqlite3, or set OS_DATABASE_URL / OS_DATABASE_DRIVER for SQL fidelity.'; + +/** `:memory:` and other `:`-prefixed pseudo-filenames are never persisted. */ +function isEphemeralFilename(filename: string): boolean { + return filename === ':memory:' || filename.startsWith(':'); +} + +/** + * Probe a `better-sqlite3` SQLite construction and, in dev, step down to wasm + * SQLite (then in-memory) when the native addon cannot load. + * + * @see {@link ResolveSqliteDriverOptions} + */ +export async function resolveSqliteDriver( + opts: ResolveSqliteDriverOptions, +): Promise { + const { filename } = opts; + const dev = opts.dev ?? process.env.NODE_ENV === 'development'; + const warn = + opts.warn ?? + ((message: string) => { + try { + // eslint-disable-next-line no-console + console.warn(message); + } catch { + /* ignore */ + } + }); + + const { SqlDriver } = await import('@objectstack/driver-sql'); + + const buildNative = () => + new SqlDriver({ + client: 'better-sqlite3', + connection: { filename }, + useNullAsDefault: true, + ...(opts.autoMigrate ? { autoMigrate: opts.autoMigrate } : {}), + ...(opts.schemaMode ? { schemaMode: opts.schemaMode } : {}), + } as any); + + // Production: never silently swap engines. Construct the native driver and + // hand it back UNPROBED — exactly the historical behavior. A native load + // failure surfaces loudly at first use (fail-closed). + if (!dev) { + return { driver: buildNative(), engine: 'better-sqlite3', label: 'SqlDriver(sqlite)' }; + } + + // ── Dev: probe-by-connect, step down on native ABI/load failure. ────────── + + // 1. Native better-sqlite3. + let nativeDriver: any; + let nativeOk = false; + try { + nativeDriver = buildNative(); + // connect() runs mkdir (so a SELECT on a file DB whose dir is missing does + // not false-positive as an ABI failure) + a PRAGMA whose error it swallows; + // the raw SELECT 1 below is what reliably forces the native addon to load + // and PROPAGATES an ABI mismatch. + await nativeDriver.connect(); + await nativeDriver.execute('SELECT 1'); + nativeOk = true; + } catch { + nativeOk = false; + if (typeof nativeDriver?.disconnect === 'function') { + try { + await nativeDriver.disconnect(); + } catch { + /* ignore */ + } + } + } + if (nativeOk) { + return { driver: nativeDriver, engine: 'better-sqlite3', label: 'SqlDriver(sqlite)' }; + } + + // 2. wasm SQLite — real SQL semantics + on-disk persistence, no native build. + let wasmDriver: any; + let wasmOk = false; + try { + const { SqliteWasmDriver } = await import('@objectstack/driver-sqlite-wasm'); + wasmDriver = new SqliteWasmDriver({ + filename, + // Match the existing construction sites: ephemeral DBs flush on + // disconnect; a persistent file flushes on every write so AI-authored + // data survives an unclean dev-server kill. + persist: isEphemeralFilename(filename) ? 'on-disconnect' : 'on-write', + } as any); + await wasmDriver.connect(); + wasmOk = true; + } catch { + wasmOk = false; + if (typeof wasmDriver?.disconnect === 'function') { + try { + await wasmDriver.disconnect(); + } catch { + /* ignore */ + } + } + } + if (wasmOk) { + warn(NATIVE_SQLITE_WASM_FALLBACK_WARNING); + return { driver: wasmDriver, engine: 'sqlite-wasm', label: 'SqliteWasmDriver' }; + } + + // 3. In-memory (mingo) — dev-only last resort. Not real SQL, not persistent. + const { InMemoryDriver } = await import('@objectstack/driver-memory'); + warn(NATIVE_SQLITE_MEMORY_FALLBACK_WARNING); + return { driver: new InMemoryDriver(), engine: 'memory', label: 'InMemoryDriver' }; +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6b7b854daf..f59df3c491 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1915,6 +1915,9 @@ importers: '@objectstack/driver-sql': specifier: workspace:* version: link:../../plugins/driver-sql + '@objectstack/driver-sqlite-wasm': + specifier: workspace:* + version: link:../../plugins/driver-sqlite-wasm '@objectstack/plugin-hono-server': specifier: workspace:* version: link:../../plugins/plugin-hono-server