Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 29 additions & 0 deletions .changeset/sqlite-wasm-fallback-artifact-path.md
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
---
"@objectstack/service-datasource": patch
"@objectstack/cli": patch
"@objectstack/runtime": patch
---

fix(cli): extend native better-sqlite3 → wasm SQLite auto-fallback to the persistent-file / `--artifact` dev path (#2229)

The native-`better-sqlite3` → wasm SQLite → in-memory step-down previously only
guarded the zero-config `:memory:` dev branch of `serve`. A normal
`objectstack dev` run never reaches it — `dev` injects a persistent `file:` DB
(so AI-authored data survives restarts) and `--artifact` boots resolve sqlite
through the datasource factory — both of which constructed
`better-sqlite3` directly with no probe and no fallback. An ABI mismatch (e.g.
a cached prebuilt binary built for a different Node version) was therefore not
caught at boot and surfaced later as a runtime `Find operation failed` on the
first query.

The probe-by-connect + step-down is now hoisted into a shared
`resolveSqliteDriver` helper (`@objectstack/service-datasource`) and applied to
both previously-unguarded sqlite construction sites: the explicit `sqlite` /
`file:` branch in `serve.ts` and the sqlite branch of the default datasource
driver factory. better-sqlite3 loads its native addon lazily (first query), so
the helper forces the load with a `SELECT 1` and, **in dev only**, steps down to
wasm SQLite (real SQL + on-disk persistence — the same `file:` keeps working)
then to the in-memory driver as a last resort, emitting the existing
`⚠ native better-sqlite3 unavailable …` warning. In production the native driver
is returned unprobed so a load failure surfaces loudly (fail-closed) rather than
silently degrading to a different engine.
133 changes: 39 additions & 94 deletions packages/cli/src/commands/serve.ts
Original file line numberDiff line numberDiff line change
Expand Up@@ -425,7 +425,7 @@ export default class Serve extends Command {
// "missing artifact" error and assemble a bare kernel that
// can later install marketplace apps at runtime.
const { createDefaultHostConfig } = await import('@objectstack/runtime');
const bootResult = await createDefaultHostConfig({ requireArtifact: !useEmptyBoot });
const bootResult = await createDefaultHostConfig({ requireArtifact: !useEmptyBoot, dev: isDev });
config = { ...originalConfig, ...bootResult } as any;
} else if (resolvedMode === 'standalone') {
const { createStandaloneStack } = await import('@objectstack/runtime');
Expand All@@ -435,6 +435,9 @@ export default class Serve extends Command {
const standaloneInput = {
...(config.standalone ?? {}),
projectRoot: (config.standalone?.projectRoot ?? path.dirname(absolutePath)),
// #2229: dev enables the native-better-sqlite3 → wasm → in-memory
// step-down in the shared datasource factory; prod fails loudly.
dev: isDev,
};
const bootResult = await createStandaloneStack(standaloneInput);
config = { ...originalConfig, ...bootResult } as any;
Expand DownExpand Up@@ -630,19 +633,25 @@ export default class Serve extends Command {
resolvedDriverLabel = 'MongoDBDriver';
resolvedDatabaseUrl = databaseUrl ?? 'mongodb://localhost:27017/objectstack';
} else if (driverType === 'sqlite' || driverType === 'sql') {
const { SqlDriver } = await import('@objectstack/driver-sql');
const filePath = (databaseUrl ?? ':memory:').replace(/^file:/, '').replace(/^sqlite:/, '').replace(/^sql:\/\//, '');
await kernel.use(new DriverPlugin(new SqlDriver({
client: 'better-sqlite3',
connection: { filename: filePath },
useNullAsDefault: true,
// Probe-by-connect with a dev-only native → wasm → in-memory
// step-down (#2229). better-sqlite3 loads its native addon lazily
// (first query), so an ABI mismatch is invisible here and would
// otherwise surface much later as a runtime crash. resolveSqliteDriver
// forces the load and degrades gracefully in dev / fails loudly in prod.
const { resolveSqliteDriver } = await import('@objectstack/service-datasource');
const resolved = await resolveSqliteDriver({
filename: filePath,
dev: isDev,
// #2186: in dev, self-heal a persisted DB when a metadata change
// relaxes a constraint (loosen-only; never destructive / never in prod).
autoMigrate: isDev ? 'safe' : undefined,
}) as any));
trackPlugin('SqlDriver');
resolvedDriverLabel = 'SqlDriver(sqlite)';
resolvedDatabaseUrl = databaseUrl ?? ':memory:';
warn: (m) => console.warn(chalk.yellow(m)),
});
await kernel.use(new DriverPlugin(resolved.driver));
trackPlugin(resolved.engine === 'memory' ? 'MemoryDriver' : resolved.engine === 'sqlite-wasm' ? 'SqliteWasmDriver' : 'SqlDriver');
resolvedDriverLabel = resolved.label;
resolvedDatabaseUrl = resolved.engine === 'memory' ? '(in-memory)' : (databaseUrl ?? ':memory:');
} else if (driverType === 'sqlite-wasm' || driverType === 'wasm-sqlite' || driverType === 'wasm') {
const { SqliteWasmDriver } = await import('@objectstack/driver-sqlite-wasm');
const filePath = (databaseUrl ?? ':memory:').replace(/^file:/, '').replace(/^wasm-sqlite:\/\//, '').replace(/^sqlite:/, '');
Expand DownExpand Up@@ -676,90 +685,26 @@ export default class Serve extends Command {
resolvedDriverLabel = 'SqlDriver(mysql2)';
resolvedDatabaseUrl = databaseUrl;
} else if (isDev) {
// Default in dev: prefer native SQLite for production-like SQL
// semantics at native speed. When the native `better-sqlite3`
// binary is unavailable — not built, ABI mismatch after a Node
// upgrade (e.g. Node 25 → NODE_MODULE_VERSION mismatch), or a
// blocked prebuild download — fall back to the pure-JS wasm SQLite
// driver, which keeps *real* SQL semantics (and on-disk
// persistence) without any native build step. Only if wasm also
// fails to load do we drop to the in-memory driver (mingo), which
// is neither real SQL nor persistent.
//
// knex loads its client lazily (at first query, not at construction),
// so the only reliable signal inside this registration window is to
// actually open a connection: connect() runs `SELECT 1`, which forces
// better-sqlite3 to load. If that throws we step down the chain here
// instead of letting the failure surface much later — as a
// missing-module crash on the first real query — or be swallowed by
// the silent catch below, leaving the kernel with no driver at all.
let sqliteDriver: any;
let sqliteOk = false;
try {
const { SqlDriver } = await import('@objectstack/driver-sql');
sqliteDriver = new SqlDriver({
client: 'better-sqlite3',
connection: { filename: ':memory:' },
useNullAsDefault: true,
autoMigrate: 'safe', // #2186 dev loosen-only self-heal
});
await sqliteDriver.connect();
sqliteOk = true;
} catch {
sqliteOk = false;
if (sqliteDriver?.disconnect) {
try { await sqliteDriver.disconnect(); } catch { /* ignore */ }
}
}

if (sqliteOk) {
await kernel.use(new DriverPlugin(sqliteDriver));
trackPlugin('SqlDriver');
resolvedDriverLabel = 'SqlDriver(sqlite)';
resolvedDatabaseUrl = ':memory:';
} else {
// Native unavailable → try the pure-JS wasm SQLite driver before
// giving up on SQL fidelity entirely. Same probe-by-connect
// approach: actually open the connection so a load failure is
// caught here rather than on the first real query.
let wasmDriver: any;
let wasmOk = false;
try {
const { SqliteWasmDriver } = await import('@objectstack/driver-sqlite-wasm');
wasmDriver = new SqliteWasmDriver({
filename: ':memory:',
persist: 'on-disconnect',
});
await wasmDriver.connect();
wasmOk = true;
} catch {
wasmOk = false;
if (wasmDriver?.disconnect) {
try { await wasmDriver.disconnect(); } catch { /* ignore */ }
}
}

if (wasmOk) {
await kernel.use(new DriverPlugin(wasmDriver));
trackPlugin('SqliteWasmDriver');
resolvedDriverLabel = 'SqliteWasmDriver';
resolvedDatabaseUrl = ':memory:';
console.warn(chalk.yellow(
' ⚠ native better-sqlite3 unavailable (ABI mismatch or not built) — dev using wasm SQLite (real SQL, slower).\n' +
' Rebuild better-sqlite3 for native speed, or set OS_DATABASE_DRIVER=sqlite-wasm to silence this.'
));
} else {
const { InMemoryDriver } = await import('@objectstack/driver-memory');
await kernel.use(new DriverPlugin(new InMemoryDriver()));
trackPlugin('MemoryDriver');
resolvedDriverLabel = 'InMemoryDriver';
resolvedDatabaseUrl = '(in-memory)';
console.warn(chalk.yellow(
' ⚠ neither native nor wasm SQLite available — dev falling back to InMemoryDriver (mingo, not real SQL).\n' +
' Rebuild better-sqlite3, or set OS_DATABASE_URL / OS_DATABASE_DRIVER for SQL fidelity.'
));
}
}
// Default in dev (no DB configured): prefer native SQLite for
// production-like SQL at native speed, with a graceful step-down to
// wasm SQLite (real SQL + on-disk persistence) then in-memory when the
// native better-sqlite3 binary is unavailable — not built, ABI mismatch
// after a Node upgrade (e.g. NODE_MODULE_VERSION change), or a blocked
// prebuild download. Shared with the explicit-file branch and the
// datasource factory via resolveSqliteDriver (#2229), which probes by
// actually opening a connection + running SELECT 1 (better-sqlite3 loads
// its native addon lazily at first query, not at construction).
const { resolveSqliteDriver } = await import('@objectstack/service-datasource');
const resolved = await resolveSqliteDriver({
filename: ':memory:',
dev: true,
autoMigrate: 'safe', // #2186 dev loosen-only self-heal
warn: (m) => console.warn(chalk.yellow(m)),
});
await kernel.use(new DriverPlugin(resolved.driver));
trackPlugin(resolved.engine === 'memory' ? 'MemoryDriver' : resolved.engine === 'sqlite-wasm' ? 'SqliteWasmDriver' : 'SqlDriver');
resolvedDriverLabel = resolved.label;
resolvedDatabaseUrl = resolved.engine === 'memory' ? '(in-memory)' : ':memory:';
}
} catch (e: any) {
// silent
Expand Down
13 changes: 12 additions & 1 deletion packages/runtime/src/standalone-stack.ts
Original file line numberDiff line numberDiff line change
Expand Up@@ -71,6 +71,13 @@ export const StandaloneStackConfigSchema = z.object({
* precedence over this default.
*/
projectRoot: z.string().optional(),
/**
* Dev gate for the sqlite driver factory's native-better-sqlite3 → wasm →
* in-memory step-down (#2229). When omitted, defaults to
* `process.env.NODE_ENV === 'development'`. In production a native load
* failure is NOT silently swapped for wasm/mingo (fail-closed).
*/
dev: z.boolean().optional(),
});

export type StandaloneStackConfig = z.input<typeof StandaloneStackConfigSchema>;
Expand DownExpand Up@@ -183,6 +190,10 @@ export async function createStandaloneStack(config?: StandaloneStackConfig): Pro
);
} else {
const { createDefaultDatasourceDriverFactory } = await import('@objectstack/service-datasource');
// #2229: in dev, a native better-sqlite3 ABI/load failure steps down to
// wasm SQLite (real SQL + on-disk persistence) then in-memory; in prod it
// fails loudly. Falls back to NODE_ENV when the caller did not pass `dev`.
const factoryDev = cfg.dev ?? process.env.NODE_ENV === 'development';
let driverId: string;
let driverConfig: Record<string, unknown>;
if (dbDriver === 'memory') {
Expand DownExpand Up@@ -211,7 +222,7 @@ export async function createStandaloneStack(config?: StandaloneStackConfig): Pro

let driverHandle: { driver?: unknown } | unknown;
try {
driverHandle = await createDefaultDatasourceDriverFactory().create({ driver: driverId, config: driverConfig });
driverHandle = await createDefaultDatasourceDriverFactory({ dev: factoryDev }).create({ driver: driverId, config: driverConfig });
} catch (err: any) {
// Preserve the actionable hint the bespoke path gave for the optional
// mongo peer dep (the factory throws a generic "not installed" message).
Expand Down
1 change: 1 addition & 0 deletions packages/services/service-datasource/package.json
Original file line numberDiff line numberDiff line change
Expand Up@@ -32,6 +32,7 @@
"devDependencies": {
"@objectstack/driver-memory": "workspace:*",
"@objectstack/driver-sql": "workspace:*",
"@objectstack/driver-sqlite-wasm": "workspace:*",
"@objectstack/plugin-hono-server": "workspace:*",
"@types/node": "^26.0.0",
"tsup": "^8.5.1",
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -113,7 +113,19 @@ function buildMongoUrl(spec: DatasourceConnectionSpec): string {
* lazily so a host that never builds (e.g.) a mongo connection doesn't pay for
* the mongo SDK.
*/
export function createDefaultDatasourceDriverFactory(): IDatasourceDriverFactory {
export interface DefaultDatasourceDriverFactoryOptions {
/**
* Enables the dev-only native-`better-sqlite3` → wasm → in-memory step-down
* for sqlite construction (#2229). When omitted, defaults per call to
* `process.env.NODE_ENV === 'development'`. In production a native load
* failure is NOT silently swapped for a different engine (fail-closed).
*/
dev?: boolean;
}

export function createDefaultDatasourceDriverFactory(
options: DefaultDatasourceDriverFactoryOptions = {},
): IDatasourceDriverFactory {
return {
supports(driverId: string): boolean {
return resolveKind(driverId) !== undefined;
Expand All@@ -140,14 +152,19 @@ export function createDefaultDatasourceDriverFactory(): IDatasourceDriverFactory
}

if (kind === 'sqlite') {
const { SqlDriver } = await import('@objectstack/driver-sql');
const driver = new SqlDriver({
client: 'better-sqlite3',
connection: buildSqlConnection(spec, 'better-sqlite3') as any,
useNullAsDefault: true,
...(schemaMode ? { schemaMode: schemaMode as any } : {}),
} as any);
return toHandle(driver, () => sqlServerVersion(driver, 'sqlite'));
// better-sqlite3 loads its native addon lazily (first query), so an ABI
// mismatch is invisible at construction and crashes later. resolveSqliteDriver
// probes up-front and, IN DEV ONLY, steps down to wasm SQLite (real SQL +
// on-disk persistence) then in-memory; in production it returns the native
// driver unprobed so a failure surfaces loudly (fail-closed). (#2229)
const conn = buildSqlConnection(spec, 'better-sqlite3') as { filename?: string };
const { resolveSqliteDriver } = await import('./sqlite-driver-fallback.js');
const resolved = await resolveSqliteDriver({
filename: conn.filename ?? ':memory:',
dev: options.dev,
...(schemaMode ? { schemaMode } : {}),
});
return toHandle(resolved.driver, () => sqlServerVersion(resolved.driver, 'sqlite'));
}

if (kind === 'mongodb') {
Expand Down
11 changes: 11 additions & 0 deletions packages/services/service-datasource/src/index.ts
Original file line numberDiff line numberDiff line change
Expand Up@@ -76,6 +76,17 @@ export type {

// Host glue: dev driver factory + fail-closed secret binder.
export { createDefaultDatasourceDriverFactory } from './default-datasource-driver-factory.js';
// Shared native-better-sqlite3 → wasm → in-memory step-down (#2229).
export {
resolveSqliteDriver,
NATIVE_SQLITE_WASM_FALLBACK_WARNING,
NATIVE_SQLITE_MEMORY_FALLBACK_WARNING,
} from './sqlite-driver-fallback.js';
export type {
ResolveSqliteDriverOptions,
ResolvedSqliteDriver,
SqliteFallbackEngine,
} from './sqlite-driver-fallback.js';
export {
createDatasourceSecretBinder,
toCredentialsRef,
Expand Down
Loading