From 09cd389a7c8030cca49baa57e85ad373728b38f9 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 26 Feb 2026 06:44:55 +0000 Subject: [PATCH 1/5] Initial plan From 15da7d24f2d1897c5c440e71234102ab6e8bc83f Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 26 Feb 2026 06:49:41 +0000 Subject: [PATCH 2/5] feat: redesign MemoryPersistenceConfigSchema to support file/local/custom adapter union type Co-authored-by: hotlong <50353452+hotlong@users.noreply.github.com> --- packages/spec/src/data/driver/memory.test.ts | 212 ++++++++++++++++--- packages/spec/src/data/driver/memory.zod.ts | 115 ++++++++-- 2 files changed, 286 insertions(+), 41 deletions(-) diff --git a/packages/spec/src/data/driver/memory.test.ts b/packages/spec/src/data/driver/memory.test.ts index 7c6332ba39..f7768769e7 100644 --- a/packages/spec/src/data/driver/memory.test.ts +++ b/packages/spec/src/data/driver/memory.test.ts @@ -1,5 +1,14 @@ import { describe, it, expect } from 'vitest'; -import { MemoryConfigSchema, MemoryPersistenceConfigSchema, MemoryDriverSpec } from './memory.zod'; +import { + MemoryConfigSchema, + MemoryPersistenceConfigSchema, + MemoryDriverSpec, + PersistenceTypeSchema, + FilePersistenceConfigSchema, + LocalStoragePersistenceConfigSchema, + CustomPersistenceConfigSchema, + PersistenceAdapterSchema, +} from './memory.zod'; describe('MemoryConfigSchema', () => { it('should accept empty config (all optional)', () => { @@ -38,27 +47,78 @@ describe('MemoryConfigSchema', () => { expect(config.strictMode).toBe(true); }); - it('should accept config with persistence', () => { + it('should accept persistence shorthand "file"', () => { + const config = MemoryConfigSchema.parse({ + persistence: 'file', + }); + + expect(config.persistence).toBe('file'); + }); + + it('should accept persistence shorthand "local"', () => { + const config = MemoryConfigSchema.parse({ + persistence: 'local', + }); + + expect(config.persistence).toBe('local'); + }); + + it('should accept persistence with file object config', () => { const config = MemoryConfigSchema.parse({ persistence: { - filePath: '/tmp/data.json', + type: 'file', + path: '/tmp/data.json', autoSaveInterval: 10000, }, }); expect(config.persistence).toBeDefined(); - expect(config.persistence!.filePath).toBe('/tmp/data.json'); - expect(config.persistence!.autoSaveInterval).toBe(10000); + const p = config.persistence as { type: 'file'; path?: string; autoSaveInterval: number }; + expect(p.type).toBe('file'); + expect(p.path).toBe('/tmp/data.json'); + expect(p.autoSaveInterval).toBe(10000); + }); + + it('should apply file persistence autoSaveInterval default', () => { + const config = MemoryConfigSchema.parse({ + persistence: { + type: 'file', + path: '/tmp/data.json', + }, + }); + + const p = config.persistence as { type: 'file'; autoSaveInterval: number }; + expect(p.autoSaveInterval).toBe(2000); }); - it('should apply persistence autoSaveInterval default', () => { + it('should accept persistence with local object config', () => { const config = MemoryConfigSchema.parse({ persistence: { - filePath: '/tmp/data.json', + type: 'local', + key: 'myapp:db', }, }); - expect(config.persistence!.autoSaveInterval).toBe(5000); + const p = config.persistence as { type: 'local'; key?: string }; + expect(p.type).toBe('local'); + expect(p.key).toBe('myapp:db'); + }); + + it('should accept persistence with custom adapter', () => { + const mockAdapter = { + load: async () => null, + save: async () => {}, + flush: async () => {}, + }; + const config = MemoryConfigSchema.parse({ + persistence: { adapter: mockAdapter }, + }); + + expect(config.persistence).toBeDefined(); + const p = config.persistence as { adapter: any }; + expect(typeof p.adapter.load).toBe('function'); + expect(typeof p.adapter.save).toBe('function'); + expect(typeof p.adapter.flush).toBe('function'); }); it('should accept config with indexes', () => { @@ -89,7 +149,8 @@ describe('MemoryConfigSchema', () => { }, strictMode: true, persistence: { - filePath: '/var/data/memory.json', + type: 'file', + path: '/var/data/memory.json', autoSaveInterval: 3000, }, indexes: { @@ -100,25 +161,25 @@ describe('MemoryConfigSchema', () => { expect(config.strictMode).toBe(true); expect(config.initialData!.users).toHaveLength(1); - expect(config.persistence!.filePath).toBe('/var/data/memory.json'); + const p = config.persistence as { type: 'file'; path?: string }; + expect(p.path).toBe('/var/data/memory.json'); expect(config.indexes!.users).toEqual(['email']); expect(config.maxRecordsPerObject).toBe(50000); }); - it('should reject persistence with invalid autoSaveInterval', () => { + it('should reject file persistence with invalid autoSaveInterval', () => { expect(() => MemoryConfigSchema.parse({ persistence: { - filePath: '/tmp/data.json', + type: 'file', + path: '/tmp/data.json', autoSaveInterval: 50, // Below minimum of 100 }, })).toThrow(); }); - it('should reject persistence without filePath', () => { + it('should reject invalid persistence string', () => { expect(() => MemoryConfigSchema.parse({ - persistence: { - autoSaveInterval: 5000, - }, + persistence: 'indexeddb', })).toThrow(); }); @@ -135,27 +196,128 @@ describe('MemoryConfigSchema', () => { }); }); -describe('MemoryPersistenceConfigSchema', () => { - it('should accept valid persistence config', () => { - const config = MemoryPersistenceConfigSchema.parse({ - filePath: '/data/store.json', +describe('PersistenceTypeSchema', () => { + it('should accept file type', () => { + expect(PersistenceTypeSchema.parse('file')).toBe('file'); + }); + + it('should accept local type', () => { + expect(PersistenceTypeSchema.parse('local')).toBe('local'); + }); + + it('should reject invalid type', () => { + expect(() => PersistenceTypeSchema.parse('indexeddb')).toThrow(); + }); +}); + +describe('FilePersistenceConfigSchema', () => { + it('should accept valid file persistence config', () => { + const config = FilePersistenceConfigSchema.parse({ + type: 'file', + path: '/data/store.json', autoSaveInterval: 10000, }); - expect(config.filePath).toBe('/data/store.json'); + expect(config.type).toBe('file'); + expect(config.path).toBe('/data/store.json'); expect(config.autoSaveInterval).toBe(10000); }); it('should apply default autoSaveInterval', () => { + const config = FilePersistenceConfigSchema.parse({ + type: 'file', + path: '/data/store.json', + }); + + expect(config.autoSaveInterval).toBe(2000); + }); + + it('should accept without path (uses default)', () => { + const config = FilePersistenceConfigSchema.parse({ + type: 'file', + }); + + expect(config.type).toBe('file'); + expect(config.path).toBeUndefined(); + }); +}); + +describe('LocalStoragePersistenceConfigSchema', () => { + it('should accept valid localStorage persistence config', () => { + const config = LocalStoragePersistenceConfigSchema.parse({ + type: 'local', + key: 'myapp:db', + }); + + expect(config.type).toBe('local'); + expect(config.key).toBe('myapp:db'); + }); + + it('should accept without key (uses default)', () => { + const config = LocalStoragePersistenceConfigSchema.parse({ + type: 'local', + }); + + expect(config.type).toBe('local'); + expect(config.key).toBeUndefined(); + }); +}); + +describe('CustomPersistenceConfigSchema', () => { + it('should accept valid custom adapter', () => { + const config = CustomPersistenceConfigSchema.parse({ + adapter: { + load: async () => null, + save: async () => {}, + flush: async () => {}, + }, + }); + + expect(typeof config.adapter.load).toBe('function'); + expect(typeof config.adapter.save).toBe('function'); + expect(typeof config.adapter.flush).toBe('function'); + }); +}); + +describe('MemoryPersistenceConfigSchema', () => { + it('should accept shorthand "file"', () => { + const config = MemoryPersistenceConfigSchema.parse('file'); + expect(config).toBe('file'); + }); + + it('should accept shorthand "local"', () => { + const config = MemoryPersistenceConfigSchema.parse('local'); + expect(config).toBe('local'); + }); + + it('should accept file object config', () => { + const config = MemoryPersistenceConfigSchema.parse({ + type: 'file', + path: '/tmp/data.json', + }); + expect(config).toEqual({ type: 'file', path: '/tmp/data.json', autoSaveInterval: 2000 }); + }); + + it('should accept local object config', () => { const config = MemoryPersistenceConfigSchema.parse({ - filePath: '/data/store.json', + type: 'local', + key: 'myapp:db', }); + expect(config).toEqual({ type: 'local', key: 'myapp:db' }); + }); - expect(config.autoSaveInterval).toBe(5000); + it('should accept custom adapter', () => { + const adapter = { + load: async () => null, + save: async () => {}, + flush: async () => {}, + }; + const config = MemoryPersistenceConfigSchema.parse({ adapter }); + expect((config as any).adapter).toBeDefined(); }); - it('should reject without filePath', () => { - expect(() => MemoryPersistenceConfigSchema.parse({})).toThrow(); + it('should reject invalid string', () => { + expect(() => MemoryPersistenceConfigSchema.parse('redis')).toThrow(); }); }); diff --git a/packages/spec/src/data/driver/memory.zod.ts b/packages/spec/src/data/driver/memory.zod.ts index b33acf4a9e..5f30338bf1 100644 --- a/packages/spec/src/data/driver/memory.zod.ts +++ b/packages/spec/src/data/driver/memory.zod.ts @@ -23,22 +23,92 @@ import { DriverDefinitionSchema } from '../datasource.zod'; // ========================================================================== /** - * Optional file-system persistence for the memory store. - * Enables data survival across process restarts. + * Persistence adapter interface for custom persistence implementations. + * Adapters must implement load/save/flush lifecycle methods. */ -export const MemoryPersistenceConfigSchema = z.object({ - /** - * File path to persist data (JSON format). - */ - filePath: z.string().describe('File path to persist data'), +export const PersistenceAdapterSchema = z.object({ + load: z.function().describe('Load persisted data on startup. Returns Promise | null>'), + save: z.function().describe('Save data to persistent storage. Accepts Record, returns Promise'), + flush: z.function().describe('Flush pending writes and ensure data is persisted. Returns Promise'), +}).describe('Custom persistence adapter interface'); - /** - * Auto-save interval in milliseconds. - * Data is written to disk on this cadence. - */ - autoSaveInterval: z.number().min(100).default(5000).describe('Auto-save interval in ms'), +export type PersistenceAdapter = z.infer; + +/** + * Persistence type enum. + * - `file`: Persist to disk file (Node.js only). + * - `local`: Persist to localStorage (Browser only). + */ +export const PersistenceTypeSchema = z.enum(['file', 'local']).describe('Persistence backend type'); + +export type PersistenceType = z.infer; + +/** + * File-system persistence configuration. + * Used in Node.js environments to save data to a JSON file. + */ +export const FilePersistenceConfigSchema = z.object({ + type: z.literal('file'), + /** File path to persist data (JSON format). Defaults to `.objectstack/data/memory-driver.json`. */ + path: z.string().optional().describe('File path to persist data'), + /** Auto-save interval in milliseconds. Default: 2000ms. */ + autoSaveInterval: z.number().min(100).default(2000).describe('Auto-save interval in ms'), }).describe('File-system persistence configuration'); +export type FilePersistenceConfig = z.infer; + +/** + * localStorage persistence configuration. + * Used in browser environments to save data to localStorage. + */ +export const LocalStoragePersistenceConfigSchema = z.object({ + type: z.literal('local'), + /** localStorage key. Defaults to `objectstack:memory-db`. */ + key: z.string().optional().describe('localStorage key for persisted data'), +}).describe('localStorage persistence configuration'); + +export type LocalStoragePersistenceConfig = z.infer; + +/** + * Custom adapter persistence configuration. + * Allows injecting a custom PersistenceAdapter implementation. + */ +export const CustomPersistenceConfigSchema = z.object({ + adapter: PersistenceAdapterSchema, +}).describe('Custom adapter persistence configuration'); + +export type CustomPersistenceConfig = z.infer; + +/** + * Unified persistence configuration. + * + * Supports shorthand strings and detailed object configs: + * - `'file'` — File-system persistence with defaults (Node.js) + * - `'local'` — localStorage persistence with defaults (Browser) + * - `{ type: 'file', path?: string }` — File-system with custom path + * - `{ type: 'local', key?: string }` — localStorage with custom key + * - `{ adapter: PersistenceAdapter }` — Custom adapter + * + * @example + * // Node.js with defaults + * persistence: 'file' + * + * // Browser with defaults + * persistence: 'local' + * + * // Custom file path + * persistence: { type: 'file', path: '/var/data/memory.json' } + * + * // Custom localStorage key + * persistence: { type: 'local', key: 'myapp:db' } + */ +export const MemoryPersistenceConfigSchema = z.union([ + PersistenceTypeSchema, + FilePersistenceConfigSchema, + LocalStoragePersistenceConfigSchema, + CustomPersistenceConfigSchema, +]).describe('Persistence configuration for the memory driver'); + // ========================================================================== // 2. Connection Configuration // ========================================================================== @@ -72,11 +142,24 @@ export const MemoryConfigSchema = z.object({ strictMode: z.boolean().default(false).describe('Throw on missing records instead of returning null'), /** - * Optional file-system persistence. - * When configured, the memory store periodically saves to disk - * and loads existing data on startup. + * Optional persistence configuration. + * When configured, the memory store automatically saves and restores data. + * + * - `'file'`: Persist to disk file (Node.js only, default path: `.objectstack/data/memory-driver.json`) + * - `'local'`: Persist to localStorage (Browser only, default key: `objectstack:memory-db`) + * - `{ type: 'file', path?: string }`: File-system with custom path + * - `{ type: 'local', key?: string }`: localStorage with custom key + * - `{ adapter: PersistenceAdapter }`: Custom persistence adapter + * + * @example + * // Node.js + * new InMemoryDriver({ persistence: 'file' }) + * // Browser + * new InMemoryDriver({ persistence: 'local' }) + * // Pure memory (default) + * new InMemoryDriver() */ - persistence: MemoryPersistenceConfigSchema.optional().describe('File-system persistence'), + persistence: MemoryPersistenceConfigSchema.optional().describe('Persistence configuration'), /** * Fields to index for faster lookups. From 833426fc35980528505b8bd41f2376d41ecf9349 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 26 Feb 2026 06:54:49 +0000 Subject: [PATCH 3/5] feat: implement FileSystem and LocalStorage persistence adapters with driver integration and tests Co-authored-by: hotlong <50353452+hotlong@users.noreply.github.com> --- .gitignore | 3 + packages/plugins/driver-memory/src/index.ts | 5 +- .../driver-memory/src/memory-driver.ts | 136 +++++++++++++ .../src/persistence/file-adapter.ts | 103 ++++++++++ .../driver-memory/src/persistence/index.ts | 4 + .../src/persistence/local-storage-adapter.ts | 60 ++++++ .../src/persistence/persistence.test.ts | 181 ++++++++++++++++++ 7 files changed, 491 insertions(+), 1 deletion(-) create mode 100644 packages/plugins/driver-memory/src/persistence/file-adapter.ts create mode 100644 packages/plugins/driver-memory/src/persistence/index.ts create mode 100644 packages/plugins/driver-memory/src/persistence/local-storage-adapter.ts create mode 100644 packages/plugins/driver-memory/src/persistence/persistence.test.ts diff --git a/.gitignore b/.gitignore index 92f25c2a86..36f8f40176 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,9 @@ package-lock.json dist/ *.tsbuildinfo +# ObjectStack data directory (persistence) +.objectstack/data/ + # IDE .vscode/ .idea/ diff --git a/packages/plugins/driver-memory/src/index.ts b/packages/plugins/driver-memory/src/index.ts index 1e0adc13c0..40a850961d 100644 --- a/packages/plugins/driver-memory/src/index.ts +++ b/packages/plugins/driver-memory/src/index.ts @@ -3,7 +3,10 @@ import { InMemoryDriver } from './memory-driver.js'; export { InMemoryDriver }; // Export class for direct usage -export type { InMemoryDriverConfig } from './memory-driver.js'; +export type { InMemoryDriverConfig, PersistenceAdapterInterface } from './memory-driver.js'; + +export { FileSystemPersistenceAdapter } from './persistence/file-adapter.js'; +export { LocalStoragePersistenceAdapter } from './persistence/local-storage-adapter.js'; export { MemoryAnalyticsService } from './memory-analytics.js'; export type { MemoryAnalyticsConfig } from './memory-analytics.js'; diff --git a/packages/plugins/driver-memory/src/memory-driver.ts b/packages/plugins/driver-memory/src/memory-driver.ts index d84e60808e..40d29aa833 100644 --- a/packages/plugins/driver-memory/src/memory-driver.ts +++ b/packages/plugins/driver-memory/src/memory-driver.ts @@ -6,6 +6,16 @@ import { Logger, createLogger } from '@objectstack/core'; import { Query, Aggregator } from 'mingo'; import { getValueByPath } from './memory-matcher.js'; +/** + * Persistence adapter interface. + * Matches the PersistenceAdapterSchema contract from @objectstack/spec. + */ +export interface PersistenceAdapterInterface { + load(): Promise | null>; + save(db: Record): Promise; + flush(): Promise; +} + /** * Configuration options for the InMemory driver. * Aligned with @objectstack/spec MemoryConfigSchema. @@ -17,6 +27,21 @@ export interface InMemoryDriverConfig { strictMode?: boolean; /** Optional: Logger instance */ logger?: Logger; + /** + * Optional persistence configuration. + * - `'file'` — File-system persistence with defaults (Node.js only) + * - `'local'` — localStorage persistence with defaults (Browser only) + * - `{ type: 'file', path?: string, autoSaveInterval?: number }` — File-system with options + * - `{ type: 'local', key?: string }` — localStorage with options + * - `{ adapter: PersistenceAdapterInterface }` — Custom adapter + */ + persistence?: string | { + type?: 'file' | 'local'; + path?: string; + key?: string; + autoSaveInterval?: number; + adapter?: PersistenceAdapterInterface; + }; } /** @@ -51,6 +76,7 @@ export class InMemoryDriver implements DriverInterface { private logger: Logger; private idCounters: Map = new Map(); private transactions: Map = new Map(); + private persistenceAdapter: PersistenceAdapterInterface | null = null; constructor(config?: InMemoryDriverConfig) { this.config = config || {}; @@ -100,6 +126,35 @@ export class InMemoryDriver implements DriverInterface { // =================================== async connect() { + // Initialize persistence adapter if configured + await this.initPersistence(); + + // Load persisted data if available + if (this.persistenceAdapter) { + const persisted = await this.persistenceAdapter.load(); + if (persisted) { + for (const [objectName, records] of Object.entries(persisted)) { + this.db[objectName] = records; + // Update ID counters based on persisted data + for (const record of records) { + if (record.id && typeof record.id === 'string') { + const match = record.id.match(/-(\d+)$/); + if (match) { + const counter = parseInt(match[1], 10); + const current = this.idCounters.get(objectName) || 0; + if (counter > current) { + this.idCounters.set(objectName, counter); + } + } + } + } + } + this.logger.info('InMemory Database restored from persistence', { + tables: Object.keys(persisted).length, + }); + } + } + // Load initial data if provided if (this.config.initialData) { for (const [objectName, records] of Object.entries(this.config.initialData)) { @@ -115,9 +170,22 @@ export class InMemoryDriver implements DriverInterface { } else { this.logger.info('InMemory Database Connected (Virtual)'); } + + // Start auto-save if using file adapter + if (this.persistenceAdapter && 'startAutoSave' in this.persistenceAdapter) { + (this.persistenceAdapter as any).startAutoSave(); + } } async disconnect() { + // Stop auto-save and flush pending writes + if (this.persistenceAdapter) { + if ('stopAutoSave' in this.persistenceAdapter) { + await (this.persistenceAdapter as any).stopAutoSave(); + } + await this.persistenceAdapter.flush(); + } + const tableCount = Object.keys(this.db).length; const recordCount = Object.values(this.db).reduce((sum, table) => sum + table.length, 0); @@ -226,6 +294,7 @@ export class InMemoryDriver implements DriverInterface { }; table.push(newRecord); + this.markDirty(); this.logger.debug('Record created', { object, id: newRecord.id, tableSize: table.length }); return { ...newRecord }; } @@ -253,6 +322,7 @@ export class InMemoryDriver implements DriverInterface { }; table[index] = updatedRecord; + this.markDirty(); this.logger.debug('Record updated', { object, id }); return { ...updatedRecord }; } @@ -293,6 +363,7 @@ export class InMemoryDriver implements DriverInterface { } table.splice(index, 1); + this.markDirty(); this.logger.debug('Record deleted', { object, id, tableSize: table.length }); return true; } @@ -350,6 +421,7 @@ export class InMemoryDriver implements DriverInterface { } } + if (count > 0) this.markDirty(); this.logger.debug('UpdateMany completed', { object, count }); return { count }; } @@ -377,6 +449,7 @@ export class InMemoryDriver implements DriverInterface { } const count = initialLength - this.db[object].length; + if (count > 0) this.markDirty(); this.logger.debug('DeleteMany completed', { object, count }); return { count }; } @@ -435,6 +508,7 @@ export class InMemoryDriver implements DriverInterface { // Restore the snapshot this.db = tx.snapshot; this.transactions.delete(txId); + this.markDirty(); this.logger.debug('Transaction rolled back', { txId }); } @@ -448,6 +522,7 @@ export class InMemoryDriver implements DriverInterface { async clear() { this.db = {}; this.idCounters.clear(); + this.markDirty(); this.logger.debug('All data cleared'); } @@ -818,4 +893,65 @@ export class InMemoryDriver implements DriverInterface { const timestamp = Date.now(); return `${key}-${timestamp}-${counter}`; } + + // =================================== + // Persistence + // =================================== + + /** + * Mark the database as dirty, triggering persistence save. + */ + private markDirty(): void { + if (this.persistenceAdapter) { + this.persistenceAdapter.save(this.db); + } + } + + /** + * Flush pending persistence writes to ensure data is safely stored. + */ + async flush(): Promise { + if (this.persistenceAdapter) { + await this.persistenceAdapter.flush(); + } + } + + /** + * Initialize the persistence adapter based on configuration. + */ + private async initPersistence(): Promise { + const persistence = this.config.persistence; + if (!persistence) return; + + if (typeof persistence === 'string') { + if (persistence === 'file') { + const { FileSystemPersistenceAdapter } = await import('./persistence/file-adapter.js'); + this.persistenceAdapter = new FileSystemPersistenceAdapter(); + } else if (persistence === 'local') { + const { LocalStoragePersistenceAdapter } = await import('./persistence/local-storage-adapter.js'); + this.persistenceAdapter = new LocalStoragePersistenceAdapter(); + } else { + throw new Error(`Unknown persistence type: "${persistence}". Use 'file' or 'local'.`); + } + } else if ('adapter' in persistence && persistence.adapter) { + this.persistenceAdapter = persistence.adapter; + } else if ('type' in persistence) { + if (persistence.type === 'file') { + const { FileSystemPersistenceAdapter } = await import('./persistence/file-adapter.js'); + this.persistenceAdapter = new FileSystemPersistenceAdapter({ + path: persistence.path, + autoSaveInterval: persistence.autoSaveInterval, + }); + } else if (persistence.type === 'local') { + const { LocalStoragePersistenceAdapter } = await import('./persistence/local-storage-adapter.js'); + this.persistenceAdapter = new LocalStoragePersistenceAdapter({ + key: persistence.key, + }); + } + } + + if (this.persistenceAdapter) { + this.logger.debug('Persistence adapter initialized'); + } + } } diff --git a/packages/plugins/driver-memory/src/persistence/file-adapter.ts b/packages/plugins/driver-memory/src/persistence/file-adapter.ts new file mode 100644 index 0000000000..256fbe8e9a --- /dev/null +++ b/packages/plugins/driver-memory/src/persistence/file-adapter.ts @@ -0,0 +1,103 @@ +// Copyright (c) 2025 ObjectStack. Licensed under the Apache-2.0 license. + +import * as fs from 'node:fs'; +import * as path from 'node:path'; + +/** + * FileSystemPersistenceAdapter + * + * Persists the in-memory database to a JSON file on disk. + * Supports atomic writes (write to temp file then rename) and auto-save with dirty tracking. + * + * Node.js only — will throw if used in non-Node.js environments. + */ +export class FileSystemPersistenceAdapter { + private readonly filePath: string; + private readonly autoSaveInterval: number; + private dirty = false; + private timer: ReturnType | null = null; + private currentDb: Record | null = null; + + constructor(options?: { path?: string; autoSaveInterval?: number }) { + this.filePath = options?.path || path.join('.objectstack', 'data', 'memory-driver.json'); + this.autoSaveInterval = options?.autoSaveInterval ?? 2000; + } + + /** + * Load persisted data from disk. + * Returns null if no file exists. + */ + async load(): Promise | null> { + try { + if (!fs.existsSync(this.filePath)) { + return null; + } + const raw = fs.readFileSync(this.filePath, 'utf-8'); + const data = JSON.parse(raw); + return data as Record; + } catch { + return null; + } + } + + /** + * Save data to disk using atomic write (temp file + rename). + */ + async save(db: Record): Promise { + this.currentDb = db; + this.dirty = true; + } + + /** + * Flush pending writes to disk immediately. + */ + async flush(): Promise { + if (!this.dirty || !this.currentDb) return; + await this.writeToDisk(this.currentDb); + this.dirty = false; + } + + /** + * Start the auto-save timer. + */ + startAutoSave(): void { + if (this.timer) return; + this.timer = setInterval(async () => { + if (this.dirty && this.currentDb) { + await this.writeToDisk(this.currentDb); + this.dirty = false; + } + }, this.autoSaveInterval); + + // Allow process to exit even if timer is running + if (this.timer && typeof this.timer === 'object' && 'unref' in this.timer) { + this.timer.unref(); + } + } + + /** + * Stop the auto-save timer and flush pending writes. + */ + async stopAutoSave(): Promise { + if (this.timer) { + clearInterval(this.timer); + this.timer = null; + } + await this.flush(); + } + + /** + * Atomic write: write to temp file, then rename. + */ + private async writeToDisk(db: Record): Promise { + const dir = path.dirname(this.filePath); + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir, { recursive: true }); + } + + const tmpPath = this.filePath + '.tmp'; + const json = JSON.stringify(db, null, 2); + fs.writeFileSync(tmpPath, json, 'utf-8'); + fs.renameSync(tmpPath, this.filePath); + } +} diff --git a/packages/plugins/driver-memory/src/persistence/index.ts b/packages/plugins/driver-memory/src/persistence/index.ts new file mode 100644 index 0000000000..c49d9f8706 --- /dev/null +++ b/packages/plugins/driver-memory/src/persistence/index.ts @@ -0,0 +1,4 @@ +// Copyright (c) 2025 ObjectStack. Licensed under the Apache-2.0 license. + +export { FileSystemPersistenceAdapter } from './file-adapter.js'; +export { LocalStoragePersistenceAdapter } from './local-storage-adapter.js'; diff --git a/packages/plugins/driver-memory/src/persistence/local-storage-adapter.ts b/packages/plugins/driver-memory/src/persistence/local-storage-adapter.ts new file mode 100644 index 0000000000..6421f4320e --- /dev/null +++ b/packages/plugins/driver-memory/src/persistence/local-storage-adapter.ts @@ -0,0 +1,60 @@ +// Copyright (c) 2025 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * LocalStoragePersistenceAdapter + * + * Persists the in-memory database to browser localStorage. + * Synchronous storage with a ~5MB size limit warning. + * + * Browser only — will throw if used in non-browser environments. + */ +export class LocalStoragePersistenceAdapter { + private readonly storageKey: string; + private static readonly SIZE_WARNING_BYTES = 4.5 * 1024 * 1024; // 4.5MB warning threshold + + constructor(options?: { key?: string }) { + this.storageKey = options?.key || 'objectstack:memory-db'; + } + + /** + * Load persisted data from localStorage. + * Returns null if no data exists. + */ + async load(): Promise | null> { + try { + const raw = localStorage.getItem(this.storageKey); + if (!raw) return null; + return JSON.parse(raw) as Record; + } catch { + return null; + } + } + + /** + * Save data to localStorage. + * Warns if data size approaches the ~5MB localStorage limit. + */ + async save(db: Record): Promise { + const json = JSON.stringify(db); + + if (json.length > LocalStoragePersistenceAdapter.SIZE_WARNING_BYTES) { + console.warn( + `[ObjectStack] localStorage persistence data size (${(json.length / 1024 / 1024).toFixed(2)}MB) ` + + `is approaching the ~5MB limit. Consider using a different persistence strategy.` + ); + } + + try { + localStorage.setItem(this.storageKey, json); + } catch (e: any) { + console.error('[ObjectStack] Failed to persist data to localStorage:', e?.message || e); + } + } + + /** + * Flush is a no-op for localStorage (writes are synchronous). + */ + async flush(): Promise { + // localStorage writes are synchronous, no flushing needed + } +} diff --git a/packages/plugins/driver-memory/src/persistence/persistence.test.ts b/packages/plugins/driver-memory/src/persistence/persistence.test.ts new file mode 100644 index 0000000000..f5a789d152 --- /dev/null +++ b/packages/plugins/driver-memory/src/persistence/persistence.test.ts @@ -0,0 +1,181 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; +import { InMemoryDriver } from '../memory-driver.js'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; + +const TEST_DATA_DIR = path.join('/tmp', 'objectstack-test-persistence'); +const TEST_FILE_PATH = path.join(TEST_DATA_DIR, 'test-db.json'); + +describe('InMemoryDriver Persistence', () => { + beforeEach(() => { + // Clean up test directory + if (fs.existsSync(TEST_DATA_DIR)) { + fs.rmSync(TEST_DATA_DIR, { recursive: true }); + } + }); + + afterEach(() => { + if (fs.existsSync(TEST_DATA_DIR)) { + fs.rmSync(TEST_DATA_DIR, { recursive: true }); + } + }); + + describe('File Persistence', () => { + it('should persist and restore data via file adapter', async () => { + // Create and populate driver with file persistence + const driver1 = new InMemoryDriver({ + persistence: { type: 'file', path: TEST_FILE_PATH, autoSaveInterval: 100 }, + }); + await driver1.connect(); + await driver1.create('users', { id: '1', name: 'Alice' }); + await driver1.create('users', { id: '2', name: 'Bob' }); + + // Flush and disconnect + await driver1.flush(); + await driver1.disconnect(); + + // Verify file was created + expect(fs.existsSync(TEST_FILE_PATH)).toBe(true); + + // Create a new driver and verify data is restored + const driver2 = new InMemoryDriver({ + persistence: { type: 'file', path: TEST_FILE_PATH, autoSaveInterval: 100 }, + }); + await driver2.connect(); + + const users = await driver2.find('users', { object: 'users' }); + expect(users).toHaveLength(2); + expect(users[0].name).toBe('Alice'); + expect(users[1].name).toBe('Bob'); + + await driver2.disconnect(); + }); + + it('should support shorthand "file" persistence string', async () => { + // Use shorthand — just verifies no error is thrown with 'file' + const driver = new InMemoryDriver({ persistence: 'file' }); + await driver.connect(); + await driver.create('items', { id: '1', name: 'Widget' }); + await driver.disconnect(); + }); + + it('should persist updates and deletes', async () => { + const driver1 = new InMemoryDriver({ + persistence: { type: 'file', path: TEST_FILE_PATH, autoSaveInterval: 100 }, + }); + await driver1.connect(); + + // Create, update, and delete + await driver1.create('tasks', { id: '1', title: 'Task A', done: false }); + await driver1.create('tasks', { id: '2', title: 'Task B', done: false }); + await driver1.update('tasks', '1', { done: true }); + await driver1.delete('tasks', '2'); + + await driver1.flush(); + await driver1.disconnect(); + + // Restore + const driver2 = new InMemoryDriver({ + persistence: { type: 'file', path: TEST_FILE_PATH, autoSaveInterval: 100 }, + }); + await driver2.connect(); + + const tasks = await driver2.find('tasks', { object: 'tasks' }); + expect(tasks).toHaveLength(1); + expect(tasks[0].id).toBe('1'); + expect(tasks[0].done).toBe(true); + + await driver2.disconnect(); + }); + + it('should handle missing persistence file gracefully', async () => { + const driver = new InMemoryDriver({ + persistence: { type: 'file', path: '/tmp/nonexistent/path/db.json' }, + }); + await driver.connect(); + + const users = await driver.find('users', { object: 'users' }); + expect(users).toHaveLength(0); + + await driver.disconnect(); + }); + }); + + describe('Custom Adapter Persistence', () => { + it('should use a custom adapter for persistence', async () => { + const stored: Record = {}; + const customAdapter = { + load: async () => Object.keys(stored).length > 0 ? { ...stored } : null, + save: async (db: Record) => { + for (const [k, v] of Object.entries(db)) { + stored[k] = [...v]; + } + }, + flush: async () => {}, + }; + + const driver1 = new InMemoryDriver({ + persistence: { adapter: customAdapter }, + }); + await driver1.connect(); + await driver1.create('projects', { id: '1', name: 'Alpha' }); + await driver1.disconnect(); + + // Verify data was saved via custom adapter + expect(stored.projects).toBeDefined(); + expect(stored.projects).toHaveLength(1); + + // Restore from custom adapter + const driver2 = new InMemoryDriver({ + persistence: { adapter: customAdapter }, + }); + await driver2.connect(); + const projects = await driver2.find('projects', { object: 'projects' }); + expect(projects).toHaveLength(1); + expect(projects[0].name).toBe('Alpha'); + + await driver2.disconnect(); + }); + }); + + describe('Pure Memory (No Persistence)', () => { + it('should work without persistence (default behavior)', async () => { + const driver = new InMemoryDriver(); + await driver.connect(); + + await driver.create('items', { id: '1', name: 'Widget' }); + const items = await driver.find('items', { object: 'items' }); + expect(items).toHaveLength(1); + + await driver.disconnect(); + + // After disconnect, data is gone + const itemsAfter = await driver.find('items', { object: 'items' }); + expect(itemsAfter).toHaveLength(0); + }); + }); + + describe('Bulk Operations with Persistence', () => { + it('should persist bulk creates', async () => { + const driver1 = new InMemoryDriver({ + persistence: { type: 'file', path: TEST_FILE_PATH, autoSaveInterval: 100 }, + }); + await driver1.connect(); + await driver1.bulkCreate('items', [ + { id: '1', name: 'A' }, + { id: '2', name: 'B' }, + { id: '3', name: 'C' }, + ]); + await driver1.flush(); + await driver1.disconnect(); + + const driver2 = new InMemoryDriver({ + persistence: { type: 'file', path: TEST_FILE_PATH, autoSaveInterval: 100 }, + }); + await driver2.connect(); + const items = await driver2.find('items', { object: 'items' }); + expect(items).toHaveLength(3); + await driver2.disconnect(); + }); + }); +}); From c23a9471d304fbc596f7d03d3501ff7778a0f634 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 26 Feb 2026 06:57:02 +0000 Subject: [PATCH 4/5] fix: address code review - fix ID counter restoration, remove as-any casts, simplify unref Co-authored-by: hotlong <50353452+hotlong@users.noreply.github.com> --- ROADMAP.md | 1 + .../.objectstack/data/memory-driver.json | 10 ++++++++++ .../driver-memory/src/memory-driver.ts | 20 ++++++++++++------- .../src/persistence/file-adapter.ts | 2 +- packages/spec/src/data/driver/memory.zod.ts | 4 ++++ 5 files changed, 29 insertions(+), 8 deletions(-) create mode 100644 packages/plugins/driver-memory/.objectstack/data/memory-driver.json diff --git a/ROADMAP.md b/ROADMAP.md index b30b6b1d3b..1c4be1f021 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -395,6 +395,7 @@ business/custom objects, aligning with industry best practices (e.g., ServiceNow - [x] **ObjectQL Engine** — CRUD, hooks (before/after), middleware chain, action registry - [x] **Schema Registry** — FQN namespacing, multi-package contribution, priority resolution - [x] **In-Memory Driver** — Full CRUD, bulk ops, transactions, aggregation pipeline (Mingo), streaming +- [x] **In-Memory Driver Persistence** — File-system (Node.js) and localStorage (Browser) persistence adapters with auto-save, custom adapter support - [x] **Metadata Service** — CRUD, query, bulk ops, overlay system, dependency tracking, import/export, file watching - [x] **Serializers** — JSON, YAML, TypeScript format support - [x] **Loaders** — Memory, Filesystem, Remote (HTTP) loaders diff --git a/packages/plugins/driver-memory/.objectstack/data/memory-driver.json b/packages/plugins/driver-memory/.objectstack/data/memory-driver.json new file mode 100644 index 0000000000..14e51b553f --- /dev/null +++ b/packages/plugins/driver-memory/.objectstack/data/memory-driver.json @@ -0,0 +1,10 @@ +{ + "items": [ + { + "id": "1", + "name": "Widget", + "created_at": "2026-02-26T06:56:49.563Z", + "updated_at": "2026-02-26T06:56:49.563Z" + } + ] +} \ No newline at end of file diff --git a/packages/plugins/driver-memory/src/memory-driver.ts b/packages/plugins/driver-memory/src/memory-driver.ts index 40d29aa833..e5b03b1027 100644 --- a/packages/plugins/driver-memory/src/memory-driver.ts +++ b/packages/plugins/driver-memory/src/memory-driver.ts @@ -14,6 +14,10 @@ export interface PersistenceAdapterInterface { load(): Promise | null>; save(db: Record): Promise; flush(): Promise; + /** Optional: Start periodic auto-save (used by FileSystemPersistenceAdapter). */ + startAutoSave?(): void; + /** Optional: Stop auto-save timer and flush pending writes. */ + stopAutoSave?(): Promise; } /** @@ -138,9 +142,11 @@ export class InMemoryDriver implements DriverInterface { // Update ID counters based on persisted data for (const record of records) { if (record.id && typeof record.id === 'string') { - const match = record.id.match(/-(\d+)$/); - if (match) { - const counter = parseInt(match[1], 10); + // ID format: {objectName}-{timestamp}-{counter} + const parts = record.id.split('-'); + const lastPart = parts[parts.length - 1]; + const counter = parseInt(lastPart, 10); + if (!isNaN(counter)) { const current = this.idCounters.get(objectName) || 0; if (counter > current) { this.idCounters.set(objectName, counter); @@ -172,16 +178,16 @@ export class InMemoryDriver implements DriverInterface { } // Start auto-save if using file adapter - if (this.persistenceAdapter && 'startAutoSave' in this.persistenceAdapter) { - (this.persistenceAdapter as any).startAutoSave(); + if (this.persistenceAdapter?.startAutoSave) { + this.persistenceAdapter.startAutoSave(); } } async disconnect() { // Stop auto-save and flush pending writes if (this.persistenceAdapter) { - if ('stopAutoSave' in this.persistenceAdapter) { - await (this.persistenceAdapter as any).stopAutoSave(); + if (this.persistenceAdapter.stopAutoSave) { + await this.persistenceAdapter.stopAutoSave(); } await this.persistenceAdapter.flush(); } diff --git a/packages/plugins/driver-memory/src/persistence/file-adapter.ts b/packages/plugins/driver-memory/src/persistence/file-adapter.ts index 256fbe8e9a..391e4debd4 100644 --- a/packages/plugins/driver-memory/src/persistence/file-adapter.ts +++ b/packages/plugins/driver-memory/src/persistence/file-adapter.ts @@ -70,7 +70,7 @@ export class FileSystemPersistenceAdapter { }, this.autoSaveInterval); // Allow process to exit even if timer is running - if (this.timer && typeof this.timer === 'object' && 'unref' in this.timer) { + if (this.timer) { this.timer.unref(); } } diff --git a/packages/spec/src/data/driver/memory.zod.ts b/packages/spec/src/data/driver/memory.zod.ts index 5f30338bf1..42710911c6 100644 --- a/packages/spec/src/data/driver/memory.zod.ts +++ b/packages/spec/src/data/driver/memory.zod.ts @@ -25,6 +25,10 @@ import { DriverDefinitionSchema } from '../datasource.zod'; /** * Persistence adapter interface for custom persistence implementations. * Adapters must implement load/save/flush lifecycle methods. + * + * Note: This schema validates presence of function properties only. + * Actual function signature enforcement is done at the TypeScript level + * via `PersistenceAdapterInterface` in the driver implementation. */ export const PersistenceAdapterSchema = z.object({ load: z.function().describe('Load persisted data on startup. Returns Promise | null>'), From 3da92273bd810ca4c96dcaf5193e0c88fbafbe8c Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 26 Feb 2026 06:57:46 +0000 Subject: [PATCH 5/5] fix: remove test artifact and fix .gitignore pattern for nested .objectstack/data/ Co-authored-by: hotlong <50353452+hotlong@users.noreply.github.com> --- .gitignore | 2 +- .../driver-memory/.objectstack/data/memory-driver.json | 10 ---------- 2 files changed, 1 insertion(+), 11 deletions(-) delete mode 100644 packages/plugins/driver-memory/.objectstack/data/memory-driver.json diff --git a/.gitignore b/.gitignore index 36f8f40176..8e28c713e6 100644 --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,7 @@ dist/ *.tsbuildinfo # ObjectStack data directory (persistence) -.objectstack/data/ +**/.objectstack/data/ # IDE .vscode/ diff --git a/packages/plugins/driver-memory/.objectstack/data/memory-driver.json b/packages/plugins/driver-memory/.objectstack/data/memory-driver.json deleted file mode 100644 index 14e51b553f..0000000000 --- a/packages/plugins/driver-memory/.objectstack/data/memory-driver.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "items": [ - { - "id": "1", - "name": "Widget", - "created_at": "2026-02-26T06:56:49.563Z", - "updated_at": "2026-02-26T06:56:49.563Z" - } - ] -} \ No newline at end of file