From c89dcb417768fb0f192b735d7ad2fea0ee07a6df Mon Sep 17 00:00:00 2001 From: Jack Zhuang <277994282+os-zhuang@users.noreply.github.com> Date: Mon, 22 Jun 2026 19:12:46 +0800 Subject: [PATCH 1/4] feat(datasource): DatasourceConnectionService + declared auto-connect (ADR-0062 Phase 1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Declare an external datasource → it auto-connects to a live ObjectQL driver and its federated objects are queryable with ZERO app code (no onEnable). Implements ADR-0062 Phase 1 (D1/D2/D5) toward epic #2163. D1 — one connect path: new DatasourceConnectionService owns the single "definition → live driver" path (factory build → credentialsRef resolve → connect → registerDriver under the datasource name → registerDatasourceDef → DDL-free syncObjectSchema per bound object). The runtime-admin registerPool now delegates to it; AppPlugin auto-connects code-defined datasources. Exposed as the 'datasource-connection' kernel service. D2 — opt-in-safe gate: connect only when external, an object explicitly binds via object.datasource, or autoConnect:true. Managed datasources referenced only by a datasourceMapping rule (e.g. app-crm's :memory: datasources) stay metadata-only — existing apps byte-for-byte unchanged. Adds datasource.autoConnect to the spec. D5 — lifecycle/ordering/policy: connect in AppPlugin.start() before the kernel:ready validation gate (init-all-then-start-all). Fail-fast for declared external + onMismatch:'fail'; degrade otherwise (always for runtime-admin/ rehydrate). New host-injectable DatasourceConnectPolicy (open-core default allows; multi-tenant host binds a stricter fail-closed policy) consulted before connect. Tests: 15 connection-service unit tests + 5 runtime integration tests (auto-connect, managed-unrouted stays metadata-only, queryable end-to-end, deny policy). onEnable + ctx.drivers.register remains a supported, idempotent escape hatch. Co-Authored-By: Claude Opus 4.8 --- .../adr-0062-datasource-connection-service.md | 38 ++ docs/adr/0062-external-datasource-runtime.md | 4 + packages/runtime/src/app-plugin.ts | 66 ++++ .../src/datasource-autoconnect.test.ts | 150 ++++++++ .../datasource-connection-service.test.ts | 232 +++++++++++++ .../src/contracts/connect-policy.ts | 69 ++++ .../service-datasource/src/contracts/index.ts | 11 + .../src/datasource-admin-plugin.ts | 77 ++-- .../src/datasource-admin-service.ts | 2 + .../src/datasource-connection-service.ts | 328 ++++++++++++++++++ .../services/service-datasource/src/index.ts | 18 + .../services/service-datasource/src/logger.ts | 2 + packages/spec/src/data/datasource.zod.ts | 12 + 13 files changed, 969 insertions(+), 40 deletions(-) create mode 100644 .changeset/adr-0062-datasource-connection-service.md create mode 100644 packages/runtime/src/datasource-autoconnect.test.ts create mode 100644 packages/services/service-datasource/src/__tests__/datasource-connection-service.test.ts create mode 100644 packages/services/service-datasource/src/contracts/connect-policy.ts create mode 100644 packages/services/service-datasource/src/datasource-connection-service.ts diff --git a/.changeset/adr-0062-datasource-connection-service.md b/.changeset/adr-0062-datasource-connection-service.md new file mode 100644 index 0000000000..32c6a08f9e --- /dev/null +++ b/.changeset/adr-0062-datasource-connection-service.md @@ -0,0 +1,38 @@ +--- +"@objectstack/service-datasource": minor +"@objectstack/runtime": minor +"@objectstack/spec": minor +--- + +feat(datasource): auto-connect declared external datasources (ADR-0062 Phase 1, D1/D2/D5) + +A declared external datasource is now connected to a live ObjectQL driver and its +federated objects are queryable **with zero app code** — no `onEnable` driver +wiring. Implements ADR-0062 Phase 1. + +- **D1 — one connect path.** New `DatasourceConnectionService` in + `@objectstack/service-datasource` owns the single "definition → live driver" + path: build via the injected driver factory → resolve `external.credentialsRef` + via the `SecretBinder` → connect → `engine.registerDriver` under the datasource + name → register the datasource def → sync each bound federated object's read + metadata (DDL-free). Both origins converge on it: the runtime-admin + `registerPool` now delegates here, and `AppPlugin` auto-connects code-defined + datasources. Exposed as the `'datasource-connection'` kernel service. +- **D2 — opt-in-safe gate.** A declared datasource auto-connects only when it is + `external`, an object **explicitly** binds to it via `object.datasource`, or it + sets the new `autoConnect: true` flag. A managed datasource that nothing + explicitly binds (incl. ones referenced only by a `datasourceMapping` rule, e.g. + `examples/app-crm`'s `:memory:` datasources) stays metadata-only — existing apps + are byte-for-byte unchanged. See the ADR-0062 D2 implementation note. +- **D5 — lifecycle, ordering & policy.** Connect happens in `AppPlugin.start()` + (before the `kernel:ready` validation gate, relying on the kernel's + init-all-then-start-all ordering). Fail-fast for a declared `external` datasource + with `validation.onMismatch: 'fail'`; degrade-with-warning otherwise (and always + for runtime-admin/rehydrate, so a UI action or replica blip never bricks the + server). Adds a host-injectable `DatasourceConnectPolicy` (open-core default + allows; a multi-tenant host binds a stricter fail-closed policy for egress + isolation) consulted before every connect — one connect path, no cloud fork. + +Adds `datasource.autoConnect` to the spec. The legacy `onEnable` + +`ctx.drivers.register` bridge remains supported as an escape hatch (idempotent vs. +auto-connect). No behavior change for managed apps. diff --git a/docs/adr/0062-external-datasource-runtime.md b/docs/adr/0062-external-datasource-runtime.md index 637b9d682f..f93544f828 100644 --- a/docs/adr/0062-external-datasource-runtime.md +++ b/docs/adr/0062-external-datasource-runtime.md @@ -57,6 +57,8 @@ Introduce a single service that, given a datasource definition, builds a driver Auto-connect must not change apps that today declare datasources that are *decorative* or routed via `datasourceMapping` (e.g. `examples/app-crm`'s `crm_primary`/`crm_analytics`). Gate auto-connect so a declared datasource is only connected when it is meaningfully addressed: **(a)** it is `external` (`schemaMode !== 'managed'`), or **(b)** an object/`datasourceMapping` actually routes to it, or **(c)** it sets an explicit `autoConnect: true`. A managed datasource that nothing routes to stays metadata-only (today's behavior). The `default` datasource keeps its current dedicated bootstrap. This is the load-bearing backward-compat decision. +> **Phase 1 implementation note (#2163) — gate (b) is "explicit `object.datasource`", not "mapped".** Implementing D2 against `examples/app-crm` surfaced a conflict between "an object/`datasourceMapping` routes to it" and the "byte-for-byte unchanged" mandate. `app-crm`'s `crm_primary` (`:memory:`, `managed`) *is* referenced by a `datasourceMapping` rule (and is the `default:true` fallback) but has **no** `onEnable` driver, so today `engine.getDriver` finds no `crm_primary` driver and its objects fall through to the `default` driver. Auto-connecting it on the strength of the mapping rule would build a fresh, empty `:memory:` driver and silently divert those objects — a behavior change. So the gate **does not** auto-connect on a `datasourceMapping` rule alone: a *managed* datasource that is only mapped (namespace/package/`default`) is treated as decorative and left metadata-only. Gate (b) fires only when an object **explicitly** binds via `object.datasource === ` — a binding that today *throws* when the driver is unregistered, so auto-connecting it is a strict improvement, never a change. External datasources (a) and `autoConnect:true` (c) are unaffected. See `isDatasourceAddressed()` in `@objectstack/service-datasource`. + ### D3 — Credentials resolved at connect via `SecretBinder`/`ICryptoProvider` `DatasourceConnectionService` resolves `external.credentialsRef` (and any `secret` config fields) through the host-provided `SecretBinder` over `ICryptoProvider` **before** building the driver. Open-core default is `InMemoryCryptoProvider`; a datasource that needs a secret the host cannot decrypt **fails closed** (clear boot error, datasource left unconnected — not a silent skip). This reuses the exact mechanism the runtime-admin "Add Datasource" wizard already uses, so code- and runtime-origin secrets converge. @@ -69,6 +71,8 @@ Code-defined datasources surface in `GET /api/v1/datasources`, `GET /api/v1/meta `DatasourceConnectionService` owns connect/disconnect (graceful shutdown), pool config per datasource, and an optional health probe surfaced in the admin list (`status`). **Ordering**: all declared datasources connect **before** the `kernel:ready` external-validation gate (ADR-0015 §5.2) and before first query — i.e. during plugin init/start, not in a `kernel:ready` handler. Connect failure policy is **fail-fast for `external` with `validation.onMismatch: 'fail'`**, **degrade-with-warning** otherwise (a connectivity blip on an optional analytics replica should not brick boot). +> **Phase 1 implementation notes (#2163).** *Ordering* is satisfied by the kernel's two-phase boot (init-all → start-all): the connection service is registered as the `'datasource-connection'` kernel service during the datasource-admin plugin's `init()`, and declared datasources are auto-connected from `AppPlugin.start()` — which runs before the `kernel:ready` validation gate. Because boot schema-sync runs before the external driver exists, `connect()` calls `engine.syncObjectSchema()` for each bound federated object (DDL-free), so they are queryable with zero app code. *Fail-fast* is scoped to the **declared-auto** trigger; the runtime-admin create/update + boot-rehydration triggers always degrade-with-warning, preserving the pre-ADR-0062 admin behavior (a UI action never bricks the running server). *Connect policy* (the epic #2163 seam): a host-injectable `DatasourceConnectPolicy` is consulted before every connect; the open-core default allows (subject to the D2 gate), and a multi-tenant host binds a stricter, fail-closed policy for egress isolation — one connect path, no cloud fork. + ### D6 — Native-analytics SQL honors the remote table/columns The analytics native-SQL strategy compiles its own `FROM ""` / column references outside the driver (ADR-0015 §18 noted this). It must resolve an external object's physical table (`remoteName`/`remoteSchema`) and columns (`columnMap`) the same way `SqlDriver` now does — reusing the driver's resolution (e.g. an exposed `physicalTableFor(object)` / `physicalColumnFor(object, field)`), not a second copy. Until then, analytics over external objects stays disabled rather than silently querying the wrong table. diff --git a/packages/runtime/src/app-plugin.ts b/packages/runtime/src/app-plugin.ts index 34820e1bd8..4bdaa669f7 100644 --- a/packages/runtime/src/app-plugin.ts +++ b/packages/runtime/src/app-plugin.ts @@ -240,6 +240,72 @@ export class AppPlugin implements Plugin { }); } + // Auto-connect declared datasources (ADR-0062 D1/D2/D5). The metadata + // registration above only makes a datasource *visible*; to make its + // federated objects *queryable* with zero app boilerplate, build + open + // + register a live driver via the shared `'datasource-connection'` + // service (when present — wired by the datasource-admin plugin). The + // service applies the D2 gate (connect only when `external`, an object + // explicitly binds via `object.datasource`, or `autoConnect:true`) and + // the host connect policy, so managed+unrouted datasources stay + // metadata-only (e.g. app-crm's `:memory:` datasources — byte-for-byte + // unchanged). Idempotent vs. a legacy `onEnable` driver registration. + // + // Runs in `start()` (before the `kernel:ready` external-validation gate) + // so the kernel's init-all-then-start-all ordering guarantees the + // connection service was already registered during init. + try { + const dsDefs = this.bundle.datasources; + const dsList: any[] = Array.isArray(dsDefs) + ? dsDefs + : dsDefs && typeof dsDefs === 'object' + ? Object.entries(dsDefs).map(([name, def]) => ({ name, ...(def as any) })) + : []; + if (dsList.length > 0) { + // `ctx.getService` throws when a service is absent, so resolve + // defensively — a runtime without the datasource-admin plugin + // simply has no connection service, and declared datasources + // stay metadata-only (the legacy `onEnable` escape hatch still + // works). This must NOT fall into the fail-fast catch below. + let connection: + | { + connectDeclared?: (input: { + datasources: any[]; + objects?: Array<{ name?: string; datasource?: string }>; + }) => Promise>; + } + | undefined; + try { + connection = ctx.getService('datasource-connection'); + } catch { + connection = undefined; + } + if (typeof connection?.connectDeclared === 'function') { + const objects = Array.isArray(this.bundle.objects) ? this.bundle.objects : []; + const results = await connection.connectDeclared({ datasources: dsList, objects }); + const connected = results.filter((r) => r.status === 'connected'); + if (connected.length > 0) { + ctx.logger.info('Auto-connected declared datasources', { + appId, + connected: connected.map((r) => r.name), + }); + } + } else { + ctx.logger.debug('No datasource-connection service — declared datasources stay metadata-only', { appId }); + } + } + } catch (err) { + // A fail-fast (external + onMismatch:'fail') connect error propagates + // to brick boot as intended (ADR-0062 D5); other errors are already + // degraded inside the connection service. Re-throw so the kernel + // surfaces the real cause. + ctx.logger.error('[AppPlugin] declared-datasource auto-connect failed', { + appId, + error: (err as Error)?.message ?? String(err), + }); + throw err; + } + // [ADR-0057 / #2077] Surface stack-declared SECURITY metadata (roles, // permission sets, sharing rules, policies) in the metadata registry so // the boot seeders (plugin-security / plugin-sharing) and runtime diff --git a/packages/runtime/src/datasource-autoconnect.test.ts b/packages/runtime/src/datasource-autoconnect.test.ts new file mode 100644 index 0000000000..9af1e640d8 --- /dev/null +++ b/packages/runtime/src/datasource-autoconnect.test.ts @@ -0,0 +1,150 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. +// +// ADR-0062 Phase 1 acceptance (D1/D2/D5): a stack that only *declares* an +// external datasource — with NO `onEnable` driver wiring — auto-connects it to +// a live ObjectQL driver and its federated objects become queryable, while a +// managed + unrouted datasource stays metadata-only (existing apps unchanged). +// +// This boots the host-config shape (instantiated plugins, no MetadataPlugin — +// the same shape `examples/app-showcase` runs under `os dev`) with the REAL +// driver factory (`createDefaultDatasourceDriverFactory`) building an in-memory +// driver, so the full AppPlugin → `datasource-connection` → engine path runs +// without any native driver dependency. + +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { Runtime } from './runtime.js'; +import { DriverPlugin } from './driver-plugin.js'; +import { AppPlugin } from './app-plugin.js'; +import type { DatasourceConnectPolicy } from '@objectstack/service-datasource'; + +const BOOT_TIMEOUT = 60_000; + +// One external datasource (auto-connect target) + one managed, unrouted +// datasource (must stay metadata-only). NO `onEnable` anywhere. +function artifact() { + return { + manifest: { id: 'com.test.ds-autoconnect', name: 'DS AutoConnect', version: '1.0.0' }, + objects: [ + // Federated object bound to the external datasource (ADR-0015). + { + name: 'ext_note', + label: 'External Note', + datasource: 'autoconn_ext', + external: {}, + fields: { id: { type: 'text' }, title: { type: 'text' } }, + }, + // A normal object on the default datasource. + { name: 'note', label: 'Note', fields: { title: { type: 'text' } } }, + ], + datasources: [ + { + name: 'autoconn_ext', + label: 'External (in-memory)', + driver: 'memory', + schemaMode: 'external', + origin: 'code', + config: {}, + external: { allowWrites: false, validation: { onMismatch: 'warn', checkOnBoot: false } }, + active: true, + }, + // Managed + unrouted: nothing binds to it, not external, no autoConnect. + // Mirrors app-crm's decorative `:memory:` datasources — must NOT connect. + { + name: 'decorative', + label: 'Decorative (unrouted)', + driver: 'memory', + schemaMode: 'managed', + origin: 'code', + config: {}, + active: true, + }, + ], + }; +} + +async function boot(opts: { connectPolicy?: DatasourceConnectPolicy } = {}) { + const { ObjectQLPlugin } = await import('@objectstack/objectql'); + const { InMemoryDriver } = await import('@objectstack/driver-memory'); + const { DatasourceAdminServicePlugin, createDefaultDatasourceDriverFactory } = await import( + '@objectstack/service-datasource' + ); + + const runtime = new Runtime({ cluster: false }); + const kernel = runtime.getKernel(); + await kernel.use(new DriverPlugin(new InMemoryDriver())); // default driver + await kernel.use(new ObjectQLPlugin()); + await kernel.use(new AppPlugin(artifact())); + await kernel.use( + new DatasourceAdminServicePlugin({ + driverFactory: createDefaultDatasourceDriverFactory(), + connectPolicy: opts.connectPolicy, + }), + ); + await kernel.bootstrap(); + return kernel; +} + +describe('ADR-0062 declared-datasource auto-connect', () => { + let kernel: Awaited>; + + beforeAll(async () => { + kernel = await boot(); + }, BOOT_TIMEOUT); + + afterAll(async () => { + try { await (kernel as any)?.stop?.(); } catch { /* noop */ } + }); + + it('auto-connects the declared EXTERNAL datasource as a live driver (no onEnable)', () => { + const engine = kernel.getService<{ getDriverByName(n: string): unknown }>('data'); + expect(engine.getDriverByName('autoconn_ext')).toBeDefined(); + }); + + it('leaves a managed + unrouted datasource metadata-only (app-crm byte-for-byte unchanged)', () => { + const engine = kernel.getService<{ getDriverByName(n: string): unknown }>('data'); + expect(engine.getDriverByName('decorative')).toBeUndefined(); + // …but it is still VISIBLE in the metadata registry. + // (visibility is asserted via the admin service below) + }); + + it('still surfaces BOTH datasources in the metadata registry (visibility unchanged)', async () => { + const metadata = kernel.getService<{ list(t: string): Promise }>('metadata'); + const names = (await metadata.list('datasource')).map((d) => d?.name); + expect(names).toContain('autoconn_ext'); + expect(names).toContain('decorative'); + }); + + it('makes the federated object queryable through the engine with zero app code', async () => { + const engine = kernel.getService<{ + getDriverByName(n: string): any; + find(object: string, query?: any): Promise; + }>('data'); + // Seed the live external driver directly (bypassing the read-only write gate, + // exactly as a real remote DB would already hold the rows). + const driver = engine.getDriverByName('autoconn_ext'); + await driver.bulkCreate('ext_note', [ + { id: 'n1', title: 'first' }, + { id: 'n2', title: 'second' }, + ]); + const rows = await engine.find('ext_note'); + expect(rows.map((r) => r.title).sort()).toEqual(['first', 'second']); + }); +}); + +describe('ADR-0062 connect policy seam', () => { + it('a deny policy leaves the external datasource unconnected (cloud egress isolation)', async () => { + const denyExternal: DatasourceConnectPolicy = { + canConnect: (ds) => (ds.schemaMode === 'external' ? { allow: false, reason: 'egress blocked' } : { allow: true }), + }; + const kernel = await boot({ connectPolicy: denyExternal }); + try { + const engine = kernel.getService<{ getDriverByName(n: string): unknown }>('data'); + expect(engine.getDriverByName('autoconn_ext')).toBeUndefined(); + // Still visible — denied means metadata-only, not invisible. + const metadata = kernel.getService<{ list(t: string): Promise }>('metadata'); + expect((await metadata.list('datasource')).map((d) => d?.name)).toContain('autoconn_ext'); + } finally { + try { await (kernel as any)?.stop?.(); } catch { /* noop */ } + } + }, BOOT_TIMEOUT); +}); diff --git a/packages/services/service-datasource/src/__tests__/datasource-connection-service.test.ts b/packages/services/service-datasource/src/__tests__/datasource-connection-service.test.ts new file mode 100644 index 0000000000..d2e186e526 --- /dev/null +++ b/packages/services/service-datasource/src/__tests__/datasource-connection-service.test.ts @@ -0,0 +1,232 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +import { describe, it, expect, vi } from 'vitest'; +import { + DatasourceConnectionService, + isDatasourceAddressed, + type ConnectableDatasource, + type ConnectionEngineLike, +} from '../datasource-connection-service.js'; +import type { IDatasourceDriverFactory } from '../contracts/datasource-driver-factory.js'; +import type { DatasourceConnectPolicy } from '../contracts/connect-policy.js'; + +/** A fake engine recording driver registration + schema syncs. */ +function fakeEngine() { + const drivers = new Map(); + const defs: Array<{ name: string; schemaMode?: string }> = []; + const synced: string[] = []; + const engine: ConnectionEngineLike & { drivers: typeof drivers; defs: typeof defs; synced: string[] } = { + drivers, + defs, + synced, + registerDriver: (driver: any) => { + if (drivers.has(driver.name)) return; // mirror engine's skip-if-present + drivers.set(driver.name, driver); + }, + registerDatasourceDef: (def) => { + defs.push(def); + }, + getDriverByName: (name) => drivers.get(name), + syncObjectSchema: async (name) => { + synced.push(name); + }, + }; + return engine; +} + +/** A fake factory that builds a trivial connectable handle. */ +function fakeFactory(opts: { supports?: (id: string) => boolean; connectThrows?: boolean } = {}): IDatasourceDriverFactory { + return { + supports: opts.supports ?? (() => true), + create: vi.fn(async () => { + const driver: any = { name: 'com.fake.driver' }; + return { + driver, + connect: opts.connectThrows + ? async () => { + throw new Error('connection refused'); + } + : async () => { + driver.connected = true; + }, + }; + }), + }; +} + +function svc(over: { + factory?: IDatasourceDriverFactory | undefined; + engine?: ConnectionEngineLike | undefined; + policy?: DatasourceConnectPolicy; + secrets?: { resolve?: (ref: string) => Promise }; +} = {}) { + const engine = over.engine === undefined ? fakeEngine() : over.engine; + const factory = over.factory === undefined ? fakeFactory() : over.factory; + const service = new DatasourceConnectionService({ + factory: () => factory ?? undefined, + engine: () => engine ?? undefined, + policy: over.policy, + secrets: over.secrets, + }); + return { service, engine: engine as ReturnType | undefined, factory }; +} + +const externalDs: ConnectableDatasource = { + name: 'warehouse', + driver: 'sqlite', + schemaMode: 'external', + config: { filename: '/tmp/w.db' }, + external: { allowWrites: false, validation: { onMismatch: 'warn' } }, +}; + +describe('isDatasourceAddressed (ADR-0062 D2 gate)', () => { + it('connects external datasources (a)', () => { + expect(isDatasourceAddressed({ name: 'x', schemaMode: 'external' }, { objects: [] })).toBe(true); + expect(isDatasourceAddressed({ name: 'x', schemaMode: 'validate-only' }, { objects: [] })).toBe(true); + }); + + it('connects when an object explicitly binds via object.datasource (b)', () => { + expect( + isDatasourceAddressed({ name: 'reporting', schemaMode: 'managed' }, { objects: [{ name: 'o', datasource: 'reporting' }] }), + ).toBe(true); + }); + + it('connects when autoConnect:true (c)', () => { + expect(isDatasourceAddressed({ name: 'x', schemaMode: 'managed', autoConnect: true }, { objects: [] })).toBe(true); + }); + + it('does NOT connect a managed datasource that is only mapped / unrouted (app-crm byte-for-byte unchanged)', () => { + // app-crm: crm_primary is managed + referenced by datasourceMapping only, + // crm_analytics is managed + unrouted. Neither has an object binding. + expect(isDatasourceAddressed({ name: 'crm_primary', schemaMode: 'managed' }, { objects: [] })).toBe(false); + expect(isDatasourceAddressed({ name: 'crm_analytics', schemaMode: 'managed' }, { objects: [] })).toBe(false); + // An object bound to a DIFFERENT datasource must not flip the gate. + expect( + isDatasourceAddressed({ name: 'crm_primary', schemaMode: 'managed' }, { objects: [{ name: 'acct', datasource: 'default' }] }), + ).toBe(false); + }); +}); + +describe('DatasourceConnectionService.connect', () => { + it('builds, connects, stamps the datasource name, and registers the driver + def', async () => { + const { service, engine, factory } = svc(); + const result = await service.connect(externalDs, { objects: ['ext_customer'] }); + expect(result.status).toBe('connected'); + expect(factory.create).toHaveBeenCalledOnce(); + // Driver registered under the DATASOURCE name (engine routes by driver.name). + expect(engine!.drivers.has('warehouse')).toBe(true); + expect(engine!.drivers.get('warehouse')!.name).toBe('warehouse'); + // Datasource definition recorded for the write gate. + expect(engine!.defs).toEqual([{ name: 'warehouse', schemaMode: 'external', external: externalDs.external }]); + // Bound external objects got read metadata synced (DDL-free). + expect(engine!.synced).toEqual(['ext_customer']); + }); + + it('is idempotent — an already-registered driver is skipped (onEnable escape hatch)', async () => { + const { service, engine, factory } = svc(); + engine!.drivers.set('warehouse', { name: 'warehouse' }); // pretend onEnable registered it + const result = await service.connect(externalDs, { objects: ['ext_customer'] }); + expect(result.status).toBe('already-registered'); + expect(factory.create).not.toHaveBeenCalled(); + expect(engine!.synced).toEqual([]); // no double sync + }); + + it('resolves external.credentialsRef via the secret resolver before building', async () => { + const resolve = vi.fn(async () => 's3cr3t'); + const create = vi.fn(async () => ({ driver: { name: 'd' }, connect: async () => {} })); + const factory: IDatasourceDriverFactory = { supports: () => true, create }; + const { service } = svc({ factory, secrets: { resolve } }); + await service.connect({ ...externalDs, external: { ...externalDs.external, credentialsRef: 'secret:wh/pw' } }); + expect(resolve).toHaveBeenCalledWith('secret:wh/pw'); + expect(create).toHaveBeenCalledWith(expect.objectContaining({ secret: 's3cr3t' })); + }); + + it('respects a deny policy — left unconnected, metadata-only', async () => { + const policy: DatasourceConnectPolicy = { canConnect: () => ({ allow: false, reason: 'egress blocked' }) }; + const { service, engine, factory } = svc({ policy }); + const result = await service.connect(externalDs); + expect(result.status).toBe('skipped-policy'); + expect(result.reason).toBe('egress blocked'); + expect(factory.create).not.toHaveBeenCalled(); + expect(engine!.drivers.size).toBe(0); + }); + + it('treats a throwing policy as a denial (fail-closed)', async () => { + const policy: DatasourceConnectPolicy = { + canConnect: () => { + throw new Error('policy backend down'); + }, + }; + const { service, engine } = svc({ policy }); + const result = await service.connect(externalDs); + expect(result.status).toBe('skipped-policy'); + expect(engine!.drivers.size).toBe(0); + }); + + it('degrades (no throw) when there is no factory / engine', async () => { + const noFactory = new DatasourceConnectionService({ factory: () => undefined, engine: () => fakeEngine() }); + expect((await noFactory.connect(externalDs)).status).toBe('skipped-no-infra'); + const noEngine = new DatasourceConnectionService({ factory: () => fakeFactory(), engine: () => undefined }); + expect((await noEngine.connect(externalDs)).status).toBe('skipped-no-infra'); + }); + + describe('D5 connect-failure policy', () => { + const failExternal: ConnectableDatasource = { + ...externalDs, + external: { allowWrites: false, validation: { onMismatch: 'fail' } }, + }; + + it('fail-fast: a declared-auto external + onMismatch:fail re-throws (bricks boot)', async () => { + const { service } = svc({ factory: fakeFactory({ connectThrows: true }) }); + await expect( + service.connect(failExternal, { context: { trigger: 'declared-auto' } }), + ).rejects.toThrow(/fail-fast/); + }); + + it('degrade: the SAME datasource connected via runtime-admin never bricks the server', async () => { + const { service } = svc({ factory: fakeFactory({ connectThrows: true }) }); + const result = await service.connect(failExternal, { context: { trigger: 'runtime-admin' } }); + expect(result.status).toBe('failed-degraded'); + }); + + it('degrade: external + onMismatch:warn degrades even at boot', async () => { + const { service } = svc({ factory: fakeFactory({ connectThrows: true }) }); + const result = await service.connect(externalDs, { context: { trigger: 'declared-auto' } }); + expect(result.status).toBe('failed-degraded'); + }); + }); +}); + +describe('DatasourceConnectionService.connectDeclared', () => { + it('connects only the gated datasources and syncs each one’s bound objects', async () => { + const { service, engine, factory } = svc(); + const datasources: ConnectableDatasource[] = [ + externalDs, // external → connect + { name: 'crm_primary', driver: 'sqlite', schemaMode: 'managed', config: { filename: ':memory:' } }, // managed+unrouted → skip + { name: 'reporting', driver: 'sqlite', schemaMode: 'managed', config: {} }, // managed but object-bound → connect + ]; + const objects = [ + { name: 'ext_customer', datasource: 'warehouse' }, + { name: 'report_row', datasource: 'reporting' }, + { name: 'account', datasource: 'default' }, // routes to default → no effect + ]; + const results = await service.connectDeclared({ datasources, objects }); + const byName = Object.fromEntries(results.map((r) => [r.name, r.status])); + expect(byName).toEqual({ warehouse: 'connected', reporting: 'connected' }); + expect(engine!.drivers.has('crm_primary')).toBe(false); // unchanged + expect(engine!.drivers.has('warehouse')).toBe(true); + expect(engine!.drivers.has('reporting')).toBe(true); + expect(engine!.synced.sort()).toEqual(['ext_customer', 'report_row']); + expect(factory.create).toHaveBeenCalledTimes(2); + }); + + it('skips inactive datasources', async () => { + const { service, engine } = svc(); + const results = await service.connectDeclared({ + datasources: [{ ...externalDs, active: false }], + objects: [], + }); + expect(results).toEqual([]); + expect(engine!.drivers.size).toBe(0); + }); +}); diff --git a/packages/services/service-datasource/src/contracts/connect-policy.ts b/packages/services/service-datasource/src/contracts/connect-policy.ts new file mode 100644 index 0000000000..cf933e7cfe --- /dev/null +++ b/packages/services/service-datasource/src/contracts/connect-policy.ts @@ -0,0 +1,69 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * DatasourceConnectPolicy — host-injectable gate consulted *before* + * {@link DatasourceConnectionService} builds and registers a live driver + * (ADR-0062 D1/D5, and the epic #2163 "connect-policy seam" note). + * + * The framework ships a permissive default ({@link allowAllConnectPolicy}) so a + * self-hosted single-environment runtime connects external datasources out of + * the box (subject to the D2 auto-connect gate, which is applied separately by + * {@link DatasourceConnectionService.connectDeclared}). A multi-tenant host + * (shared-container cloud) binds a stricter policy that can *fail-close* on the + * shared runtime — e.g. checking `sys_environment.plan`, an egress allow-list, + * and per-tenant quota — to enforce SSRF / egress isolation. + * + * This keeps a single connect path for code- and runtime-origin datasources + * (D1): the host injects a policy rather than forking a second connect path. + * No plan coupling lives in the open framework. + */ + +/** Why a connect is being attempted — lets a policy treat origins differently. */ +export interface DatasourceConnectContext { + /** Provenance of the datasource being connected. */ + origin?: 'code' | 'runtime'; + /** + * What triggered this connect: + * - `declared-auto` — code-defined datasource auto-connected at boot (D2 gate passed). + * - `runtime-admin` — UI "Add/Update Datasource" hot pool registration. + * - `rehydrate` — boot rehydration of a persisted runtime datasource. + */ + trigger?: 'declared-auto' | 'runtime-admin' | 'rehydrate'; +} + +/** A policy verdict. `allow:false` leaves the datasource unconnected (metadata-only). */ +export interface DatasourceConnectDecision { + allow: boolean; + /** Human-readable reason, surfaced in logs when a connect is denied. */ + reason?: string; +} + +/** The minimal datasource shape a policy inspects (never a secret). */ +export interface DatasourceConnectSubject { + name: string; + driver: string; + schemaMode?: 'managed' | 'external' | 'validate-only'; + external?: Record; +} + +/** Host-provided policy gate consulted before opening a connection. */ +export interface DatasourceConnectPolicy { + /** + * Decide whether `ds` may be connected. Sync or async. Throwing is treated + * as a denial (fail-closed) by {@link DatasourceConnectionService}. + */ + canConnect( + ds: DatasourceConnectSubject, + ctx?: DatasourceConnectContext, + ): DatasourceConnectDecision | Promise; +} + +/** + * Open-core default: allow every connect. The D2 auto-connect gate (external / + * explicitly-routed / `autoConnect:true`) still applies on top of this for + * code-defined datasources, so a managed, unrouted datasource is never + * connected even under the permissive policy. + */ +export const allowAllConnectPolicy: DatasourceConnectPolicy = { + canConnect: () => ({ allow: true }), +}; diff --git a/packages/services/service-datasource/src/contracts/index.ts b/packages/services/service-datasource/src/contracts/index.ts index 1f4ed2c767..0a2bb390e8 100644 --- a/packages/services/service-datasource/src/contracts/index.ts +++ b/packages/services/service-datasource/src/contracts/index.ts @@ -16,3 +16,14 @@ export type { DatasourceDriverHandle, IDatasourceDriverFactory, } from './datasource-driver-factory.js'; + +// Host-injectable connect policy (ADR-0062 D5 / epic #2163 seam). +export { + allowAllConnectPolicy, +} from './connect-policy.js'; +export type { + DatasourceConnectPolicy, + DatasourceConnectDecision, + DatasourceConnectContext, + DatasourceConnectSubject, +} from './connect-policy.js'; diff --git a/packages/services/service-datasource/src/datasource-admin-plugin.ts b/packages/services/service-datasource/src/datasource-admin-plugin.ts index 6e7f68001b..36186bd2b8 100644 --- a/packages/services/service-datasource/src/datasource-admin-plugin.ts +++ b/packages/services/service-datasource/src/datasource-admin-plugin.ts @@ -4,7 +4,6 @@ import type { Plugin, PluginContext } from '@objectstack/core'; import { registerMetadataTypeActions } from '@objectstack/spec/kernel'; import type { IDatasourceDriverFactory, - DatasourceConnectionSpec, TestConnectionResult, } from './contracts/index.js'; import { @@ -13,6 +12,11 @@ import { type StoredDatasource, type ProbeInput, } from './datasource-admin-service.js'; +import { + DatasourceConnectionService, + type ConnectionEngineLike, +} from './datasource-connection-service.js'; +import type { DatasourceConnectPolicy } from './contracts/connect-policy.js'; import type { Logger } from './logger.js'; /** @@ -135,6 +139,13 @@ export interface DatasourceAdminServicePluginOptions { secrets?: SecretBinder; /** Override the driver factory (defaults to the `'datasource-driver-factory'` service). */ driverFactory?: IDatasourceDriverFactory; + /** + * Host-injectable connect policy consulted before opening any datasource + * connection (ADR-0062 D5 / epic #2163 seam). Open-core default is permissive + * (allow); a multi-tenant host binds a stricter, fail-closed policy. Shared by + * both code-defined auto-connect and runtime-admin pool registration. + */ + connectPolicy?: DatasourceConnectPolicy; logger?: Logger; } @@ -162,6 +173,8 @@ export class DatasourceAdminServicePlugin implements Plugin { private service?: DatasourceAdminService; private config?: DatasourceAdminServiceConfig; + /** Shared "definition → live driver" path (ADR-0062 D1); also exposed as the `'datasource-connection'` service. */ + private connection?: DatasourceConnectionService; private readonly options: DatasourceAdminServicePluginOptions; constructor(options: DatasourceAdminServicePluginOptions = {}) { @@ -204,6 +217,19 @@ export class DatasourceAdminServicePlugin implements Plugin { const factory = (): IDatasourceDriverFactory | undefined => this.options.driverFactory ?? safeGetService(ctx, 'datasource-driver-factory'); + // The single "definition → live driver" path (ADR-0062 D1). Built here so + // the admin pool registration (runtime origin) and the app-plugin + // auto-connect (code origin) share one connect + lifecycle + policy path. + // Registered as a kernel service so `AppPlugin.start()` can resolve it. + this.connection = new DatasourceConnectionService({ + factory, + engine: () => engineOf() as ConnectionEngineLike | undefined, + secrets: { resolve: (ref) => this.options.secrets?.resolve?.(ref) ?? Promise.resolve(undefined) }, + policy: this.options.connectPolicy, + logger: this.options.logger, + }); + ctx.registerService('datasource-connection', this.connection); + const config: DatasourceAdminServiceConfig = { probe: (input) => this.probe(factory(), input), @@ -261,40 +287,21 @@ export class DatasourceAdminServicePlugin implements Plugin { return objects.filter((o) => o?.datasource === datasource).length; }, + // Hot pool (de)registration converges on the shared + // DatasourceConnectionService (ADR-0062 D1) — one connect path for code- + // and runtime-origin datasources. `connect()` builds the driver via the + // factory, dereferences `external.credentialsRef` through the SecretBinder, + // opens the connection, and registers the live driver + datasource def. + // Runtime-admin connects always degrade-with-warning on failure (never + // fail-fast), preserving the pre-ADR-0062 admin behavior. registerPool: async (record) => { - const f = factory(); - const engine = engineOf(); - if (!f || !engine?.registerDriver || !f.supports(record.driver)) return; - // Recover the cleartext credential from `sys_secret` so the pool opens - // with the real password. The cleartext is never persisted on the - // record (only `credentialsRef`), so it must be dereferenced here — - // both on create/update and on boot rehydration. Credential-less - // drivers (sqlite/memory) simply have no ref and skip this. - const credentialsRef = record.external?.credentialsRef; - const secret = credentialsRef ? await this.options.secrets?.resolve?.(credentialsRef) : undefined; - const handle = await f.create({ ...this.toSpec(record), ...(secret ? { secret } : {}) }); - if (typeof handle?.connect === 'function') await handle.connect(); - // The engine routes a datasource to a driver by `driver.name === ` - // (see ObjectQL engine.getDriver). Prefer the factory's underlying engine - // driver (the `driver` escape hatch); fall back to the handle itself. Stamp - // the name so routing resolves to this pool. - const engineDriver = (handle.driver ?? handle) as { name?: string }; - try { - engineDriver.name = record.name; - } catch { - /* frozen driver — registration may still work if name already matches */ - } - engine.registerDriver(engineDriver); - engine.registerDatasourceDef?.({ - name: record.name, - schemaMode: record.schemaMode, - external: record.external as { allowWrites?: boolean } | undefined, + await this.connection?.connect(record, { + context: { origin: record.origin ?? 'runtime', trigger: 'runtime-admin' }, }); }, unregisterPool: async (name) => { - const driver = engineOf()?.getDriverByName?.(name) as { disconnect?: () => Promise } | undefined; - if (typeof driver?.disconnect === 'function') await driver.disconnect(); + await this.connection?.disconnect(name); }, logger, @@ -426,16 +433,6 @@ export class DatasourceAdminServicePlugin implements Plugin { // --- internals ----------------------------------------------------------- - private toSpec(record: StoredDatasource): DatasourceConnectionSpec { - return { - name: record.name, - driver: record.driver, - config: record.config ?? {}, - external: record.external, - pool: record.pool, - }; - } - /** Probe a connection via the driver factory: build → connect → ping → close. */ private async probe( factory: IDatasourceDriverFactory | undefined, diff --git a/packages/services/service-datasource/src/datasource-admin-service.ts b/packages/services/service-datasource/src/datasource-admin-service.ts index d8573f3fdb..97ad89c50b 100644 --- a/packages/services/service-datasource/src/datasource-admin-service.ts +++ b/packages/services/service-datasource/src/datasource-admin-service.ts @@ -47,6 +47,8 @@ export interface StoredDatasource { external?: (Record & { credentialsRef?: string }) | undefined; pool?: Record; active?: boolean; + /** Force a live connection at boot even when managed + unrouted (ADR-0062 D2(c)). */ + autoConnect?: boolean; origin?: 'code' | 'runtime'; /** Package that defines a code-origin datasource, when known. */ definedIn?: string; diff --git a/packages/services/service-datasource/src/datasource-connection-service.ts b/packages/services/service-datasource/src/datasource-connection-service.ts new file mode 100644 index 0000000000..f8f7002134 --- /dev/null +++ b/packages/services/service-datasource/src/datasource-connection-service.ts @@ -0,0 +1,328 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * DatasourceConnectionService — the single "definition → live driver" path + * (ADR-0062 D1). + * + * Given a datasource definition, it: consults the injectable connect policy + * (D5/epic seam), builds a driver via the host-provided driver factory, + * resolves any `external.credentialsRef` to a cleartext secret via the + * `SecretBinder` (D3, wired in Phase 2), opens the connection, and registers + * the live driver + the datasource *definition* into the ObjectQL engine under + * the datasource name (the engine routes by `driver.name === `). + * + * Both origins converge here (D1): + * - **code-defined** datasources auto-connect at boot via + * {@link connectDeclared} (gated per D2 — see {@link isDatasourceAddressed}), + * called from `AppPlugin.start()`. + * - **runtime** (UI-created) datasources connect via {@link connect}, called + * from `DatasourceAdminServicePlugin`'s `registerPool` (create/update + boot + * rehydration). + * + * Idempotent: a datasource already registered as a live driver is skipped, so + * an app's legacy `onEnable` driver registration (the escape hatch, ADR-0062 + * D8) and auto-connect never double-register. + */ + +import type { + IDatasourceDriverFactory, + DatasourceConnectionSpec, +} from './contracts/datasource-driver-factory.js'; +import { + allowAllConnectPolicy, + type DatasourceConnectPolicy, + type DatasourceConnectContext, +} from './contracts/connect-policy.js'; +import type { Logger } from './logger.js'; + +/** A datasource definition this service can connect (code- or runtime-origin). */ +export interface ConnectableDatasource { + name: string; + label?: string; + driver: string; + schemaMode?: 'managed' | 'external' | 'validate-only'; + config?: Record; + external?: (Record & { + credentialsRef?: string; + validation?: { onMismatch?: 'fail' | 'warn' | 'ignore' }; + }) | undefined; + pool?: Record; + active?: boolean; + origin?: 'code' | 'runtime'; + /** + * ADR-0062 D2(c): explicit opt-in to auto-connect even for a managed, + * unrouted datasource. Defaults to false. + */ + autoConnect?: boolean; +} + +/** Minimal object shape used for the D2 routing gate + post-connect schema sync. */ +export interface DatasourceBoundObject { + name?: string; + /** The object's explicit `datasource` binding (ADR-0015 federation). */ + datasource?: string; +} + +/** Engine surface this service drives (the ObjectQL `'data'` engine). */ +export interface ConnectionEngineLike { + registerDriver?: (driver: unknown, isDefault?: boolean) => void; + registerDatasourceDef?: (def: { + name: string; + schemaMode?: string; + external?: { allowWrites?: boolean }; + }) => void; + getDriverByName?: (name: string) => unknown; + /** + * Register read metadata (DDL-free) for a federated object so its physical + * remote table/columns resolve for queries. Idempotent; called per bound + * external object after the driver is registered, because boot schema-sync + * ran before this driver existed (ADR-0015 §18; matches what the legacy + * `onEnable` bridge does manually). + */ + syncObjectSchema?: (objectName: string) => Promise; +} + +/** Secret dereference surface (the `SecretBinder.resolve`, Phase 2 / D3). */ +export interface ConnectionSecretResolver { + resolve?: (credentialsRef: string) => Promise; +} + +export interface DatasourceConnectionServiceConfig { + /** Resolve the host driver factory (lazy — may be registered after init). */ + factory: () => IDatasourceDriverFactory | undefined; + /** Resolve the ObjectQL engine (lazy). */ + engine: () => ConnectionEngineLike | undefined; + /** Dereference `credentialsRef` → cleartext (Phase 2). Optional in Phase 1. */ + secrets?: ConnectionSecretResolver; + /** Injectable connect policy. Defaults to {@link allowAllConnectPolicy}. */ + policy?: DatasourceConnectPolicy; + logger?: Logger; +} + +/** Outcome of a single {@link DatasourceConnectionService.connect} attempt. */ +export type ConnectStatus = + | 'connected' + | 'already-registered' + | 'skipped-policy' + | 'skipped-no-infra' + | 'skipped-unsupported' + | 'failed-degraded'; + +export interface ConnectResult { + name: string; + status: ConnectStatus; + reason?: string; +} + +/** + * ADR-0062 D2 — is this declared datasource "meaningfully addressed", such that + * auto-connecting it is safe and intended? + * + * Returns true when: + * - (a) it is external (`schemaMode !== 'managed'`), OR + * - (b) some object **explicitly** binds to it (`object.datasource === name`), OR + * - (c) it sets `autoConnect: true`. + * + * Deliberately NOT triggered by a `datasourceMapping` rule alone. A managed + * datasource that is only *mapped* (namespace/package/default) but has no live + * driver historically falls through to the `default` driver at query time + * (`engine.getDriver` step 4) — e.g. `examples/app-crm`'s `crm_primary` + * (`:memory:`, mapped + default-fallback, no `onEnable`). Connecting it would + * divert those objects to a fresh, empty connection and silently change app + * behavior. So mapping-only routing to a *managed* datasource is treated as + * decorative, keeping existing apps byte-for-byte unchanged (D2's load-bearing + * backward-compat guarantee). External datasources and explicit + * `object.datasource` bindings never resolved to `default` (they throw when + * unregistered), so auto-connecting them is a strict improvement, not a change. + */ +export function isDatasourceAddressed( + ds: Pick, + ctx: { objects?: readonly DatasourceBoundObject[] }, +): boolean { + if (ds.schemaMode && ds.schemaMode !== 'managed') return true; // (a) + if (ds.autoConnect === true) return true; // (c) + if (ctx.objects?.some((o) => o?.datasource === ds.name)) return true; // (b) + return false; +} + +export class DatasourceConnectionService { + private readonly cfg: DatasourceConnectionServiceConfig; + private readonly policy: DatasourceConnectPolicy; + private readonly logger?: Logger; + + constructor(cfg: DatasourceConnectionServiceConfig) { + this.cfg = cfg; + this.policy = cfg.policy ?? allowAllConnectPolicy; + this.logger = cfg.logger; + } + + /** + * Auto-connect the declared (code-defined) datasources that pass the D2 gate. + * Called from `AppPlugin.start()` with the app bundle's datasources + objects. + * Each connected external datasource also has its bound objects' read metadata + * synced so they are immediately queryable with zero app code. + */ + async connectDeclared(input: { + datasources: readonly ConnectableDatasource[]; + objects?: readonly DatasourceBoundObject[]; + }): Promise { + const objects = input.objects ?? []; + const results: ConnectResult[] = []; + for (const ds of input.datasources) { + if (!ds?.name) continue; + if (ds.active === false) continue; + if (!isDatasourceAddressed(ds, { objects })) continue; // D2 gate + const bound = objects + .filter((o) => o?.datasource === ds.name && typeof o?.name === 'string') + .map((o) => o.name as string); + results.push( + await this.connect(ds, { objects: bound, context: { origin: ds.origin ?? 'code', trigger: 'declared-auto' } }), + ); + } + return results; + } + + /** + * Build + connect + register a single datasource's live driver. The shared + * core used by both auto-connect and the runtime-admin pool registration. + * + * Failure policy (ADR-0062 D5): an `external` datasource with + * `validation.onMismatch: 'fail'` fails fast (re-throws, bricking boot as + * intended); everything else degrades with a warning so an optional replica's + * connectivity blip never bricks boot. + */ + async connect( + record: ConnectableDatasource, + opts: { objects?: readonly string[]; context?: DatasourceConnectContext } = {}, + ): Promise { + const name = record.name; + const engine = this.cfg.engine(); + const factory = this.cfg.factory(); + + // Idempotent: never double-register (e.g. a legacy `onEnable` bridge already + // registered this driver — the D8 escape hatch). + if (engine?.getDriverByName?.(name)) { + return { name, status: 'already-registered' }; + } + + // Policy gate (fail-closed on throw). + let decision; + try { + decision = await this.policy.canConnect( + { name, driver: record.driver, schemaMode: record.schemaMode, external: record.external }, + opts.context, + ); + } catch (err) { + decision = { allow: false, reason: `connect policy threw: ${errMsg(err)}` }; + } + if (!decision.allow) { + this.logger?.info?.(`datasource '${name}': connect denied by policy${decision.reason ? ` (${decision.reason})` : ''}`); + return { name, status: 'skipped-policy', reason: decision.reason }; + } + + if (!factory || !engine?.registerDriver) { + this.logger?.debug?.(`datasource '${name}': no driver factory / engine — left metadata-only`); + return { name, status: 'skipped-no-infra' }; + } + if (!factory.supports(record.driver)) { + return this.handleFailure( + record, + 'skipped-unsupported', + `no driver factory supports driver '${record.driver}'`, + opts.context, + ); + } + + try { + const credentialsRef = record.external?.credentialsRef; + const secret = credentialsRef ? await this.cfg.secrets?.resolve?.(credentialsRef) : undefined; + const handle = await factory.create({ ...toSpec(record), ...(secret ? { secret } : {}) }); + if (typeof handle?.connect === 'function') await handle.connect(); + + // The engine routes a datasource to a driver by `driver.name === `. + // Prefer the factory's underlying engine driver (the `driver` escape hatch); + // fall back to the handle. Stamp the name so routing resolves to this pool. + const engineDriver = (handle.driver ?? handle) as { name?: string }; + try { + engineDriver.name = name; + } catch { + /* frozen driver — registration may still work if name already matches */ + } + engine.registerDriver(engineDriver); + engine.registerDatasourceDef?.({ + name, + schemaMode: record.schemaMode, + external: record.external as { allowWrites?: boolean } | undefined, + }); + + // Register read metadata for bound federated objects (DDL-free). Boot + // schema-sync ran before this driver existed, so do it on-demand now. + for (const objectName of opts.objects ?? []) { + try { + await engine.syncObjectSchema?.(objectName); + } catch (err) { + this.logger?.warn?.(`datasource '${name}': syncObjectSchema('${objectName}') failed: ${errMsg(err)}`); + } + } + + this.logger?.info?.(`datasource '${name}': connected (driver=${record.driver}, schemaMode=${record.schemaMode ?? 'managed'})`); + return { name, status: 'connected' }; + } catch (err) { + return this.handleFailure(record, 'failed-degraded', errMsg(err), opts.context); + } + } + + /** Gracefully disconnect a previously-registered datasource pool. */ + async disconnect(name: string): Promise { + const driver = this.cfg.engine()?.getDriverByName?.(name) as { disconnect?: () => Promise } | undefined; + if (typeof driver?.disconnect === 'function') { + try { + await driver.disconnect(); + } catch (err) { + this.logger?.warn?.(`datasource '${name}': disconnect failed: ${errMsg(err)}`); + } + } + } + + /** + * Apply the D5 connect-failure policy. A code-defined `external` datasource + * with `onMismatch:'fail'` auto-connected at boot re-throws (fail-fast, + * bricking boot as intended). Runtime-admin create/update + boot rehydration + * always degrade-with-warning — a UI action or a replica blip must never + * brick the running server (preserves the pre-ADR-0062 admin behavior). + */ + private handleFailure( + record: ConnectableDatasource, + status: ConnectStatus, + reason: string, + context?: DatasourceConnectContext, + ): ConnectResult { + const isExternal = record.schemaMode && record.schemaMode !== 'managed'; + const failFast = + context?.trigger === 'declared-auto' && + isExternal && + record.external?.validation?.onMismatch === 'fail'; + const msg = `datasource '${record.name}': connect failed — ${reason}`; + if (failFast) { + throw new Error( + `${msg}. (schemaMode=${record.schemaMode}, validation.onMismatch='fail' ⇒ fail-fast per ADR-0062 D5)`, + ); + } + this.logger?.warn?.(`${msg} — degrading (datasource left unconnected)`); + return { name: record.name, status, reason }; + } +} + +function toSpec(record: ConnectableDatasource): DatasourceConnectionSpec { + return { + name: record.name, + driver: record.driver, + config: record.config ?? {}, + external: record.external, + pool: record.pool, + }; +} + +function errMsg(err: unknown): string { + return err instanceof Error ? err.message : String(err); +} diff --git a/packages/services/service-datasource/src/index.ts b/packages/services/service-datasource/src/index.ts index b54ee4f576..264291bbc2 100644 --- a/packages/services/service-datasource/src/index.ts +++ b/packages/services/service-datasource/src/index.ts @@ -39,7 +39,25 @@ export type { DatasourceConnectionSpec, DatasourceDriverHandle, IDatasourceDriverFactory, + // Connect policy (ADR-0062 D5 / epic #2163 seam). + DatasourceConnectPolicy, + DatasourceConnectDecision, + DatasourceConnectContext, + DatasourceConnectSubject, } from './contracts/index.js'; +export { allowAllConnectPolicy } from './contracts/index.js'; + +// Shared "definition → live driver" path (ADR-0062 D1). +export { DatasourceConnectionService, isDatasourceAddressed } from './datasource-connection-service.js'; +export type { + DatasourceConnectionServiceConfig, + ConnectableDatasource, + DatasourceBoundObject, + ConnectionEngineLike, + ConnectionSecretResolver, + ConnectResult, + ConnectStatus, +} from './datasource-connection-service.js'; // Decoupled lifecycle service + injected-config shape. export { DatasourceAdminService } from './datasource-admin-service.js'; diff --git a/packages/services/service-datasource/src/logger.ts b/packages/services/service-datasource/src/logger.ts index d6292e8bbf..5bc07225b6 100644 --- a/packages/services/service-datasource/src/logger.ts +++ b/packages/services/service-datasource/src/logger.ts @@ -8,4 +8,6 @@ export interface Logger { warn: (message: string, meta?: unknown) => void; info?: (message: string, meta?: unknown) => void; + debug?: (message: string, meta?: unknown) => void; + error?: (message: string, meta?: unknown) => void; } diff --git a/packages/spec/src/data/datasource.zod.ts b/packages/spec/src/data/datasource.zod.ts index 2ef55ea3f6..f77ceee130 100644 --- a/packages/spec/src/data/datasource.zod.ts +++ b/packages/spec/src/data/datasource.zod.ts @@ -212,6 +212,18 @@ export const DatasourceSchema = lazySchema(() => z.object({ /** Is enabled? */ active: z.boolean().default(true).describe('Is datasource enabled'), + /** + * Auto-connect opt-in (ADR-0062 D2(c)). + * + * Forces the runtime to build a live driver for this datasource at boot even + * when it is `managed` and nothing routes to it. By default a declared + * datasource only auto-connects when it is `external` or an object explicitly + * binds to it via `object.datasource` (see ADR-0062 D2). Set this to opt a + * managed, unrouted datasource into the live-connection lifecycle. + */ + autoConnect: z.boolean().default(false) + .describe('Force a live driver connection at boot even when managed + unrouted (ADR-0062 D2).'), + /** * Schema Ownership Mode (ADR-0015) * Declares whether ObjectStack owns this schema (`managed`, default) or From 68a8318a14a76298df6b67c67c4c089fdf607f6a Mon Sep 17 00:00:00 2001 From: Jack Zhuang <277994282+os-zhuang@users.noreply.github.com> Date: Mon, 22 Jun 2026 19:23:30 +0800 Subject: [PATCH 2/4] fix(runtime): use single-string logger.error in auto-connect catch (DTS build) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The context logger interface types error(message, error?: Error) — passing a meta object {appId, error} tripped the tsup DTS build (TS2353) even though tsc --noEmit passed. Switch to a single interpolated message; the rethrow still surfaces the real cause to the kernel. Co-Authored-By: Claude Opus 4.8 --- content/docs/references/ai/agent.mdx | 1 + content/docs/references/ai/skill.mdx | 1 + content/docs/references/api/contract.mdx | 2 +- content/docs/references/data/data-engine.mdx | 1 + content/docs/references/data/datasource.mdx | 1 + content/docs/references/data/driver.mdx | 1 + content/docs/references/data/hook-body.mdx | 13 ++++- content/docs/references/data/object.mdx | 17 +------ content/docs/references/data/query.mdx | 2 +- .../references/kernel/execution-context.mdx | 6 +++ .../docs/references/security/permission.mdx | 20 +++++++- content/docs/references/security/rls.mdx | 38 ++++++++------- content/docs/references/ui/component.mdx | 20 +++++++- content/docs/references/ui/dataset.mdx | 1 + content/docs/references/ui/page.mdx | 8 +++- content/docs/references/ui/view.mdx | 47 +++++++++++++++++-- packages/runtime/src/app-plugin.ts | 10 ++-- 17 files changed, 140 insertions(+), 49 deletions(-) diff --git a/content/docs/references/ai/agent.mdx b/content/docs/references/ai/agent.mdx index 8f204d40b1..2f0c530656 100644 --- a/content/docs/references/ai/agent.mdx +++ b/content/docs/references/ai/agent.mdx @@ -76,6 +76,7 @@ const result = AIKnowledge.parse(data); | **instructions** | `string` | ✅ | System Prompt / Prime Directives | | **model** | `Object` | optional | | | **lifecycle** | `Object` | optional | State machine defining the agent conversation follow and constraints | +| **surface** | `Enum<'ask' \| 'build'>` | ✅ | Product surface this agent binds ('ask' | 'build') — ADR-0063 §1 | | **skills** | `string[]` | optional | Skill names to attach (Agent→Skill→Tool architecture) | | **tools** | `Object[]` | optional | Direct tool references (legacy fallback) | | **knowledge** | `Object` | optional | RAG access | diff --git a/content/docs/references/ai/skill.mdx b/content/docs/references/ai/skill.mdx index 8b56199f8e..39039601a0 100644 --- a/content/docs/references/ai/skill.mdx +++ b/content/docs/references/ai/skill.mdx @@ -36,6 +36,7 @@ const result = Skill.parse(data); | **name** | `string` | ✅ | Skill unique identifier (snake_case) | | **label** | `string` | ✅ | Skill display name | | **description** | `string` | optional | Skill description | +| **surface** | `Enum<'ask' \| 'build' \| 'both'>` | ✅ | Agent surface this skill binds to ('ask' | 'build' | 'both') — ADR-0063 §3 | | **instructions** | `string` | optional | LLM instructions when skill is active | | **tools** | `string[]` | ✅ | Tool names belonging to this skill (supports trailing wildcard, e.g. `action_*`) | | **triggerPhrases** | `string[]` | optional | Phrases that activate this skill | diff --git a/content/docs/references/api/contract.mdx b/content/docs/references/api/contract.mdx index 3b48a7d113..835c34ae63 100644 --- a/content/docs/references/api/contract.mdx +++ b/content/docs/references/api/contract.mdx @@ -154,7 +154,7 @@ const result = ApiError.parse(data); | **having** | `[__schema2](./__schema2)` | optional | HAVING clause for aggregation filtering | | **windowFunctions** | `Object[]` | optional | Window functions with OVER clause | | **distinct** | `boolean` | optional | SELECT DISTINCT flag | -| **expand** | `Record` | optional | Recursive relation loading map. Keys are lookup/master_detail field names; values are nested QueryAST objects that control select, filter, sort, and further expansion on the related object. The engine resolves expand via batch $in queries (driver-agnostic) with a default max depth of 3. | +| **expand** | `Record` | optional | Recursive relation loading map. Keys are lookup/master_detail field names; values are nested QueryAST objects that control select (`fields`) and filter (`where`, AND-merged with the batch $in), plus further expansion on the related object. The engine resolves expand via batch $in queries (driver-agnostic) with a default max depth of 3; per-parent `limit`/`offset`/`orderBy` are NOT applied on this path. | --- diff --git a/content/docs/references/data/data-engine.mdx b/content/docs/references/data/data-engine.mdx index ce45126874..dfc16767fd 100644 --- a/content/docs/references/data/data-engine.mdx +++ b/content/docs/references/data/data-engine.mdx @@ -481,6 +481,7 @@ QueryAST-aligned options for DataEngine.aggregate operations | **where** | `Record \| [__schema0](./__schema0)` | optional | | | **groupBy** | `string[]` | optional | | | **aggregations** | `Object[]` | optional | | +| **timezone** | `string` | optional | | --- diff --git a/content/docs/references/data/datasource.mdx b/content/docs/references/data/datasource.mdx index 46dbdacd32..1ce8de7212 100644 --- a/content/docs/references/data/datasource.mdx +++ b/content/docs/references/data/datasource.mdx @@ -43,6 +43,7 @@ const result = Datasource.parse(data); | **retryPolicy** | `Object` | optional | Connection retry policy for transient failures | | **description** | `string` | optional | Internal description | | **active** | `boolean` | ✅ | Is datasource enabled | +| **autoConnect** | `boolean` | ✅ | Force a live driver connection at boot even when managed + unrouted (ADR-0062 D2). | | **schemaMode** | `Enum<'managed' \| 'external' \| 'validate-only'>` | ✅ | Schema ownership mode | | **external** | `Object` | optional | External datasource federation settings (schemaMode != "managed") | | **origin** | `Enum<'code' \| 'runtime'>` | ✅ | Datasource provenance (server-managed, read-only) | diff --git a/content/docs/references/data/driver.mdx b/content/docs/references/data/driver.mdx index e78c206330..d2037791fa 100644 --- a/content/docs/references/data/driver.mdx +++ b/content/docs/references/data/driver.mdx @@ -95,6 +95,7 @@ const result = DriverCapabilities.parse(data); | **skipCache** | `boolean` | optional | Bypass cache | | **traceContext** | `Record` | optional | OpenTelemetry context or request ID | | **tenantId** | `string` | optional | Tenant Isolation identifier | +| **timezone** | `string` | optional | Business reference timezone (IANA) for date-dependent generation, e.g. autonumber date tokens | --- diff --git a/content/docs/references/data/hook-body.mdx b/content/docs/references/data/hook-body.mdx index d3ff7fdd47..c9e4b466ee 100644 --- a/content/docs/references/data/hook-body.mdx +++ b/content/docs/references/data/hook-body.mdx @@ -15,6 +15,14 @@ a capability it did not declare, the call throws at invocation time. - `api.write` — `ctx.api.object(...).insert / update / delete` +- `api.transaction` — `ctx.api.transaction(async () => \{ … \})` — runs the + +callback's `ctx.api` writes/reads inside one driver transaction, committed + +on return and rolled back if the callback throws. Requires `api.write` + +alongside it to be useful (the transaction body still needs write access). + - `crypto.uuid` — `ctx.crypto.randomUUID()` - `crypto.hash` — `ctx.crypto.hash(algo, data)` @@ -86,7 +94,7 @@ L2 sandboxed JS body — runs inside an isolated VM with declared capabilities | :--- | :--- | :--- | :--- | | **language** | `string` | ✅ | | | **source** | `string` | ✅ | Function body source | -| **capabilities** | `Enum<'api.read' \| 'api.write' \| 'crypto.uuid' \| 'crypto.hash' \| 'log'>[]` | ✅ | Granted capability tokens | +| **capabilities** | `Enum<'api.read' \| 'api.write' \| 'api.transaction' \| 'crypto.uuid' \| 'crypto.hash' \| 'log'>[]` | ✅ | Granted capability tokens | | **timeoutMs** | `integer` | optional | Per-invocation timeout (ms) | | **memoryMb** | `integer` | optional | Per-invocation memory cap (MB) | @@ -101,6 +109,7 @@ L2 sandboxed JS body — runs inside an isolated VM with declared capabilities * `api.read` * `api.write` +* `api.transaction` * `crypto.uuid` * `crypto.hash` * `log` @@ -118,7 +127,7 @@ L2 sandboxed JS body — runs inside an isolated VM with declared capabilities | :--- | :--- | :--- | :--- | | **language** | `string` | ✅ | | | **source** | `string` | ✅ | Function body source | -| **capabilities** | `Enum<'api.read' \| 'api.write' \| 'crypto.uuid' \| 'crypto.hash' \| 'log'>[]` | ✅ | Granted capability tokens | +| **capabilities** | `Enum<'api.read' \| 'api.write' \| 'api.transaction' \| 'crypto.uuid' \| 'crypto.hash' \| 'log'>[]` | ✅ | Granted capability tokens | | **timeoutMs** | `integer` | optional | Per-invocation timeout (ms) | | **memoryMb** | `integer` | optional | Per-invocation memory cap (MB) | diff --git a/content/docs/references/data/object.mdx b/content/docs/references/data/object.mdx index 059d0bfdac..32ed1caea5 100644 --- a/content/docs/references/data/object.mdx +++ b/content/docs/references/data/object.mdx @@ -14,8 +14,8 @@ API Operations Enum ## TypeScript Usage ```typescript -import { ApiMethod, CDCConfig, Index, ObjectCapabilities, ObjectExternalBinding, ObjectOwnershipEnum, PartitioningConfig, SoftDeleteConfig, TenancyConfig, VersioningConfig } from '@objectstack/spec/data'; -import type { ApiMethod, CDCConfig, Index, ObjectCapabilities, ObjectExternalBinding, ObjectOwnershipEnum, PartitioningConfig, SoftDeleteConfig, TenancyConfig, VersioningConfig } from '@objectstack/spec/data'; +import { ApiMethod, Index, ObjectCapabilities, ObjectExternalBinding, ObjectOwnershipEnum, PartitioningConfig, SoftDeleteConfig, TenancyConfig, VersioningConfig } from '@objectstack/spec/data'; +import type { ApiMethod, Index, ObjectCapabilities, ObjectExternalBinding, ObjectOwnershipEnum, PartitioningConfig, SoftDeleteConfig, TenancyConfig, VersioningConfig } from '@objectstack/spec/data'; // Validate data const result = ApiMethod.parse(data); @@ -43,19 +43,6 @@ const result = ApiMethod.parse(data); * `export` ---- - -## CDCConfig - -### Properties - -| Property | Type | Required | Description | -| :--- | :--- | :--- | :--- | -| **enabled** | `boolean` | ✅ | Enable Change Data Capture | -| **events** | `Enum<'insert' \| 'update' \| 'delete'>[]` | ✅ | Event types to capture | -| **destination** | `string` | ✅ | Destination endpoint (e.g., "kafka://topic", "webhook://url") | - - --- ## Index diff --git a/content/docs/references/data/query.mdx b/content/docs/references/data/query.mdx index 2ae0511aad..68ba187d3f 100644 --- a/content/docs/references/data/query.mdx +++ b/content/docs/references/data/query.mdx @@ -202,7 +202,7 @@ Type: `string` | **having** | `[__schema1](./__schema1)` | optional | HAVING clause for aggregation filtering | | **windowFunctions** | `Object[]` | optional | Window functions with OVER clause | | **distinct** | `boolean` | optional | SELECT DISTINCT flag | -| **expand** | `Record` | optional | Recursive relation loading map. Keys are lookup/master_detail field names; values are nested QueryAST objects that control select, filter, sort, and further expansion on the related object. The engine resolves expand via batch $in queries (driver-agnostic) with a default max depth of 3. | +| **expand** | `Record` | optional | Recursive relation loading map. Keys are lookup/master_detail field names; values are nested QueryAST objects that control select (`fields`) and filter (`where`, AND-merged with the batch $in), plus further expansion on the related object. The engine resolves expand via batch $in queries (driver-agnostic) with a default max depth of 3; per-parent `limit`/`offset`/`orderBy` are NOT applied on this path. | --- diff --git a/content/docs/references/kernel/execution-context.mdx b/content/docs/references/kernel/execution-context.mdx index 64d5ec27c2..aac3ec6025 100644 --- a/content/docs/references/kernel/execution-context.mdx +++ b/content/docs/references/kernel/execution-context.mdx @@ -48,12 +48,18 @@ const result = ExecutionContext.parse(data); | Property | Type | Required | Description | | :--- | :--- | :--- | :--- | | **userId** | `string` | optional | | +| **actor** | `string` | optional | | +| **email** | `string` | optional | | | **tenantId** | `string` | optional | | +| **timezone** | `string` | optional | | +| **locale** | `string` | optional | | +| **currency** | `string` | optional | | | **roles** | `string[]` | ✅ | | | **permissions** | `string[]` | ✅ | | | **systemPermissions** | `string[]` | optional | | | **tabPermissions** | `Record>` | optional | | | **org_user_ids** | `string[]` | optional | | +| **rlsMembership** | `Record` | optional | | | **isSystem** | `boolean` | ✅ | | | **accessToken** | `string` | optional | | | **transaction** | `any` | optional | | diff --git a/content/docs/references/security/permission.mdx b/content/docs/references/security/permission.mdx index 8017aa287a..e62a73b4cc 100644 --- a/content/docs/references/security/permission.mdx +++ b/content/docs/references/security/permission.mdx @@ -24,8 +24,8 @@ Refined with enterprise data lifecycle controls: ## TypeScript Usage ```typescript -import { FieldPermission, ObjectPermission, PermissionSet } from '@objectstack/spec/security'; -import type { FieldPermission, ObjectPermission, PermissionSet } from '@objectstack/spec/security'; +import { FieldPermission, ObjectAccessScope, ObjectPermission, PermissionSet } from '@objectstack/spec/security'; +import type { FieldPermission, ObjectAccessScope, ObjectPermission, PermissionSet } from '@objectstack/spec/security'; // Validate data const result = FieldPermission.parse(data); @@ -43,6 +43,19 @@ const result = FieldPermission.parse(data); | **editable** | `boolean` | ✅ | Field edit access | +--- + +## ObjectAccessScope + +### Allowed Values + +* `own` +* `own_and_reports` +* `unit` +* `unit_and_below` +* `org` + + --- ## ObjectPermission @@ -60,6 +73,8 @@ const result = FieldPermission.parse(data); | **allowPurge** | `boolean` | ✅ | [EXPERIMENTAL — not enforced] Permanently delete (Hard Delete/GDPR) | | **viewAllRecords** | `boolean` | ✅ | View All Data (Bypass Sharing) | | **modifyAllRecords** | `boolean` | ✅ | Modify All Data (Bypass Sharing) | +| **readScope** | `Enum<'own' \| 'own_and_reports' \| 'unit' \| 'unit_and_below' \| 'org'>` | optional | [ADR-0057 D1] Read depth: own|unit|unit_and_below|org | +| **writeScope** | `Enum<'own' \| 'own_and_reports' \| 'unit' \| 'unit_and_below' \| 'org'>` | optional | [ADR-0057 D1] Write depth: own|unit|unit_and_below|org | --- @@ -73,6 +88,7 @@ const result = FieldPermission.parse(data); | **name** | `string` | ✅ | Permission set unique name (lowercase snake_case) | | **label** | `string` | optional | Display label | | **isProfile** | `boolean` | ✅ | Whether this is a user profile | +| **isDefault** | `boolean` | ✅ | [ADR-0056 D7] When true, this profile is the FALLBACK assigned to authenticated users who have no explicit grants — an app declares its default access posture here instead of relying on the built-in member_default. Foundation for SSO/JIT provisioning. | | **objects** | `Record` | ✅ | Entity permissions | | **fields** | `Record` | optional | Field level security | | **systemPermissions** | `string[]` | optional | System level capabilities | diff --git a/content/docs/references/security/rls.mdx b/content/docs/references/security/rls.mdx index 75ef228999..a0e8a5f277 100644 --- a/content/docs/references/security/rls.mdx +++ b/content/docs/references/security/rls.mdx @@ -25,31 +25,37 @@ permissions (CRUD), RLS provides record-level filtering. - Users only see records from their organization -- `using: "organization_id = current_user.organization_id"` +- `using: "organization_id == current_user.organization_id"` 2. **Ownership-Based Access** - Users only see records they own -- `using: "owner_id = current_user.id"` +- `using: "owner_id == current_user.id"` -3. **Department-Based Access** +3. **Organization Member Visibility** -- Users only see records from their department +- Users see fellow members of their active organization -- `using: "department = current_user.department"` +- `using: "id in current_user.org_user_ids"` -4. **Regional Access Control** +(`org_user_ids` is pre-resolved by the runtime) -- Sales reps only see accounts in their territory +4. **Territory / Regional Access (§7.3.1 dynamic membership)** -- `using: "region IN (current_user.assigned_regions)"` +- Sales reps only see accounts in their assigned territories -5. **Time-Based Access** +- `using: "account_id in current_user.territory_account_ids"` -- Users can only access active records +(the runtime stages `territory_account_ids` in `ExecutionContext.rlsMembership`) -- `using: "status = 'active' AND expiry_date > NOW()"` +5. **Manager / Hierarchy Access (§7.3.1 dynamic membership)** + +- Managers see records assigned to anyone they manage + +- `using: "assigned_to_id in current_user.team_member_ids"` + +(the runtime pre-resolves `team_member_ids`, no subquery needed) ## PostgreSQL RLS Comparison @@ -83,7 +89,7 @@ object: 'account', operation: 'select', -using: 'organization_id = current_user.organization_id' +using: 'organization_id == current_user.organization_id' \} @@ -105,11 +111,11 @@ Salesforce: ObjectStack RLS: -- More flexible formula-based conditions +- A small, fixed expression grammar (equality, set-membership, always-true) -- Direct SQL-like syntax +- Subquery-shaped needs are pre-resolved by the runtime (§7.3.1) -- Supports complex logic with AND/OR/NOT +- Multiple policies OR-combine for union (any-match-allows) semantics ## Best Practices @@ -265,7 +271,7 @@ const result = RLSAuditConfig.parse(data); | **description** | `string` | optional | Policy description and business justification | | **object** | `string` | ✅ | Target object name | | **operation** | `Enum<'select' \| 'insert' \| 'update' \| 'delete' \| 'all'>` | ✅ | Database operation this policy applies to | -| **using** | `string` | optional | Filter condition for SELECT/UPDATE/DELETE (PostgreSQL SQL WHERE clause syntax with parameterized context variables). Optional for INSERT-only policies. | +| **using** | `string` | optional | Filter condition for SELECT/UPDATE/DELETE. One of the four compiler-supported forms: `field = current_user.`, `field = 'literal'`, `field IN (current_user.)`, or `1 = 1`. Optional for INSERT-only policies. | | **check** | `string` | optional | Validation condition for INSERT/UPDATE (defaults to USING clause if not specified - enforced at application level) | | **roles** | `string[]` | optional | Roles this policy applies to (omit for all roles) | | **enabled** | `boolean` | ✅ | Whether this policy is active | diff --git a/content/docs/references/ui/component.mdx b/content/docs/references/ui/component.mdx index 8b162f0523..03e9807722 100644 --- a/content/docs/references/ui/component.mdx +++ b/content/docs/references/ui/component.mdx @@ -14,8 +14,8 @@ Empty Properties Schema ## TypeScript Usage ```typescript -import { AIChatWindowProps, ElementButtonProps, ElementFilterProps, ElementImageProps, ElementNumberProps, ElementRecordPickerProps, ElementTextProps, PageAccordionProps, PageCardProps, PageHeaderProps, PageTabsProps, RecordActivityProps, RecordChatterProps, RecordDetailsProps, RecordHighlightsField, RecordHighlightsProps, RecordPathProps, RecordRelatedListProps } from '@objectstack/spec/ui'; -import type { AIChatWindowProps, ElementButtonProps, ElementFilterProps, ElementImageProps, ElementNumberProps, ElementRecordPickerProps, ElementTextProps, PageAccordionProps, PageCardProps, PageHeaderProps, PageTabsProps, RecordActivityProps, RecordChatterProps, RecordDetailsProps, RecordHighlightsField, RecordHighlightsProps, RecordPathProps, RecordRelatedListProps } from '@objectstack/spec/ui'; +import { AIChatWindowProps, ElementButtonProps, ElementFilterProps, ElementImageProps, ElementMetadataViewerProps, ElementNumberProps, ElementRecordPickerProps, ElementTextProps, PageAccordionProps, PageCardProps, PageHeaderProps, PageTabsProps, RecordActivityProps, RecordChatterProps, RecordDetailsProps, RecordHighlightsField, RecordHighlightsProps, RecordPathProps, RecordRelatedListProps } from '@objectstack/spec/ui'; +import type { AIChatWindowProps, ElementButtonProps, ElementFilterProps, ElementImageProps, ElementMetadataViewerProps, ElementNumberProps, ElementRecordPickerProps, ElementTextProps, PageAccordionProps, PageCardProps, PageHeaderProps, PageTabsProps, RecordActivityProps, RecordChatterProps, RecordDetailsProps, RecordHighlightsField, RecordHighlightsProps, RecordPathProps, RecordRelatedListProps } from '@objectstack/spec/ui'; // Validate data const result = AIChatWindowProps.parse(data); @@ -83,6 +83,22 @@ const result = AIChatWindowProps.parse(data); | **aria** | `Object` | optional | ARIA accessibility attributes | +--- + +## ElementMetadataViewerProps + +### Properties + +| Property | Type | Required | Description | +| :--- | :--- | :--- | :--- | +| **type** | `Enum<'state_machine' \| 'flow' \| 'permission'>` | ✅ | Metadata view kind (ADR-0051): state_machine | flow | permission | +| **name** | `string` | ✅ | Target metadata item name; resolved package-scoped (ADR-0048), then dependencies (ADR-0046 §3.3) | +| **object** | `string` | optional | Owning object — required for object-scoped kinds: state_machine is a rule ON an object (ADR-0020), permission renders a matrix FOR one; omit for top-level flow | +| **mode** | `Enum<'diagram' \| 'matrix' \| 'summary'>` | optional | Render form; defaults per type (diagram for flow/state_machine, matrix for permission) | +| **detail** | `Enum<'business' \| 'technical'>` | ✅ | Authoring altitude (ADR-0051 §3.4): business collapses technical flow nodes to business steps + approvals. NOT access (cf. book.audience); permission projection is automatic and render-time, never set here | +| **aria** | `Object` | optional | ARIA accessibility attributes | + + --- ## ElementNumberProps diff --git a/content/docs/references/ui/dataset.mdx b/content/docs/references/ui/dataset.mdx index 422fa3f5c6..4b7954c990 100644 --- a/content/docs/references/ui/dataset.mdx +++ b/content/docs/references/ui/dataset.mdx @@ -108,6 +108,7 @@ const result = Dataset.parse(data); | **field** | `string` | optional | Aggregated field; optional for count(*) | | **filter** | `[__schema0](./__schema0)` | optional | | | **format** | `string` | optional | | +| **currency** | `string` | optional | Display currency code (ISO 4217) | | **certified** | `boolean` | ✅ | Blessed metric (governance checkpoint) | | **derived** | `Object` | optional | | diff --git a/content/docs/references/ui/page.mdx b/content/docs/references/ui/page.mdx index 7e18ec5e47..cdce2d1b59 100644 --- a/content/docs/references/ui/page.mdx +++ b/content/docs/references/ui/page.mdx @@ -78,13 +78,17 @@ Interface-level page configuration (Airtable parity) | Property | Type | Required | Description | | :--- | :--- | :--- | :--- | | **source** | `string` | optional | Source object name for the page | -| **sourceView** | `string` | optional | Named list view on the source object to inherit columns/filter/sort from (ADR-0047 iron rule: the page adds presentation policy only). Omit to use the object default view | +| **columns** | `string[] \| Object[]` | optional | Columns shown by the page. Blank = all object fields. Defined directly on the page (no view inheritance). | +| **sort** | `Object[]` | optional | Default sort order for the page, defined directly on the page. | +| **filterBy** | `Object[]` | optional | Always-on page filter (base filter). | | **levels** | `integer` | optional | Number of hierarchy levels to display | -| **filterBy** | `Object[]` | optional | Page-level filter criteria | +| **sourceView** | `string` | optional | @deprecated Legacy named-view inheritance. Define columns/sort/filterBy on the page instead. | | **appearance** | `Object` | optional | Appearance and visualization configuration | | **userFilters** | `Object` | optional | End-user quick-filter bar for this page (overrides the source view's userFilters) | | **userActions** | `Object` | optional | User action toggles | | **addRecord** | `Object` | optional | Add record entry point configuration | +| **buttons** | `string[]` | optional | Toolbar buttons — names of the source object's actions to surface in the page toolbar | +| **recordAction** | `Enum<'drawer' \| 'page' \| 'modal' \| 'none'>` | optional | How clicking a record opens its detail (drawer | page | modal | none). Default: drawer | | **showRecordCount** | `boolean` | optional | Show record count at page bottom | | **allowPrinting** | `boolean` | optional | Allow users to print the page | diff --git a/content/docs/references/ui/view.mdx b/content/docs/references/ui/view.mdx index b7481744de..78d87ea58b 100644 --- a/content/docs/references/ui/view.mdx +++ b/content/docs/references/ui/view.mdx @@ -16,8 +16,8 @@ Migrated to shared/http.zod.ts. Re-exported here for backward compatibility. ## TypeScript Usage ```typescript -import { AddRecordConfig, AppearanceConfig, CalendarConfig, ColumnSummary, GalleryConfig, GanttConfig, GroupingConfig, GroupingField, KanbanConfig, ListChartConfig, ListColumn, NavigationConfig, NavigationMode, PaginationConfig, RowColorConfig, RowHeight, SelectionConfig, TimelineConfig, UserActionsConfig, UserFilterField, UserFilters, ViewData, ViewFilterRule, ViewItemName, ViewKind, ViewScope, ViewSharing, ViewTab, VisualizationType } from '@objectstack/spec/ui'; -import type { AddRecordConfig, AppearanceConfig, CalendarConfig, ColumnSummary, GalleryConfig, GanttConfig, GroupingConfig, GroupingField, KanbanConfig, ListChartConfig, ListColumn, NavigationConfig, NavigationMode, PaginationConfig, RowColorConfig, RowHeight, SelectionConfig, TimelineConfig, UserActionsConfig, UserFilterField, UserFilters, ViewData, ViewFilterRule, ViewItemName, ViewKind, ViewScope, ViewSharing, ViewTab, VisualizationType } from '@objectstack/spec/ui'; +import { AddRecordConfig, AppearanceConfig, CalendarConfig, ColumnSummary, GalleryConfig, GanttConfig, GanttQuickFilter, GroupingConfig, GroupingField, KanbanConfig, ListChartConfig, ListColumn, NavigationConfig, NavigationMode, PaginationConfig, RowColorConfig, RowHeight, SelectionConfig, TimelineConfig, TreeConfig, UserActionsConfig, UserFilterField, UserFilters, ViewData, ViewFilterRule, ViewItemName, ViewKind, ViewScope, ViewSharing, ViewTab, VisualizationType } from '@objectstack/spec/ui'; +import type { AddRecordConfig, AppearanceConfig, CalendarConfig, ColumnSummary, GalleryConfig, GanttConfig, GanttQuickFilter, GroupingConfig, GroupingField, KanbanConfig, ListChartConfig, ListColumn, NavigationConfig, NavigationMode, PaginationConfig, RowColorConfig, RowHeight, SelectionConfig, TimelineConfig, TreeConfig, UserActionsConfig, UserFilterField, UserFilters, ViewData, ViewFilterRule, ViewItemName, ViewKind, ViewScope, ViewSharing, ViewTab, VisualizationType } from '@objectstack/spec/ui'; // Validate data const result = AddRecordConfig.parse(data); @@ -50,7 +50,7 @@ Appearance and visualization configuration | Property | Type | Required | Description | | :--- | :--- | :--- | :--- | | **showDescription** | `boolean` | ✅ | Show the view description text | -| **allowedVisualizations** | `Enum<'grid' \| 'kanban' \| 'gallery' \| 'calendar' \| 'timeline' \| 'gantt' \| 'map' \| 'chart'>[]` | optional | Whitelist of visualization types users can switch between (e.g. ["grid", "gallery", "kanban"]) | +| **allowedVisualizations** | `Enum<'grid' \| 'kanban' \| 'gallery' \| 'calendar' \| 'timeline' \| 'gantt' \| 'map' \| 'chart' \| 'tree'>[]` | optional | Whitelist of visualization types users can switch between (e.g. ["grid", "gallery", "kanban"]) | --- @@ -118,6 +118,32 @@ Gallery/card view configuration | **titleField** | `string` | ✅ | | | **progressField** | `string` | optional | | | **dependenciesField** | `string` | optional | | +| **colorField** | `string` | optional | Field that drives the bar color | +| **parentField** | `string` | optional | Field holding the parent task id (builds the summary → step tree) | +| **typeField** | `string` | optional | Field whose value maps to task/summary/milestone | +| **baselineStartField** | `string` | optional | Baseline (planned) start field | +| **baselineEndField** | `string` | optional | Baseline (planned) end field | +| **groupByField** | `string` | optional | Field to group leaf tasks by (synthesized summary rows) | +| **resourceView** | `boolean` | optional | Render a per-resource workload histogram instead of the timeline | +| **assigneeField** | `string` | optional | Resource field to bucket load by (resource view) | +| **effortField** | `string` | optional | Per-task load units (resource view; default 1) | +| **capacity** | `number` | optional | Per-resource capacity ceiling; loads above this flag overload | +| **tooltipFields** | `string \| Object[]` | optional | Fields to surface in the hover tooltip, in display order | +| **quickFilters** | `Object[]` | optional | Multi-select filter dropdowns rendered above the chart | +| **autoZoomToFilter** | `boolean` | optional | When true (default), filtering zooms the range to the filtered tasks | + + +--- + +## GanttQuickFilter + +### Properties + +| Property | Type | Required | Description | +| :--- | :--- | :--- | :--- | +| **field** | `string` | ✅ | Record field / dot-path the dimension filters on | +| **label** | `string` | optional | Trigger label (falls back to the field label) | +| **options** | `string \| Object[]` | optional | Explicit option override for fixed enums | --- @@ -298,6 +324,20 @@ Timeline view configuration | **scale** | `Enum<'hour' \| 'day' \| 'week' \| 'month' \| 'quarter' \| 'year'>` | ✅ | Default timeline scale | +--- + +## TreeConfig + +### Properties + +| Property | Type | Required | Description | +| :--- | :--- | :--- | :--- | +| **parentField** | `string` | optional | Single-parent pointer field (auto-detected from the object schema when omitted) | +| **labelField** | `string` | optional | Field rendered indented in the first column (defaults to "name") | +| **fields** | `string[]` | optional | Additional fields rendered as flat columns alongside the label | +| **defaultExpandedDepth** | `integer` | optional | Initial expansion depth (0 = roots only; omit = expand all) | + + --- ## UserActionsConfig @@ -499,6 +539,7 @@ Visualization type that users can switch to * `gantt` * `map` * `chart` +* `tree` --- diff --git a/packages/runtime/src/app-plugin.ts b/packages/runtime/src/app-plugin.ts index 4bdaa669f7..e65acebce8 100644 --- a/packages/runtime/src/app-plugin.ts +++ b/packages/runtime/src/app-plugin.ts @@ -298,11 +298,11 @@ export class AppPlugin implements Plugin { // A fail-fast (external + onMismatch:'fail') connect error propagates // to brick boot as intended (ADR-0062 D5); other errors are already // degraded inside the connection service. Re-throw so the kernel - // surfaces the real cause. - ctx.logger.error('[AppPlugin] declared-datasource auto-connect failed', { - appId, - error: (err as Error)?.message ?? String(err), - }); + // surfaces the real cause. (Single-string message: the context + // logger types `error(message, error?)`, not a meta object.) + ctx.logger.error( + `[AppPlugin] declared-datasource auto-connect failed for app '${appId}': ${(err as Error)?.message ?? String(err)}`, + ); throw err; } From 688a28b768ac5c79eec11df037a6f66734b9075e Mon Sep 17 00:00:00 2001 From: Jack Zhuang <277994282+os-zhuang@users.noreply.github.com> Date: Mon, 22 Jun 2026 19:20:44 +0800 Subject: [PATCH 3/4] feat(datasource): fail-closed credential resolution at connect (ADR-0062 Phase 2, D3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A declared external.credentialsRef MUST resolve to a cleartext secret before the driver is built — an absent secret store or an unresolvable/undecryptable ref now fails closed (clear message, datasource left unconnected) instead of silently building a driver without the credential. Follows the same fail-fast (declared external + onMismatch:fail) vs degrade policy as connect failures. Converges with the runtime-admin secret path (same SecretBinder threaded through the shared connection service). Co-Authored-By: Claude Opus 4.8 --- .../adr-0062-credentials-fail-closed.md | 20 ++++++ .../src/datasource-autoconnect.test.ts | 49 +++++++++++++++ .../datasource-connection-service.test.ts | 62 +++++++++++++++++++ .../src/datasource-connection-service.ts | 50 ++++++++++++--- 4 files changed, 174 insertions(+), 7 deletions(-) create mode 100644 .changeset/adr-0062-credentials-fail-closed.md diff --git a/.changeset/adr-0062-credentials-fail-closed.md b/.changeset/adr-0062-credentials-fail-closed.md new file mode 100644 index 0000000000..d80305fdfd --- /dev/null +++ b/.changeset/adr-0062-credentials-fail-closed.md @@ -0,0 +1,20 @@ +--- +"@objectstack/service-datasource": minor +--- + +feat(datasource): fail-closed credential resolution at connect (ADR-0062 Phase 2, D3) + +`DatasourceConnectionService` now treats a declared `external.credentialsRef` as +**fail-closed**: the credential must resolve to a cleartext secret (via the +host's `SecretBinder` over `ICryptoProvider`) *before* the driver is built. An +absent secret store, or a ref that cannot be resolved/decrypted (missing +`sys_secret` row, rotated key, or a throwing resolver), leaves the datasource +**unconnected with a clear message** — never a silent build-without-secret that +would connect with no/wrong auth or fail later with a confusing driver error. + +The same policy as connect failures applies: a code-defined `external` datasource +with `validation.onMismatch: 'fail'` auto-connected at boot fails fast (bricks +boot); runtime-admin create/update + boot rehydration degrade-with-warning. Code- +and runtime-origin secrets converge on the one connection path (the same +`SecretBinder` is threaded through the shared service). New `failed-credentials` +connect status. diff --git a/packages/runtime/src/datasource-autoconnect.test.ts b/packages/runtime/src/datasource-autoconnect.test.ts index 9af1e640d8..30c177fb22 100644 --- a/packages/runtime/src/datasource-autoconnect.test.ts +++ b/packages/runtime/src/datasource-autoconnect.test.ts @@ -131,6 +131,55 @@ describe('ADR-0062 declared-datasource auto-connect', () => { }); }); +describe('ADR-0062 credentials fail-closed (D3)', () => { + // An external datasource that declares a credentialsRef the host cannot + // resolve (no matching sys_secret row) must FAIL CLOSED — never connect with + // a missing credential. With onMismatch:'fail' that bricks boot (fail-fast). + function credArtifact() { + return { + manifest: { id: 'com.test.ds-cred', name: 'DS Cred', version: '1.0.0' }, + objects: [{ name: 'note', label: 'Note', fields: { title: { type: 'text' } } }], + datasources: [ + { + name: 'needs_secret', + driver: 'memory', + schemaMode: 'external', + origin: 'code', + config: {}, + external: { + allowWrites: false, + credentialsRef: 'sys_secret:does-not-exist', + validation: { onMismatch: 'fail', checkOnBoot: false }, + }, + active: true, + }, + ], + }; + } + + it('bricks boot with a clear message when a required credential cannot be resolved', async () => { + const { ObjectQLPlugin } = await import('@objectstack/objectql'); + const { InMemoryDriver } = await import('@objectstack/driver-memory'); + const { DatasourceAdminServicePlugin, createDefaultDatasourceDriverFactory } = await import( + '@objectstack/service-datasource' + ); + const runtime = new Runtime({ cluster: false }); + const kernel = runtime.getKernel(); + await kernel.use(new DriverPlugin(new InMemoryDriver())); + await kernel.use(new ObjectQLPlugin()); + await kernel.use(new AppPlugin(credArtifact())); + await kernel.use( + new DatasourceAdminServicePlugin({ + driverFactory: createDefaultDatasourceDriverFactory(), + // A binder whose resolve never finds the secret (rotated key / missing row). + secrets: { bind: async () => 'sys_secret:x', resolve: async () => undefined }, + }), + ); + await expect(kernel.bootstrap()).rejects.toThrow(/needs_secret|credential|fail-fast/i); + try { await (kernel as any)?.stop?.(); } catch { /* noop */ } + }, BOOT_TIMEOUT); +}); + describe('ADR-0062 connect policy seam', () => { it('a deny policy leaves the external datasource unconnected (cloud egress isolation)', async () => { const denyExternal: DatasourceConnectPolicy = { diff --git a/packages/services/service-datasource/src/__tests__/datasource-connection-service.test.ts b/packages/services/service-datasource/src/__tests__/datasource-connection-service.test.ts index d2e186e526..5ae82da386 100644 --- a/packages/services/service-datasource/src/__tests__/datasource-connection-service.test.ts +++ b/packages/services/service-datasource/src/__tests__/datasource-connection-service.test.ts @@ -197,6 +197,68 @@ describe('DatasourceConnectionService.connect', () => { }); }); +describe('D3 credential resolution — fail-closed', () => { + const credExternal: ConnectableDatasource = { + name: 'warehouse', + driver: 'sqlite', + schemaMode: 'external', + config: {}, + external: { allowWrites: false, credentialsRef: 'sys_secret:abc', validation: { onMismatch: 'warn' } }, + }; + + it('fails closed when a credentialsRef is declared but NO secret store is configured', async () => { + const factory = fakeFactory(); + const { service, engine } = svc({ factory }); // no `secrets` + const result = await service.connect(credExternal, { context: { trigger: 'runtime-admin' } }); + expect(result.status).toBe('failed-credentials'); + expect(result.reason).toMatch(/no secret store/); + expect(factory.create).not.toHaveBeenCalled(); // never built without the secret + expect(engine!.drivers.size).toBe(0); + }); + + it('fails closed when the credentialsRef cannot be resolved/decrypted (undefined)', async () => { + const factory = fakeFactory(); + const { service } = svc({ factory, secrets: { resolve: async () => undefined } }); + const result = await service.connect(credExternal, { context: { trigger: 'runtime-admin' } }); + expect(result.status).toBe('failed-credentials'); + expect(result.reason).toMatch(/could not be resolved or decrypted/); + expect(factory.create).not.toHaveBeenCalled(); + }); + + it('fails closed when the resolver throws', async () => { + const factory = fakeFactory(); + const { service } = svc({ + factory, + secrets: { + resolve: async () => { + throw new Error('kms unreachable'); + }, + }, + }); + const result = await service.connect(credExternal, { context: { trigger: 'runtime-admin' } }); + expect(result.status).toBe('failed-credentials'); + expect(factory.create).not.toHaveBeenCalled(); + }); + + it('fail-fast: a declared-auto external + onMismatch:fail with an unresolvable credential re-throws', async () => { + const failCred: ConnectableDatasource = { + ...credExternal, + external: { allowWrites: false, credentialsRef: 'sys_secret:abc', validation: { onMismatch: 'fail' } }, + }; + const { service } = svc({ secrets: { resolve: async () => undefined } }); + await expect(service.connect(failCred, { context: { trigger: 'declared-auto' } })).rejects.toThrow(/fail-fast/); + }); + + it('connects when the credential resolves to a secret', async () => { + const create = vi.fn(async () => ({ driver: { name: 'd' }, connect: async () => {} })); + const factory: IDatasourceDriverFactory = { supports: () => true, create }; + const { service } = svc({ factory, secrets: { resolve: async () => 's3cr3t' } }); + const result = await service.connect(credExternal); + expect(result.status).toBe('connected'); + expect(create).toHaveBeenCalledWith(expect.objectContaining({ secret: 's3cr3t' })); + }); +}); + describe('DatasourceConnectionService.connectDeclared', () => { it('connects only the gated datasources and syncs each one’s bound objects', async () => { const { service, engine, factory } = svc(); diff --git a/packages/services/service-datasource/src/datasource-connection-service.ts b/packages/services/service-datasource/src/datasource-connection-service.ts index f8f7002134..db0357149a 100644 --- a/packages/services/service-datasource/src/datasource-connection-service.ts +++ b/packages/services/service-datasource/src/datasource-connection-service.ts @@ -106,6 +106,7 @@ export type ConnectStatus = | 'skipped-policy' | 'skipped-no-infra' | 'skipped-unsupported' + | 'failed-credentials' | 'failed-degraded'; export interface ConnectResult { @@ -233,9 +234,42 @@ export class DatasourceConnectionService { ); } + // Credential resolution (ADR-0062 D3) — FAIL-CLOSED, and done *before* the + // build try-block so a fail-fast verdict propagates (rather than being + // swallowed and re-classified by the catch below). A declared + // `external.credentialsRef` MUST resolve to a cleartext secret before we + // open a connection: building a driver without it would silently connect + // with no/wrong auth (or fail later with a confusing driver error). So an + // absent secret store, or an unresolvable/undecryptable ref, leaves the + // datasource unconnected with a clear message — never a silent skip. + let secret: string | undefined; + const credentialsRef = record.external?.credentialsRef; + if (credentialsRef) { + const resolver = this.cfg.secrets?.resolve; + if (!resolver) { + return this.handleFailure( + record, + 'failed-credentials', + `requires credential '${credentialsRef}' but no secret store (SecretBinder/ICryptoProvider) is configured`, + opts.context, + ); + } + try { + secret = await resolver(credentialsRef); + } catch (err) { + return this.handleFailure(record, 'failed-credentials', `resolving credential '${credentialsRef}' threw: ${errMsg(err)}`, opts.context); + } + if (secret == null || secret === '') { + return this.handleFailure( + record, + 'failed-credentials', + `credential '${credentialsRef}' could not be resolved or decrypted (missing sys_secret row, or the encryption key changed)`, + opts.context, + ); + } + } + try { - const credentialsRef = record.external?.credentialsRef; - const secret = credentialsRef ? await this.cfg.secrets?.resolve?.(credentialsRef) : undefined; const handle = await factory.create({ ...toSpec(record), ...(secret ? { secret } : {}) }); if (typeof handle?.connect === 'function') await handle.connect(); @@ -285,11 +319,13 @@ export class DatasourceConnectionService { } /** - * Apply the D5 connect-failure policy. A code-defined `external` datasource - * with `onMismatch:'fail'` auto-connected at boot re-throws (fail-fast, - * bricking boot as intended). Runtime-admin create/update + boot rehydration - * always degrade-with-warning — a UI action or a replica blip must never - * brick the running server (preserves the pre-ADR-0062 admin behavior). + * Apply the D5 connect-failure policy (also covers D3 credential failures). A + * code-defined `external` datasource with `onMismatch:'fail'` auto-connected at + * boot re-throws (fail-fast, bricking boot as intended). Runtime-admin + * create/update + boot rehydration always degrade-with-warning — a UI action + * or a replica blip must never brick the running server (preserves the + * pre-ADR-0062 admin behavior). Either way the datasource is left unconnected + * with a clear message — never a silent skip. */ private handleFailure( record: ConnectableDatasource, From c7e5f4b92ed654f828fd859dcf8a751ceaa57ce6 Mon Sep 17 00:00:00 2001 From: Jack Zhuang <277994282+os-zhuang@users.noreply.github.com> Date: Mon, 22 Jun 2026 19:33:07 +0800 Subject: [PATCH 4/4] feat(analytics): route external objects through driver-correct path (ADR-0062 Phase 3, D6) NativeSQLStrategy hand-compiles FROM/columns that bypass the driver's physical resolution (remoteName/remoteSchema/columnMap), so it would aggregate against the wrong table for a federated object. It now declines any query whose base or joined object is external (new optional StrategyContext.isExternalObject hook, reported by the analytics plugin from the object's external block), routing it to the ObjectQLStrategy whose engine.aggregate goes through the driver's getBuilder (#2138/#2149). Reuses the driver's resolution rather than re-implementing it; until a native-SQL fast path exists, external analytics is correct (via ObjectQL) instead of silently wrong. Co-Authored-By: Claude Opus 4.8 --- .changeset/adr-0062-analytics-external.md | 20 +++++++ docs/adr/0062-external-datasource-runtime.md | 2 + .../native-sql-strategy-external.test.ts | 52 +++++++++++++++++++ .../src/analytics-service.ts | 9 ++++ .../services/service-analytics/src/plugin.ts | 11 ++++ .../src/strategies/native-sql-strategy.ts | 20 +++++++ .../spec/src/contracts/analytics-service.ts | 20 +++++++ 7 files changed, 134 insertions(+) create mode 100644 .changeset/adr-0062-analytics-external.md create mode 100644 packages/services/service-analytics/src/__tests__/native-sql-strategy-external.test.ts diff --git a/.changeset/adr-0062-analytics-external.md b/.changeset/adr-0062-analytics-external.md new file mode 100644 index 0000000000..c9e82fa909 --- /dev/null +++ b/.changeset/adr-0062-analytics-external.md @@ -0,0 +1,20 @@ +--- +"@objectstack/spec": minor +"@objectstack/service-analytics": minor +--- + +feat(analytics): correct analytics over federated objects (ADR-0062 Phase 3, D6) + +Analytics over an external (federated) object now aggregates against the +**correct** remote table instead of silently querying the wrong one. The +`NativeSQLStrategy` hand-compiles `FROM ""` and bare column references, +which bypass the driver's physical-table resolution (`external.remoteName` / +`remoteSchema` / `columnMap`). It now **declines** any query whose base or joined +object is federated, routing it to the `ObjectQLStrategy` — whose +`engine.aggregate()` goes through the driver's `getBuilder` and already honours +`remoteName`/`remoteSchema` (#2138/#2149). This "reuses the driver's resolution" +(D6) rather than re-implementing it. + +Adds an optional `StrategyContext.isExternalObject(objectName)` hook (reported by +the analytics plugin from the object's `external` block). Purely additive — with +no hook, behavior is unchanged for managed objects. diff --git a/docs/adr/0062-external-datasource-runtime.md b/docs/adr/0062-external-datasource-runtime.md index f93544f828..cb49907403 100644 --- a/docs/adr/0062-external-datasource-runtime.md +++ b/docs/adr/0062-external-datasource-runtime.md @@ -77,6 +77,8 @@ Code-defined datasources surface in `GET /api/v1/datasources`, `GET /api/v1/meta The analytics native-SQL strategy compiles its own `FROM "
"` / column references outside the driver (ADR-0015 §18 noted this). It must resolve an external object's physical table (`remoteName`/`remoteSchema`) and columns (`columnMap`) the same way `SqlDriver` now does — reusing the driver's resolution (e.g. an exposed `physicalTableFor(object)` / `physicalColumnFor(object, field)`), not a second copy. Until then, analytics over external objects stays disabled rather than silently querying the wrong table. +> **Phase 3 implementation note (#2163) — "reuse the driver's resolution" = route external objects to the ObjectQL path.** Rather than re-implement `remoteName`/`remoteSchema`/`columnMap` resolution inside the native-SQL strategy (a second copy — explicitly rejected), `NativeSQLStrategy.canHandle` now **declines** any query whose base or joined object is federated (new optional `StrategyContext.isExternalObject` hook, reported by the analytics plugin from the object's `external` block). Declining routes the query to the lower-priority `ObjectQLStrategy`, whose `engine.aggregate()` already goes through the driver's `getBuilder` — which honours `remoteName`/`remoteSchema` (#2138/#2149). So external analytics aggregates against the **correct** remote table via the single source of truth (the driver), and native-SQL never queries the wrong table. (A native-SQL fast path for external objects can be added later by exposing `physicalTableFor`/`physicalColumnFor` on the driver; deeper `columnMap`-in-`GROUP BY` support is a separate driver concern.) + ### D7 — `columnMap` is the external mechanism; reconcile `field.columnName` `external.columnMap` ({ remoteColumn → localField }) is the supported way to map external columns (shipped #2149). `field.columnName` (localField → physicalColumn) is its inverse and is **not** applied by the driver's query pipeline for external objects. Decision: for external objects, `columnMap` is authoritative; `field.columnName` on an external object is rejected at validation (no silent dual-source) until a unified column-resolution model is designed. Managed objects' `field.columnName` semantics are untouched. diff --git a/packages/services/service-analytics/src/__tests__/native-sql-strategy-external.test.ts b/packages/services/service-analytics/src/__tests__/native-sql-strategy-external.test.ts new file mode 100644 index 0000000000..ad3f3d5d1b --- /dev/null +++ b/packages/services/service-analytics/src/__tests__/native-sql-strategy-external.test.ts @@ -0,0 +1,52 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. +// +// ADR-0062 D6 — the NativeSQLStrategy must DECLINE federated (external-datasource) +// objects: its hand-compiled `FROM ""` / bare column refs bypass the +// driver's physical-table resolution (remoteName/remoteSchema/columnMap) and +// would query the wrong table. Declining routes the query to the lower-priority +// ObjectQL aggregate path, which goes through the driver's getBuilder (correct). + +import { describe, it, expect } from 'vitest'; +import { NativeSQLStrategy } from '../strategies/native-sql-strategy.js'; +import type { StrategyContext } from '../strategies/types.js'; + +const baseCaps = { nativeSql: true, objectqlAggregate: true, inMemory: false }; + +function ctxFor(cube: any, isExternalObject?: (o: string) => boolean): StrategyContext { + return { + getCube: () => cube, + queryCapabilities: () => baseCaps, + executeRawSql: async () => [], + ...(isExternalObject ? { isExternalObject } : {}), + } as unknown as StrategyContext; +} + +const query = { cube: 'c', measures: ['cnt'], dimensions: ['region'] } as any; + +describe('NativeSQLStrategy external-object gate (ADR-0062 D6)', () => { + const strategy = new NativeSQLStrategy(); + + it('DECLINES when the base object is external', () => { + const cube = { name: 'c', sql: 'ext_customer', dimensions: {}, measures: {} }; + const ctx = ctxFor(cube, (o) => o === 'ext_customer'); + expect(strategy.canHandle(query, ctx)).toBe(false); + }); + + it('ACCEPTS a managed object (native-SQL still used)', () => { + const cube = { name: 'c', sql: 'account', dimensions: {}, measures: {} }; + const ctx = ctxFor(cube, () => false); + expect(strategy.canHandle(query, ctx)).toBe(true); + }); + + it('DECLINES when a JOINED object is external (the join would hit the wrong table)', () => { + const cube = { name: 'c', sql: 'orders', joins: { customer: { name: 'ext_customer' } }, dimensions: {}, measures: {} }; + const ctx = ctxFor(cube, (o) => o === 'ext_customer'); + expect(strategy.canHandle(query, ctx)).toBe(false); + }); + + it('is purely additive — with no isExternalObject hook it behaves as before (accept)', () => { + const cube = { name: 'c', sql: 'ext_customer', dimensions: {}, measures: {} }; + const ctx = ctxFor(cube); // no hook + expect(strategy.canHandle(query, ctx)).toBe(true); + }); +}); diff --git a/packages/services/service-analytics/src/analytics-service.ts b/packages/services/service-analytics/src/analytics-service.ts index 30e4201a3f..c9ac0f90c6 100644 --- a/packages/services/service-analytics/src/analytics-service.ts +++ b/packages/services/service-analytics/src/analytics-service.ts @@ -135,6 +135,14 @@ export interface AnalyticsServiceConfig { * `StrategyContext.coerceTemporalFilterValue` for the full rationale. */ coerceTemporalFilterValue?: (objectName: string, fieldName: string, value: unknown) => unknown; + /** + * ADR-0062 D6 — report whether an object is federated (external datasource). + * Threaded into the StrategyContext so `NativeSQLStrategy` declines external + * objects (which it would otherwise query against the wrong physical table), + * routing them to the driver-correct ObjectQL aggregate path instead. See + * `StrategyContext.isExternalObject`. + */ + isExternalObject?: (objectName: string) => boolean; /** * ADR-0021 — optional object-graph resolver used when compiling datasets: * `(baseObject, relationshipName) => relatedObjectName | undefined`. When @@ -260,6 +268,7 @@ export class AnalyticsService implements IAnalyticsService { this.datasetRegistry.get(cubeName)?.allowedRelationships ?? config.getAllowedRelationships?.(cubeName), coerceTemporalFilterValue: config.coerceTemporalFilterValue, + isExternalObject: config.isExternalObject, }; // Build strategy chain (built-in + custom, sorted by priority) diff --git a/packages/services/service-analytics/src/plugin.ts b/packages/services/service-analytics/src/plugin.ts index d9fb93f46b..bcb2d3438a 100644 --- a/packages/services/service-analytics/src/plugin.ts +++ b/packages/services/service-analytics/src/plugin.ts @@ -32,6 +32,10 @@ interface DataEngineLike { reference?: string; options?: Array<{ value: unknown; label?: string }>; }>; + /** Federation marker (ADR-0015): set on objects bound to an external datasource. */ + external?: unknown; + /** The datasource this object is bound to (ADR-0062 D6 external detection). */ + datasource?: string; } | undefined; /** * Resolve the storage driver backing an object (public ObjectQL accessor). @@ -443,6 +447,13 @@ export class AnalyticsServicePlugin implements Plugin { | undefined; return f ? { type: f.type, defaultCurrency: f.currencyConfig?.defaultCurrency } : undefined; }, + // ADR-0062 D6 — a federated object carries an `external` block (ADR-0015). + // Reported so NativeSQLStrategy declines it (its hand-compiled FROM would + // hit the wrong physical table) and the driver-correct ObjectQL path runs. + isExternalObject: (objectName: string) => { + const obj = dataEngine()?.getObject?.(objectName); + return !!(obj && obj.external != null); + }, draftRowsResolver, }; diff --git a/packages/services/service-analytics/src/strategies/native-sql-strategy.ts b/packages/services/service-analytics/src/strategies/native-sql-strategy.ts index bf5f521574..03668fa328 100644 --- a/packages/services/service-analytics/src/strategies/native-sql-strategy.ts +++ b/packages/services/service-analytics/src/strategies/native-sql-strategy.ts @@ -28,6 +28,26 @@ export class NativeSQLStrategy implements AnalyticsStrategy { // query silently grouped by the raw timestamp — one bucket per row — and a // non-UTC reference timezone was ignored entirely (ADR-0053 Phase 2, #1982). if (query.timeDimensions?.some((td) => !!td.granularity)) return false; + // ADR-0062 D6 — DECLINE federated (external-datasource) objects. This + // strategy hand-compiles `FROM ""` and bare column references, which + // bypass the driver's physical-table resolution (`external.remoteName` / + // `remoteSchema` / `columnMap`) and would query the WRONG table. Routing the + // query to the lower-priority ObjectQL aggregate path keeps it correct — + // that path goes through the driver's `getBuilder` (#2138/#2149). Applies to + // the base object AND any joined object (a join would also hit the wrong + // table). Until native-SQL learns the driver's resolution, "disabled" beats + // "silently wrong". + if (typeof ctx.isExternalObject === 'function') { + const cube = ctx.getCube(query.cube); + if (cube) { + if (ctx.isExternalObject(this.extractObjectName(cube))) return false; + const joinTargets = cube.joins ? Object.values(cube.joins) : []; + for (const j of joinTargets) { + const joinedObject = (j as { name?: string })?.name; + if (joinedObject && ctx.isExternalObject(joinedObject)) return false; + } + } + } const caps = ctx.queryCapabilities(query.cube); return caps.nativeSql && typeof ctx.executeRawSql === 'function'; } diff --git a/packages/spec/src/contracts/analytics-service.ts b/packages/spec/src/contracts/analytics-service.ts index d7c9dab12f..4a92559e1a 100644 --- a/packages/spec/src/contracts/analytics-service.ts +++ b/packages/spec/src/contracts/analytics-service.ts @@ -322,6 +322,26 @@ export interface StrategyContext { * @param value The stringified comparand from the normalized filter. */ coerceTemporalFilterValue?(objectName: string, fieldName: string, value: unknown): unknown; + + /** + * ADR-0062 D6 — is `objectName` a federated (external-datasource) object? + * + * The `NativeSQLStrategy` compiles its own `FROM ""` and column + * references, which bypass the driver's physical-table resolution and so + * would query the WRONG table for a federated object whose `external.remoteName` + * / `remoteSchema` / `columnMap` differ from the logical object/field names. + * Until native-SQL learns the driver's physical resolution, the strategy + * DECLINES external objects (see its `canHandle`), so they fall through to the + * ObjectQL aggregate path — which routes through the driver's `getBuilder` + * (honouring `remoteName`/`remoteSchema`, #2138/#2149). This keeps external + * analytics correct ("reuse the driver's resolution") rather than silently + * querying the wrong table. + * + * Returns `true` for a federated object, `false`/`undefined` otherwise. When + * the hook is absent (legacy wiring) the strategy assumes non-external — + * purely additive, no behavior change for managed objects. + */ + isExternalObject?(objectName: string): boolean; } /**