From f51c6aeab19f6b2a0123871d44b5fd8529f2e47a Mon Sep 17 00:00:00 2001 From: os-elon Date: Wed, 19 Aug 2026 00:32:09 +0000 Subject: [PATCH 1/3] fix(objectql): discriminate a failed sys_organization probe from a genuinely empty one (#9261) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `probeInstallOrganizations` answered every read failure with `[]`, which `resolveSystemWriteOrganization` reads as `no-organization-yet` — so one transient failure silently skipped both halves of the #8844 ruling (the single-organization stamp and the multi-organization refusal), and the memo cached the invented answer past the outage. Only the measured benign cause — an unprovisioned `sys_organization` table, asked through the shared `isMissingTableError` predicate — still answers the empty probe. Everything else propagates, envelope intact, and is not memoised. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_019yDEhPBC3tcGkW9bkce1HM --- .../engine-organization-probe-outage.test.ts | 269 ++++++++++++++++++ packages/objectql/src/engine.ts | 53 +++- 2 files changed, 318 insertions(+), 4 deletions(-) create mode 100644 packages/objectql/src/engine-organization-probe-outage.test.ts diff --git a/packages/objectql/src/engine-organization-probe-outage.test.ts b/packages/objectql/src/engine-organization-probe-outage.test.ts new file mode 100644 index 0000000000..2e2228454a --- /dev/null +++ b/packages/objectql/src/engine-organization-probe-outage.test.ts @@ -0,0 +1,269 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * #9261 — a FAILED `sys_organization` probe must not be indistinguishable from + * "this install has no organizations yet". + * + * `probeInstallOrganizations` answers the one question + * `resolveSystemWriteOrganization` decides on (#8844): 0 organizations ⇒ the + * system insert proceeds UNSTAMPED, 1 ⇒ stamp the derived id, 2+ ⇒ REFUSE with + * `ERR_SYSTEM_WRITE_ORGANIZATION_REQUIRED`. Its read used to sit behind a bare + * `} catch { ids = [] }`, so every failure — connection drop, pool exhaustion, + * a timeout mid-boot — was answered with the count that means "none". Both + * halves of the ruling then became silently skippable: on a `single` install + * with one organization the rows land untenanted and fork the autonumber + * counter the ruling exists to protect, and on a multi-organization install the + * mandated refusal never fires. ADR-0110 D3: the probe finding nothing and the + * probe being unable to run are different facts. + * + * ⚠️ Aggravator, pinned separately below: the answer is MEMOISED. One transient + * failure pinned "no organizations" for every later system write until an + * organization write happened to clear it — so the outage's consequence + * outlived the outage. + * + * ## What is benign here, MEASURED rather than assumed + * + * The card and the old comment both name "`sys_organization` may not be + * registered at all (a lean embedding, a bare-kernel test)" as the benign + * reason, and the dispatch expected the benign case to be TWO distinct errors. + * Measured on this seam it is ONE, and it is not the one they name — so this + * file pins the measurement, not the expectation: + * + * - an object missing from the REGISTRY does not fail the read at all when the + * driver tolerates an unknown table — `find` returns `[]` through the normal + * path and never reaches the catch (pinned below, so a future predicate is + * not written against a case that cannot occur); + * - a strict driver surfaces that same install as a MISSING TABLE, which IS the + * benign cause, named by the shared `isMissingTableError` predicate + * (`@objectstack/metadata/errors`, #4825); + * - and the "no driver at all" install cannot reach the probe. `getDriver` + * answers every object from the default driver, which the FIRST + * `registerDriver` always sets (`isDefault || drivers.size === 1`) and which + * nothing ever clears — no driver is ever removed — so the only engine whose + * routing fails for `sys_organization` is one with no drivers, where the + * write that would have asked already failed on its own object. Pinned below + * with that error as the positive control, so "we did not need a second + * predicate" is a measurement rather than an omission. + * + * Both directions are pinned in the same cases deliberately. A file that pinned + * only the propagation would stay green if the probe threw on every install, + * and one that pinned only the benign empties would stay green if the fix had + * never landed. + */ + +import { describe, it, expect } from 'vitest'; +import type { ExecutionContext } from '@objectstack/spec/kernel'; +import { ObjectQL } from './engine.js'; + +const ORG_ID = 'org_msokm9oaz0cal87q'; +const SECOND_ORG_ID = 'org_second'; +const PACKAGE_ID = '#9261'; + +/** The card's producer: a hook / cron write, elevated, carrying no organization. */ +const SYSTEM_CTX: ExecutionContext = { isSystem: true } as ExecutionContext; + +/** An application object the #8844 ruling judges: tenant-scoped, org-scoped counter. */ +const DISPATCH_ORDER = { + name: 'dispatch_order', + fields: { + subject: { type: 'text' }, + document_no: { type: 'autonumber', format: 'WI-{00000}', unique: 'organization' }, + }, +} as any; + +const ORG_OBJECT = { name: 'sys_organization', fields: { name: { type: 'text' } } } as any; + +interface ObservedWrite { + object: string; + method: string; + data: any; + options: Record | undefined; +} + +/** A transient database outage: the shape `isMissingTableError` must answer `false` for. */ +const outage = () => + Object.assign(new Error('connect ECONNREFUSED 127.0.0.1:5432'), { code: 'ECONNREFUSED' }); + +/** The benign unprovisioned table, in the SQLite-family spelling #4825 recognises. */ +const missingTable = () => new Error('no such table: sys_organization'); + +function makeDriver( + observed: ObservedWrite[], + organizationFind: () => any[], + name = 'memory', +) { + const record = (object: string, method: string, data: any, options: any) => + observed.push({ object, method, data, options }); + return { + name, + version: '0.0.0', + supports: {}, + async connect() {}, async disconnect() {}, async checkHealth() { return true; }, + async execute() { return null; }, + async find(object: string, _ast: any, options: any) { + record(object, 'find', undefined, options); + return object === 'sys_organization' ? organizationFind() : []; + }, + async findOne() { return null; }, + async count() { return 0; }, + async create(object: string, data: any, options: any) { + record(object, 'create', data, options); + return { id: 'r_1', ...data }; + }, + async update(object: string, id: string, data: any, options: any) { + record(object, 'update', data, options); return { id, ...data }; + }, + async delete() { return true; }, + async bulkCreate(object: string, rows: any[], options: any) { + record(object, 'bulkCreate', rows, options); + return rows.map((r, i) => ({ id: `r_${i + 1}`, ...r })); + }, + async bulkUpdate() { return []; }, async bulkDelete() {}, + async syncSchema() {}, + } as any; +} + +async function makeEngine(opts: { + organizationFind?: () => any[]; + registerOrganizationObject?: boolean; +} = {}) { + const observed: ObservedWrite[] = []; + const engine = new ObjectQL(); + engine.registerDriver(makeDriver(observed, opts.organizationFind ?? (() => [{ id: ORG_ID }])), true); + await engine.init(); + engine.registry.registerObject(DISPATCH_ORDER, PACKAGE_ID); + if (opts.registerOrganizationObject !== false) engine.registry.registerObject(ORG_OBJECT, PACKAGE_ID); + // The seam under test only runs on the `single` posture — a walled install is + // refused without asking the database anything. + engine.setTenancyPostureProvider(() => 'single'); + return { engine, observed }; +} + +const lastWrite = (observed: ObservedWrite[], object: string) => + [...observed].reverse().find((c) => c.object === object && c.method !== 'find'); + +const systemInsert = (engine: ObjectQL, subject: string) => + engine.insert('dispatch_order', { subject }, { context: SYSTEM_CTX } as any); + +describe('#9261 a non-benign probe failure no longer becomes "no organizations"', () => { + it('propagates the outage instead of proceeding unstamped, and writes nothing', async () => { + // The `single` install that HAS one organization — the ruling requires the + // derived stamp. Before the fix the outage answered 0 and the row landed + // untenanted on the `__global__` counter partition. + const { engine, observed } = await makeEngine({ organizationFind: () => { throw outage(); } }); + + const failure = await systemInsert(engine, 'during the outage').catch((e) => e); + + // The probe's OWN failure, envelope intact — no new code, no new field. + expect(failure).toBeInstanceOf(Error); + expect((failure as any).code).toBe('ECONNREFUSED'); + expect((failure as Error).message).toContain('ECONNREFUSED'); + // ⛔ Not filed under a guessed topology: nothing reached the driver. + expect(observed.filter((c) => c.object === 'dispatch_order')).toEqual([]); + }); + + it('does not silently skip the multi-organization REFUSAL', async () => { + // The #8895 shape: fail-open on a guard the ruling says must be loud. A + // walled/ambiguous install whose probe fails must not be told "0, proceed". + const { engine, observed } = await makeEngine({ organizationFind: () => { throw outage(); } }); + const failure = await systemInsert(engine, 'ambiguous install, outage').catch((e) => e); + + // It fails — and it fails as the OUTAGE, not as the refusal, because the + // topology was never measured. Either way the write does not proceed. + expect((failure as any).code).toBe('ECONNREFUSED'); + expect((failure as any).code).not.toBe('ERR_SYSTEM_WRITE_ORGANIZATION_REQUIRED'); + expect(observed.filter((c) => c.object === 'dispatch_order')).toEqual([]); + + // DISCRIMINATING CONTROL — the same install with a probe that WORKS still + // reaches the ruling's refusal, so the case above is about the outage and + // not about the refusal having been broken. + const healthy = await makeEngine({ organizationFind: () => [{ id: ORG_ID }, { id: SECOND_ORG_ID }] }); + const refusal = await systemInsert(healthy.engine, 'ambiguous install, healthy').catch((e) => e); + expect((refusal as any).code).toBe('ERR_SYSTEM_WRITE_ORGANIZATION_REQUIRED'); + expect((refusal as any).status).toBe(500); + expect((refusal as any).reason).toBe('ambiguous-organization'); + }); + + it('⛔ does NOT memoise a failure — the outage does not outlive itself', async () => { + // The aggravator. One transient failure used to pin "no organizations" for + // every later system write until an organization write cleared the memo. + let failing = true; + const { engine, observed } = await makeEngine({ + organizationFind: () => { if (failing) throw outage(); return [{ id: ORG_ID }]; }, + }); + + await expect(systemInsert(engine, 'during')).rejects.toThrow(/ECONNREFUSED/); + + failing = false; + await systemInsert(engine, 'after recovery'); + + // The very next write re-probes and gets the truth — the stamp the #8844 + // ruling requires, not the guess the outage would have cached. + expect(lastWrite(observed, 'dispatch_order')?.options?.tenantId).toBe(ORG_ID); + // Two probes: the failed one and the repair. A memoised failure would show one. + expect(observed.filter((c) => c.object === 'sys_organization' && c.method === 'find')).toHaveLength(2); + }); +}); + +describe('#9261 the benign causes still answer the empty probe', () => { + it('[cause 2] an unprovisioned `sys_organization` table proceeds unstamped', async () => { + // Schema sync has not run; the table cannot hold a row, so zero IS the + // measurement and first boot must not be refused. + const { engine, observed } = await makeEngine({ organizationFind: () => { throw missingTable(); } }); + await systemInsert(engine, 'first boot'); + expect(lastWrite(observed, 'dispatch_order')?.method).toBe('create'); + expect(lastWrite(observed, 'dispatch_order')?.options?.tenantId).toBeUndefined(); + }); + + it('an UNROUTABLE organization object never reaches the probe, so it needs no predicate', async () => { + // Why `isMissingTableError` alone is not a fail-CLOSED regression. An engine + // that cannot route `sys_organization` raises a bare `No driver available`, + // which that predicate answers `false` for — it would propagate. It is not + // discriminated because it cannot occur on this path: the ONLY engine whose + // routing fails for `sys_organization` is one with no drivers at all, and + // there the write that would have asked fails on its OWN object first. + const engine = new ObjectQL(); + await engine.init(); + engine.registry.registerObject(DISPATCH_ORDER, PACKAGE_ID); + engine.registry.registerObject(ORG_OBJECT, PACKAGE_ID); + engine.setTenancyPostureProvider(() => 'single'); + + // POSITIVE CONTROL — the unroutable read really does raise that error, and + // it really is outside the benign predicate. Without this the case below + // would be a zero-hit measurement of nothing. + const unroutable = await engine + .find('sys_organization', { fields: ['id'], limit: 2, context: { isSystem: true } } as any) + .catch((e) => e); + expect((unroutable as Error).message).toContain("No driver available for object 'sys_organization'"); + + // …and the system insert stops on `dispatch_order`, one frame BEFORE the + // probe would have run. + const failure = await engine + .insert('dispatch_order', { subject: 'bare kernel' }, { context: SYSTEM_CTX } as any) + .catch((e) => e); + expect((failure as Error).message).toContain("No driver available for object 'dispatch_order'"); + }); + + it('an object missing from the REGISTRY never reaches the catch on a tolerant driver', async () => { + // Pinned so the next author does not write a predicate against a case that + // cannot occur: the read succeeds and returns `[]` through the NORMAL path. + const { engine, observed } = await makeEngine({ + registerOrganizationObject: false, + organizationFind: () => [], + }); + await systemInsert(engine, 'unregistered organization object'); + expect(lastWrite(observed, 'dispatch_order')?.options?.tenantId).toBeUndefined(); + // The control: the read really did happen (it is not the structural branch). + expect(observed.filter((c) => c.object === 'sys_organization' && c.method === 'find')).toHaveLength(1); + }); + + it('a healthy probe is unchanged — one organization is still stamped, and memoised', async () => { + // The discriminating control for this whole file: the fix must not have + // narrowed the ordinary path. + const { engine, observed } = await makeEngine({ organizationFind: () => [{ id: ORG_ID }] }); + await systemInsert(engine, 'a'); + await systemInsert(engine, 'b'); + expect(lastWrite(observed, 'dispatch_order')?.options?.tenantId).toBe(ORG_ID); + expect(observed.filter((c) => c.object === 'sys_organization' && c.method === 'find')).toHaveLength(1); + }); +}); diff --git a/packages/objectql/src/engine.ts b/packages/objectql/src/engine.ts index 782d989967..340f7e549b 100644 --- a/packages/objectql/src/engine.ts +++ b/packages/objectql/src/engine.ts @@ -3023,6 +3023,51 @@ export class ObjectQL implements IObjectQLEngine { * `isSystem` on the read: resolving the install's organization must not * depend on the caller's own reach, and this runs on writes that have no * caller reach at all. + * + * [#9261] "The probe found no organizations" and "the probe could not run" + * are different facts (ADR-0110 D3), and this returns the FIRST one only when + * it measured it. The `catch {}` this replaces answered both with `[]`, which + * `resolveSystemWriteOrganization` reads as `no-organization-yet` — so one + * transient failure silently skipped both halves of the #8844 ruling: the + * single-organization stamp (rows land untenanted and fork exactly the + * counter the ruling exists to protect) and the multi-organization REFUSAL + * (fail-open on a guard that must be loud, the #8895 shape). The memo made + * the damage outlive the outage — the invented answer was cached until an + * organization write happened to clear it. + * + * **Exactly one cause is benign, and it was measured rather than assumed.** + * `sys_organization` ROUTES but its TABLE was never provisioned (schema sync + * not run yet): it cannot hold a row, so zero really is the count. Asked + * through the shared `isMissingTableError` predicate + * (`@objectstack/metadata/errors`, #4825) — the same call + * `resolveFileReferences` and `cascadeDeleteRelations` make — never a + * hand-rolled code test. + * + * ⚠️ The old comment's "`sys_organization` may not be registered at all (a + * lean embedding, a bare-kernel test)" is NOT a second benign cause, and a + * predicate written for it would have guarded a case that cannot reach this + * catch. Measured on this seam: + * + * - an object missing from the REGISTRY does not fail the read at all on a + * driver that tolerates an unknown table — {@link find} returns `[]` + * through the normal path, never entering the catch; + * - a strict driver surfaces that same install as a MISSING TABLE, i.e. as + * the one benign cause above; + * - and "no driver at all" cannot reach here: {@link getDriver} answers every + * object from the default driver, which the first {@link registerDriver} + * always sets and nothing ever clears, so the only engine whose routing + * fails for `sys_organization` is one with no drivers — where the write + * that would have asked already failed on its OWN object. + * + * Everything else — connection loss, pool exhaustion, a timeout mid-boot, a + * datasource that never connected ({@link DatasourceUnavailableError}), a + * permission denial — means organizations may well exist and simply were not + * seen. It PROPAGATES, envelope intact, and ⛔ nothing is memoised on that + * path: the write that asked fails loudly instead of being filed under a + * guessed topology, and the next write re-probes rather than inheriting the + * guess. No new error code and no new response field — the caller receives + * the probe's own failure, the same disposition #8895 gave the + * referential-integrity probe. */ private async probeInstallOrganizations(): Promise { if (this.organizationProbeMemo) return this.organizationProbeMemo; @@ -3037,10 +3082,10 @@ export class ObjectQL implements IObjectQLEngine { .map((r: any) => (r?.id ?? r?._id)) .filter((id: unknown) => id != null && String(id) !== '') .map((id: unknown) => String(id)); - } catch { - // `sys_organization` may not be registered at all (a lean embedding, a - // bare-kernel test). No organizations is the honest reading, and it is - // the branch that changes nothing. + } catch (error) { + // The one benign cause only — anything else is an outage, not an + // emptiness, and must not be memoised as one. + if (!isMissingTableError(error)) throw error; ids = []; } this.organizationProbeMemo = ids; From c808b3ef296d0ce1d3b30e7fb08b0c037aaee46d Mon Sep 17 00:00:00 2001 From: os-elon Date: Wed, 19 Aug 2026 00:40:53 +0000 Subject: [PATCH 2/3] docs(changeset): organization probe discriminates outage from emptiness (#9261) Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_019yDEhPBC3tcGkW9bkce1HM --- .changeset/organization-probe-discriminate.md | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 .changeset/organization-probe-discriminate.md diff --git a/.changeset/organization-probe-discriminate.md b/.changeset/organization-probe-discriminate.md new file mode 100644 index 0000000000..4fcb24ac1d --- /dev/null +++ b/.changeset/organization-probe-discriminate.md @@ -0,0 +1,47 @@ +--- +"@objectstack/objectql": patch +--- + +fix(objectql): a failed `sys_organization` probe stops reading as "this install has no organizations" (#9261) + +`probeInstallOrganizations` — the read the #8844 system-write organization +resolution decides on — sat behind a bare `} catch { ids = [] }`. Every failure +answered with the count that means *none*, and `resolveSystemWriteOrganization` +maps 0 / 1 / 2+ organizations to *proceed unstamped* / *stamp the derived id* / +*refuse*. So one transient probe failure silently skipped **both** halves of the +2026-08-15 ruling: + +- on a `single`-posture install that really has one organization, system-context + inserts (a hook, a cron tick, a `runAs: system` flow) landed **unstamped** — + filing the row under the `__global__` pseudo-tenant and forking exactly the + per-organization autonumber counter and partitioned unique index the ruling + exists to protect; +- on a multi-organization install, the refusal the ruling mandates + (`ERR_SYSTEM_WRITE_ORGANIZATION_REQUIRED`) **never fired** — fail-open on a + guard that must be loud. + +Aggravated by the memo: the invented answer was cached in +`organizationProbeMemo`, so the outage's consequence outlived the outage — every +later system write inherited "no organizations" until an organization write +happened to clear it. + +The probe now discriminates by error TYPE, the disposition ADR-0110 D3 requires +("the probe found nothing" and "the probe could not run" are different facts): + +- **benign** — `sys_organization` routes but its table was never provisioned + (schema sync not run yet). It cannot hold a row, so zero really is the count, + and first boot still proceeds unstamped. Asked through the shared + `isMissingTableError` predicate (`@objectstack/metadata/errors`), the same call + the file's sibling read seams make, never a hand-rolled code test. +- **everything else** — connection loss, pool exhaustion, a timeout mid-boot, a + datasource that never connected, a permission denial — propagates with its + envelope intact, and is **not memoised**. The write that asked fails loudly + instead of being filed under a guessed topology, and the next write re-probes + rather than inheriting the guess. No new error code and no new response field. + +Measured rather than assumed: the old comment's "`sys_organization` may not be +registered at all (a lean embedding, a bare-kernel test)" is not a second benign +cause. An object missing from the registry does not fail the read at all on a +driver that tolerates an unknown table (`find` returns `[]` through the normal +path), a strict driver surfaces it as the missing table above, and an engine +with no driver fails the write on its own object one frame before the probe runs. From 049d6e651059b785b748db0cb950323d319c12a8 Mon Sep 17 00:00:00 2001 From: os-elon Date: Wed, 19 Aug 2026 01:27:23 +0000 Subject: [PATCH 3/3] test(objectql): type the probe read as EngineQueryOptions, keeping the erasure ratchet at 240 (#9261) Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_019yDEhPBC3tcGkW9bkce1HM --- .../objectql/src/engine-organization-probe-outage.test.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/objectql/src/engine-organization-probe-outage.test.ts b/packages/objectql/src/engine-organization-probe-outage.test.ts index 2e2228454a..773f86f5c3 100644 --- a/packages/objectql/src/engine-organization-probe-outage.test.ts +++ b/packages/objectql/src/engine-organization-probe-outage.test.ts @@ -52,6 +52,7 @@ */ import { describe, it, expect } from 'vitest'; +import type { EngineQueryOptions } from '@objectstack/spec/data'; import type { ExecutionContext } from '@objectstack/spec/kernel'; import { ObjectQL } from './engine.js'; @@ -145,6 +146,9 @@ const lastWrite = (observed: ObservedWrite[], object: string) => const systemInsert = (engine: ObjectQL, subject: string) => engine.insert('dispatch_order', { subject }, { context: SYSTEM_CTX } as any); +/** The probe's own read, spelled exactly as `probeInstallOrganizations` issues it. */ +const PROBE_QUERY: EngineQueryOptions = { fields: ['id'], limit: 2, context: { isSystem: true } }; + describe('#9261 a non-benign probe failure no longer becomes "no organizations"', () => { it('propagates the outage instead of proceeding unstamped, and writes nothing', async () => { // The `single` install that HAS one organization — the ruling requires the @@ -231,9 +235,7 @@ describe('#9261 the benign causes still answer the empty probe', () => { // POSITIVE CONTROL — the unroutable read really does raise that error, and // it really is outside the benign predicate. Without this the case below // would be a zero-hit measurement of nothing. - const unroutable = await engine - .find('sys_organization', { fields: ['id'], limit: 2, context: { isSystem: true } } as any) - .catch((e) => e); + const unroutable = await engine.find('sys_organization', PROBE_QUERY).catch((e) => e); expect((unroutable as Error).message).toContain("No driver available for object 'sys_organization'"); // …and the system insert stops on `dispatch_order`, one frame BEFORE the