From 4e5989b9a47b240e5dc3e7ebf72109af96108f73 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 17 Aug 2026 09:25:15 +0000 Subject: [PATCH] fix(metadata-protocol): derive the reference graph from the type schemas (#9190) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `findReferencesToMeta` — the admin "Used by" panel behind `GET /api/v1/meta/:type/:name/references` — was driven by a hand-curated table of 7 target types and 40 dotted paths. Measured against the schemas it claimed to describe, 34 of the 40 named properties no metadata type declares, leaving 5 of its 7 target keys answering `{ references: [] }` unconditionally while appearing covered. The panel's empty state reads "Nothing in the metadata graph points at this item. Safe to delete." Coverage is now derived at boot from `DEFAULT_METADATA_TYPE_REGISTRY` and each type's schema, in the shape #7894 used for the URL-spelling map, so a newly declared type arrives covered. The unit of derivation is a PROPERTY, not a path: recursive containers (app navigation) make an exhaustive path list unbounded, and the walk reports where the name was actually found. No wire change: response shape, status codes and error envelope untouched. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01NTKPDRoynY8i3HmdSFUxFj --- .changeset/reference-paths-derivation.md | 39 ++ ...otocol.read-seam-empty-accumulator.test.ts | 53 ++- .../protocol.read-verb-canonical-fold.test.ts | 50 ++- .../src/protocol.reference-derivation.test.ts | 196 +++++++++ packages/metadata-protocol/src/protocol.ts | 271 ++++++------ .../src/reference-sites.derivation.test.ts | 156 +++++++ .../metadata-protocol/src/reference-sites.ts | 392 ++++++++++++++++++ .../objectql/src/protocol-references.test.ts | 57 ++- 8 files changed, 1036 insertions(+), 178 deletions(-) create mode 100644 .changeset/reference-paths-derivation.md create mode 100644 packages/metadata-protocol/src/protocol.reference-derivation.test.ts create mode 100644 packages/metadata-protocol/src/reference-sites.derivation.test.ts create mode 100644 packages/metadata-protocol/src/reference-sites.ts diff --git a/.changeset/reference-paths-derivation.md b/.changeset/reference-paths-derivation.md new file mode 100644 index 0000000000..c22a0562a2 --- /dev/null +++ b/.changeset/reference-paths-derivation.md @@ -0,0 +1,39 @@ +--- +'@objectstack/metadata-protocol': minor +--- + +Derive the metadata reference graph from the type schemas instead of curating it by hand + +`GET /api/v1/meta/:type/:name/references` — the admin "Used by" panel, rendered +immediately before a rename or a delete — was driven by a hand-written table of +seven target types and forty dotted paths. Measured against the schemas it was +supposed to describe, **34 of those 40 paths named properties no metadata type +declares**: `app.navItems[]` / `app.tabs[]` (the schema declares `navigation` +and `areas`), `agent.tools[]` (removed in `@objectstack/spec` 17), +`permission.objects[].name` (a name-keyed record, not an array), +`object.fields{}.referenceTo` (the field property is `reference`), +`dashboard.widgets[].view`, `page.viewName`, and every path the table listed for +`flow`. Five of its seven target types therefore answered `{ references: [] }` +unconditionally, on every deployment, while appearing to be covered — and an +empty panel reads as "nothing depends on this, safe to delete". + +Coverage is now derived at boot from `DEFAULT_METADATA_TYPE_REGISTRY` and each +type's Zod schema, so a newly declared metadata type arrives covered instead of +waiting for someone to remember it. Seventeen target types now resolve real +reference sites, including `permission`-to-object grants (through the record +key, which the old path grammar could not express), `translation`, `dataset`, +`action`, `report`, `doc` and `datasource`, plus flow-node references such as +`subflow`. References nested inside recursive containers — a view named from a +third-level app navigation group — are found at any depth, which no finite path +list could do. + +No wire change: the response shape, status codes and error envelope are +untouched. The `path` and `kind` values now describe where the reference was +actually found rather than which table row matched. + +Two gaps are deliberately declared rather than papered over: `external_catalog` +resolves no schema, so its references are not computable and it is named in the +derivation's `unwalkableSourceTypes` (pinned by a test, so the set cannot grow +silently), and reference properties whose name does not spell their target — +`FieldSchema.reference` is the one carried — need a producer-side annotation to +become derivable. diff --git a/packages/metadata-protocol/src/protocol.read-seam-empty-accumulator.test.ts b/packages/metadata-protocol/src/protocol.read-seam-empty-accumulator.test.ts index bff6108f25..f9aa33d6cb 100644 --- a/packages/metadata-protocol/src/protocol.read-seam-empty-accumulator.test.ts +++ b/packages/metadata-protocol/src/protocol.read-seam-empty-accumulator.test.ts @@ -210,12 +210,23 @@ describe('[#8896] searchAll — an object that could not be READ is not an objec describe('[#8896] findReferencesToMeta — a source type that could not be READ is not a source type with no references', () => { /** - * `view` has three matchers (`dashboard`, `app`, `page`), so a single - * failing source type leaves the other two answering — which is exactly the - * pre-fix trap: a SHORT list that looks complete. `page` carries a real - * reference to `my_view`, so the healthy half is observable. + * `view` is reachable from four source types (`app`, `object`, `page`, + * `view`), so a single failing source type leaves the others answering — + * which is exactly the pre-fix trap: a SHORT list that looks complete. + * `page` carries a real reference to `my_view`, so the healthy half is + * observable. + * + * [#9190] The fixture used to spell that reference `page.viewName`, which + * `PageSchema` does not declare — it agreed with the hand-curated path + * table, and the table was wrong. The real site is `view`, reached through + * a `dataSource`, and the derived walk finds it wherever the document puts + * it rather than at one memorised path. */ - const pageReferencingTheView = { name: 'home_page', label: 'Home', viewName: 'my_view' }; + const pageReferencingTheView = { + name: 'home_page', + label: 'Home', + slots: { header: { dataSource: { view: 'my_view' } } }, + }; function engineWhereTypeFails(failingType: string | null, error?: unknown) { const typeReads: string[] = []; @@ -239,18 +250,24 @@ describe('[#8896] findReferencesToMeta — a source type that could not be READ const result = await protocol.findReferencesToMeta({ type: 'view', name: 'my_view' }); expect(result.references).toEqual([ - { type: 'page', name: 'home_page', label: 'Home', path: 'viewName', kind: 'page' }, + { + type: 'page', + name: 'home_page', + label: 'Home', + path: 'slots.header.dataSource.view', + kind: 'page view', + }, ]); - // All three source types were really consulted — this is what makes - // "one of them failed" a meaningful condition below. - expect(typeReads).toContain('dashboard'); + // Every source type that can name a view was really consulted — this is + // what makes "one of them failed" a meaningful condition below. expect(typeReads).toContain('app'); + expect(typeReads).toContain('object'); expect(typeReads).toContain('page'); }); it('a source type whose read FAILS fails the whole scan, envelope intact', async () => { const injected = connectionDropped(); - const { engine, typeReads } = engineWhereTypeFails('dashboard', injected); + const { engine, typeReads } = engineWhereTypeFails('app', injected); const protocol = new ObjectStackProtocolImplementation(engine as never); const caught = await rejection( @@ -267,7 +284,7 @@ describe('[#8896] findReferencesToMeta — a source type that could not be READ expect(ErrorCode.safeParse(caught.code).success).toBe(true); // The driver's own error is not lost — it rides as `cause`. expect(caught.cause).toBe(injected); - expect(typeReads).toContain('dashboard'); + expect(typeReads).toContain('app'); // Pre-fix this resolved `{ references: [ …the page hit… ] }` — one real // reference presented as the complete dependency list, which an admin // reads as "safe to delete". @@ -286,7 +303,7 @@ describe('[#8896] findReferencesToMeta — a source type that could not be READ expect(result.references).toEqual([]); }); - it('a target type absent from REFERENCE_PATHS still returns an empty list without reading anything', async () => { + it('a target type with no derived reference site still returns an empty list without reading anything', async () => { const { engine, typeReads } = engineWhereTypeFails(null); const protocol = new ObjectStackProtocolImplementation(engine as never); @@ -300,15 +317,21 @@ describe('[#8896] findReferencesToMeta — a source type that could not be READ // The benign discrimination lives in `getMetaItems`, one layer down — // this seam inherits it rather than repeating it, and this pin is what // proves the inheritance still holds through the removed `catch`. - const { engine, typeReads } = engineWhereTypeFails('dashboard', tableNotProvisioned('sys_metadata')); + const { engine, typeReads } = engineWhereTypeFails('app', tableNotProvisioned('sys_metadata')); const protocol = new ObjectStackProtocolImplementation(engine as never); const result = await protocol.findReferencesToMeta({ type: 'view', name: 'my_view' }); expect(result.references).toEqual([ - { type: 'page', name: 'home_page', label: 'Home', path: 'viewName', kind: 'page' }, + { + type: 'page', + name: 'home_page', + label: 'Home', + path: 'slots.header.dataSource.view', + kind: 'page view', + }, ]); // Proof the benign branch was actually EXERCISED. - expect(typeReads).toContain('dashboard'); + expect(typeReads).toContain('app'); }); }); diff --git a/packages/metadata-protocol/src/protocol.read-verb-canonical-fold.test.ts b/packages/metadata-protocol/src/protocol.read-verb-canonical-fold.test.ts index b0cce428df..ecb230a52b 100644 --- a/packages/metadata-protocol/src/protocol.read-verb-canonical-fold.test.ts +++ b/packages/metadata-protocol/src/protocol.read-verb-canonical-fold.test.ts @@ -122,6 +122,12 @@ function makeStubEngine() { getPackage: () => undefined, registerItem: () => {}, registerObject: () => {}, + // [#9190] `getMetaItems({ type: 'app' })` decorates each app with + // its contributed nav groups. The double gained this the day a + // reference scan first read `app` — identity is the right stub, + // because the fold under test is about the TYPE KEY a read used, + // not about nav contribution. + applyNavContributions: (app: unknown) => app, }, }; return { engine, tables, reads, items }; @@ -434,37 +440,49 @@ describe('#9157 — findReferencesToMeta', () => { }); it('CONTROL: a manifest-PRESENT plural still resolves its real dependents', async () => { + // [#9190] The fixture moved from `dashboard.widgets[].view` to + // `app.navigation[].viewName`, and the move is the point rather than a + // detail: `DashboardSchema` declares no `view` property anywhere, so + // the old fixture agreed with the old hand-curated path table and BOTH + // described a document the platform cannot store. `AppSchema.navigation` + // is real, and the site the walk uses for it is derived from that + // schema. const { engine, items } = makeStubEngine(); - items.dashboard = [{ name: 'sales_dash', label: 'Sales', widgets: [{ id: 'w1', view: 'all_leads' }] }]; + items.app = [{ name: 'sales_app', label: 'Sales', navigation: [{ viewName: 'all_leads' }] }]; const p = new ObjectStackProtocolImplementation(engine); const res = await p.findReferencesToMeta({ type: PRESENT_PLURAL, name: 'all_leads' }); expect(res.references).toEqual([ - { type: 'dashboard', name: 'sales_dash', label: 'Sales', path: 'widgets[].view', kind: 'dashboard widget' }, + { type: 'app', name: 'sales_app', label: 'Sales', path: 'navigation[].viewName', kind: 'app viewName' }, ]); }); - it('the manifest-ABSENT class is NOT closed here, and that is stated rather than implied', async () => { - // ⚠️ Honest scope pin. The card's `translations` example claims this verb - // answers `{ references: [] }` for a manifest-absent type — true, and the - // fold does not change it: every `REFERENCE_PATHS` key (`object`, `view`, - // `tool`, `skill`, `flow`, `dashboard`, `page`) is manifest-PRESENT, so - // `translation` has no registry entry either. This method's own doc calls - // an unregistered target a legitimate no-hit rather than an error. + it('[#9190] the manifest-ABSENT residue #9157 pinned here is CLOSED, and both spellings reach the same real hits', async () => { + // ⚠️ This pin has MOVED, deliberately. #9157 asserted that + // `translation` answers `{ references: [] }` whichever spelling you use + // — true then, because the hand-curated table had no `translation` key + // and this method's doc called that a legitimate no-hit. #9190 closed + // it the way the ruling required: by DERIVATION, not by adding a key. + // `DocSchema.translations` is a real, schema-declared reference site, so + // the walk finds it without anyone having written `translation` down. // - // Closing it is a `REFERENCE_PATHS` COVERAGE question, not a spelling - // one, and it is a different card. Asserted so a reader cannot over-read - // this PR's claim, and so the day `translation` gains a matcher this - // test goes red and asks to be re-read. - const { engine } = makeStubEngine(); + // What #9157 owns is UNCHANGED and is what this test still proves: the + // two spellings fold to one answer. What changed is that the answer is + // no longer vacuously empty, so the test can prove the fold on a + // non-trivial result — which is a stronger assertion than the empty one + // it replaces. + const { engine, items } = makeStubEngine(); + items.doc = [{ name: 'intro', label: 'Intro', translations: { greeting: { title: 'Hallo' } } }]; const p = new ObjectStackProtocolImplementation(engine); const viaPlural = await p.findReferencesToMeta({ type: OVERLAY_ABSENT_PLURAL, name: 'greeting' }); const viaCanonical = await p.findReferencesToMeta({ type: OVERLAY_ABSENT_TYPE, name: 'greeting' }); - expect(viaPlural.references).toEqual([]); - expect(viaCanonical.references).toEqual([]); + expect(viaCanonical.references).toEqual([ + { type: 'doc', name: 'intro', label: 'Intro', path: 'translations{key}', kind: 'doc translations' }, + ]); + expect(viaPlural.references).toEqual(viaCanonical.references); }); it('CONTROL: a spelling that reaches for no declared type is served, not refused', async () => { diff --git a/packages/metadata-protocol/src/protocol.reference-derivation.test.ts b/packages/metadata-protocol/src/protocol.reference-derivation.test.ts new file mode 100644 index 0000000000..f818c112ad --- /dev/null +++ b/packages/metadata-protocol/src/protocol.reference-derivation.test.ts @@ -0,0 +1,196 @@ +// Copyright (c) 2025 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#9190] `findReferencesToMeta` end-to-end, over the DERIVED site index. + * + * The unit pins next door prove the index is derived. These prove the thing an + * operator actually experiences: target types that answered "nothing depends on + * this item — safe to delete" on every deployment, whatever was stored, now + * answer with the dependents that exist. + * + * The empty state is not a metaphor. `objectui`'s metadata-admin renders it + * verbatim as *"Nothing in the metadata graph points at this item. Safe to + * delete."*, immediately before the rename or delete the panel exists to gate + * (ADR-0110 D3, the #8896 harm shape). + */ + +import { describe, expect, it } from 'vitest'; +import { ObjectStackProtocolImplementation } from './protocol.js'; + +/** + * Registry-backed stub. Reference scanning reads through `getMetaItems`, which + * falls back to the registry when `sys_metadata` holds nothing — so seeding the + * registry is enough, and nothing here needs a store. + */ +function protocolWith(items: Record>>) { + const engine: any = { + async find() { return []; }, + async findOne() { return null; }, + async count() { return 0; }, + registry: { + listItems: (type: string) => items[type] ?? [], + getItem: () => undefined, + getObject: () => undefined, + isPackageDisabled: () => false, + getPackage: () => undefined, + registerItem: () => {}, + registerObject: () => {}, + applyNavContributions: (app: unknown) => app, + }, + }; + return new ObjectStackProtocolImplementation(engine as never); +} + +describe('[#9190] target types that were silent by construction now answer', () => { + it('a permission set that grants on an object is found — through the record KEY', async () => { + // `PermissionSetSchema.objects` is `z.record(objectName, …)`. The + // curated table spelled it `objects[].name`, an array shape the schema + // has never had, so `GET /meta/object/account/references` never once + // mentioned a permission set. Deleting `account` looked free. + const protocol = protocolWith({ + permission: [{ + name: 'sales_admin', + label: 'Sales Admin', + objects: { account: { allowRead: true }, contact: { allowRead: true } }, + }], + }); + + const result = await protocol.findReferencesToMeta({ type: 'object', name: 'account' }); + + expect(result.references).toEqual([ + { + type: 'permission', + name: 'sales_admin', + label: 'Sales Admin', + path: 'objects{key}', + kind: 'permission objects', + }, + ]); + }); + + it("the card's own example — a `translation` target — resolves its doc", async () => { + // #9157 pinned this answering `[]` and named it a COVERAGE question + // rather than a spelling one. This is that question, closed: nobody + // wrote `translation` into a table; `DocSchema.translations` is a + // declared property and the walk reads it. + const protocol = protocolWith({ + doc: [{ name: 'getting_started', label: 'Getting Started', translations: { greeting: { title: 'Hallo' } } }], + }); + + const result = await protocol.findReferencesToMeta({ type: 'translation', name: 'greeting' }); + + expect(result.references).toEqual([ + { + type: 'doc', + name: 'getting_started', + label: 'Getting Started', + path: 'translations{key}', + kind: 'doc translations', + }, + ]); + }); + + it('a `dataset` target — never a curated key at all — resolves the widgets that chart it', async () => { + const protocol = protocolWith({ + dashboard: [{ name: 'revenue', label: 'Revenue', widgets: [{ id: 'w1', dataset: 'orders_by_month' }] }], + report: [{ name: 'q3', dataset: 'orders_by_month' }], + }); + + const result = await protocol.findReferencesToMeta({ type: 'dataset', name: 'orders_by_month' }); + + expect(result.references.map((r) => `${r.type}:${r.name}:${r.path}`)).toEqual([ + 'dashboard:revenue:widgets[].dataset', + 'report:q3:dataset', + ]); + }); + + it('a reference nested inside a RECURSIVE container is found at any depth', async () => { + // The reason the unit of derivation is a property and not a path. + // `AppSchema.navigation[].children[]` is self-recursive, so no finite + // path list can cover it — the curated table stopped at the top level + // even for the paths it spelled correctly, and a view referenced from a + // nested nav group was invisible. + const protocol = protocolWith({ + app: [{ + name: 'crm', + label: 'CRM', + navigation: [{ label: 'Sales', children: [{ label: 'Pipeline', children: [{ viewName: 'deal_board' }] }] }], + }], + }); + + const result = await protocol.findReferencesToMeta({ type: 'view', name: 'deal_board' }); + + expect(result.references).toEqual([ + { + type: 'app', + name: 'crm', + label: 'CRM', + path: 'navigation[].children[].children[].viewName', + kind: 'app viewName', + }, + ]); + }); + + it('the highest-value edge survives the rewrite: an object whose field points at another object', async () => { + // `FieldSchema.reference` — carried by `SEMANTIC_REFERENCE_SITES` + // because its NAME does not spell its target. One of only six curated + // paths that were live, and the one an admin most needs before a + // delete. ⚠️ The curated table ALSO spelled it `fields{}.referenceTo`, + // which `FieldSchema` does not declare; that limb is gone rather than + // tolerated, because a consumer that accepts both spellings is how the + // wrong one survives (contract-first). + const protocol = protocolWith({ + object: [{ + name: 'task', + label: 'Task', + fields: { account_id: { name: 'account_id', type: 'lookup', reference: 'account' } }, + }], + }); + + const result = await protocol.findReferencesToMeta({ type: 'object', name: 'account' }); + + expect(result.references).toEqual([ + { + type: 'object', + name: 'task', + label: 'Task', + path: 'fields.account_id.reference', + kind: 'object reference', + }, + ]); + }); +}); + +describe('[#9190] the widened scan does not start inventing dependents', () => { + it('an item nothing points at still answers with an empty list', async () => { + const protocol = protocolWith({ + view: [{ name: 'lead_list', object: 'lead' }], + permission: [{ name: 'sales_admin', objects: { lead: { allowRead: true } } }], + }); + + const result = await protocol.findReferencesToMeta({ type: 'object', name: 'orphan' }); + + expect(result.references).toEqual([]); + }); + + it('a value that merely LOOKS like the target name under a non-reference key is ignored', async () => { + // The scan reads named properties, not every string in the document — + // over-reporting is the safe direction here, but it is not a licence to + // report a description that happens to contain the word. + const protocol = protocolWith({ + view: [{ name: 'lead_list', label: 'account', description: 'account', object: 'lead' }], + }); + + const result = await protocol.findReferencesToMeta({ type: 'object', name: 'account' }); + + expect(result.references).toEqual([]); + }); + + it('an item is not listed as its own dependent through a scalar self-reference', async () => { + const protocol = protocolWith({ view: [{ name: 'account_list', object: 'account' }] }); + + const result = await protocol.findReferencesToMeta({ type: 'view', name: 'account_list' }); + + expect(result.references).toEqual([]); + }); +}); diff --git a/packages/metadata-protocol/src/protocol.ts b/packages/metadata-protocol/src/protocol.ts index 6617594ed9..720d4b1d44 100644 --- a/packages/metadata-protocol/src/protocol.ts +++ b/packages/metadata-protocol/src/protocol.ts @@ -77,6 +77,7 @@ import { } from '@objectstack/spec/kernel'; import { validateObjectNamespacePrefix, deriveNamespaceFromPackageId } from '@objectstack/spec/kernel'; import { stripReadDecorations } from '@objectstack/spec/kernel'; +import { REFERENCE_SITES } from './reference-sites.js'; // [#5488] The `@objectstack/spec/api` import that stood here — `ApiEndpointSchema`, // `validateApiEndpointDeclarations`, `type ApiEndpoint` — went with // `gateApiDraftsForPublish` (see its retirement note in `publishPackageDrafts`). @@ -2952,121 +2953,105 @@ const SERVICE_CONFIG: Record> = { - object: [ - { fromType: 'view', paths: ['object', 'objectName'], kind: 'view' }, - { fromType: 'dashboard', paths: ['widgets[].object', 'widgets[].objectName'], kind: 'dashboard widget' }, - { fromType: 'flow', paths: ['object', 'context.object', 'trigger.object', 'targetObject'], kind: 'flow' }, - // fromType 'workflow' removed (#4451): no such metadata type is - // registered, so the row scanned nothing. - { fromType: 'permission', paths: ['objects[].name', 'objects[].object'], kind: 'permission' }, - { fromType: 'app', paths: ['navItems[].objectName', 'navItems[].object', 'tabs[].objectName', 'tabs[].object'], kind: 'app nav' }, - { fromType: 'page', paths: ['object', 'objectName'], kind: 'page' }, - { fromType: 'report', paths: ['object', 'objectName'], kind: 'report' }, - { fromType: 'action', paths: ['object', 'objectName'], kind: 'action' }, - // fromType 'validation' removed (#4509): it scanned `object`/`objectName` - // on a schema that has neither — every variant is strict, so parse would - // have stripped such a key anyway — and the kind is now retired - // (ADR-0088). Rules travel with their object, so a rule's dependency on - // that object needs no row: deleting the object takes them with it. - { fromType: 'hook', paths: ['object', 'objectName'], kind: 'hook' }, - { fromType: 'object', paths: ['fields[].referenceTo', 'fields{}.referenceTo', 'fields{}.reference'], kind: 'field reference' }, - ], - view: [ - { fromType: 'dashboard', paths: ['widgets[].view', 'widgets[].viewName'], kind: 'dashboard widget' }, - { fromType: 'app', paths: ['navItems[].viewName', 'tabs[].viewName'], kind: 'app nav' }, - { fromType: 'page', paths: ['viewName'], kind: 'page' }, - ], - tool: [ - { fromType: 'agent', paths: ['tools[]', 'tools[].name'], kind: 'agent tool' }, - ], - skill: [ - { fromType: 'agent', paths: ['skills[]', 'skills[].name'], kind: 'agent skill' }, - ], - flow: [ - { fromType: 'app', paths: ['navItems[].flowName', 'tabs[].flowName'], kind: 'app nav' }, - ], - dashboard: [ - { fromType: 'app', paths: ['navItems[].dashboardName', 'tabs[].dashboardName'], kind: 'app nav' }, - ], - page: [ - { fromType: 'app', paths: ['navItems[].pageName', 'tabs[].pageName'], kind: 'app nav' }, - ], -}; /** - * Extract one or more string values from `item` at `path`. Supports - * `'a.b'` (nested object access) and `'a[].b'` (array element access). - * Returns an empty array if any segment is missing. + * Every name a reference-bearing property value may be carrying, tolerant of + * the four shapes a name can arrive in. + * + * Deliberately shape-TOLERANT at read time while the derivation is shape-STRICT + * at build time: the derivation decides whether a property means "a reference" + * (and rejects `enum`-constrained lookalikes such as a chart axis `position`), + * while this function only has to read a value whose meaning is already + * settled. Record KEYS are included because `z.record(name, …)` is how the + * platform spells a name-keyed collection — `PermissionSetSchema.objects` is + * exactly that, and the old table's `objects[].name` could not express it. */ -function extractPathValues(item: unknown, path: string): string[] { - if (!item || typeof item !== 'object') return []; - const segments = path.split('.'); - let current: unknown[] = [item]; - for (const rawSeg of segments) { - let kind: 'value' | 'array' | 'record' = 'value'; - let seg = rawSeg; - if (seg.endsWith('[]')) { - kind = 'array'; - seg = seg.slice(0, -2); - } else if (seg.endsWith('{}')) { - kind = 'record'; - seg = seg.slice(0, -2); - } - const next: unknown[] = []; - for (const node of current) { - if (!node || typeof node !== 'object') continue; - let value: unknown; - if (seg === '') { - value = node; - } else { - value = (node as Record)[seg]; - } - if (value === undefined || value === null) continue; - if (kind === 'array') { - if (Array.isArray(value)) { - for (const v of value) next.push(v); - } - } else if (kind === 'record') { - if (Array.isArray(value)) { - for (const v of value) next.push(v); - } else if (typeof value === 'object') { - for (const v of Object.values(value as Record)) next.push(v); - } - } else { - next.push(value); +function referencedNamesAt(value: unknown): Array<{ name: string; suffix: string }> { + if (typeof value === 'string') { + return value.length > 0 ? [{ name: value, suffix: '' }] : []; + } + if (Array.isArray(value)) { + const out: Array<{ name: string; suffix: string }> = []; + for (const element of value) { + if (typeof element === 'string' && element.length > 0) out.push({ name: element, suffix: '[]' }); + else if (element && typeof element === 'object') { + const named = (element as Record).name; + if (typeof named === 'string' && named.length > 0) out.push({ name: named, suffix: '[].name' }); } } - current = next; - if (current.length === 0) return []; + return out; } - // Coerce final values to strings, dropping non-string non-object leaves. - const out: string[] = []; - for (const v of current) { - if (typeof v === 'string' && v.length > 0) out.push(v); - else if (v && typeof v === 'object' && 'name' in (v as any) && typeof (v as any).name === 'string') { - out.push((v as any).name); + if (value && typeof value === 'object') { + const out: Array<{ name: string; suffix: string }> = []; + const record = value as Record; + for (const key of Object.keys(record)) { + if (key.length > 0) out.push({ name: key, suffix: '{key}' }); } + const named = record.name; + if (typeof named === 'string' && named.length > 0) out.push({ name: named, suffix: '.name' }); + return out; } - return out; + return []; +} + +/** + * Walk a stored metadata document and report every place it names + * `targetName` through one of `properties`. + * + * `properties` is the derived property → target binding for THIS source type, + * so the walk never guesses; it only looks where the schema said a name lives. + * Paths are reported with array indices and record keys elided (`a.b[].c`, + * `a{}.b`), matching the grammar the previous table used, so the value the + * "Used by" panel renders keeps its established shape. + */ +function collectReferenceHits( + item: unknown, + properties: ReadonlyMap, + targetName: string, +): Array<{ path: string; property: string }> { + const hits: Array<{ path: string; property: string }> = []; + const seenNodes = new Set(); + + const walk = (node: unknown, trail: string, depth: number): void => { + if (!node || typeof node !== 'object' || depth > 16) return; + // Cycle guard. Stored documents are JSON, but a runtime-assembled item + // can still be self-referential, and a scan that hangs is a worse + // answer than a short one. + if (seenNodes.has(node)) return; + seenNodes.add(node); + + if (Array.isArray(node)) { + for (const element of node) walk(element, `${trail}[]`, depth + 1); + return; + } + for (const [key, value] of Object.entries(node as Record)) { + const here = trail ? `${trail}.${key}` : key; + if (properties.has(key)) { + for (const hit of referencedNamesAt(value)) { + if (hit.name === targetName) hits.push({ path: `${here}${hit.suffix}`, property: key }); + } + } + walk(value, here, depth + 1); + } + }; + + walk(item, '', 0); + return hits; } /** @@ -17855,10 +17840,23 @@ export class ObjectStackProtocolImplementation implements * "Used by" panel before destructive actions (rename / delete / * type-narrowing). * - * Coverage is driven by the hand-curated {@link REFERENCE_PATHS} - * registry. A target type not present in the registry, and a SOURCE type - * this deployment does not declare, both simply produce no hits — neither - * is an error. + * [#9190] Coverage is DERIVED from the metadata type schemas + * ({@link REFERENCE_SITES}), not curated. The table that used to drive this + * had drifted until 34 of its 40 paths named properties no schema declares + * and five of its seven target keys answered `{ references: [] }` + * unconditionally — while this method's own doc called that a legitimate + * no-hit. It is no longer legitimate and it is no longer what happens: an + * empty answer now means the walk read every declared source type's shape + * and found nothing that can name this target. A SOURCE type this + * deployment does not declare still simply produces no hits, which is a + * fact about the deployment rather than about the platform. + * + * ⚠️ What an empty answer still cannot promise is bounded and recorded + * rather than implied: {@link REFERENCE_SITES.unwalkableSourceTypes} names + * every declared type whose shape could not be read (pinned by + * `reference-sites.derivation.test.ts`, so it cannot grow silently), and + * `SEMANTIC_REFERENCE_SITES` carries the properties whose name does not + * spell their target. Both are argued in `reference-sites.ts`. * * [#8896] A source type that could not be READ is a different fact and is * no longer answered the same way. This list is what an admin consults @@ -17889,33 +17887,47 @@ export class ObjectStackProtocolImplementation implements // than on the two verbs above, and the difference is stated rather than // implied because a future reader will otherwise assume symmetry: // - // • The REFUSAL is the whole wire-visible change. `viewes` used to miss - // `REFERENCE_PATHS` and answer 200 `{ references: [] }` — read by an + // • The REFUSAL is the whole wire-visible change #9157 made. `viewes` + // used to miss the lookup and answer 200 `{ references: [] }` — read by an // operator as "nothing depends on this", immediately before the // rename or delete this panel exists to gate (ADR-0110 D3). It is now // 400, naming `view` / `views`. - // • The manifest-ABSENT class changes NOTHING here, measured rather - // than assumed: every {@link REFERENCE_PATHS} key (`object`, `view`, - // `tool`, `skill`, `flow`, `dashboard`, `page`) is manifest-PRESENT, - // so the old map already folded each of them. `translations` folded - // to `translation` answers `{ references: [] }` exactly as - // `translations` did, because `translation` is not a registry key - // either — which this method's own doc calls a legitimate no-hit - // rather than an error. Closing THAT gap is a `REFERENCE_PATHS` - // coverage question and not a spelling one; it is not this card. + // • The manifest-ABSENT class changed NOTHING here at the time, and + // #9157 pinned that residue rather than implying it: every curated + // key was manifest-PRESENT, so `translations` folded to + // `translation` answered `{ references: [] }` exactly as + // `translations` had. [#9190] That residue is now CLOSED by + // derivation rather than by folding — `translation` has real + // reference sites (`doc.translations`, `book.groups[].translations`) + // because the walk reads the schemas that declare them. The spelling + // contract below is untouched; what changed is what the lookup finds. // - // ⛔ Do not "improve" this by teaching `REFERENCE_PATHS` a plural key. + // ⛔ Do not "improve" this by teaching the site index a plural key. // That is the spelling-tolerant lookup one layer down which // {@link canonicalMetaType} has rejected since #4432. request = canonicalizeMetaRequestType(request); // Canonical by construction from the fold above — NOT a second fold. const singularTarget = request.type; const targetName = request.name; - const matchers = REFERENCE_PATHS[singularTarget]; - if (!matchers || matchers.length === 0) { + const sites = REFERENCE_SITES.byTarget.get(singularTarget); + if (!sites || sites.length === 0) { return { references: [] }; } + // One read per source type, not one per site: several properties of the + // same source type routinely name the same target, and the previous + // shape re-read the store once per matcher row. + const propertiesByFromType = new Map>(); + for (const site of sites) { + let properties = propertiesByFromType.get(site.fromType); + if (!properties) { + properties = new Map(); + propertiesByFromType.set(site.fromType, properties); + } + properties.set(site.property, site.target); + } + const matchers = [...propertiesByFromType].map(([fromType, properties]) => ({ fromType, properties })); + const seen = new Set(); // dedup key: `${fromType}|${itemName}|${path}` const out: Array<{ type: string; name: string; label?: string; path: string; kind: string }> = []; @@ -17962,20 +17974,27 @@ export class ObjectStackProtocolImplementation implements // Don't list an item as a reference to itself unless the // self-reference is meaningful (e.g. object→field path). const isSelfReference = matcher.fromType === singularTarget && sourceName === targetName; - for (const path of matcher.paths) { - const values = extractPathValues(raw, path); - if (!values.includes(targetName)) continue; + for (const { path, property } of collectReferenceHits(raw, matcher.properties, targetName)) { + // An item naming ITSELF through a scalar property is + // the item, not a dependent. Collection-valued paths + // are kept, because an item that lists itself among + // many (an object's own field pointing back at it) is + // a real edge an admin needs to see before a delete. if (isSelfReference && !path.includes('[]') && !path.includes('{}')) continue; const key = `${matcher.fromType}|${sourceName}|${path}`; if (seen.has(key)) continue; seen.add(key); const label = (raw as any).label as string | undefined; + // `kind` is a human label for the edge, derived from the + // property that produced it rather than hand-written per + // row — a curated label is the same maintenance debt as + // a curated path, one field over. out.push({ type: matcher.fromType, name: sourceName, ...(label ? { label } : {}), path, - kind: matcher.kind, + kind: `${matcher.fromType} ${property}`, }); } } diff --git a/packages/metadata-protocol/src/reference-sites.derivation.test.ts b/packages/metadata-protocol/src/reference-sites.derivation.test.ts new file mode 100644 index 0000000000..45a7d81286 --- /dev/null +++ b/packages/metadata-protocol/src/reference-sites.derivation.test.ts @@ -0,0 +1,156 @@ +// Copyright (c) 2025 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#9190] The reference-site index is DERIVED — these are the pins that keep it + * derived, and the pins that make its remaining gaps loud. + * + * Three jobs, and they are different: + * + * 1. **Regression** — every target type the old hand-curated table CLAIMED + * must resolve at least one live site. Five of its seven claimed keys + * resolved none, which is the defect this card closed. + * 2. **Anti-guesser** — the naming rule must stay the small total rule it is. + * A suffix rule was measured and rejected; these pins are what a future + * "let's also match `endsWith(Cap(T))`" change trips over. + * 3. **Honest gaps** — `unwalkableSourceTypes` is where "not computable" + * lives now that it no longer hides inside `{ references: [] }`. It is + * pinned exactly, so a type that stops being walkable turns this test red + * instead of silently shortening a "Used by" panel. + */ + +import { describe, expect, it } from 'vitest'; +import { DEFAULT_METADATA_TYPE_REGISTRY } from '@objectstack/spec/kernel'; +import { REFERENCE_SITES, deriveReferenceSites } from './reference-sites.js'; + +/** The seven keys `REFERENCE_PATHS` carried on `origin/main` before this card. */ +const PREVIOUSLY_CLAIMED_TARGETS = ['object', 'view', 'tool', 'skill', 'flow', 'dashboard', 'page'] as const; + +/** Site lookup that reads as the sentence it is asserting. */ +function sitesFor(target: string): Array<{ fromType: string; property: string }> { + return (REFERENCE_SITES.byTarget.get(target) ?? []).map((s) => ({ fromType: s.fromType, property: s.property })); +} + +function hasSite(target: string, fromType: string, property: string): boolean { + return sitesFor(target).some((s) => s.fromType === fromType && s.property === property); +} + +describe('[#9190] derived reference sites — regression: every target the old table claimed now resolves', () => { + it.each(PREVIOUSLY_CLAIMED_TARGETS)('%s has at least one derived site', (target) => { + expect(sitesFor(target).length).toBeGreaterThan(0); + }); + + it('the five targets whose every curated path was DEAD now resolve real ones', () => { + // Measured on `origin/main` @ `739fe5b79`: `view`, `tool`, `flow`, + // `dashboard` and `page` were registry keys whose only rows named + // properties no schema declares — `app.navItems[]` / `app.tabs[]` + // (`AppSchema` declares `navigation` / `areas`), `agent.tools[]` + // (removed in spec 17, #3894), `dashboard.widgets[].view`. Each + // answered `{ references: [] }` unconditionally while LOOKING covered, + // which is the exact shape the "Used by" panel reads as "safe to + // delete". + expect(hasSite('view', 'app', 'viewName')).toBe(true); + expect(hasSite('view', 'page', 'view')).toBe(true); + expect(hasSite('tool', 'skill', 'tools')).toBe(true); + expect(hasSite('dashboard', 'app', 'dashboardName')).toBe(true); + expect(hasSite('page', 'app', 'pageName')).toBe(true); + // `flow` is reachable only through a flow NODE config, which + // `FlowSchema` declares as `additionalProperties: {}` — limb 2 of the + // derivation exists for exactly this, and this is its proof. + expect(hasSite('flow', 'flow', 'flowName')).toBe(true); + }); + + it('the curated table is beaten on coverage, not merely matched', () => { + // The old table claimed 7 targets and served 2. Anything at or below + // its CLAIM would mean derivation bought nothing. + expect(REFERENCE_SITES.byTarget.size).toBeGreaterThan(PREVIOUSLY_CLAIMED_TARGETS.length); + }); + + it('a newly DECLARED type cannot arrive uncovered — the index is keyed off the declared universe', () => { + // The property that makes the defect non-recurring (#7894's shape): the + // walk enumerates `DEFAULT_METADATA_TYPE_REGISTRY`, so every declared + // type is either walked or named in `unwalkableSourceTypes`. There is + // no third state, and no list anyone has to remember to extend. + const declared = DEFAULT_METADATA_TYPE_REGISTRY.map((e) => e.type); + const walked = declared.filter((t) => !REFERENCE_SITES.unwalkableSourceTypes.includes(t)); + expect(walked.length + REFERENCE_SITES.unwalkableSourceTypes.length).toBe(declared.length); + }); +}); + +describe('[#9190] derived reference sites — the naming rule stays total and unclever', () => { + it('a name that merely ENDS WITH a type name is NOT a reference to it', () => { + // ⛔ The rejected rule, pinned so it cannot come back by accident. A + // suffix rule was measured against the real schemas and is ~15% signal: + // it reads `displayField`, `nameField`, `startDateField`, `stageField` + // and thirty siblings as references to the `field` METADATA TYPE, when + // every one of them names a field INSIDE an object. + expect(hasSite('field', 'object', 'displayField')).toBe(false); + expect(hasSite('field', 'object', 'nameField')).toBe(false); + expect(hasSite('mapping', 'mapping', 'fieldMapping')).toBe(false); + expect(hasSite('agent', 'app', 'defaultAgent')).toBe(false); + }); + + it('an ENUM-constrained value that shares a type name is NOT a reference', () => { + // `chartConfig.xAxis.position` is `'left' | 'right'` and + // `flow.nodes[].position` is `{ x, y }`. Neither names a `position` + // artifact, and the shape half of the rule is what keeps them out — + // while the real one stays in. + expect(hasSite('position', 'dashboard', 'position')).toBe(false); + expect(hasSite('position', 'flow', 'position')).toBe(false); + expect(hasSite('position', 'permission', 'positions')).toBe(true); + }); + + it('a name-keyed RECORD is a reference site, which the curated path grammar could not express', () => { + // `PermissionSetSchema.objects` is `z.record(objectName, …)`. The old + // table spelled it `objects[].name` — an ARRAY of `{ name }`, which the + // schema has never declared — so a permission set was invisible to + // every "what depends on this object?" question an admin asked. + expect(hasSite('object', 'permission', 'objects')).toBe(true); + }); +}); + +describe('[#9190] derived reference sites — the gaps are named, not hidden', () => { + it('THE PIN: exactly one declared type cannot be walked, and it is named', () => { + // This is where "not computable" lives now. `external_catalog` resolves + // no schema at all (runtime-created by the datasource Sync wizard, + // ADR-0062/0088), so nothing can be said about what it references — + // which is a different fact from "it references nothing". + // + // ⚠️ If this set GROWS, a source type stopped being readable and every + // "Used by" panel silently got shorter. That is the #8896 harm shape + // arriving through the back door, and it must be a red test rather than + // a quiet one. Do not "fix" a failure here by widening the expectation. + expect(REFERENCE_SITES.unwalkableSourceTypes).toEqual(['external_catalog']); + }); + + it('the un-derivable residue is exactly one property, so growing it is a conscious act', () => { + // `FieldSchema.reference` names an object in PROSE ("Target object name + // (snake_case) for lookup/master_detail fields") and nowhere a machine + // can read. It is carried because dropping it would regress the + // highest-value edge in the graph; it is pinned at ONE because a + // hand-written list is exactly what this card removed. Other members of + // the same class are measured and deliberately EXCLUDED + // (`AppSchema.homePageId` → page, `AppSchema.defaultAgent` → agent), so + // the incompleteness stays visible rather than looking handled. + // + // Closing this class properly is a producer-side annotation in + // `packages/spec`, not another row here. + expect(hasSite('object', 'object', 'reference')).toBe(true); + expect(hasSite('page', 'app', 'homePageId')).toBe(false); + }); +}); + +describe('[#9190] derived reference sites — derivation is a pure function of the schemas', () => { + it('two derivations of the same schemas agree exactly', () => { + // The module-load singleton must not be doing anything a caller cannot + // reproduce; a derived registry that depends on WHEN it ran is a + // curated one wearing a function. + const a = deriveReferenceSites(); + const b = deriveReferenceSites(); + const flatten = (index: ReturnType) => + [...index.byTarget.entries()] + .map(([target, sites]) => `${target}=${sites.map((s) => `${s.fromType}.${s.property}`).join(',')}`) + .sort(); + expect(flatten(a)).toEqual(flatten(b)); + expect(flatten(a)).toEqual(flatten(REFERENCE_SITES)); + }); +}); diff --git a/packages/metadata-protocol/src/reference-sites.ts b/packages/metadata-protocol/src/reference-sites.ts new file mode 100644 index 0000000000..7941a48c48 --- /dev/null +++ b/packages/metadata-protocol/src/reference-sites.ts @@ -0,0 +1,392 @@ +// Copyright (c) 2025 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * DERIVED reference-site index — what may point at a metadata item, worked out + * from the metadata type schemas instead of written down by hand (#9190). + * + * ## The defect this replaces + * + * `findReferencesToMeta` used to consult a hand-curated `REFERENCE_PATHS` + * table: seven target keys, forty `fromType` + dotted-path tuples, maintained + * by whoever last remembered to. The table drove the admin UI's "Used by" + * panel, whose empty state reads — verbatim, `objectui` + * `metadata-admin/i18n.ts` — *"Nothing in the metadata graph points at this + * item. Safe to delete."* So an incomplete table is not a cosmetic gap; it is + * a green light for a destructive action (ADR-0110 D3, the #8896 harm shape). + * + * It had drifted exactly as a hand-written list does. Measured against the + * schemas on `origin/main` @ `739fe5b79`, **34 of the 40 curated paths did not + * exist in the source type's own schema**: + * + * - `app.navItems[]` / `app.tabs[]` — `AppSchema` declares `navigation[]` and + * `areas[].navigation[]`. Fourteen paths across six target types, dead. As + * those were the ONLY rows for `flow`, `dashboard` and `page`, all three + * were registry keys that answered `{ references: [] }` unconditionally — + * indistinguishable from a key that was never there; + * - `agent.tools[]` — the key was REMOVED in `@objectstack/spec` 17 (#3894); + * it converts to `{ not: {} }`, i.e. a refusal. Tools reach an agent through + * its skills (ADR-0064), so `tool` had no live row either; + * - `permission.objects[].name` — `PermissionSetSchema.objects` is a RECORD + * keyed by object name, not an array of `{ name }`; + * - `object.fields{}.referenceTo` — `FieldSchema` spells it `reference`; + * - `dashboard.widgets[].object` / `.view`, `page.viewName`, `view.objectName`, + * `flow.object` / `.context.object` / `.trigger.object` / `.targetObject` — + * none of these properties exist. + * + * Only SIX of the forty were live, reaching two target types (`object`, + * `skill`). The endpoint advertised seven and answered for two. ⛔ Adding the + * missing keys by hand reproduces the defect one generation later — #8908 + * measured a hand-written list of "four types" that shipped two members short. + * + * ## What is derived, and from what + * + * The one authority on which metadata types exist is + * `DEFAULT_METADATA_TYPE_REGISTRY` (#8586: "the TOTAL universe of declared + * metadata types"), and the one authority on an item's shape is + * `getMetadataTypeSchema()`. This module joins them. Same shape #7894 used to + * make the URL-spelling map non-recurring: a newly DECLARED type arrives + * covered, because the walk reads its schema rather than a list someone has to + * remember to extend. + * + * A property is a reference SITE for target `T` when BOTH hold: + * + * 1. **Its name spells `T`** — `T`, its camelCase form, `T + 'Name'`, + * `T + '_name'`, `'target' + Cap(T)`, or a plural of any of those. This is + * the same total, deliberately unclever rule `restPluralOfMetaType` uses + * one package over; ⛔ do NOT make it cleverer. A suffix rule (`endsWith + * Cap(T)`) was measured and REJECTED: it is ~15% signal. It reads + * `displayField`, `nameField`, `startDateField`, `stageField` and thirty + * more as references to the `field` METADATA TYPE when every one of them + * names a field inside an object, plus `fieldMapping`/`inputMapping` as + * `mapping` references and `tabPosition` as a `position` reference. A + * boundary that guesses is what #7894 and #4432 both refuse. + * 2. **Its value is name-shaped** — an unconstrained `string`, an array or + * record of those, or objects carrying a `name`. The constraint half is + * load-bearing: `chartConfig.xAxis.position` is an `enum` of `'left' | + * 'right'` and `flow.nodes[].position` is `{ x, y }`, so neither is read as + * a reference to a `position` item, while + * `permission.rowLevelSecurity[].positions[]` is. + * + * Three limbs feed the walk: + * + * 1. every declared type's own schema; + * 2. `SCHEMALESS_NODE_CONFIG_SCHEMAS` attributed to `flow`. `FlowSchema` + * declares `nodes[].config` as `additionalProperties: {}` — wide open — so + * a flow's real references (`create_record` → object, `subflow` → flow) + * are invisible to limb 1 and reachable only through that separate + * registry; + * 3. {@link SEMANTIC_REFERENCE_SITES}, the residue — read its header. + * + * ## What "no references" now means, and what it still cannot mean + * + * After derivation an empty answer is a DERIVED statement for every declared + * type whose schema resolves: no declared source type carries a property that + * names this type. That is a real answer, not a gap, and it is what collapses + * the "no references" / "not computable" ambiguity the card names — without a + * wire change, because the honest discriminator moved OFF the response and + * INTO the build. {@link ReferenceSiteIndex.unwalkableSourceTypes} names every + * declared type whose shape could not be read, and + * `reference-sites.derivation.test.ts` pins that set, so a type that stops + * being walkable is a red test rather than a silently shorter list. + * + * ⚠️ Two residues remain open and are NOT closed here — see this module's + * card. Neither can be closed inside this package. + * + * @module + */ + +import { z } from 'zod'; +import { DEFAULT_METADATA_TYPE_REGISTRY, getMetadataTypeSchema } from '@objectstack/spec/kernel'; +import { SCHEMALESS_NODE_CONFIG_SCHEMAS } from '@objectstack/spec/automation'; + +/** + * One derived fact: an item of `fromType` may name a `target` item through a + * property called `property`, wherever that property occurs in the document. + * + * Deliberately a PROPERTY and not a PATH. The hand-written table enumerated + * fully-qualified paths, which is why it rotted: `AppSchema`'s navigation is + * recursive (`navigation[].children[].children[]…`), so an exhaustive path list + * is both unbounded and stale the moment a wrapper moves. The runtime walk + * finds the property wherever it sits and reports the path it actually found it + * at, so a nesting change cannot silently shorten the answer. + */ +export interface ReferenceSite { + /** Metadata type of the item that may hold the reference. */ + readonly fromType: string; + /** Property name that carries the target's name. */ + readonly property: string; + /** Metadata type being referenced. */ + readonly target: string; +} + +/** The derived index, plus the honest record of what could not be derived. */ +export interface ReferenceSiteIndex { + /** Target metadata type → every site that may point at it. */ + readonly byTarget: ReadonlyMap; + /** + * Declared types whose shape could NOT be read, so their references are + * genuinely not computable rather than absent. Pinned by a test — see the + * module header on why this lives here and not on the response. + */ + readonly unwalkableSourceTypes: readonly string[]; +} + +/** + * The RESIDUE: reference properties whose NAME does not spell their target, so + * no naming rule can derive them. + * + * ⚠️ This map is hand-written and therefore, by #8908's measurement, incomplete + * — that is a statement about the map, not an excuse for it. It exists for one + * reason: dropping it would REGRESS the single highest-value edge in the graph + * (which objects point at this object), which is one of the six curated paths + * that were actually live. `FieldSchema.reference` is described as *"Target + * object name (snake_case) for lookup/master_detail fields"* — the binding is + * real, and it is stated in prose, which is not a machine-readable surface. + * + * ⛔ Do NOT grow this map to "improve coverage". Growing it IS the defect this + * module exists to end. Other sites in the same class are already measured and + * deliberately left out rather than curated in — `AppSchema.homePageId` names a + * `page`, `AppSchema.defaultAgent` names an `agent` — precisely so that the + * incompleteness stays visible instead of looking handled. + * + * The durable close is an annotation at the PRODUCER, so the binding is + * declared where the property is declared and derivation reads it like any + * other schema fact — the shape `flow-node-expression-paths.ts` already uses + * for `.meta({ xExpression })`, and the "declared = enforced" side of ADR-0049. + * That is a `packages/spec` change and is out of this card's scope by ruling. + */ +const SEMANTIC_REFERENCE_SITES: readonly ReferenceSite[] = [ + { fromType: 'object', property: 'reference', target: 'object' }, +]; + +/** `external_catalog` → `externalCatalog`. Identity for a type with no underscore. */ +function camelCaseOf(type: string): string { + return type.replace(/_([a-z])/g, (_m, c: string) => c.toUpperCase()); +} + +function capitalize(s: string): string { + return s.charAt(0).toUpperCase() + s.slice(1); +} + +/** + * The ONE pluralization rule, kept byte-identical to `restPluralOfMetaType` in + * `@objectstack/spec/shared` on purpose: two spellings of "plural of a metadata + * type" that can disagree is the #4432 defect wearing a different key. + */ +function pluralOf(type: string): string { + if (/[^aeiou]y$/.test(type)) return `${type.slice(0, -1)}ies`; + if (/(s|x|z|ch|sh)$/.test(type)) return `${type}es`; + return `${type}s`; +} + +/** Property spellings that name a single item of `type`. */ +function singularSpellings(type: string): string[] { + const camel = camelCaseOf(type); + return [type, camel, `${type}_name`, `${camel}Name`, `target${capitalize(camel)}`]; +} + +/** Property spellings that name a COLLECTION of items of `type`. */ +function pluralSpellings(type: string): string[] { + const camel = camelCaseOf(type); + return [pluralOf(type), pluralOf(camel), `${camel}Names`, `${pluralOf(camel)}Names`]; +} + +type JsonSchemaNode = Record; + +/** + * Resolve `$ref`s and flatten `anyOf`/`oneOf`/`allOf` into the concrete + * alternatives a node may take. + * + * The `$ref` cycle guard is per-branch rather than global: a recursive schema + * (`AppSchema.navigation[].children[]`) must be enterable once on each branch + * it appears on, and must not loop. + */ +function alternatives( + node: unknown, + defs: Record, + seenRefs: ReadonlySet, + depth: number, +): JsonSchemaNode[] { + if (!node || typeof node !== 'object' || depth > 40) return []; + const n = node as JsonSchemaNode; + const ref = n.$ref; + if (typeof ref === 'string') { + const key = ref.replace('#/$defs/', ''); + if (seenRefs.has(key)) return []; + return alternatives(defs[key], defs, new Set([...seenRefs, key]), depth + 1); + } + const out: JsonSchemaNode[] = [n]; + for (const combinator of ['anyOf', 'oneOf', 'allOf'] as const) { + const members = n[combinator]; + if (Array.isArray(members)) { + for (const member of members) out.push(...alternatives(member, defs, seenRefs, depth + 1)); + } + } + return out; +} + +/** + * Is this node an unconstrained name string? + * + * `enum` / `const` / `format` all mean the value is drawn from a closed or + * typed vocabulary rather than naming an artifact — this is the half of the + * rule that keeps `xAxis.position: 'left' | 'right'` out of the `position` + * target's site list. + */ +function isNameString(node: JsonSchemaNode): boolean { + return node.type === 'string' && node.enum === undefined && node.const === undefined && node.format === undefined; +} + +/** Does this property's declared shape carry one or more item NAMES? */ +function carriesNames( + node: unknown, + defs: Record, + expectCollection: boolean, +): boolean { + for (const alt of alternatives(node, defs, new Set(), 0)) { + if (!expectCollection && isNameString(alt)) return true; + if (alt.type === 'array') { + for (const item of alternatives(alt.items, defs, new Set(), 0)) { + if (isNameString(item)) return true; + const props = item.properties as Record | undefined; + if (props?.name && alternatives(props.name, defs, new Set(), 0).some(isNameString)) return true; + } + } + // A record is name-keyed by construction (`z.record(name, …)`), so the + // KEYS are the reference — `PermissionSetSchema.objects` is exactly + // this, and the curated table's `objects[].name` could not express it. + if (alt.type === 'object' && alt.additionalProperties && typeof alt.additionalProperties === 'object') return true; + if (!expectCollection && alt.type === 'object') { + const props = alt.properties as Record | undefined; + if (props?.name && alternatives(props.name, defs, new Set(), 0).some(isNameString)) return true; + } + } + return false; +} + +/** + * Walk one source schema, recording every (property → target) binding it + * declares. Depth-bounded and branch-cycle-guarded; the bound is on the SCHEMA + * walk only, and cannot shorten a runtime answer, because what is collected is + * a property NAME rather than a path. + */ +function collectSitesFromSchema( + fromType: string, + root: JsonSchemaNode, + singular: ReadonlyMap, + plural: ReadonlyMap, + into: Map, +): void { + const defs = (root.$defs as Record | undefined) ?? {}; + const visited = new Set(); + + const walk = (node: unknown, trail: string, depth: number): void => { + if (depth > 12) return; + for (const alt of alternatives(node, defs, new Set(), 0)) { + const props = alt.properties as Record | undefined; + if (props) { + for (const [key, child] of Object.entries(props)) { + const singularTarget = singular.get(key); + const pluralTarget = singularTarget ? undefined : plural.get(key); + const target = singularTarget ?? pluralTarget; + if (target && carriesNames(child, defs, pluralTarget !== undefined)) { + const site: ReferenceSite = { fromType, property: key, target }; + into.set(`${fromType}|${key}|${target}`, site); + } + const branch = `${trail}.${key}`; + if (visited.has(branch)) continue; + visited.add(branch); + walk(child, branch, depth + 1); + } + } + if (alt.items) walk(alt.items, `${trail}[]`, depth + 1); + if (alt.additionalProperties && typeof alt.additionalProperties === 'object') { + walk(alt.additionalProperties, `${trail}{}`, depth + 1); + } + } + }; + + walk(root, '', 0); +} + +/** + * Build the reference-site index from the metadata type schemas. + * + * Pure and side-effect free; {@link REFERENCE_SITES} memoizes one call of it at + * module load. Exported so the derivation can be tested against the real + * schemas rather than against a snapshot of itself. + */ +export function deriveReferenceSites(): ReferenceSiteIndex { + const declaredTypes = DEFAULT_METADATA_TYPE_REGISTRY.map((entry) => entry.type); + + // Property spelling → target type. First writer wins, so a type never + // steals a spelling another type already owns. + const singular = new Map(); + const plural = new Map(); + for (const type of declaredTypes) { + for (const spelling of singularSpellings(type)) if (!singular.has(spelling)) singular.set(spelling, type); + for (const spelling of pluralSpellings(type)) if (!plural.has(spelling)) plural.set(spelling, type); + } + + const sites = new Map(); + const unwalkable: string[] = []; + + // Limb 1 — every declared type's own schema. + for (const type of declaredTypes) { + const schema = getMetadataTypeSchema(type); + if (!schema) { + unwalkable.push(type); + continue; + } + let json: JsonSchemaNode; + try { + json = z.toJSONSchema(schema, { unrepresentable: 'any', io: 'input' }) as JsonSchemaNode; + } catch { + unwalkable.push(type); + continue; + } + collectSitesFromSchema(type, json, singular, plural, sites); + } + + // Limb 2 — flow node configs. See the module header: `FlowSchema` declares + // `nodes[].config` as `additionalProperties: {}`, so limb 1 sees nothing + // inside it and a flow's real references live only in this registry. + for (const schema of Object.values(SCHEMALESS_NODE_CONFIG_SCHEMAS)) { + try { + const json = z.toJSONSchema(schema as z.ZodType, { unrepresentable: 'any', io: 'input' }) as JsonSchemaNode; + collectSitesFromSchema('flow', json, singular, plural, sites); + } catch { + // A node config that cannot be converted contributes no sites. It + // is not a declared metadata type, so it does not belong in + // `unwalkableSourceTypes` — that set is about TYPES, and `flow` + // itself remains walkable through limb 1. + } + } + + // Limb 3 — the residue. See {@link SEMANTIC_REFERENCE_SITES}. + for (const site of SEMANTIC_REFERENCE_SITES) { + sites.set(`${site.fromType}|${site.property}|${site.target}`, site); + } + + const byTarget = new Map(); + for (const site of sites.values()) { + const bucket = byTarget.get(site.target); + if (bucket) bucket.push(site); + else byTarget.set(site.target, [site]); + } + // Stable order so the derived index reads the same on every boot. + for (const bucket of byTarget.values()) { + bucket.sort((a, b) => a.fromType.localeCompare(b.fromType) || a.property.localeCompare(b.property)); + } + + return { byTarget, unwalkableSourceTypes: unwalkable.sort() }; +} + +/** + * The derived index, computed once per process. + * + * Module-load derivation is what makes the answer impossible to forget to + * update: there is no list to edit, so a newly declared type is covered by the + * next boot rather than by the next person who notices. + */ +export const REFERENCE_SITES: ReferenceSiteIndex = deriveReferenceSites(); diff --git a/packages/objectql/src/protocol-references.test.ts b/packages/objectql/src/protocol-references.test.ts index 4c5bb4cd84..ac479536b9 100644 --- a/packages/objectql/src/protocol-references.test.ts +++ b/packages/objectql/src/protocol-references.test.ts @@ -7,8 +7,18 @@ import { SchemaRegistry } from './registry.js'; /** * Phase 3a-references tests. * - * Validates that findReferencesToMeta walks the hand-curated path registry - * across all loaded metadata and surfaces "what depends on this artifact". + * Validates that findReferencesToMeta walks all loaded metadata and surfaces + * "what depends on this artifact". + * + * [#9190] The registry it walks is now DERIVED from the metadata type schemas + * rather than hand-curated, and the fixtures below were re-spelled to match the + * schemas because they did not. `FieldSchema` spells the lookup target + * `reference`, not `referenceTo`; `PermissionSetSchema.objects` is a + * name-keyed RECORD, not an array of `{ name }`; `DashboardSchema` declares no + * `view` property at any depth. Each of those fixtures agreed with a curated + * path and therefore described a document the platform cannot store — the + * hand-written list and the hand-written fixture confirming each other is + * exactly how the drift stayed invisible. */ describe('ObjectStackProtocolImplementation - findReferencesToMeta', () => { let protocol: ObjectStackProtocolImplementation; @@ -26,29 +36,33 @@ describe('ObjectStackProtocolImplementation - findReferencesToMeta', () => { label: 'Task', fields: { name: { name: 'name', type: 'text' }, - account_id: { name: 'account_id', type: 'lookup', referenceTo: 'account' }, + account_id: { name: 'account_id', type: 'lookup', reference: 'account' }, }, } as any, 'pkg'); // Views pointing at account. registry.registerItem('view', { name: 'account_list', type: 'grid', object: 'account', label: 'Account List' }, 'name'); registry.registerItem('view', { name: 'task_list', type: 'grid', object: 'task' }, 'name'); - // Permission listing the account object. + // Permission granting on the account object — a name-keyed record, + // which is what `PermissionSetSchema.objects` actually is. registry.registerItem('permission', { name: 'sales_admin', label: 'Sales Admin', - objects: [{ name: 'account', allowRead: true }, { name: 'task', allowRead: true }], + objects: { account: { allowRead: true }, task: { allowRead: true } }, }, 'name'); - // Dashboard widget referencing the account view. - registry.registerItem('dashboard', { - name: 'sales_dash', - label: 'Sales Dashboard', - widgets: [{ id: 'w1', view: 'account_list' }], + // App navigation referencing the account view. (This used to be a + // dashboard widget, which `DashboardSchema` cannot carry.) + registry.registerItem('app', { + name: 'sales_app', + label: 'Sales App', + navigation: [{ label: 'Accounts', viewName: 'account_list' }], }, 'name'); - // Agent referencing a tool. + // Skill referencing a tool. (This used to be an agent, whose `tools` + // key was REMOVED in spec 17 — an agent reaches tools through its + // skills, ADR-0064.) registry.registerItem('tool', { name: 'crm_query', label: 'CRM Query' }, 'name'); - registry.registerItem('agent', { - name: 'sdr', - label: 'SDR Agent', + registry.registerItem('skill', { + name: 'lookup_account', + label: 'Look up account', tools: ['crm_query'], }, 'name'); @@ -72,23 +86,24 @@ describe('ObjectStackProtocolImplementation - findReferencesToMeta', () => { expect(byTypeName.has('view:account_list')).toBe(true); expect(byTypeName.has('object:task')).toBe(true); expect(byTypeName.has('permission:sales_admin')).toBe(true); - // Path is reported. + // Path is reported — as the place in the document the name was found. expect(byTypeName.get('view:account_list')!.path).toBe('object'); - expect(byTypeName.get('object:task')!.path).toBe('fields{}.referenceTo'); + expect(byTypeName.get('object:task')!.path).toBe('fields.account_id.reference'); + expect(byTypeName.get('permission:sales_admin')!.path).toBe('objects{key}'); }); - it('finds dashboards that reference a view', async () => { + it('finds apps that reference a view', async () => { const result = await protocol.findReferencesToMeta({ type: 'view', name: 'account_list' }); const names = result.references.map((r) => `${r.type}:${r.name}`); - expect(names).toContain('dashboard:sales_dash'); + expect(names).toContain('app:sales_app'); }); - it('finds agents that reference a tool', async () => { + it('finds skills that reference a tool', async () => { const result = await protocol.findReferencesToMeta({ type: 'tool', name: 'crm_query' }); - expect(result.references.some((r) => r.type === 'agent' && r.name === 'sdr')).toBe(true); + expect(result.references.some((r) => r.type === 'skill' && r.name === 'lookup_account')).toBe(true); }); - it('returns empty array for unknown target type', async () => { + it('returns empty array for a target type no declared schema can name', async () => { const result = await protocol.findReferencesToMeta({ type: 'unknown_kind', name: 'foo' }); expect(result.references).toEqual([]); });