diff --git a/.changeset/dotted-fields-prose-corrected.md b/.changeset/dotted-fields-prose-corrected.md new file mode 100644 index 0000000000..934de08bfc --- /dev/null +++ b/.changeset/dotted-fields-prose-corrected.md @@ -0,0 +1,40 @@ +--- +"@objectstack/spec": patch +--- + +**docs(spec): `fields` stops prescribing a dotted path no driver resolves (#7601)** + +Six in-repo surfaces offered `fields: ['owner.name']` as the supported way to read +one related column. No driver ever implemented it — measured on a real `SqlDriver`, +a dotted projection is byte-identical to no projection at all, because Knex renders +`"account"."name"` against a table that was never joined and the #3821 recovery +ladder retries `select('*')`. Since #7532 those surfaces are additionally +contradicted by a `400 INVALID_FIELD` at the ingress gate +(`assertProjectionFieldsExist`). The migration tooling was the sharpest case: both +protocol-17 upgrade prescriptions routed authors off `query.joins` and off the +retired `{ field, fields, alias }` form directly into the refused spelling. + +This aligns the declaration to the enforcement. The normative `fields` `.describe()` +now names `expand` as the sanctioned mechanism for related data — its nested +`QueryAST` both filters (`where`) and selects (`fields`) the related record's +columns — and carries the sharp edge that was pinned but never documented: **the +projection must retain the foreign-key column.** `fields: ['title']` with +`expand: 'project_id'` resolves nothing, because the relation is carried by that +key; adding `'project_id'` makes it work. Where the value is wanted on the queried +object itself, the honest remedy is to denormalise it onto that object (a stored +field, written when the source changes) — the same remedy the sort axis prescribes +(#6924). Both retirement prescriptions and the two tombstone rejection messages now +say the same thing, and the JSON Schema artifacts and reference docs regenerate from +the source. + +**No schema change.** `FieldNodeSchema` stays `z.string()`: the refusal of dotted +projections is a *semantic* verdict, made at the ingress gate where the field map is +available to judge against — not a *shape* check. Narrowing the type would duplicate +that gate and refuse the registry-less internal callers the ingress deliberately +tolerates. Every input that parsed before this change parses byte-identically after +it, and the type/runtime pins that assert so are kept and renamed +(`fieldNodeDottedNotNarrowed`) so they read as the non-narrowing guard they are +rather than as an endorsement of a feature that does not exist. + +Prose, prescriptions and generated artifacts only — no wire, stored-data or +validation behaviour changes. diff --git a/.changeset/query-field-node-object-form-removed.md b/.changeset/query-field-node-object-form-removed.md index 333ed2f09c..db57173f36 100644 --- a/.changeset/query-field-node-object-form-removed.md +++ b/.changeset/query-field-node-object-form-removed.md @@ -40,7 +40,7 @@ Directive #12: one capability, one contract. | :--- | :--- | | `fields: [{ field: 'owner', fields: ['name'] }]` | `expand: { owner: { object: 'user', fields: ['name'] } }` | | `fields: [{ field: 'owner' }]` | `fields: ['owner']` | -| `fields: [{ field: 'owner', fields: ['name'] }]`, one column only | `fields: ['owner.name']` (dotted path) | +| `fields: [{ field: 'owner', fields: ['name'] }]`, one column only | the same `expand`, keeping the FK in your own projection (`fields: ['title', 'owner_id']`) — **not** a dotted `fields` path, which no driver resolves and the ingress refuses (#7532) | | `fields: [{ field: 'total', alias: 't' }]` | `aggregations` / `windowFunctions` — they carry the live `alias` | The one-line fix: **a `fields[]` entry is a string.** Move nested selection to diff --git a/content/docs/kernel/contracts/data-engine.mdx b/content/docs/kernel/contracts/data-engine.mdx index 024586cea2..276f2ea853 100644 --- a/content/docs/kernel/contracts/data-engine.mdx +++ b/content/docs/kernel/contracts/data-engine.mdx @@ -105,7 +105,8 @@ Defined by `EngineQueryOptionsSchema` in `@objectstack/spec`: ```typescript interface EngineQueryOptions { where?: FilterCondition; // WHERE clause — MongoDB-style $op - fields?: FieldNode[]; // SELECT — field names ('name', 'owner.name') + fields?: FieldNode[]; // SELECT — the object's OWN column names; related + // data comes from `expand`, not a dotted path (#7532) orderBy?: SortNode[]; // ORDER BY limit?: number; // LIMIT offset?: number; // OFFSET diff --git a/content/docs/protocol/objectql/index.mdx b/content/docs/protocol/objectql/index.mdx index 4e13af181f..4ead928243 100644 --- a/content/docs/protocol/objectql/index.mdx +++ b/content/docs/protocol/objectql/index.mdx @@ -135,7 +135,12 @@ const query: QueryAST = { industry: 'tech', annual_revenue: { $gt: 1000000 } }, - fields: ['company_name', 'industry', 'owner.name'], + // `fields` names the object's OWN columns — a dotted path ('owner.name') is + // refused by the ingress (400 INVALID_FIELD, #7532). Related data comes from + // `expand`, which resolves THROUGH the foreign key, so `owner_id` has to stay + // in the projection. + fields: ['company_name', 'industry', 'owner_id'], + expand: { owner_id: { object: 'user', fields: ['name'] } }, orderBy: [{ field: 'created_at', order: 'desc' }], limit: 10 }; @@ -144,35 +149,27 @@ const query: QueryAST = { **Runtime compilation to different databases:** ```sql --- PostgreSQL (with JOIN) -SELECT c.company_name, c.industry, u.name AS "owner.name" -FROM customer c -LEFT JOIN user u ON c.owner_id = u.id -WHERE c.industry = 'tech' AND c.annual_revenue > 1000000 -ORDER BY c.created_at DESC +-- PostgreSQL +SELECT company_name, industry, owner_id +FROM customer +WHERE industry = 'tech' AND annual_revenue > 1000000 +ORDER BY created_at DESC LIMIT 10; + +-- …then `expand` is a second, batched read on the related object — driver-agnostic, +-- not a JOIN the driver renders: +SELECT id, name FROM "user" WHERE id IN (…the owner_ids of the page above); ``` ```javascript // MongoDB -db.customer.aggregate([ - { - $match: { - industry: 'tech', - annual_revenue: { $gt: 1000000 } - } - }, - { - $lookup: { - from: 'user', - localField: 'owner_id', - foreignField: '_id', - as: 'owner' - } - }, - { $sort: { created_at: -1 } }, - { $limit: 10 } -]); +db.customer.find( + { industry: 'tech', annual_revenue: { $gt: 1000000 } }, + { company_name: 1, industry: 1, owner_id: 1 } +).sort({ created_at: -1 }).limit(10); + +// …then the same batched expand read, spelled $in: +db.user.find({ _id: { $in: [/* the owner_ids of the page above */] } }, { name: 1 }); ``` ### 4. Validation: Business Rules as Data diff --git a/content/docs/protocol/objectql/query-syntax.mdx b/content/docs/protocol/objectql/query-syntax.mdx index e92cd92606..83e84b48d8 100644 --- a/content/docs/protocol/objectql/query-syntax.mdx +++ b/content/docs/protocol/objectql/query-syntax.mdx @@ -137,9 +137,12 @@ interface AggregationNode { } // `distinct?: boolean` was REMOVED in protocol 17 (#6815) — see the callout above. -// FieldNode — one entry of the select list. A field name, optionally dotted to -// reach through a relationship ('owner.name'). Related *records* come from -// `expand`, not from inside this list. +// FieldNode — one entry of the select list. One of the queried object's OWN +// column names. The type is `string`, so a dotted path ('owner.name') still +// PARSES, but it resolves nothing: no driver ever implemented dotted +// projection, and the ingress refuses it (400 INVALID_FIELD, #7532). Related +// data — whole records and single related columns alike — comes from `expand`, +// not from inside this list. // // The `{ field, fields, alias }` nested-select member this union used to carry // was REMOVED in protocol 17 (#4196): nothing produced it and nothing read @@ -1003,9 +1006,30 @@ as a single-table query. The key is tombstoned — authoring it is a `tsc` error query that still carries it (even as an empty array) fails to parse with the upgrade prescription. The `JoinNode` / `JoinType` / `JoinStrategy` exports left with it. -Use `expand` (§4) for relationship loading — the live spelling for related records — -a dotted `fields` path (`'owner.name'`) for a single related column, or two queries -joined in application code. +Use `expand` (§4) for relationship loading — the live spelling for related records, +and for single related columns too, since its nested `QueryAST` both filters (`where`) +and selects (`fields`) the related record's columns. Otherwise, two queries joined in +application code. + +**A dotted `fields` path is not the alternative.** `'owner.name'` still *parses* — +`FieldNode` is `string`, a shape check — but no driver ever resolved one, and the +ingress refuses it with `400 INVALID_FIELD` (#7532). Where the value is wanted on the +queried object itself, denormalise it onto that object (a stored field, written when +the source changes) — the same remedy the sort axis prescribes (#6924). + + +**`expand` needs the foreign key in the projection.** The relation is carried by the +FK column, so a narrowed projection that projects it away leaves expansion nothing to +resolve: + +```typescript +{ fields: ['title'], expand: { project_id: { object: 'project' } } } +// -> nothing to resolve; no related record comes back + +{ fields: ['title', 'project_id'], expand: { project_id: { object: 'project' } } } +// -> works +``` + ### Window Functions — removed from the request surface (#4286) diff --git a/content/docs/references/api/contract.mdx b/content/docs/references/api/contract.mdx index 72f71dad0c..8b45a70174 100644 --- a/content/docs/references/api/contract.mdx +++ b/content/docs/references/api/contract.mdx @@ -404,7 +404,7 @@ const result = ApiErrorSchema.parse(data); | Property | Type | Required | Description | | :--- | :--- | :--- | :--- | | **object** | `string` | ✅ | Object name (e.g. account) | -| **fields** | `string[]` | optional | Fields to retrieve — field names, optionally dotted to reach through a relationship (`owner.name`). Related *records* are selected with `expand`, not from inside this list. | +| **fields** | `string[]` | optional | Fields to retrieve — names of the queried object's OWN columns. A dotted path (`owner.name`) is not a projection: no driver resolves one, and the ingress refuses it with `400 INVALID_FIELD` (#7532). Related data is read with `expand`, whose nested QueryAST both filters (`where`) and selects (`fields`) the related record's columns. The projection must RETAIN the foreign-key column: `fields: ['title']` with `expand: 'project_id'` resolves nothing, because the relation is carried by that key — add `'project_id'` and it works. Where the value is wanted on the queried object itself, denormalise it onto that object (a stored field, written when the source changes), the same remedy the sort axis prescribes (#6924). | | **where** | `any` | optional | Filtering criteria (WHERE) | | **search** | `string \| { query: string; fields?: string[]; fuzzy: boolean; operator: Enum<'and' \| 'or'>; … }` | optional | Full-text search — the query text (canonical, ADR-0061 D1), or a structured FullTextSearch configuration | | **searchFields** | `string[]` | optional | Narrow the search to these fields (server-intersected with the allowed searchable set — can only narrow, never widen; ADR-0061 D1) | @@ -413,7 +413,7 @@ const result = ApiErrorSchema.parse(data); | **offset** | `number` | optional | Records to skip (OFFSET) | | **top** | `number` | optional | Alias for limit (OData compatibility) | | **cursor** | `never` | optional | [REMOVED] `query.cursor` was removed in @objectstack/spec 17 (#4286, ADR-0049) — no driver ever implemented keyset pagination, so the cursor was accepted and ignored and every page came back identical (a caller looping "until hasMore is false" never terminates). Delete the key; `QueryBuilder.cursor()` was removed with it. Express the keyset as an ordinary `where` predicate on your sort key — `where: { created_at: { $gt: last.created_at } }` with the matching `orderBy` — which every driver executes with canonicalised comparands. A first-class cursor, if ever built, will be a response-minted opaque token, not this caller-built record. | -| **joins** | `never` | optional | [REMOVED] `query.joins` was removed in @objectstack/spec 17 (#4286, ADR-0049) — no engine or driver ever read it: a query carrying `joins` behaved exactly as if the key were absent, while its name squatted on the reserved REST parameter set. Delete the key. Related records are read through `expand` — `expand: { owner: { object: 'user', fields: ['name'] } }` — which the engine resolves via batch $in queries, and a single related column is a dotted `fields` path (`fields: ['owner.name']`). | +| **joins** | `never` | optional | [REMOVED] `query.joins` was removed in @objectstack/spec 17 (#4286, ADR-0049) — no engine or driver ever read it: a query carrying `joins` behaved exactly as if the key were absent, while its name squatted on the reserved REST parameter set. Delete the key. Related records are read through `expand` — `expand: { owner_id: { object: 'user', fields: ['name'] } }` — which the engine resolves via batch $in queries, and whose nested query selects the related record's own columns. Keep the foreign key in your own projection (`fields: ['title', 'owner_id']`): the relation is carried by that column, so projecting it away leaves expansion nothing to resolve. A dotted `fields` path is NOT a replacement — no driver ever resolved one and the ingress refuses it (`400 INVALID_FIELD`, #7532). | | **aggregations** | `{ function: Enum<'count' \| 'sum' \| 'avg' \| 'min' \| 'max' \| 'count_distinct'>; field?: string; alias: string; filter?: any }[]` | optional | Aggregation functions | | **groupBy** | `(string \| { field: string; dateGranularity?: Enum<'day' \| 'week' \| 'month' \| 'quarter' \| 'year'>; alias?: string })[]` | optional | GROUP BY targets (strings or `{field, dateGranularity?}` objects for date bucketing) | | **having** | `any` | optional | HAVING — filter over the AGGREGATED rows (aggregation aliases + groupBy projections); applied engine-side after aggregation | diff --git a/content/docs/references/data/query.mdx b/content/docs/references/data/query.mdx index 08477bfe7c..5175c97754 100644 --- a/content/docs/references/data/query.mdx +++ b/content/docs/references/data/query.mdx @@ -122,7 +122,7 @@ Type: `string` | Property | Type | Required | Description | | :--- | :--- | :--- | :--- | | **object** | `string` | ✅ | Object name (e.g. account) | -| **fields** | `string[]` | optional | Fields to retrieve — field names, optionally dotted to reach through a relationship (`owner.name`). Related *records* are selected with `expand`, not from inside this list. | +| **fields** | `string[]` | optional | Fields to retrieve — names of the queried object's OWN columns. A dotted path (`owner.name`) is not a projection: no driver resolves one, and the ingress refuses it with `400 INVALID_FIELD` (#7532). Related data is read with `expand`, whose nested QueryAST both filters (`where`) and selects (`fields`) the related record's columns. The projection must RETAIN the foreign-key column: `fields: ['title']` with `expand: 'project_id'` resolves nothing, because the relation is carried by that key — add `'project_id'` and it works. Where the value is wanted on the queried object itself, denormalise it onto that object (a stored field, written when the source changes), the same remedy the sort axis prescribes (#6924). | | **where** | `any` | optional | Filtering criteria (WHERE) | | **search** | `string \| { query: string; fields?: string[]; fuzzy: boolean; operator: Enum<'and' \| 'or'>; … }` | optional | Full-text search — the query text (canonical, ADR-0061 D1), or a structured FullTextSearch configuration | | **searchFields** | `string[]` | optional | Narrow the search to these fields (server-intersected with the allowed searchable set — can only narrow, never widen; ADR-0061 D1) | @@ -131,7 +131,7 @@ Type: `string` | **offset** | `number` | optional | Records to skip (OFFSET) | | **top** | `number` | optional | Alias for limit (OData compatibility) | | **cursor** | `never` | optional | [REMOVED] `query.cursor` was removed in @objectstack/spec 17 (#4286, ADR-0049) — no driver ever implemented keyset pagination, so the cursor was accepted and ignored and every page came back identical (a caller looping "until hasMore is false" never terminates). Delete the key; `QueryBuilder.cursor()` was removed with it. Express the keyset as an ordinary `where` predicate on your sort key — `where: { created_at: { $gt: last.created_at } }` with the matching `orderBy` — which every driver executes with canonicalised comparands. A first-class cursor, if ever built, will be a response-minted opaque token, not this caller-built record. | -| **joins** | `never` | optional | [REMOVED] `query.joins` was removed in @objectstack/spec 17 (#4286, ADR-0049) — no engine or driver ever read it: a query carrying `joins` behaved exactly as if the key were absent, while its name squatted on the reserved REST parameter set. Delete the key. Related records are read through `expand` — `expand: { owner: { object: 'user', fields: ['name'] } }` — which the engine resolves via batch $in queries, and a single related column is a dotted `fields` path (`fields: ['owner.name']`). | +| **joins** | `never` | optional | [REMOVED] `query.joins` was removed in @objectstack/spec 17 (#4286, ADR-0049) — no engine or driver ever read it: a query carrying `joins` behaved exactly as if the key were absent, while its name squatted on the reserved REST parameter set. Delete the key. Related records are read through `expand` — `expand: { owner_id: { object: 'user', fields: ['name'] } }` — which the engine resolves via batch $in queries, and whose nested query selects the related record's own columns. Keep the foreign key in your own projection (`fields: ['title', 'owner_id']`): the relation is carried by that column, so projecting it away leaves expansion nothing to resolve. A dotted `fields` path is NOT a replacement — no driver ever resolved one and the ingress refuses it (`400 INVALID_FIELD`, #7532). | | **aggregations** | `{ function: Enum<'count' \| 'sum' \| 'avg' \| 'min' \| 'max' \| 'count_distinct'>; field?: string; alias: string; filter?: any }[]` | optional | Aggregation functions | | **groupBy** | `(string \| { field: string; dateGranularity?: Enum<'day' \| 'week' \| 'month' \| 'quarter' \| 'year'>; alias?: string })[]` | optional | GROUP BY targets (strings or `{field, dateGranularity?}` objects for date bucketing) | | **having** | `any` | optional | HAVING — filter over the AGGREGATED rows (aggregation aliases + groupBy projections); applied engine-side after aggregation | diff --git a/docs/protocol-upgrade-guide.md b/docs/protocol-upgrade-guide.md index 31cc8df101..a9eec1fb8f 100644 --- a/docs/protocol-upgrade-guide.md +++ b/docs/protocol-upgrade-guide.md @@ -431,12 +431,12 @@ This is a RUNTIME registration API, not stored metadata, so — like `hook-conte - **`query-distinct-retired`** — `data.query.distinct` → `groupBy` for unique combinations; the `count_distinct` aggregation for deduplicated counts; the SQL/memory drivers' `distinct(object, field)` door for one column's values - Why not automatic: The `distinct` flag promised SELECT DISTINCT and no driver ever rendered it — but it was MIS-WIRED rather than merely dead (the harsher ADR-0078 class): the REST list path treated a distinct query as not countable and silently degraded `total`/`hasMore` to a page-local estimate, so the caller got duplicate rows AND worse pagination metadata, and a side effect that "confirmed" the flag was doing something. It had a shipped public producer (`QueryBuilder.distinct()`, removed with the key). The count suppression is deleted in the same change — `total` is truthful for those queries again. A REQUEST surface, never stored; nothing to rewrite. ADR-0049 / ADR-0078, #4286. - Done when: No caller sends `distinct` and no SDK call site uses `QueryBuilder.distinct()`; deduplication goes through `groupBy` / `count_distinct` / the drivers' `distinct()` door. A query still carrying the key fails to parse with the removal prescription, and the REST list response reports a real `total` for queries that used to send it. -- **`query-field-node-object-form-retired`** — `data.query.fields` → expand (`expand: { owner: { object: 'user', fields: ['name'] } }`), or a dotted path for a single related column (`fields: ['owner.name']`) +- **`query-field-node-object-form-retired`** — `data.query.fields` → expand (`expand: { owner_id: { object: 'user', fields: ['name'] } }`), whose nested query selects the related record's own columns — keeping the foreign key in your own projection (`fields: ['title', 'owner_id']`), because the relation is carried by that column and projecting it away leaves expansion nothing to resolve. A dotted `fields` path is NOT a replacement: no driver ever resolved one, and the ingress refuses it (`400 INVALID_FIELD`, #7532). Where the value is wanted on the queried object itself, denormalise it onto that object (a stored field, written when the source changes) — the same remedy the sort axis prescribes (#6924) - Why not automatic: The `FieldNode` union declared a nested-select object form `{ field, fields, alias }` that was inert end to end: no producer emitted it, and no consumer read `.fields` or `.alias` — objectql's formula projection and known-field filters, driver-sql's `select()` and driver-memory's projection all treat the list as `string[]`, driver-mongodb keyed its projection with the entry itself, and the REST ingress stringified it. Nested selection is `expand`, which the engine resolves via batch `$in` queries. This is a REQUEST surface — `QueryAST` is never stored in stack metadata (no view, dataset or report authors one), so there is no source for the chain to rewrite: the schema narrows to `z.string()` and callers move their own select lists. ADR-0049 / ADR-0078, #4196. - - Done when: No caller puts an object in `fields[]`; related records are read through `expand` and single related columns through dotted paths. A `fields` entry that is not a string fails to parse with the removal prescription, and the list/query/export routes answer 400 INVALID_FIELD naming the retired form instead of the field `"[object Object]"`. -- **`query-joins-retired`** — `data.query.joins` → expand (`expand: { owner: { object: 'user', fields: ['name'] } }`), or a dotted `fields` path for a single related column (`fields: ['owner.name']`) + - Done when: No caller puts an object in `fields[]`; related records AND single related columns are read through `expand`, with the foreign-key column retained in the projection so expansion has something to resolve. A `fields` entry that is not a string fails to parse with the removal prescription, and the list/query/export routes answer 400 INVALID_FIELD naming the retired form instead of the field `"[object Object]"`. +- **`query-joins-retired`** — `data.query.joins` → expand (`expand: { owner_id: { object: 'user', fields: ['name'] } }`), whose nested query selects the related record's own columns — keeping the foreign key in your own projection (`fields: ['title', 'owner_id']`), because the relation is carried by that column and projecting it away leaves expansion nothing to resolve. A dotted `fields` path is NOT a replacement: no driver ever resolved one, and the ingress refuses it (`400 INVALID_FIELD`, #7532). Where the value is wanted on the queried object itself, denormalise it onto that object (a stored field, written when the source changes) — the same remedy the sort axis prescribes (#6924) - Why not automatic: The `joins` array was declared-but-inert: no engine or driver read `query.joins` anywhere on the query path, so a query carrying it behaved exactly as if the key were absent — while the name squatted on the reserved REST parameter set. Related-record retrieval already has a live spelling (`expand`, resolved by the engine via batch `$in` queries), so the removal deletes the second, broken spelling rather than the capability, and the orphaned `JoinNode`/`JoinType`/`JoinStrategy` cluster goes with the key. A REQUEST surface — `QueryAST` is never stored in stack metadata — so there is no source for the chain to rewrite; callers move their own queries. ADR-0049 / ADR-0078, #4286. - - Done when: No caller sends `joins`; related records are read through `expand` and single related columns through dotted `fields` paths. A query that still carries `joins` fails to parse with the removal prescription (even as an empty array), and authoring it is a `tsc` error at the call site. + - Done when: No caller sends `joins`; related records AND single related columns are read through `expand`, with the foreign-key column retained in the projection so expansion has something to resolve. A query that still carries `joins` fails to parse with the removal prescription (even as an empty array), and authoring it is a `tsc` error at the call site. - **`query-window-functions-retired`** — `data.query.windowFunctions` → `aggregations` + `groupBy` for request-level analytics; `SqlDriver.findWithWindowFunctions(object, query)` for embedders on a SQL datasource - Why not automatic: The `windowFunctions` array was declared-but-inert on the query path: `find()` never applied a window function, so every OVER clause a caller declared was silently dropped. The capability only ever ran behind `SqlDriver.findWithWindowFunctions()`, a driver-level door that is not on the `IDataDriver` contract and whose flat input shape (`{ function, alias, partitionBy?, orderBy? }`) the spec vocabulary never matched — `WindowFunctionNodeSchema` declared `field`/`over`/`frame` members the door never read, so that cluster is removed with the key rather than left as a false affordance. A REQUEST surface, never stored; no source to rewrite. ADR-0049 / ADR-0078, #4286. - Done when: No caller sends `windowFunctions` in a query; request-level analytics use `aggregations` + `groupBy`, and embedders needing OVER-clause SQL call the SQL driver's `findWithWindowFunctions` door directly. A query that still carries the key fails to parse with the removal prescription naming that door. diff --git a/packages/spec/spec-changes.json b/packages/spec/spec-changes.json index a3d64057f5..469b8d7dba 100644 --- a/packages/spec/spec-changes.json +++ b/packages/spec/spec-changes.json @@ -744,14 +744,14 @@ }, { "surface": "data.query.fields", - "replacement": "expand (`expand: { owner: { object: 'user', fields: ['name'] } }`), or a dotted path for a single related column (`fields: ['owner.name']`)", + "replacement": "expand (`expand: { owner_id: { object: 'user', fields: ['name'] } }`), whose nested query selects the related record's own columns — keeping the foreign key in your own projection (`fields: ['title', 'owner_id']`), because the relation is carried by that column and projecting it away leaves expansion nothing to resolve. A dotted `fields` path is NOT a replacement: no driver ever resolved one, and the ingress refuses it (`400 INVALID_FIELD`, #7532). Where the value is wanted on the queried object itself, denormalise it onto that object (a stored field, written when the source changes) — the same remedy the sort axis prescribes (#6924)", "migrationId": "query-field-node-object-form-retired", "toMajor": 17, "rationale": "The `FieldNode` union declared a nested-select object form `{ field, fields, alias }` that was inert end to end: no producer emitted it, and no consumer read `.fields` or `.alias` — objectql's formula projection and known-field filters, driver-sql's `select()` and driver-memory's projection all treat the list as `string[]`, driver-mongodb keyed its projection with the entry itself, and the REST ingress stringified it. Nested selection is `expand`, which the engine resolves via batch `$in` queries. This is a REQUEST surface — `QueryAST` is never stored in stack metadata (no view, dataset or report authors one), so there is no source for the chain to rewrite: the schema narrows to `z.string()` and callers move their own select lists. ADR-0049 / ADR-0078, #4196." }, { "surface": "data.query.joins", - "replacement": "expand (`expand: { owner: { object: 'user', fields: ['name'] } }`), or a dotted `fields` path for a single related column (`fields: ['owner.name']`)", + "replacement": "expand (`expand: { owner_id: { object: 'user', fields: ['name'] } }`), whose nested query selects the related record's own columns — keeping the foreign key in your own projection (`fields: ['title', 'owner_id']`), because the relation is carried by that column and projecting it away leaves expansion nothing to resolve. A dotted `fields` path is NOT a replacement: no driver ever resolved one, and the ingress refuses it (`400 INVALID_FIELD`, #7532). Where the value is wanted on the queried object itself, denormalise it onto that object (a stored field, written when the source changes) — the same remedy the sort axis prescribes (#6924)", "migrationId": "query-joins-retired", "toMajor": 17, "rationale": "The `joins` array was declared-but-inert: no engine or driver read `query.joins` anywhere on the query path, so a query carrying it behaved exactly as if the key were absent — while the name squatted on the reserved REST parameter set. Related-record retrieval already has a live spelling (`expand`, resolved by the engine via batch `$in` queries), so the removal deletes the second, broken spelling rather than the capability, and the orphaned `JoinNode`/`JoinType`/`JoinStrategy` cluster goes with the key. A REQUEST surface — `QueryAST` is never stored in stack metadata — so there is no source for the chain to rewrite; callers move their own queries. ADR-0049 / ADR-0078, #4286." @@ -1635,14 +1635,14 @@ }, { "surface": "data.query.fields", - "replacement": "expand (`expand: { owner: { object: 'user', fields: ['name'] } }`), or a dotted path for a single related column (`fields: ['owner.name']`)", + "replacement": "expand (`expand: { owner_id: { object: 'user', fields: ['name'] } }`), whose nested query selects the related record's own columns — keeping the foreign key in your own projection (`fields: ['title', 'owner_id']`), because the relation is carried by that column and projecting it away leaves expansion nothing to resolve. A dotted `fields` path is NOT a replacement: no driver ever resolved one, and the ingress refuses it (`400 INVALID_FIELD`, #7532). Where the value is wanted on the queried object itself, denormalise it onto that object (a stored field, written when the source changes) — the same remedy the sort axis prescribes (#6924)", "migrationId": "query-field-node-object-form-retired", "toMajor": 17, "rationale": "The `FieldNode` union declared a nested-select object form `{ field, fields, alias }` that was inert end to end: no producer emitted it, and no consumer read `.fields` or `.alias` — objectql's formula projection and known-field filters, driver-sql's `select()` and driver-memory's projection all treat the list as `string[]`, driver-mongodb keyed its projection with the entry itself, and the REST ingress stringified it. Nested selection is `expand`, which the engine resolves via batch `$in` queries. This is a REQUEST surface — `QueryAST` is never stored in stack metadata (no view, dataset or report authors one), so there is no source for the chain to rewrite: the schema narrows to `z.string()` and callers move their own select lists. ADR-0049 / ADR-0078, #4196." }, { "surface": "data.query.joins", - "replacement": "expand (`expand: { owner: { object: 'user', fields: ['name'] } }`), or a dotted `fields` path for a single related column (`fields: ['owner.name']`)", + "replacement": "expand (`expand: { owner_id: { object: 'user', fields: ['name'] } }`), whose nested query selects the related record's own columns — keeping the foreign key in your own projection (`fields: ['title', 'owner_id']`), because the relation is carried by that column and projecting it away leaves expansion nothing to resolve. A dotted `fields` path is NOT a replacement: no driver ever resolved one, and the ingress refuses it (`400 INVALID_FIELD`, #7532). Where the value is wanted on the queried object itself, denormalise it onto that object (a stored field, written when the source changes) — the same remedy the sort axis prescribes (#6924)", "migrationId": "query-joins-retired", "toMajor": 17, "rationale": "The `joins` array was declared-but-inert: no engine or driver read `query.joins` anywhere on the query path, so a query carrying it behaved exactly as if the key were absent — while the name squatted on the reserved REST parameter set. Related-record retrieval already has a live spelling (`expand`, resolved by the engine via batch `$in` queries), so the removal deletes the second, broken spelling rather than the capability, and the orphaned `JoinNode`/`JoinType`/`JoinStrategy` cluster goes with the key. A REQUEST surface — `QueryAST` is never stored in stack metadata — so there is no source for the chain to rewrite; callers move their own queries. ADR-0049 / ADR-0078, #4286." diff --git a/packages/spec/src/data/query.test.ts b/packages/spec/src/data/query.test.ts index a8f72ba15b..3346a2ce1b 100644 --- a/packages/spec/src/data/query.test.ts +++ b/packages/spec/src/data/query.test.ts @@ -126,7 +126,13 @@ describe('QuerySchema - Basic', () => { * the parse error is the channel an upgrading consumer actually hits. */ describe('FieldNode — the nested-select object form is REMOVED (#4196)', () => { - it('accepts a field name, and a dotted path through a relationship', () => { + // The dotted half is a NON-NARROWING guard, not a feature pin (#7601): the + // refusal of dotted projections (#7532) is a SEMANTIC verdict at the ingress + // gate (`assertProjectionFieldsExist`, `400 INVALID_FIELD`), where the field + // map is available to judge against — so `FieldNodeSchema` stays `z.string()` + // and every input valid before #7601 still parses byte-identically after it. + // Parsing is not resolving: no driver ever resolved a dotted projection. + it('accepts a field name, and still parses a dotted string (shape, not semantics)', () => { expect(FieldNodeSchema.parse('name')).toBe('name'); expect(FieldNodeSchema.parse('owner.name')).toBe('owner.name'); expect(() => QuerySchema.parse({ diff --git a/packages/spec/src/data/query.zod.ts b/packages/spec/src/data/query.zod.ts index 0b0b49c7ab..77fc40f4cb 100644 --- a/packages/spec/src/data/query.zod.ts +++ b/packages/spec/src/data/query.zod.ts @@ -277,9 +277,10 @@ export const AggregationNodeSchema = lazySchema(() => z.object({ // `JoinNodeBaseSchema`, `JoinNodeSchema` and the `JoinNode`/`JoinNodeInput` // types — was deleted together with the `query.joins` tombstone below: no // engine or driver ever read a query's `joins`, and an exported schema with no -// consumer reads as a capability (the #3950 precedent). Related records are -// read through `expand`; a single related column is a dotted `fields` path -// (`fields: ['owner.name']`). +// consumer reads as a capability (the #3950 precedent). Related records — and +// single related columns, via the nested query's own `fields` — are read +// through `expand`. NOT through a dotted `fields` path: no driver ever resolved +// one, and the ingress refuses it (`400 INVALID_FIELD`, #7532). // ─── Window functions: REMOVED (#4286, ADR-0049) ───────────────────────────── // The window cluster — `WindowFunction`, `WindowSpecSchema`, @@ -295,9 +296,13 @@ export const AggregationNodeSchema = lazySchema(() => z.object({ /** * One entry of a select list: a field name. * - * The whole vocabulary is a column (`'name'`) or a dotted path the engine - * resolves through a relationship field (`'owner.name'`). Related *records* are - * selected with {@link QueryAST.expand}, not from inside this list. + * The whole vocabulary is one of the queried object's OWN columns (`'name'`). + * The type is `string`, so a dotted path (`'owner.name'`) still PARSES — that + * is a shape check — but it resolves nothing: no driver ever implemented dotted + * projection, and the ingress refuses it semantically + * (`assertProjectionFieldsExist`, `400 INVALID_FIELD`, #7532). Related data — + * whole records and single related columns alike — comes from + * {@link QueryAST.expand}, not from inside this list. * * The TYPE half of {@link FieldNodeSchema} — it used to be that schema's * recursion annotation, back when the union carried a second @@ -318,10 +323,14 @@ const FIELD_NODE_OBJECT_FORM_REMOVED = + 'ever produced it and nothing ever read `.fields`/`.alias`: every consumer on this path ' + 'treats the list as `string[]`, so the object form was dropped by the SQL and memory drivers, ' + 'projected as a column literally named "[object Object]" by MongoDB, and refused as an unknown ' - + 'field by the REST ingress. Select related records with `expand` — ' - + "`expand: { owner: { object: 'user', fields: ['name'] } }` — or name a single related column " - + "with a dotted path (`fields: ['owner.name']`). `alias` has no replacement here; an aliased " - + 'projection is an `aggregations` or `windowFunctions` entry, which carry their own `alias`.'; + + 'field by the REST ingress. Select related data with `expand` — ' + + "`expand: { owner_id: { object: 'user', fields: ['name'] } }` — whose nested query names the " + + 'related columns you want. A dotted `fields` path is NOT the replacement: no driver ever ' + + 'resolved one and the ingress refuses it (`400 INVALID_FIELD`, #7532). Keep the foreign-key ' + + "column in your own projection (`fields: ['title', 'owner_id']`) — the relation is carried by " + + 'that key, so projecting it away leaves expansion nothing to resolve. `alias` has no ' + + 'replacement here; an aliased projection is an `aggregations` or `windowFunctions` entry, ' + + 'which carry their own `alias`.'; /** * Field Selection Node @@ -356,9 +365,12 @@ const QUERY_JOINS_REMOVED = '`query.joins` was removed in @objectstack/spec 17 (#4286, ADR-0049) — no engine or driver ' + 'ever read it: a query carrying `joins` behaved exactly as if the key were absent, while ' + 'its name squatted on the reserved REST parameter set. Delete the key. Related records are ' - + "read through `expand` — `expand: { owner: { object: 'user', fields: ['name'] } }` — which " - + 'the engine resolves via batch $in queries, and a single related column is a dotted ' - + "`fields` path (`fields: ['owner.name']`)."; + + "read through `expand` — `expand: { owner_id: { object: 'user', fields: ['name'] } }` — which " + + 'the engine resolves via batch $in queries, and whose nested query selects the related ' + + "record's own columns. Keep the foreign key in your own projection (`fields: ['title', " + + "'owner_id']`): the relation is carried by that column, so projecting it away leaves " + + 'expansion nothing to resolve. A dotted `fields` path is NOT a replacement — no driver ever ' + + 'resolved one and the ingress refuses it (`400 INVALID_FIELD`, #7532).'; /** * Exported (unlike the two above) because `EngineQueryOptionsSchema` @@ -481,7 +493,7 @@ const BaseQuerySchema = z.object({ object: z.string().describe('Object name (e.g. account)'), /** Select Clause */ - fields: z.array(FieldNodeSchema).optional().describe('Fields to retrieve — field names, optionally dotted to reach through a relationship (`owner.name`). Related *records* are selected with `expand`, not from inside this list.'), + fields: z.array(FieldNodeSchema).optional().describe("Fields to retrieve — names of the queried object's OWN columns. A dotted path (`owner.name`) is not a projection: no driver resolves one, and the ingress refuses it with `400 INVALID_FIELD` (#7532). Related data is read with `expand`, whose nested QueryAST both filters (`where`) and selects (`fields`) the related record's columns. The projection must RETAIN the foreign-key column: `fields: ['title']` with `expand: 'project_id'` resolves nothing, because the relation is carried by that key — add `'project_id'` and it works. Where the value is wanted on the queried object itself, denormalise it onto that object (a stored field, written when the source changes), the same remedy the sort axis prescribes (#6924)."), /** Where Clause (Filtering) */ where: FilterConditionSchema.optional().describe('Filtering criteria (WHERE)'), diff --git a/packages/spec/src/migrations/entries/semantic/17.query-field-node-object-form-retired.ts b/packages/spec/src/migrations/entries/semantic/17.query-field-node-object-form-retired.ts index c548e6dc78..fbc658667c 100644 --- a/packages/spec/src/migrations/entries/semantic/17.query-field-node-object-form-retired.ts +++ b/packages/spec/src/migrations/entries/semantic/17.query-field-node-object-form-retired.ts @@ -5,7 +5,15 @@ import type { SemanticMigration } from '../../types.js'; export const entry: SemanticMigration = { id: 'query-field-node-object-form-retired', surface: 'data.query.fields', - replacement: "expand (`expand: { owner: { object: 'user', fields: ['name'] } }`), or a dotted path for a single related column (`fields: ['owner.name']`)", + replacement: + "expand (`expand: { owner_id: { object: 'user', fields: ['name'] } }`), whose nested " + + "query selects the related record's own columns — keeping the foreign key in your own " + + "projection (`fields: ['title', 'owner_id']`), because the relation is carried by that " + + 'column and projecting it away leaves expansion nothing to resolve. A dotted `fields` ' + + 'path is NOT a replacement: no driver ever resolved one, and the ingress refuses it ' + + '(`400 INVALID_FIELD`, #7532). Where the value is wanted on the queried object itself, ' + + 'denormalise it onto that object (a stored field, written when the source changes) — the ' + + 'same remedy the sort axis prescribes (#6924)', reason: 'The `FieldNode` union declared a nested-select object form `{ field, fields, alias }` that ' + 'was inert end to end: no producer emitted it, and no consumer read `.fields` or `.alias` ' @@ -17,8 +25,9 @@ export const entry: SemanticMigration = { + 'authors one), so there is no source for the chain to rewrite: the schema narrows to ' + '`z.string()` and callers move their own select lists. ADR-0049 / ADR-0078, #4196.', acceptanceCriteria: - 'No caller puts an object in `fields[]`; related records are read through `expand` and ' - + 'single related columns through dotted paths. A `fields` entry that is not a string ' + 'No caller puts an object in `fields[]`; related records AND single related columns are ' + + 'read through `expand`, with the foreign-key column retained in the projection so ' + + 'expansion has something to resolve. A `fields` entry that is not a string ' + 'fails to parse with the removal prescription, and the list/query/export routes answer ' + '400 INVALID_FIELD naming the retired form instead of the field `"[object Object]"`.', }; diff --git a/packages/spec/src/migrations/entries/semantic/17.query-joins-retired.ts b/packages/spec/src/migrations/entries/semantic/17.query-joins-retired.ts index ca248d3ec7..6f520b5a24 100644 --- a/packages/spec/src/migrations/entries/semantic/17.query-joins-retired.ts +++ b/packages/spec/src/migrations/entries/semantic/17.query-joins-retired.ts @@ -6,8 +6,14 @@ export const entry: SemanticMigration = { id: 'query-joins-retired', surface: 'data.query.joins', replacement: - "expand (`expand: { owner: { object: 'user', fields: ['name'] } }`), or a dotted " - + "`fields` path for a single related column (`fields: ['owner.name']`)", + "expand (`expand: { owner_id: { object: 'user', fields: ['name'] } }`), whose nested " + + "query selects the related record's own columns — keeping the foreign key in your own " + + "projection (`fields: ['title', 'owner_id']`), because the relation is carried by that " + + 'column and projecting it away leaves expansion nothing to resolve. A dotted `fields` ' + + 'path is NOT a replacement: no driver ever resolved one, and the ingress refuses it ' + + '(`400 INVALID_FIELD`, #7532). Where the value is wanted on the queried object itself, ' + + 'denormalise it onto that object (a stored field, written when the source changes) — the ' + + 'same remedy the sort axis prescribes (#6924)', reason: 'The `joins` array was declared-but-inert: no engine or driver read `query.joins` ' + 'anywhere on the query path, so a query carrying it behaved exactly as if the key were ' @@ -19,8 +25,8 @@ export const entry: SemanticMigration = { + 'is no source for the chain to rewrite; callers move their own queries. ' + 'ADR-0049 / ADR-0078, #4286.', acceptanceCriteria: - 'No caller sends `joins`; related records are read through `expand` and single related ' - + 'columns through dotted `fields` paths. A query that still carries `joins` fails to ' - + 'parse with the removal prescription (even as an empty array), and authoring it is a ' - + '`tsc` error at the call site.', + 'No caller sends `joins`; related records AND single related columns are read through ' + + '`expand`, with the foreign-key column retained in the projection so expansion has ' + + 'something to resolve. A query that still carries `joins` fails to parse with the removal ' + + 'prescription (even as an empty array), and authoring it is a `tsc` error at the call site.', }; diff --git a/packages/spec/src/migrations/registry.ts b/packages/spec/src/migrations/registry.ts index c402ec9739..18269e671f 100644 --- a/packages/spec/src/migrations/registry.ts +++ b/packages/spec/src/migrations/registry.ts @@ -3059,7 +3059,15 @@ const step17: MigrationStep = { { id: 'query-field-node-object-form-retired', surface: 'data.query.fields', - replacement: "expand (`expand: { owner: { object: 'user', fields: ['name'] } }`), or a dotted path for a single related column (`fields: ['owner.name']`)", + replacement: + "expand (`expand: { owner_id: { object: 'user', fields: ['name'] } }`), whose nested " + + "query selects the related record's own columns — keeping the foreign key in your own " + + "projection (`fields: ['title', 'owner_id']`), because the relation is carried by that " + + 'column and projecting it away leaves expansion nothing to resolve. A dotted `fields` ' + + 'path is NOT a replacement: no driver ever resolved one, and the ingress refuses it ' + + '(`400 INVALID_FIELD`, #7532). Where the value is wanted on the queried object itself, ' + + 'denormalise it onto that object (a stored field, written when the source changes) — the ' + + 'same remedy the sort axis prescribes (#6924)', reason: 'The `FieldNode` union declared a nested-select object form `{ field, fields, alias }` that ' + 'was inert end to end: no producer emitted it, and no consumer read `.fields` or `.alias` ' @@ -3071,8 +3079,9 @@ const step17: MigrationStep = { + 'authors one), so there is no source for the chain to rewrite: the schema narrows to ' + '`z.string()` and callers move their own select lists. ADR-0049 / ADR-0078, #4196.', acceptanceCriteria: - 'No caller puts an object in `fields[]`; related records are read through `expand` and ' - + 'single related columns through dotted paths. A `fields` entry that is not a string ' + 'No caller puts an object in `fields[]`; related records AND single related columns are ' + + 'read through `expand`, with the foreign-key column retained in the projection so ' + + 'expansion has something to resolve. A `fields` entry that is not a string ' + 'fails to parse with the removal prescription, and the list/query/export routes answer ' + '400 INVALID_FIELD naming the retired form instead of the field `"[object Object]"`.', }, @@ -3080,8 +3089,14 @@ const step17: MigrationStep = { id: 'query-joins-retired', surface: 'data.query.joins', replacement: - "expand (`expand: { owner: { object: 'user', fields: ['name'] } }`), or a dotted " - + "`fields` path for a single related column (`fields: ['owner.name']`)", + "expand (`expand: { owner_id: { object: 'user', fields: ['name'] } }`), whose nested " + + "query selects the related record's own columns — keeping the foreign key in your own " + + "projection (`fields: ['title', 'owner_id']`), because the relation is carried by that " + + 'column and projecting it away leaves expansion nothing to resolve. A dotted `fields` ' + + 'path is NOT a replacement: no driver ever resolved one, and the ingress refuses it ' + + '(`400 INVALID_FIELD`, #7532). Where the value is wanted on the queried object itself, ' + + 'denormalise it onto that object (a stored field, written when the source changes) — the ' + + 'same remedy the sort axis prescribes (#6924)', reason: 'The `joins` array was declared-but-inert: no engine or driver read `query.joins` ' + 'anywhere on the query path, so a query carrying it behaved exactly as if the key were ' @@ -3093,10 +3108,10 @@ const step17: MigrationStep = { + 'is no source for the chain to rewrite; callers move their own queries. ' + 'ADR-0049 / ADR-0078, #4286.', acceptanceCriteria: - 'No caller sends `joins`; related records are read through `expand` and single related ' - + 'columns through dotted `fields` paths. A query that still carries `joins` fails to ' - + 'parse with the removal prescription (even as an empty array), and authoring it is a ' - + '`tsc` error at the call site.', + 'No caller sends `joins`; related records AND single related columns are read through ' + + '`expand`, with the foreign-key column retained in the projection so expansion has ' + + 'something to resolve. A query that still carries `joins` fails to parse with the removal ' + + 'prescription (even as an empty array), and authoring it is a `tsc` error at the call site.', }, { id: 'query-window-functions-retired', diff --git a/packages/spec/src/recursive-schema-input-assertions.ts b/packages/spec/src/recursive-schema-input-assertions.ts index 0e9490dc61..dafba36560 100644 --- a/packages/spec/src/recursive-schema-input-assertions.ts +++ b/packages/spec/src/recursive-schema-input-assertions.ts @@ -68,11 +68,18 @@ import type { /* ── data/query.zod.ts ─────────────────────────────────────────────────────── */ -/** The authoring shape: only `object` is required, `expand` recurses. */ +/** + * The authoring shape: only `object` is required, `expand` recurses. + * + * The projection names the object's OWN columns, and keeps the foreign key + * (`owner_id`) that `expand` resolves through — this pin doubles as the one + * canonical query in the file, so it spells the shape the spec prescribes + * (#7601) rather than the dotted path the ingress refuses (#7532). + */ export const queryInput: QueryInput = { object: 'account', - fields: ['name', 'owner.email'], - expand: { owner: { object: 'user', fields: ['name'] } }, + fields: ['name', 'owner_id'], + expand: { owner_id: { object: 'user', fields: ['name'] } }, }; // @ts-expect-error — a query is not a string (`unknown` would take it) @@ -86,7 +93,7 @@ export const queryNeedsObject: QueryInput = { fields: ['name'] }; // `QuerySchema` — is gone, so there is no input half left to pin. /** - * A select entry is a field name, optionally dotted through a relationship. + * A select entry is a field name — one of the queried object's own columns. * * `FieldNode` stopped being recursive in #4196 — the `{ field, fields, alias }` * member it used to carry was declared-but-inert and is gone, so @@ -94,9 +101,23 @@ export const queryNeedsObject: QueryInput = { fields: ['name'] }; * `z.ZodType` annotation. The probes stay: this file's job is that * the input side is CHECKED, and a `z.string()` that someone re-widens to * `z.ZodType` fails here exactly as a dropped type parameter would. + * + * ## The dotted probe is a NON-NARROWING guard, not an endorsement (#7601) + * + * A dotted string still satisfies `FieldNode`, and that must stay true: the + * refusal of dotted projections (#7532) is a SEMANTIC verdict at the ingress + * gate (`assertProjectionFieldsExist`, `400 INVALID_FIELD`), where the field map + * is available to judge against — not a SHAPE check. `FieldNodeSchema` stays + * `z.string()`, so every input valid before #7601 parses byte-identically after + * it. The probe below pins exactly that: it fails if someone "fixes" the docs + * defect by narrowing the type, which would move a semantic verdict into the + * shape check and refuse the registry-less internal callers the ingress + * deliberately tolerates. It does NOT assert that a dotted path RESOLVES — + * no driver ever implemented that, which is the defect #7601 corrected in the + * prose. Named for what it guards so it stops reading as a feature pin. */ export const fieldNodeString: FieldNode = 'name'; -export const fieldNodeDotted: FieldNode = 'owner.email'; +export const fieldNodeDottedNotNarrowed: FieldNode = 'owner.email'; // @ts-expect-error — a select entry is not a number export const fieldNodeNotANumber: FieldNode = 42;