diff --git a/.claude/workflows/docs-accuracy-audit.js b/.claude/workflows/docs-accuracy-audit.js index 849c878e9c..e4bc13c6f6 100644 --- a/.claude/workflows/docs-accuracy-audit.js +++ b/.claude/workflows/docs-accuracy-audit.js @@ -85,8 +85,10 @@ const ALL_HANDWRITTEN = [ "content/docs/data-modeling/field-types.mdx", "content/docs/data-modeling/fields.mdx", "content/docs/data-modeling/formulas.mdx", + "content/docs/data-modeling/import-mappings.mdx", "content/docs/data-modeling/index.mdx", "content/docs/data-modeling/indexing.mdx", + "content/docs/data-modeling/object-extensions.mdx", "content/docs/data-modeling/objects.mdx", "content/docs/data-modeling/queries.mdx", "content/docs/data-modeling/relationships.mdx", diff --git a/content/docs/data-modeling/import-mappings.mdx b/content/docs/data-modeling/import-mappings.mdx new file mode 100644 index 0000000000..712a37b734 --- /dev/null +++ b/content/docs/data-modeling/import-mappings.mdx @@ -0,0 +1,298 @@ +--- +title: Import Mappings +description: Named, reusable source-column to field projections for CSV/JSON/xlsx import, and the mappingName request that applies one. +--- + +# Import Mappings + +An **import mapping** is a named, reusable projection from the columns of a source +file onto the fields of one object. `mapping` is a first-class metadata kind, so a +mapping either ships inside a package or is saved at runtime — and either way +`POST /api/v1/data/:object/import` applies it by name with `mappingName`. + +Reach for one when the same file shape keeps arriving: a weekly export from another +system, whose column headers and codes are not yours. For a single ad-hoc file the +import wizard's inline column rename is enough — the two are different mechanisms with +different semantics, compared in [Inline `mapping` vs `mappingName`](#inline-mapping-vs-mappingname). + +--- + +## Declaring one in a package + +{/* os:check */} +```typescript +import { defineMapping } from '@objectstack/spec/data'; + +export const InquiryFeedMapping = defineMapping({ + name: 'showcase_inquiry_feed', // machine id — lowercase snake_case + label: 'Inquiry feed (marketing CSV)', + sourceFormat: 'csv', + targetObject: 'showcase_inquiry', + fieldMapping: [ + { source: 'Full Name', target: 'name' }, + { source: 'E-mail', target: 'email' }, + { source: 'Company', target: 'company' }, + { + source: 'Channel', + target: 'source', + transform: 'map', + params: { valueMap: { Webform: 'website', 'Partner Referral': 'referral' } }, + }, + ], + mode: 'upsert', + upsertKey: ['email'], +}); +``` + +Register it on the stack alongside the objects it targets: + +```typescript +export default defineStack({ + manifest: { /* … */ }, + objects: [Inquiry], + mappings: [InquiryFeedMapping], +}); +``` + +`defineStack()` validates the mapping while you build, not at first use: + +- `targetObject` must name an object the stack defines, otherwise the build fails with + `Mapping '' targets object '' which is not defined in objects.`; +- a `javascript` transform fails the build outright — see + [Transformations](#transformations-what-the-server-actually-does). + + + `defineMapping()` parses the config at import time, so a misspelled key + (`fields:` instead of `fieldMapping:`, `column:` instead of `source:`) is rejected + where you wrote it, with the canonical spelling named in the error. A bare + `: Mapping` annotation gets none of that. + + +--- + +## The two ways a mapping comes into being + +| Origin | How | Notes | +|:---|:---|:---| +| **Shipped in a package** | `defineMapping()` + `defineStack({ mappings })` | Versioned with the package; validated at build time. Packaged mappings are locked against tenant edits. | +| **Saved at runtime** | `PUT /api/v1/meta/mapping/` | The `mapping` kind is runtime-creatable, which is what lets an import wizard save the column choices a user just made as a reusable artifact. | + +Both end up resolvable by the same name. When the import endpoint resolves +`mappingName` it reads the runtime metadata rows first (org-scoped, then env-wide) and +falls back to the packaged registry — so a runtime-saved mapping and a packaged one are +addressed identically by the request. + +The `mapping` kind carries no per-organization overlay: there is no tenant-customized +variant of a packaged mapping. A tenant that needs different columns saves its own +mapping under its own name. + +--- + +## The shape + +| Key | Type | What it does | +|:---|:---|:---| +| `name` | `string` (snake_case) | The id `mappingName` resolves. Missing artifact → `404 MAPPING_NOT_FOUND`. | +| `label` | `string` | Display text in a saved-mapping picker. Falls back to `name`. | +| `sourceFormat` | `'csv' \| 'json' \| 'xml' \| 'sql'` | Declared payload format, checked against what the request actually sent. Defaults to `csv`. See [the format gate](#rejections-before-any-row-is-read). | +| `targetObject` | `string` | The object this mapping is for. Must equal the object in the URL. | +| `fieldMapping` | `ImportFieldMapping[]` | The projection itself — one entry per target field. | +| `mode` | `'insert' \| 'update' \| 'upsert'` | Supplies the request's `writeMode` when the request omits it. Defaults to `insert`. | +| `upsertKey` | `string[]` | Supplies the request's `matchFields` when the request omits them. | + +Each `fieldMapping` entry: + +| Key | Type | What it does | +|:---|:---|:---| +| `source` | `string \| string[]` | Source column header(s). An array is only meaningful for `join`. | +| `target` | `string \| string[]` | Target field name(s). An array is only meaningful for `split`. | +| `transform` | `TransformType` | Defaults to `'none'`. | +| `params` | object | Configuration for the transform — `value`, `valueMap`, `separator`. | + +The full generated property tables live in the +[Mapping schema reference](/docs/references/data/mapping). + +--- + +## Transformations: what the server actually does + +`TransformType` declares seven values. Five are executed row by row by the import +path, one is a deliberate pass-through, and one is refused: + +| `transform` | Behaviour | `params` it reads | +|:---|:---|:---| +| `none` | Copy the source cell to the target field. The default. | — | +| `constant` | Write a fixed value into the target field, ignoring the source column. | `value` | +| `map` | Translate the source system's codes into yours. A value with no entry in the table passes through unchanged. | `valueMap` | +| `split` | Split one column into several fields (`"John Doe"` into `first_name` / `last_name`). Each part is trimmed. | `separator` (default `' '`) | +| `join` | Compose one field from several columns. Empty and missing cells are dropped before joining. | `separator` (default `' '`) | +| `lookup` | **Pass-through.** The cell is copied unchanged, and the import's own reference resolution turns the display text into a record id afterwards — see [After the mapping](#after-the-mapping-cell-coercion). | — | +| `javascript` | **Refused.** There is no server-side sandbox, and silently skipping a declared transform would corrupt data. `defineStack()` fails the build; a runtime-saved mapping is rejected by the import request with `400 UNSUPPORTED_TRANSFORM`. | — | + + + The `lookup` transform's own `params` keys (`object`, `fromField`, `toField`, + `autoCreate`) parse, but the import path reads none of them: reference resolution is + driven by the **target object's field definitions**, not by the mapping. Do not write + them expecting them to steer anything — tracked as + [#10329](https://github.com/objectstack-ai/objectstack/issues/10329). + + +For logic beyond these, transform the data before you post it, or model it as a +[flow](/docs/automation) on the target object. + +--- + +## Applying it: `POST /api/v1/data/:object/import` + +The request that makes the mapping worth having is the one that names it: + +**`POST /api/v1/data/showcase_inquiry/import`** + +```json +{ + "format": "csv", + "csv": "Full Name,E-mail,Company,Channel\nAda Lovelace,ada@example.com,Analytical Engines,Webform\nAlan Turing,alan@example.com,NPL,Partner Referral\n", + "mappingName": "showcase_inquiry_feed" +} +``` + +No `mapping`, no `writeMode`, no `matchFields`: the artifact supplies all three. The +response is the standard import report — aggregate counters plus one entry per row: + +```json +{ + "object": "showcase_inquiry", + "dryRun": false, + "writeMode": "upsert", + "total": 2, + "ok": 2, + "errors": 0, + "created": 1, + "updated": 1, + "skipped": 0, + "results": [ + { "row": 1, "ok": true, "action": "created", "id": "inq_01HQ4A7B9D3F5G8J2K4L" }, + { "row": 2, "ok": true, "action": "updated", "id": "inq_01HQ3V5K8N2M4P6R7T9W" } + ] +} +``` + +`mappingName` works the same way on the three payload shapes the endpoint accepts — +`format: "csv"` with `csv` text, `format: "json"` with `rows[]`, and `format: "xlsx"` +with `xlsxBase64` — and on the asynchronous +`POST /api/v1/data/:object/import/jobs` route, which parses the identical body. + +Add `"dryRun": true` to get the same report with nothing persisted; the verdict comes +from the engine's own write-path validation. + + + `mappingName` is accepted on the wire but is **not** declared on the SDK's + `ImportRequest` type, so `client.data.import(object, { mappingName: '…' })` does not + type-check today. Issue the request directly until that is closed — + [#10330](https://github.com/objectstack-ai/objectstack/issues/10330). + + +### What the artifact contributes to the request + +| Request key | When omitted | When present | +|:---|:---|:---| +| `writeMode` | Falls back to the artifact's `mode` (when that is `update` or `upsert`). | The request wins. | +| `matchFields` | Falls back to the artifact's `upsertKey`. | The request wins. | + +Everything else on the request — `dryRun`, `runAutomations`, `treatAsHistorical`, +`trimWhitespace`, `nullValues`, `createMissingOptions`, `skipBlankMatchKey` — belongs +to the request alone. The mapping declares no error policy and no batch size; error +handling is per-row and reported per-row, and the write path sizes its own batches. + +--- + +## Inline `mapping` vs `mappingName` + +The endpoint accepts two mapping mechanisms and they are **mutually exclusive** — +sending both is `400 CONFLICTING_MAPPING`. + +| | inline `mapping` | `mappingName` | +|:---|:---|:---| +| Shape | A flat `{ "": "" }` rename (or a `sourceField`/`targetField` array) | A registered `mapping` artifact | +| Transforms | None — rename only | The `fieldMapping` pipeline | +| Unmapped columns | **Pass through** to the write path under their own header | **Dropped.** The artifact is a strict projection: only mapped targets survive | +| Lives | In the one request | In a package or in metadata, reusable | + +That projection difference is the one to keep in mind: a file from an external system +routinely carries columns that must not reach the write path, and the artifact path +guarantees they do not. + +--- + +## Rejections before any row is read + +These are whole-request failures — nothing is written, and there is no per-row report: + +| Status | `code` | Cause | +|:---|:---|:---| +| 404 | `MAPPING_NOT_FOUND` | No mapping artifact is registered under that name. | +| 400 | `MAPPING_TARGET_MISMATCH` | The artifact's `targetObject` is not the object in the URL. | +| 400 | `MAPPING_FORMAT_UNSUPPORTED` | The artifact declares `sourceFormat: 'xml'` or `'sql'`; the endpoint accepts csv, json and xlsx payloads only. | +| 400 | `MAPPING_FORMAT_MISMATCH` | The declared `sourceFormat` contradicts the payload actually sent. A `csv` mapping does apply to an `xlsx` payload — those rows are tabular in the same way — but a `json` mapping does not. | +| 400 | `UNSUPPORTED_TRANSFORM` | Some entry declares `javascript`, or a transform name the pipeline does not implement. | +| 400 | `CONFLICTING_MAPPING` | Both `mappingName` and an inline `mapping` were supplied. | +| 400 | `INVALID_REQUEST` | No recognizable payload, or `writeMode` resolved to update/upsert with no `matchFields` from either the request or the artifact. | +| 413 | `PAYLOAD_TOO_LARGE` | More than 5,000 rows on the synchronous route. | + +--- + +## Per-row outcomes + +Once the mapping is applied, every row gets its own verdict in `results[]`. A row that +fails does **not** stop the import. + +| `action` | `ok` | `code` | Meaning | +|:---|:---|:---|:---| +| `created` | `true` | — | Inserted. | +| `updated` | `true` | — | Matched an existing record and updated it. | +| `skipped` | `true` | `NO_MATCH` | `writeMode: 'update'` and the match fields matched nothing. Nothing is created. | +| `skipped` | `true` | `BLANK_MATCH_KEY` | The row's match fields are blank. Upsert creates such a row by default; `update` skips it, and `skipBlankMatchKey: true` skips it in either mode. | +| `failed` | `false` | `AMBIGUOUS_MATCH` | The match fields matched **more than one** record. Nothing is written for that row. | +| `failed` | `false` | a field error code | A cell could not be coerced, or the engine rejected the record. `field` names the column and `error` carries the message. | + +So "a row that does not match" is not one behaviour but three, chosen by `writeMode` +and by how many records matched: created (upsert), skipped (update), or failed +(more than one match). + +--- + +## After the mapping: cell coercion + +The mapping decides *which* value lands in *which* field. Converting that value into +what storage accepts is a separate step that runs afterwards, from the target object's +own field metadata: + +- **booleans** — spreadsheet spellings are accepted on both sides, including + non-English and check-mark cells; +- **numbers**, **dates** and **times** — parsed to storage form; an offset-free + datetime cell is read in the importing user's business timezone, which is the same + clock the export writes; +- **select / multiselect** — the human-visible option **label** resolves to the stored + option value (the translated label too, when the app is localized); +- **lookup / master\_detail / user** — the display text resolves to a record **id**. + This is what a `lookup` transform is relying on when it copies its cell through. + +A field the object does not know is left untouched. A cell that cannot be coerced fails +its own row with the offending column named. + +--- + +## Size limits + +| Route | Ceiling | +|:---|:---| +| `POST /api/v1/data/:object/import` | 5,000 rows — the report comes back in the response | +| `POST /api/v1/data/:object/import/jobs` | 50,000 rows — returns a `jobId`; poll progress and results, and the job can be cancelled or undone | + +--- + +## Related + +- **Schema reference:** [Mapping](/docs/references/data/mapping) — every property, generated from the spec +- **Neighbors:** [Objects](/docs/data-modeling/objects) · [Fields](/docs/data-modeling/fields) · [Seed Data & Fixtures](/docs/data-modeling/seed-data) for bundled data that is not an import +- **Wire format:** [REST wire format](/docs/api/wire-format) diff --git a/content/docs/data-modeling/index.mdx b/content/docs/data-modeling/index.mdx index 685705b262..8e8782127f 100644 --- a/content/docs/data-modeling/index.mdx +++ b/content/docs/data-modeling/index.mdx @@ -38,6 +38,8 @@ That one definition is enough to get a persisted table, CRUD + query endpoints, - **A compiled query AST** — queries are JSON documents validated against the protocol, then compiled by a driver into native queries with joins, aggregations, window functions, HAVING, and subqueries. The [query cheat sheet](/docs/data-modeling/queries) covers the syntax; the [spec](/docs/protocol/objectql/query-syntax) is normative. - **Four database drivers in this repo** — `driver-sql` (PostgreSQL / MySQL / SQLite via Knex), `driver-mongodb`, `driver-memory` (in-memory, for tests and demos), and `driver-sqlite-wasm` (SQLite in the browser / WebContainers). The same model runs unchanged on any of them. - **External datasource federation** — introspect an existing external database, import selected tables into the catalog, and query them alongside native objects ([External Datasources](/docs/data-modeling/external-datasources)). +- **Composable ownership** — one package owns an object, and any other package can merge fields, validation rules, and indexes into it without forking the definition ([Object Extensions](/docs/data-modeling/object-extensions)). +- **Repeatable bulk import** — a named mapping projects someone else's column headers and codes onto your fields, and the import endpoint applies it by name ([Import Mappings](/docs/data-modeling/import-mappings)). ## What's in this module @@ -47,11 +49,13 @@ That one definition is enough to get a persisted table, CRUD + query endpoints, + + diff --git a/content/docs/data-modeling/meta.json b/content/docs/data-modeling/meta.json index 8f867221a6..25bf32b75c 100644 --- a/content/docs/data-modeling/meta.json +++ b/content/docs/data-modeling/meta.json @@ -8,12 +8,14 @@ "field-types", "field-type-decision-tree", "relationships", + "object-extensions", "validation", "validation-rules", "formulas", "queries", "indexing", "seed-data", + "import-mappings", "external-datasources", "drivers", "analytics" diff --git a/content/docs/data-modeling/object-extensions.mdx b/content/docs/data-modeling/object-extensions.mdx new file mode 100644 index 0000000000..46bad08c7f --- /dev/null +++ b/content/docs/data-modeling/object-extensions.mdx @@ -0,0 +1,192 @@ +--- +title: Object Extensions +description: Add fields, validations and indexes to an object another package owns, without forking it — declaration, merge order, and what an extension may not contribute. +--- + +# Object Extensions + +Every object has exactly **one owning package**. That package defines the table, the +primary key and the core fields, and no second package may claim the same name — the +registry refuses it outright. + +An **object extension** is how a package contributes to an object it does not own. It +declares fields, validations and indexes that are merged into the target at boot, and +the result is indistinguishable from fields the owner had authored inline: the same +DDL, the same forms and list views, the same API. + +## When to reach for one + +| Situation | Do this | +|:---|:---| +| You need extra data on an object another package ships (`sys_user`, a CRM package's `account`) | **Object extension** | +| You need a new business entity of your own | Define your own [object](/docs/data-modeling/objects) | +| A single tenant needs a field only they use | Tenant customization, not a package extension — an extension ships in code with the package | +| You need an action, hook, view or page on someone else's object | Declare that artifact at the top level and bind it to the target object — [see below](#what-an-extension-may-not-contribute) | + +--- + +## Declaring one + +Extensions are declared on the **package**, in the `objectExtensions` collection — never +on the object schema. There is no `extends:` key on an object and no mixin mechanism. + +{/* os:check */} +```typescript +import { defineObjectExtension, Field } from '@objectstack/spec/data'; + +export const AccountSuccessExtension = defineObjectExtension({ + extend: 'showcase_account', // the target object, owned elsewhere + fields: { + loyalty_tier: Field.select(['bronze', 'silver', 'gold'], { label: 'Loyalty Tier' }), + csat_score: Field.number({ label: 'CSAT Score', min: 0, max: 100 }), + }, + priority: 210, +}); +``` + +```typescript +export default defineStack({ + manifest: { /* … */ }, + objectExtensions: [AccountSuccessExtension], +}); +``` + +The target object is owned by someone else, so `defineStack()` cannot verify it exists — +there is nothing in your package to check the name against. A typo therefore survives +the build and surfaces at boot instead, as a warning that the name has extenders but no +owner; the object is then skipped entirely. If an extension's fields never appear, +check that spelling first. + +--- + +## What an extension may contribute + +| Key | Merge behaviour | +|:---|:---| +| `fields` | **Additive.** Merged into the target's field map. A name the target already has is **replaced**, not merged. | +| `validations` | **Concatenated.** Your rules run alongside the owner's; neither replaces the other. | +| `indexes` | **Concatenated**, same as validations. | +| `label` | Replaces the target's label — with one exception, [below](#extensions-and-tenant-customization). | +| `pluralLabel` | Same as `label`. | +| `description` | Same as `label`. | +| `priority` | Not merged — it *orders* the merge. `0`–`999`, default `200`. | + +Added fields are ordinary fields, so everything on the +[field reference](/docs/data-modeling/fields) applies: types, validation, `required`, +`group`, formulas, and lookups back to your own objects. + +### What an extension may **not** contribute + +The merge carries the seven keys above and nothing else. These four are not +"unsupported yet" — there is no slot for them to arrive through, and the schema rejects +each one by name with the alternative to use instead: + +| Key you might reach for | Declare this instead | +|:---|:---| +| `actions` | A top-level action with `objectName: ''` — `defineStack()` attaches it to the object | +| `hooks` | A top-level hook bound to the target object | +| `listViews` | A top-level `view` bound to the target object | +| `fieldGroups` | Add the fields here and declare the groups on the owning object, or assign a Page for the layout | + +The same holds for any other key: the extension schema is strict, so an unknown key +fails at authoring time with a prescription rather than being dropped in silence. + +--- + +## Naming the fields you add + +Field names in an extension are **not** namespace-prefixed by the platform. The +namespace rule (`_`) governs the names of objects a package *defines*; +it does not walk `objectExtensions`, and it could not — the target's name belongs to +its owner, not to you. + +Nothing therefore stops two packages from contributing the same field name to the same +object, and nothing warns when they do: the merge is last-writer-wins by `priority`, so +one of the two silently disappears. Name defensively — prefix added fields with +something specific to your package when the name is at all generic. + +--- + +## Merge order and conflicts + +1. The registry picks the **base layer** for the object. +2. Every `extend` contribution is folded onto that base, in ascending `priority` order. +3. Within a fold, the rules in the table above apply: fields and scalars are + last-writer-wins, validations and indexes accumulate. + +So a **higher `priority` is applied later and wins** a conflict. Two extensions with the +same priority are folded in registration order, which is not a guarantee to build on — +give competing extensions distinct priorities. + +Merging is idempotent: re-registering a package (a metadata rebuild, a dev-server +reload) replaces that package's previous contribution rather than stacking a second copy. + +--- + +## Extensions and tenant customization + +Object contributions come in three kinds, and only the first two can be authored: + +| Kind | Who writes it | What it is | +|:---|:---|:---| +| `own` | The owning package | The base definition. Exactly one per object, always. | +| `extend` | Any other package | This page. Folded on top of the base. | +| `overlay` | Nobody, directly | A tenant customization layer, hydrated from stored metadata by the loader. It replaces the base at resolution time and owns nothing. | + +The overlay is the seam where a package extension meets a tenant's own edits: when an +overlay exists it *becomes* the base the extensions fold onto, so extension fields +survive a customized object rather than being dropped by it. + + + One deliberate asymmetry, ruled 2026-08-13: an extension's `label`, `pluralLabel` and + `description` apply only while the base still carries the **packaged owner's** value. + Once a tenant has renamed the object, the extension's packaged default yields and the + tenant's name stands. Fields, validations and indexes are unaffected — they merge + either way. There is no escape hatch: a package cannot relabel an object a tenant has + deliberately renamed. + + +--- + +## Worked example + +A customer-success package adds churn tracking to an account object owned by a CRM +package — fields, a rule and an index, without touching the CRM package's source: + +{/* os:check */} +```typescript +import { defineObjectExtension, Field } from '@objectstack/spec/data'; + +export const AccountChurnExtension = defineObjectExtension({ + extend: 'crm_account', + fields: { + cs_health_score: Field.number({ label: 'Health Score', min: 0, max: 100 }), + cs_renewal_date: Field.date({ label: 'Renewal Date' }), + cs_owner: Field.text({ label: 'Success Manager' }), + }, + validations: [ + { + name: 'cs_renewal_date_required_for_at_risk', + type: 'script', + severity: 'error', + message: 'An at-risk account needs a renewal date.', + // CEL predicate — TRUE means the record is invalid. + condition: '!isBlank(record.cs_health_score) && record.cs_health_score < 40 && isBlank(record.cs_renewal_date)', + }, + ], + indexes: [{ name: 'idx_crm_account_cs_renewal', fields: ['cs_renewal_date'] }], + priority: 300, +}); +``` + +Every added name carries the `cs_` prefix, so this package cannot collide with the CRM +package's own fields or with another extension's. `priority: 300` puts it after the +default `200`, so it wins against a lower-priority extension of the same object. + +--- + +## Related + +- **Spec:** [ObjectQL — Schema, Object Extensions](/docs/protocol/objectql/schema#object-extensions) — the normative protocol text +- **Schema reference:** [Object](/docs/references/data/object) — every property, generated from the spec +- **Neighbors:** [Objects](/docs/data-modeling/objects) · [Fields](/docs/data-modeling/fields) · [Validation](/docs/data-modeling/validation) · [Database Indexing](/docs/data-modeling/indexing)