From e8de3e1ddb3d696c5c3c3dd53ef11f68da4e1a01 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 30 Jul 2026 04:46:55 +0000 Subject: [PATCH 1/3] fix(driver-sql): Field.date NOW() default reads the UTC calendar day on PG/MySQL (#4022); raise cloud-connection seed-lookup test timeout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two follow-ups from the #3994 line: 1. Field.date + defaultValue NOW() emitted a bare CURRENT_TIMESTAMP default, which resolves the calendar day in the SERVER's timezone on Postgres — measured: a UTC-12 server recorded yesterday's date; an Asia/Shanghai server records tomorrow for every default after 16:00 UTC. MySQL 8.0 additionally rejects the bare default on a DATE column outright (MariaDB is merely permissive, and the driver's UTC-pinned session masked the semantic half there). nowColumnDefault now emits a UTC expression default on both dialects — timezone('utc', now())::date on Postgres, (cast(utc_timestamp() as date)) on MySQL — the #3994 D-C3 construction one type over. Defaults only govern newly created columns (D-B3 policy). Live tests pin the expression and a defaulted-insert round-trip. 2. marketplace-install-local-seed-lookup.test.ts: 30s → 120s timeout. The test cold-imports the real @objectstack/runtime on purpose; under a fully parallel turbo run that import stalls on core contention and the 30s cap produced recurring false reds (observed 3x, passes standalone). A genuine hang still fails, just later. Closes #4022 Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01TPxNwPnjcn599ujXpU3ibJ --- .changeset/date-now-default-utc.md | 15 ++++ docs/adr/0053-date-and-datetime-semantics.md | 9 ++ ...ketplace-install-local-seed-lookup.test.ts | 7 +- .../sql-driver-date-now-default-live.test.ts | 90 +++++++++++++++++++ packages/plugins/driver-sql/src/sql-driver.ts | 12 +++ 5 files changed, 131 insertions(+), 2 deletions(-) create mode 100644 .changeset/date-now-default-utc.md create mode 100644 packages/plugins/driver-sql/src/sql-driver-date-now-default-live.test.ts diff --git a/.changeset/date-now-default-utc.md b/.changeset/date-now-default-utc.md new file mode 100644 index 0000000000..566a491d7f --- /dev/null +++ b/.changeset/date-now-default-utc.md @@ -0,0 +1,15 @@ +--- +"@objectstack/driver-sql": patch +--- + +fix(driver-sql): `Field.date` + `defaultValue: 'NOW()'` records the UTC calendar day on Postgres/MySQL (#4022) + +The bare `CURRENT_TIMESTAMP` default resolved the calendar day in the SERVER's +timezone on Postgres — measured: a UTC-12 server recorded yesterday; an +Asia/Shanghai server records tomorrow for every default after 16:00 UTC — and +MySQL 8.0 rejects it on a DATE column outright (MariaDB is merely permissive, +and the driver's UTC-pinned session masked the semantic half there). +`nowColumnDefault` now emits a UTC expression default on both dialects, the +#3994 D-C3 construction one type over. Defaults only govern newly created +columns; existing columns keep their legacy default, per the standing D-B3 +policy. diff --git a/docs/adr/0053-date-and-datetime-semantics.md b/docs/adr/0053-date-and-datetime-semantics.md index aa07c8ab5c..3fa03052a3 100644 --- a/docs/adr/0053-date-and-datetime-semantics.md +++ b/docs/adr/0053-date-and-datetime-semantics.md @@ -643,6 +643,15 @@ canonical `.000` trim on SQLite, `timezone('utc', now())::time(3)` on Postgres, `(cast(utc_timestamp(3) as time(3)))` on MySQL (8.0.13+/MariaDB 10.2+ for the expression-default syntax) — all reading the UTC clock. +The same construction closes the `Field.date` half of the gap (#4022): +`nowColumnDefault('date')` emits `timezone('utc', now())::date` on Postgres and +`(cast(utc_timestamp() as date))` on MySQL. Measured: the bare +`CURRENT_TIMESTAMP` default recorded YESTERDAY's calendar day on a UTC-12 +Postgres server, and MySQL 8.0 rejects it on a DATE column outright (the +driver's UTC-pinned session masked the semantic half on MariaDB). Legacy +columns keep their old default — defaults only govern newly created columns, +the standing policy since D-B3. + The D-B3 caveat applies unchanged: the backfill converges *shapes*; a wall clock the old path never recorded correctly (a pg `Date` bind serialised in the host's zone) is not recoverable. Regression cover: diff --git a/packages/cloud-connection/src/marketplace-install-local-seed-lookup.test.ts b/packages/cloud-connection/src/marketplace-install-local-seed-lookup.test.ts index c8edce4e78..8030616180 100644 --- a/packages/cloud-connection/src/marketplace-install-local-seed-lookup.test.ts +++ b/packages/cloud-connection/src/marketplace-install-local-seed-lookup.test.ts @@ -192,8 +192,11 @@ afterEach(() => { rmSync(dir, { recursive: true, force: true }); vi.restoreAllMo describe('marketplace install — seed lookup resolution', () => { // Generous timeout: the install handler dynamically imports the real // @objectstack/runtime (unmocked on purpose), and that cold import alone - // can eat several seconds on a fresh CI runner. - it('writes target record ids (never raw externalId strings) for package objects unknown to the metadata service', { timeout: 30_000 }, async () => { + // can eat several seconds on a fresh CI runner — and multiples of that + // under a fully parallel turbo run, where this test competes with every + // other package's suite for cores. 30s was observed flaking exactly that + // way (an import stall, not a hang); a genuine hang still fails, just later. + it('writes target record ids (never raw externalId strings) for package objects unknown to the metadata service', { timeout: 120_000 }, async () => { const { engine, store, registry } = makeEngine(); const rawApp = makeRawApp(); const { ctx, fire } = makeCtx(rawApp, { diff --git a/packages/plugins/driver-sql/src/sql-driver-date-now-default-live.test.ts b/packages/plugins/driver-sql/src/sql-driver-date-now-default-live.test.ts new file mode 100644 index 0000000000..3527249a22 --- /dev/null +++ b/packages/plugins/driver-sql/src/sql-driver-date-now-default-live.test.ts @@ -0,0 +1,90 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * #4022 — `Field.date` + `defaultValue: 'NOW()'` on live Postgres and MySQL. + * + * The bare `CURRENT_TIMESTAMP` default `knex.fn.now()` emits resolves the + * calendar day in the SERVER's timezone on Postgres — measured: with the server + * at UTC-12, a defaulted insert on UTC's 2026-07-30 stored `2026-07-29`. On an + * Asia/Shanghai production server every default after 16:00 UTC records + * TOMORROW. MySQL 8.0 rejects the bare default on a DATE column outright + * (MariaDB is merely permissive, and the driver's UTC-pinned session masked + * the semantic half there). + * + * The fix is an expression default reading the UTC clock (`nowColumnDefault`), + * the #3994 D-C3 construction one type over. The wrong-day scenario cannot be + * asserted deterministically in CI (it depends on the time of day relative to + * the server's offset), so these tests pin the two things that CAN be: + * the DDL builds — which on MySQL 8.0 is itself the compatibility fix — with a + * default expression that spells `utc`, and a defaulted insert round-trips a + * valid calendar day. + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import { SqlDriver } from '../src/index.js'; + +const PG_URL = process.env.OS_TEST_POSTGRES_URL; +const MY_URL = process.env.OS_TEST_MYSQL_URL; +const TABLE = 'os4022_probe'; + +const SHAPE = { + name: TABLE, + fields: { + label: { type: 'string' }, + due_on: { type: 'date', defaultValue: 'NOW()' }, + }, +} as any; + +function suite(dialect: 'pg' | 'mysql', url: string | undefined) { + describe.skipIf(!url)(`Field.date NOW() default on live ${dialect} (#4022)`, () => { + let driver: SqlDriver; + + beforeEach(async () => { + driver = new SqlDriver( + dialect === 'pg' + ? { client: 'pg', connection: url } + : { client: 'mysql2', connection: url }, + ); + await driver.execute(`drop table if exists ${TABLE}`).catch(() => {}); + await driver.initObjects([SHAPE]); + }); + + afterEach(async () => { + await driver.execute(`drop table if exists ${TABLE}`).catch(() => {}); + await driver.disconnect(); + }); + + it('builds with a UTC expression default, not a bare CURRENT_TIMESTAMP', async () => { + // On MySQL 8.0 reaching this line at all is the compatibility half of the + // fix — the bare default fails CREATE TABLE there. + const res: any = await driver.execute( + dialect === 'pg' + ? `select column_default as d from information_schema.columns + where table_name = '${TABLE}' and column_name = 'due_on'` + : `select column_default as d from information_schema.columns + where table_schema = database() and table_name = ? and column_name = 'due_on'`, + dialect === 'pg' ? [] : [TABLE], + ); + const rows = Array.isArray(res) && Array.isArray(res[0]) ? res[0] : (res?.rows ?? res); + const def = String((rows[0] as any).d ?? (rows[0] as any).D ?? '').toLowerCase(); + expect(def).toContain('utc'); + expect(def).not.toBe('current_timestamp'); + }); + + it('a defaulted insert round-trips a valid canonical calendar day', async () => { + await driver.create(TABLE, { id: 'x', label: 'x' }, { bypassTenantAudit: true }); + const row: any = await driver.findOne(TABLE, 'x', { bypassTenantAudit: true }); + expect(String(row.due_on)).toMatch(/^\d{4}-\d{2}-\d{2}$/); + // The UTC calendar day only ever differs from this probe's own UTC clock + // across a midnight boundary — accept today or the day either side rather + // than flake at 00:00, while still catching a ±12h zone leak on any run + // that is not within a day-boundary race. + const utcToday = Date.parse(new Date().toISOString().slice(0, 10)); + const stored = Date.parse(String(row.due_on)); + expect(Math.abs(stored - utcToday)).toBeLessThanOrEqual(24 * 3600 * 1000); + }); + }); +} + +suite('pg', PG_URL); +suite('mysql', MY_URL); diff --git a/packages/plugins/driver-sql/src/sql-driver.ts b/packages/plugins/driver-sql/src/sql-driver.ts index f3fbd919b9..5370ff7c6a 100644 --- a/packages/plugins/driver-sql/src/sql-driver.ts +++ b/packages/plugins/driver-sql/src/sql-driver.ts @@ -5107,6 +5107,18 @@ export class SqlDriver implements IDataDriver { if (this.isMysql) return this.knex.raw('(cast(utc_timestamp(3) as time(3)))'); if (this.isPostgres) return this.knex.raw("(timezone('utc', now())::time(3))"); } + // `date` has the same two problems one type over (#4022): a bare + // CURRENT_TIMESTAMP default resolves the calendar day in the SERVER's + // timezone on Postgres (measured: a UTC-12 server records YESTERDAY), + // and MySQL 8.0 rejects it on a DATE column outright (MariaDB is merely + // permissive; the driver's UTC-pinned session masked the semantic half + // there). Same fix as `time`: an expression default reading the UTC + // clock, which is also what `formatInput`'s NOW() safety-net and the + // SQLite branch below store. + if (type === 'date') { + if (this.isMysql) return this.knex.raw('(cast(utc_timestamp() as date))'); + if (this.isPostgres) return this.knex.raw("(timezone('utc', now())::date)"); + } return this.knex.fn.now(); } switch (type) { From 0c951225503995ec039645dbbc4c74690b82fd72 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 30 Jul 2026 07:08:54 +0000 Subject: [PATCH 2/3] docs: correct temporal-field storage and filter semantics in the ObjectQL reference MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The temporal line (#3912 #3928 #3942 #3954 #3979 #3994 #4022) changed what a Field.date / Field.datetime / Field.time physically IS, and three hand-written docs still described the pre-#3912 world. Audited against the implementation (sql-driver.ts canonicalUtcDatetime / canonicalTimeOfDay / toDateOnly / nowColumnDefault / createColumn) and ADR-0053 addenda D-B1..D-B4, D-C1..D-C3, with an adversarial verification pass. - protocol/objectql/types.mdx: per-type storage form and physical column type per dialect — canonical UTC ISO TEXT on SQLite, timestamptz on Postgres, DATETIME(3) on MySQL (with why NOT TIMESTAMP: 2038 ceiling, dropped milliseconds, session-tz conversion); time as HH:MM:SS gaining .fff only when the milliseconds are non-zero; the MySQL literal respelling; and a new defaultValue: 'NOW()' section stating the UTC-clock rule. - protocol/objectql/query-syntax.mdx: the comparand table (a filter value is canonicalised by the same function as a write, so both sides of a comparison share one shape on every dialect) and the date-vs-datetime range trap — a $between of bare dates against a datetime column stops at midnight. - data-modeling/queries.mdx: the same comparand rules at the query-authoring altitude, plus UTC bucket boundaries for date bucketing. Scope note: nine further docs were audited in the same pass but their agents ran against a stale base commit that predated this work, so their output described superseded behaviour and is deliberately NOT included. Their pristine text carries no incorrect temporal claim — only looser wording — so nothing is left wrong by omitting them. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01TPxNwPnjcn599ujXpU3ibJ --- .changeset/temporal-docs-accuracy.md | 6 + content/docs/data-modeling/queries.mdx | 115 +++++- .../docs/protocol/objectql/query-syntax.mdx | 335 ++++++++++++----- content/docs/protocol/objectql/types.mdx | 349 +++++++++++++----- 4 files changed, 601 insertions(+), 204 deletions(-) create mode 100644 .changeset/temporal-docs-accuracy.md diff --git a/.changeset/temporal-docs-accuracy.md b/.changeset/temporal-docs-accuracy.md new file mode 100644 index 0000000000..4f4747f65c --- /dev/null +++ b/.changeset/temporal-docs-accuracy.md @@ -0,0 +1,6 @@ +--- +--- + +docs: correct the temporal-field storage and filter semantics in the ObjectQL type/query reference (#3912 #3942 #3994 #4022) + +Documentation only — no package changes, so this changeset releases nothing. diff --git a/content/docs/data-modeling/queries.mdx b/content/docs/data-modeling/queries.mdx index 69d71c59fa..1a10a54d1e 100644 --- a/content/docs/data-modeling/queries.mdx +++ b/content/docs/data-modeling/queries.mdx @@ -61,6 +61,7 @@ const query = { | Operator | Description | Example | |:---|:---|:---| | `$contains` | String contains substring | `{ title: { $contains: 'urgent' } }` | +| `$notContains` | String does NOT contain substring | `{ title: { $notContains: 'spam' } }` | | `$startsWith` | String starts with | `{ email: { $startsWith: 'admin' } }` | | `$endsWith` | String ends with | `{ email: { $endsWith: '@acme.com' } }` | @@ -73,6 +74,41 @@ const query = { --- +## Date, Datetime & Time Comparands + +On SQL-backed objects, filter values on temporal fields are **canonicalised by the same +functions the driver's write path uses**, so the two sides of a comparison can never +disagree about shape: + +| Field type | Canonical comparand | Semantics | +|:---|:---|:---| +| `date` | `YYYY-MM-DD` | Timezone-naive calendar day. A `Date` collapses to its UTC day; a longer ISO string is truncated to its leading date. | +| `datetime` | `YYYY-MM-DDTHH:MM:SS.sssZ` | A UTC instant. A `Date`, an epoch-millisecond number, a bare `YYYY-MM-DD` (→ midnight UTC) and a zone-naive `YYYY-MM-DD HH:MM:SS` (→ read as UTC) all fold to this one form. | +| `time` | `HH:MM:SS`, with `.fff` **only** when the milliseconds are non-zero | Timezone-naive wall clock. `'14:30'` and `'14:30:00'` canonicalise identically, so they are the same filter. | + +```typescript +{ + object: 'meeting', + where: { + meeting_day: { $gte: '2026-01-01' }, // date → '2026-01-01' + starts_at: { $gte: '2026-01-01T00:00:00Z' }, // datetime → '2026-01-01T00:00:00.000Z' + start_time: { $between: ['09:00', '18:00'] } // time → '09:00:00' / '18:00:00' + } +} +``` + + +Canonicalisation runs on **every SQL dialect**, not just SQLite — a zone-naive string bound +into a Postgres `timestamptz` would otherwise be read in the *server's* timezone. MySQL +is the one dialect that cannot parse the `T`/`Z` spelling, so the driver binds the same +instant as a MySQL datetime literal — a physical respelling only; every layer above the +bind (filter authoring, API payloads, CEL) stays on the canonical `…Z` form. Values the +driver cannot interpret (empty strings, junk) pass through untouched rather than being +silently rewritten. + + +--- + ## Logical Operators ### `$and` — All conditions must match @@ -167,7 +203,7 @@ Sort results with `orderBy` (array of sort nodes): } ``` -### Cursor-Based Pagination (Keyset) +### Cursor-Based Pagination (Keyset) — not implemented ```typescript { @@ -176,9 +212,13 @@ Sort results with `orderBy` (array of sort nodes): } ``` - -`cursor` is an opaque record (`Record`) carrying the keyset -position from the previous page. There is no `keyset`/`after` query property. + +`cursor` is declared on `QuerySchema` as an opaque record (`Record`) +and `QueryBuilder.cursor()` will set it on a query, but **nothing on the server reads +it** — the query engine, the REST query dispatcher, and the SQL / in-memory / MongoDB +drivers all ignore it, so a cursor query silently returns the same first page every +time. Use `limit` + `offset` until keyset pagination is wired up. (There is no +`keyset`/`after` query property either.) --- @@ -211,17 +251,27 @@ position from the previous page. There is no `keyset`/`after` query property. This object form of a field node isn't wired up for top-level `fields` projection. When the object's schema is registered, the engine's unknown-field filter compares each entry against the schema's field names via `String(f)`, so an object entry never matches and is silently -dropped from the projection — the aliased field is simply missing from results, no error. Use -`expand` to pull in a relationship's fields instead. +dropped from the projection — the aliased field is simply missing from results, no error. + +Dotted relationship paths (`'owner.name'`) fare no better: the unknown-field filter validates +only the **head** segment and keeps the path, but there is no populate step anywhere in the +engine, so the SQL driver selects `owner.name` verbatim and the database rejects it. What you +get back depends on whether the driver recognises that dialect's error text (`no such column`, +or `column … does not exist`): if it does, its recovery retry re-runs the query as `SELECT *` +and you get every column; if it doesn't, the error is rethrown. Either way there is never an +`owner.name` key. + +Use `expand` to pull in a relationship's fields instead. --- ## Expand (Related Records) -Load related records through `lookup` / `master_detail` fields with `expand`. -Each key is a relationship field name; the value is a nested query that can -select fields, filter, and expand further (default max depth: 3). +Load related records through `lookup`, `master_detail`, and `user` fields with +`expand`. Each key is a relationship field name; the value is a nested query that +can select fields, filter, and expand further (max depth 3 — a fixed constant, not +configurable). ```typescript { @@ -239,8 +289,12 @@ select fields, filter, and expand further (default max depth: 3). ``` The engine resolves `expand` via batch `$in` queries (driver-agnostic), so it -works on every driver. Per-parent `limit` / `offset` / `orderBy` are **not** -applied on this path. +works on every driver. A nested `limit` / `offset` is **not forwarded at all** — +one batch query serves every parent, so a per-parent window cannot be expressed. +A nested `orderBy` *is* forwarded to that batch query, but it has no observable +effect: the expanded records are re-keyed to each parent by id, so a multi-value +relationship keeps the order stored on the parent record. Paginate or sort by +querying the related object directly. --- @@ -481,6 +535,38 @@ As noted under [Aggregations](#aggregations) above, `having` is not currently en it's dropped before reaching the aggregation engine or any driver. +### Date Bucketing in `groupBy` + +A `groupBy` entry is either a bare field name or a structured node that buckets a +`date` / `datetime` column into uniform periods: + +```typescript +{ + object: 'deal', + aggregations: [ + { function: 'sum', field: 'amount', alias: 'revenue' } + ], + groupBy: ['stage', { field: 'closed_at', dateGranularity: 'quarter' }] +} +``` + +`dateGranularity` accepts `day` | `week` | `month` | `quarter` | `year`. The engine +pushes bucketing down to the driver only when that driver advertises the granularity +via `supports.queryDateGranularity` — SQLite reports `week: false`, for instance, so a +weekly bucket falls back to fetching rows and bucketing in memory. The fallback is +transparent to the query — you get the same buckets either way. + + +Buckets are computed in **UTC**. A non-UTC reference timezone is only reachable through +`ObjectQL.aggregate(object, { …, timezone })`; the `POST /api/v1/data/:object/query` +route forwards only `where` / `groupBy` / `aggregations`, so a query sent over REST +always buckets on UTC calendar boundaries. + +The optional `alias` on a structured `groupBy` node is honoured only on the in-memory +path. The SQL driver ignores it and always projects the bucket under the **field name**, +so don't depend on the aliased key being present. + + --- ## Common Query Patterns @@ -530,9 +616,12 @@ it's dropped before reaching the aggregation engine or any driver. ```typescript { object: 'activity', - fields: ['id', 'type', 'description', 'user.name', 'created_at'], + fields: ['id', 'type', 'description', 'user', 'created_at'], + expand: { + user: { object: 'user', fields: ['name'] } + }, where: { - created_at: { $gte: '2026-01-01T00:00:00Z' } + created_at: { $gte: '2026-01-01T00:00:00Z' } // canonicalised to '…T00:00:00.000Z' }, orderBy: [{ field: 'created_at', order: 'desc' }], limit: 10 diff --git a/content/docs/protocol/objectql/query-syntax.mdx b/content/docs/protocol/objectql/query-syntax.mdx index cca3c6a3d3..4bf51d7e28 100644 --- a/content/docs/protocol/objectql/query-syntax.mdx +++ b/content/docs/protocol/objectql/query-syntax.mdx @@ -30,10 +30,12 @@ import type { QueryAST } from '@objectstack/spec/data'; const query: QueryAST = { object: 'contact', - fields: ['name', 'email', { field: 'account', fields: ['company_name'] }], - where: { - is_active: true, - 'account.industry': 'tech', + fields: ['name', 'email', 'account'], + where: { is_active: true }, + // Related records are loaded through `expand` — not through a JOIN and not + // through a dotted `account.industry` path (see §2 and §4). + expand: { + account: { object: 'account', fields: ['company_name'] }, }, orderBy: [{ field: 'created_at', order: 'desc' }], limit: 10, @@ -41,7 +43,7 @@ const query: QueryAST = { ``` **Runtime compilation:** -- PostgreSQL / MySQL → Optimized SQL with JOINs (`@objectstack/driver-sql`) +- PostgreSQL / MySQL → parameterised single-table SQL (`@objectstack/driver-sql`); related records are a second, batched `$in` read, not a JOIN - MongoDB → Native queries + aggregation pipeline (`@objectstack/driver-mongodb`) - SQLite → Portable SQL, in-process or in-browser via `@objectstack/driver-sqlite-wasm` - In-Memory → In-process evaluation, no external database (`@objectstack/driver-memory`) @@ -70,7 +72,7 @@ interface QueryAST { cursor?: Record; // Keyset pagination cursor joins?: JoinNode[]; // Explicit JOINs aggregations?: AggregationNode[]; // Aggregation functions - groupBy?: string[]; // GROUP BY fields + groupBy?: GroupByNode[]; // GROUP BY targets (string | object) having?: FilterCondition; // HAVING clause windowFunctions?: WindowFunctionNode[]; // Window functions (OVER) distinct?: boolean; // SELECT DISTINCT @@ -78,6 +80,29 @@ interface QueryAST { } ``` + +**The protocol shape is wider than what the data engine executes.** + +`QuerySchema` validates the whole structure above, but `IDataEngine.find()` plus the +shipped drivers run a subset. `SqlDriver.find()` builds only `where` / `orderBy` / +`limit` / `offset` / `fields` (`packages/plugins/driver-sql/src/sql-driver.ts`), and +`expand` is resolved afterwards by the engine as a batched `$in` read +(`packages/objectql/src/engine.ts`). These members validate but are **not executed** +on the `find()` path: + +| Member | Status | +|:-------|:-------| +| `joins` | No driver reads it — there is no `query.joins` / `ast.joins` consumer anywhere in `packages/` | +| `having` | Never read: `engine.aggregate()` forwards only `object`/`where`/`groupBy`/`aggregations`, and the in-memory fallback has no HAVING stage | +| `cursor` | Accepted by `EngineQueryOptions`, but no driver implements keyset pagination | +| `distinct` | Not applied by `find()`; the SQL and in-memory drivers expose a separate `distinct(object, field, filters?)` method instead | +| `windowFunctions` | Not applied by `find()`; `SqlDriver.findWithWindowFunctions()` is a separate method that is not on the `IDataDriver` contract | +| Object `FieldNode`s (`{ field, fields }`) | The engine drops non-string entries from `fields` — use `expand` to project related records | +| `search.fuzzy` / `boost` / `operator` / `minScore` / `language` / `highlight` | Parsed by the schema but ignored: only `query` and `fields` drive the expansion | + +`top` is the exception that *is* honored — the engine normalises it to `limit`. + + ### Key Types ```typescript @@ -93,16 +118,23 @@ interface AggregationNode { | 'count_distinct' | 'array_agg' | 'string_agg'; field?: string; // optional for COUNT(*) alias: string; // result column alias - distinct?: boolean; // DISTINCT before aggregation - filter?: FilterCondition; // FILTER WHERE clause + distinct?: boolean; // DISTINCT before aggregation — in-memory path only + filter?: FilterCondition; // FILTER WHERE clause — protocol only, never applied } // FieldNode — field selection type FieldNode = string | { field: string; - fields?: FieldNode[]; // nested select + fields?: FieldNode[]; // nested select — protocol only, dropped by the engine alias?: string; }; + +// GroupByNode — GROUP BY target +type GroupByNode = string | { + field: string; + dateGranularity?: 'day' | 'week' | 'month' | 'quarter' | 'year'; + alias?: string; // defaults to `field` +}; ``` --- @@ -202,7 +234,7 @@ const query: QueryAST = { | `$notContains` | String does not contain | `{ name: { $notContains: 'test' } }` | | `$startsWith` | String starts with | `{ email: { $startsWith: 'admin' } }` | | `$endsWith` | String ends with | `{ domain: { $endsWith: '.com' } }` | -| `$between` | Range (inclusive) | `{ created_at: { $between: ['2024-01-01', '2024-12-31'] } }` | +| `$between` | Range (inclusive) | `{ close_date: { $between: ['2024-01-01', '2024-12-31'] } }` | | `$null` | Null check | `{ manager_id: { $null: true } }` / `{ phone: { $null: false } }` | | `$exists` | Field exists (NoSQL) | `{ metadata: { $exists: true } }` | @@ -279,7 +311,7 @@ const query: QueryAST = { const query: QueryAST = { object: 'opportunity', where: { - 'account.industry': 'tech', // AND (industry = tech) + type: 'new_business', // AND (type = new_business) $or: [ // AND ( { amount: { $gt: 100000 } }, // amount > 100000 { is_strategic: true }, // OR is_strategic = true @@ -287,25 +319,57 @@ const query: QueryAST = { }, }; -// SQL: WHERE account.industry = 'tech' +// SQL: WHERE type = 'new_business' // AND (amount > 100000 OR is_strategic = true) ``` -### Date Filters +### Date, Datetime, and Time Filters + +Before a comparison is built, the driver puts the comparand into the **same canonical +form the column is stored in** (`SqlDriver.coerceFilterValue`) — the identical function +the write path uses, on every dialect, so the two sides of a comparison can never be +decided by their shapes disagreeing: + +| Field type | Canonical form | Meaning | +|:-----------|:---------------|:--------| +| `date` | `YYYY-MM-DD` | Timezone-naive calendar day | +| `datetime` | `YYYY-MM-DDTHH:MM:SS.sssZ` | A UTC instant | +| `time` | `HH:MM:SS` — `.fff` only when the milliseconds are non-zero | Timezone-naive wall-clock time of day | {/* os:check */} ```typescript -// Specific date -where: { created_at: '2024-01-15' } +import type { FilterCondition } from '@objectstack/spec/data'; + +// `date` field — a bare calendar day matches that day +const onDay: FilterCondition = { close_date: '2024-01-15' }; + +// `date` range — $between is inclusive on both ends +const inYear: FilterCondition = { + close_date: { $between: ['2024-01-01', '2024-12-31'] }, +}; + +// `datetime` field — a bare `YYYY-MM-DD` is completed to midnight UTC +// (`2024-01-15T00:00:00.000Z`), so this is an exact-instant match, not "that day" +const atMidnight: FilterCondition = { created_at: '2024-01-15' }; -// Date range with $between -where: { created_at: { $between: ['2024-01-01', '2024-12-31'] } } +// A whole UTC day on a `datetime` needs a half-open range +const duringDay: FilterCondition = { + created_at: { $gte: '2024-01-15', $lt: '2024-01-16' }, +}; -// Comparison operators on dates -where: { due_date: { $lt: '2024-06-01' } } -where: { created_at: { $gte: '2024-01-01' } } +// `time` field — `'09:00'` is completed to `'09:00:00'` +const businessHours: FilterCondition = { + start_time: { $gte: '09:00', $lte: '18:00' }, +}; ``` + +`created_at` / `updated_at` are `datetime` (UTC instants), not calendar days. A +`$between: ['2024-01-01', '2024-12-31']` against a `datetime` column stops at +`2024-12-31T00:00:00.000Z` and therefore excludes almost all of December 31 — use +`{ $gte: '2024-01-01', $lt: '2025-01-01' }` instead. + + ### Null Checks {/* os:check */} @@ -320,20 +384,29 @@ where: { phone: { $null: false } } where: { metadata: { $exists: true } } ``` -### Nested Relation Filters +### Filtering Across Relationships -Filter through relationships without explicit joins: + +**Relation traversal inside `where` is not supported.** Neither the nested form +(`where: { account: { industry: 'tech' } }`) nor a dotted path +(`where: { 'account.industry': 'tech' }`) is resolved. `SqlDriver.applyFilters()` only +recognises a nested object as an operator map when its keys start with `$`; anything +else is compiled as a comparison against a single column of the queried table, and a +dotted key is emitted verbatim, so Knex renders it as `"account"."industry"` against a +table that was never joined. + + +Filter on the local foreign key, or run two queries: ```typescript -const query: QueryAST = { - object: 'opportunity', - where: { - account: { - industry: 'tech', - annual_revenue: { $gt: 1000000 }, - }, - }, -}; +const techAccounts = await engine.find('account', { + where: { industry: 'tech', annual_revenue: { $gt: 1000000 } }, + fields: ['id'], +}); + +const opportunities = await engine.find('opportunity', { + where: { account_id: { $in: techAccounts.map((a) => a.id) } }, +}); ``` --- @@ -367,22 +440,22 @@ const query: QueryAST = { // SQL: ORDER BY priority DESC, created_at ASC ``` -### Sort on Related Fields - -```typescript -const query: QueryAST = { - object: 'contact', - orderBy: [{ field: 'account.company_name', order: 'asc' }], -}; +### Sorting on Related Fields -// SQL: ORDER BY account.company_name ASC -``` + +`orderBy` only reaches columns of the queried table. A dotted path +(`account.company_name`) is handed to Knex verbatim and renders as +`"account"."company_name"`, which the database rejects as an unknown column; +`SqlDriver.find()` then retries **without the sort**, so the rows come back unordered +rather than sorted — and no error surfaces. Denormalise the value onto the queried +object (for example with a formula or rollup field) when you need to sort by it. + --- ## 4. Relationships (Expand) -The `expand` property enables **recursive loading of related records** through lookup and master_detail fields. Each key is a relationship field name; the value is a nested `QueryAST`. +The `expand` property enables **recursive loading of related records** through `lookup`, `master_detail`, and `user` fields. Each key is a relationship field name; the value is a nested `QueryAST`. ### Basic Expand @@ -445,9 +518,9 @@ The engine resolves `expand` via batch `$in` queries (driver-agnostic) with a de ### Filtered Expand -Expansion follows **`lookup`** and **`master_detail`** fields — i.e. the foreign key -lives on the object you are querying. The nested `QueryAST` can **filter** (`where`) and -**select** (`fields`) the related records: +Expansion follows **`lookup`**, **`master_detail`**, and **`user`** fields — i.e. the +foreign key lives on the object you are querying. The nested `QueryAST` can **filter** +(`where`) and **select** (`fields`) the related records: ```typescript const query: QueryAST = { @@ -542,7 +615,14 @@ const query: QueryAST = { // { count: 100, total: 5000000, average: 50000, min_amount: 10000, max_amount: 500000 } ``` -**Supported functions:** `count`, `sum`, `avg`, `min`, `max`, `count_distinct`, `array_agg`, `string_agg` +**Schema enum:** `count`, `sum`, `avg`, `min`, `max`, `count_distinct`, `array_agg`, `string_agg`. + + +Only `count`, `sum`, `avg`, `min`, and `max` are portable. `SqlDriver.mapAggregateFunc()` +throws `Unsupported aggregate function: ` for `count_distinct`, `array_agg`, and +`string_agg`; those three are implemented by the MongoDB driver and by the engine's +in-memory aggregation fallback, but not by the SQL drivers. + ### Group By Multiple Fields @@ -558,64 +638,106 @@ const query: QueryAST = { }; ``` -### HAVING Clause +### HAVING Clause (protocol only) -Filter groups after aggregation using the `having` property: +`QuerySchema` defines a `having` property, but **nothing executes it**: +`engine.aggregate()` forwards only `object` / `where` / `groupBy` / `aggregations` to +the driver, no driver reads `query.having`, and the in-memory aggregation fallback has +no HAVING stage. Filter the aggregated rows in application code: ```typescript -const query: QueryAST = { - object: 'opportunity', +const rows = await engine.aggregate('opportunity', { groupBy: ['account_id'], aggregations: [ { function: 'sum', field: 'amount', alias: 'total' }, ], - having: { - total: { $gt: 1000000 }, // Only accounts with > $1M pipeline - }, -}; +}); -// SQL: HAVING SUM(amount) > 1000000 +// Only accounts with > $1M pipeline +const bigAccounts = rows.filter((r) => r.total > 1_000_000); ``` +### Date Bucketing + +A `groupBy` entry may be an object carrying `dateGranularity`, which buckets a +`date`/`datetime` column into uniform periods. The bucketed value is projected under +the field name (or `alias`, on the in-memory path): + +```typescript +const revenueByMonth = await engine.aggregate('order', { + where: { status: 'completed' }, + groupBy: [{ field: 'created_at', dateGranularity: 'month' }], + aggregations: [ + { function: 'sum', field: 'total_amount', alias: 'revenue' }, + ], +}); +``` + +**Granularities:** `day`, `week`, `month`, `quarter`, `year`. The engine pushes the +bucket down to the driver only when it advertises native support for that granularity +(`supports.queryDateGranularity`); otherwise it falls back to in-memory bucketing over +the driver's raw rows. + --- ## 6. Advanced Queries ### Distinct -```typescript -const query: QueryAST = { - object: 'account', - fields: ['industry'], - distinct: true, -}; +The `distinct` flag on `QueryAST` is **not applied by `find()`**. Distinct values come +from the driver's own `distinct()` method (implemented by the SQL and in-memory drivers; +it is not part of the `IDataDriver` contract): -// SQL: SELECT DISTINCT industry FROM account +```typescript +const industries = await driver.distinct('account', 'industry'); ``` +`SqlDriver.distinct()` presents each value exactly the way `find()` presents that +column — a `date` as `YYYY-MM-DD` and a `time` as `HH:MM:SS[.fff]` on every dialect, a +`datetime` folded to canonical UTC ISO on SQLite (the one dialect where storage differs +from presentation; Postgres and MySQL hand back their own native temporal value) — and +then re-deduplicates the presented values, because SQL `DISTINCT` compares the *stored* +form. + ### Full-Text Search -The `search` parameter configures full-text search: +The `search` parameter does **not** reach a full-text index. The engine expands it into +an `$or` of `$contains` predicates across the object's server-resolved searchable fields +(ADR-0061) and deletes `search` from the AST before the driver sees it — every driver +already runs `$or`/`$contains`, so no driver support is needed (`SqlDriver` reports +`supports.fullTextSearch: false`). ```typescript const query: QueryAST = { object: 'article', search: { query: 'ObjectStack tutorial', - fields: ['title', 'content', 'tags'], - fuzzy: true, - boost: { title: 2.0 }, + fields: ['title', 'content'], }, limit: 10, }; - -// The engine expands `search` into an `$or` of `$contains` across the object's -// searchable fields (ADR-0061); every driver runs it as native $or/$contains. ``` -### Joins +Field resolution is server-side and never client-trusted: `search.fields` is +**intersected** with the object's declared `searchableFields` (or, absent those, an +auto-default of the name field plus short-text/enum fields), so naming a field outside +that set does not widen the search. Multiple whitespace-separated terms are AND-ed and +fields are OR-ed. Case sensitivity is the **driver's**, not the expansion's: the +expansion emits a plain `$contains`, which `SqlDriver` compiles to a parameterised +`LIKE '%…%'` with no case folding — so the dialect's own `LIKE`/collation rules decide — +while the in-memory driver matches with a case-insensitive regex. Only `select` / +`status` option *labels* are matched case-insensitively by the expansion itself. +`fuzzy`, `boost`, `operator`, `minScore`, `language`, and `highlight` are accepted by +the schema but ignored by the expansion. -For cross-object queries beyond `expand`, use explicit joins: +### Joins (protocol only) + +`QuerySchema` defines a `joins` array, but **no driver reads it** — there is no +`query.joins` consumer anywhere in `packages/`, so a query carrying `joins` silently +runs as a single-table query. Use `expand` (§4) for relationship loading, or two +queries joined in application code. + +For reference, the protocol shape is: ```typescript const query: QueryAST = { @@ -630,19 +752,18 @@ const query: QueryAST = { }, ], }; - -// SQL: SELECT o.id, o.amount FROM orders o -// INNER JOIN customers c ON o.customer_id = c.id ``` **Join types:** `inner`, `left`, `right`, `full` ### Window Functions +`windowFunctions` on a `QueryAST` is ignored by `find()`. The SQL drivers implement +window functions through a separate `findWithWindowFunctions()` method, which is not on +the `IDataDriver` contract and is not surfaced by `IDataEngine`: + ```typescript -const query: QueryAST = { - object: 'order', - fields: ['id', 'customer_id', 'amount'], +const ranked = await driver.findWithWindowFunctions('order', { windowFunctions: [ { function: 'row_number', @@ -653,12 +774,15 @@ const query: QueryAST = { }, }, ], -}; +}); // SQL: SELECT *, ROW_NUMBER() OVER (PARTITION BY customer_id ORDER BY amount DESC) AS rank // FROM orders ``` +That method always selects `*` plus the window columns — it does not honor a `fields` +projection. + --- ## 7. Pagination @@ -681,20 +805,30 @@ const page2 = await engine.find('customer', { **Drawback:** Slow for large offsets (database still scans all skipped rows). -### Cursor-Based Pagination +### Keyset Pagination + + +`cursor` is accepted by `QuerySchema` and `EngineQueryOptions`, but **no driver +implements keyset pagination** — passing it has no effect. Express the keyset yourself +as an ordinary `where` predicate on the sort key: + ```typescript // First page -const result = await engine.find('customer', { +const page = await engine.find('customer', { limit: 10, - orderBy: [{ field: 'id', order: 'asc' }], + orderBy: [{ field: 'created_at', order: 'asc' }], }); -// Next page (use cursor) -const nextResult = await engine.find('customer', { - cursor: { id: result[result.length - 1].id }, +// Next page — seek past the last row instead of offsetting. +// The comparand is canonicalised by the same function that wrote the column +// (`coerceFilterValue` → `storageDatetimeValue`), so the range compare is an +// ordinary indexable comparison on every dialect: canonical UTC ISO text on +// SQLite, `timestamptz` on Postgres, `DATETIME(3)` on MySQL. +const next = await engine.find('customer', { + where: { created_at: { $gt: page[page.length - 1].created_at } }, limit: 10, - orderBy: [{ field: 'id', order: 'asc' }], + orderBy: [{ field: 'created_at', order: 'asc' }], }); ``` @@ -733,7 +867,6 @@ const products = await engine.find('product', { search: { query: searchTerm, fields: ['name', 'description'], - fuzzy: true, }, orderBy: [{ field: 'popularity_score', order: 'desc' }], limit: 20, @@ -742,21 +875,29 @@ const products = await engine.find('product', { ### Analytics: Revenue by Month +`month` is not a column — bucket the `created_at` instant with `dateGranularity`. Note +that `engine.aggregate()` accepts only `where` / `groupBy` / `aggregations` (plus a +`timezone` for bucketing): there is no `orderBy` or `limit` on this path, so sort the +returned rows yourself. + ```typescript -const monthlyRevenue: QueryAST = { - object: 'order', +const monthlyRevenue = await engine.aggregate('order', { where: { status: 'completed', + // `created_at` is a datetime — a bare date is read as midnight UTC created_at: { $gte: '2024-01-01' }, }, - groupBy: ['month'], + groupBy: [{ field: 'created_at', dateGranularity: 'month' }], aggregations: [ { function: 'sum', field: 'total_amount', alias: 'revenue' }, { function: 'count', alias: 'order_count' }, { function: 'avg', field: 'total_amount', alias: 'avg_order' }, ], - orderBy: [{ field: 'month', order: 'asc' }], -}; +}); + +const sorted = monthlyRevenue.sort((a, b) => + String(a.created_at).localeCompare(String(b.created_at)), +); ``` --- @@ -778,13 +919,19 @@ const rows = await engine.find('customer', { ### Security Violations +Row-level scoping is applied by *narrowing* the query — the security middleware +AND-merges its read filter into `where`, so an over-broad filter returns fewer rows +rather than throwing. What throws is an operation the caller is not permitted to run at +all, or a predicate that references a field the caller cannot read: + ```typescript try { - await engine.find('account', { - where: { owner_id: { $ne: currentUser.id } }, - }); + await engine.find('account', { where: { owner_id: currentUser.id } }); } catch (error) { - // PermissionDeniedError: [Security] Access denied to object 'account' + // PermissionDeniedError: + // [Security] Access denied: operation 'find' on object 'account' ... + // [Security] Access denied: query on 'account' references field(s) not + // readable by the caller: ... } ``` diff --git a/content/docs/protocol/objectql/types.mdx b/content/docs/protocol/objectql/types.mdx index 0f96207cd3..64b0b2957a 100644 --- a/content/docs/protocol/objectql/types.mdx +++ b/content/docs/protocol/objectql/types.mdx @@ -77,9 +77,9 @@ company_name: ``` **Database mapping:** -- PostgreSQL: `VARCHAR(maxLength)` or `TEXT` +- SQL driver: `TEXT` on every dialect. `maxLength` is enforced by record + validation, not by the column type — the DDL does not read it. - MongoDB: `String` -- Redis: `String` **UI rendering:** ```html @@ -104,7 +104,7 @@ description: ``` **Database mapping:** -- PostgreSQL: `TEXT` +- SQL driver: `TEXT` - MongoDB: `String` **UI rendering:** @@ -129,7 +129,7 @@ bio: ``` **Database mapping:** -- PostgreSQL: `TEXT` +- SQL driver: `TEXT` - MongoDB: `String` **UI rendering:** @@ -156,13 +156,13 @@ email: unique: true ``` -**Validation:** -- RFC 5322 compliant regex -- Lowercase normalization -- Domain validation (optional DNS check) +**Validation:** a deliberately permissive, ReDoS-safe shape check — a local part, +an `@`, and a dotted domain (`invalid_email` otherwise). It is **not** an RFC 5322 +parser, the value is **not** lowercased, and no DNS/MX lookup is performed. Stricter +rules belong in a validation rule or custom validator. **Database mapping:** -- PostgreSQL: `VARCHAR(255)` +- SQL driver: `VARCHAR(255)` - MongoDB: `String` **Use cases:** @@ -181,10 +181,11 @@ website: label: Website ``` -**Validation:** -- Must include protocol (`http://`, `https://`) -- Valid domain format -- Optional reachability check +**Validation:** accepts any `scheme://…` (not just `http`/`https` — `libsql://`, +`postgres://`, `s3://`, `file://` all pass), plus root-/protocol-relative refs +(`/path`, `//host/path`) and `data:` / `blob:` URIs. A bare scheme-less string +with no leading `/` is rejected (`invalid_url`). There is **no** domain-format +check and **no** reachability check. **Use cases:** - Company websites @@ -200,17 +201,15 @@ Phone number with international format. phone: type: phone label: Phone Number - format: international # E.164 format ``` -**Storage format:** E.164 (`+12025551234`) +**Storage format:** the string as entered — a `VARCHAR(255)` column. The engine +does **not** normalize to E.164 and does **not** reformat for display. -**Display format:** `(202) 555-1234` - -**Validation:** -- Valid phone number format -- Country code verification -- Optional SMS capability check +**Validation:** a shape check only — at least 5 characters drawn from digits and +`+ ( ) - . ` and whitespace (`invalid_phone` otherwise). There is no country-code +verification and no SMS-capability check. If you need E.164 at rest, normalize in +a before-save trigger or a custom validator. --- @@ -235,8 +234,10 @@ quantity: - `min`/`max`: Range validation **Database mapping:** -- PostgreSQL: `NUMERIC(precision, scale)` -- MongoDB: `Number` or `Decimal128` +- SQL driver: a floating-point column (`REAL` on PostgreSQL/SQLite, `FLOAT` on + MySQL). `precision`/`scale` are validation and display metadata — the DDL does + **not** emit `NUMERIC(precision, scale)`. +- MongoDB: `Number` **Use cases:** - Quantities, counts @@ -258,23 +259,30 @@ annual_revenue: defaultCurrency: USD ``` -**Storage:** +**Storage:** a **bare number** — the same value shape as `number`. + ```json -{ - "value": 1234.56, - "currency": "USD" -} +1234.56 ``` + +A currency value is **not** a `{ value, currency }` object. The currency *code* +lives once on the field definition (`currencyConfig.defaultCurrency`), not on +every stored value. The old per-value object shape (`CurrencyValueSchema`) was +never consumed by the validator, the driver, or import coercion and is +deprecated in the spec. + + **Features:** -- Multi-currency support -- Automatic formatting ($1,234.56) -- Exchange rate conversion (optional) -- ISO 4217 currency codes +- `currencyMode: fixed | dynamic` and a `defaultCurrency` code on the field +- Codes are validated by **length only** (3 characters), so ISO 4217 (`USD`, + `EUR`, `CNY`) and non-ISO codes (`BTC`, `ETH`) both pass +- `precision` (0–10, default 2) for decimal places **Database mapping:** -- PostgreSQL: `NUMERIC(18,2)` + `VARCHAR(3)` or `JSONB` -- MongoDB: `{ value: Decimal128, currency: String }` +- SQL driver: a floating-point column (`REAL` / `FLOAT`) — one column, no + companion currency column and no JSON blob +- MongoDB: `Number` --- @@ -292,7 +300,15 @@ discount_rate: **Display:** `25.5%` (automatically adds % symbol) -**Storage:** As decimal (0.255 for 25.5%) +**Storage:** the percentage **number itself** — `25.5` means 25.5%, matching the +`min: 0` / `max: 100` bounds above. It is *not* rescaled to a 0–1 ratio on write. +Physically it is the same floating-point column as `number`. + + +The separate `percent` **template filter** (`{{ record.rate | percent }}`) does +take a 0–1 ratio and render it as `42%`. That is a formatting choice at render +time, not the field type's storage convention — don't mix the two. + **Use cases:** - Discounts, margins @@ -312,11 +328,23 @@ birth_date: label: Date of Birth ``` -**Storage format:** ISO 8601 (`2024-01-15`) +**Storage format:** a **timezone-naive calendar day** — the `YYYY-MM-DD` string +`2024-01-15`, never an instant (ADR-0053). A `Date` collapses to its **UTC** +calendar day; a longer ISO string is truncated to its leading 10 characters. The +same normalization is applied on write, on read, and to every filter comparand, +so the two sides of a comparison can never disagree about what a date *is*. + + +A `date` is never converted to a timestamp and never timezone-shifted. Storing +UTC-midnight instants is exactly the "date-as-instant" mistake ADR-0053 removed — +it renders as the *previous* day for any viewer west of UTC. If a value genuinely +depends on a timezone, it is a `datetime`, not a `date`. + **Database mapping:** -- PostgreSQL: `DATE` -- MongoDB: `Date` (time set to 00:00:00 UTC) +- SQL driver: `DATE` on PostgreSQL/MySQL, `TEXT` on SQLite — holding + `YYYY-MM-DD` on every dialect +- MongoDB: no DDL (schemaless); the driver stores the value it is given **Use cases:** - Birthdays, anniversaries @@ -326,7 +354,7 @@ birth_date: --- #### `datetime` -Date and time with timezone. +A UTC instant. ```yaml meeting_time: @@ -334,11 +362,29 @@ meeting_time: label: Meeting Time ``` -**Storage format:** ISO 8601 with timezone (`2024-01-15T14:30:00Z`) +**Storage format:** the canonical, fixed-width, zone-explicit UTC instant +`YYYY-MM-DDTHH:MM:SS.sssZ` — e.g. `2024-01-15T14:30:00.000Z`. Milliseconds are +always present and the zone is always `Z`; an offset-bearing input such as +`…T22:30:00+08:00` is rewritten to the equivalent `…Z` instant so text ordering +stays chronological ordering. **Database mapping:** -- PostgreSQL: `TIMESTAMP WITH TIME ZONE` -- MongoDB: `Date` +- PostgreSQL: `timestamptz` +- MySQL: `DATETIME(3)` — deliberately **not** `TIMESTAMP`, which is a 32-bit + epoch (a 2038 ceiling on the column every list view sorts by), drops the + milliseconds, and converts on read/write using the session timezone. The + canonical instant is bound as a MySQL literal (`YYYY-MM-DD HH:MM:SS.sss`, + no `T`/`Z`) because MySQL rejects ISO-8601 in a datetime literal. +- SQLite: `TEXT` holding the canonical string — fixed width plus UTC means + lexicographic order is chronological order, so range filters use the index +- MongoDB: no DDL (schemaless) + + +Filter comparands go through the **same** canonicalization function as writes, on +**every** dialect. That is what makes `$gte`/`$lt` windows and `$eq` behave +identically on SQLite, PostgreSQL, and MySQL instead of depending on the shape +the caller happened to pass. + --- @@ -352,7 +398,23 @@ business_hours_start: defaultValue: "09:00:00" ``` -**Storage format:** `HH:MM:SS` — input accepts `HH:MM` or `HH:MM:SS` (with an optional fractional part and `Z`/offset). A `time` is a wall-clock value, not an instant: it is validated as a time-of-day, not parsed as a date. +**Storage format:** `HH:MM:SS`, gaining a `.fff` millisecond suffix **only** when +the milliseconds are non-zero (`14:30:00`, but `14:30:00.100`). Input accepts +`HH:MM` or `HH:MM:SS` (with an optional fractional part and `Z`/offset); `14:30` +is completed to `14:30:00`, so one wall clock can never split into several stored +values. A `Date`, an epoch, or a full timestamp folds to its **UTC** time-of-day. +A `time` is a wall-clock value, not an instant: it is validated as a time-of-day, +not parsed as a date. + +**Database mapping:** +- PostgreSQL: `time` +- MySQL: `TIME(3)` — bare `TIME` is zero-precision and *rounds* a fractional + literal (`14:30:00.500` → `14:30:01`), which would change the stored wall clock +- SQLite: `TEXT` holding the canonical string +- MongoDB: no DDL (schemaless) + +As with `date` and `datetime`, filter comparands are canonicalized by the same +function as writes, so `09:00 <= t <= 18:00` windows compare like against like. **Use cases:** - Business hours @@ -361,6 +423,40 @@ business_hours_start: --- +#### `defaultValue: 'NOW()'` on temporal fields + +`NOW()` is a framework convention meaning "use the database clock at insert +time". The driver translates it into a dialect-native default that resolves +against the **UTC** clock on every dialect, for `date`, `datetime`, and `time` +alike: + +```yaml +opened_at: + type: datetime + label: Opened At + defaultValue: "NOW()" +``` + +For `date` and `time` on PostgreSQL and MySQL the driver emits an explicit +UTC **expression** default rather than a bare `CURRENT_TIMESTAMP`, which resolves +the calendar day / wall clock in the *server's* timezone on PostgreSQL and the +*inserting session's* timezone on MySQL — one instant producing three different +stored values across the three dialects (and MySQL 8.0 rejects a bare +`CURRENT_TIMESTAMP` default on `DATE`/`TIME` columns outright). On SQLite all +three types use `strftime(…, 'now')` expressions that emit the canonical form +directly. `datetime` on PostgreSQL/MySQL keeps the native `now()`, which is +already UTC — the driver pins every MySQL connection with +`SET time_zone = '+00:00'`. + + +A DDL default only governs **newly created** columns. A column created before +this convention keeps its legacy default and can still emit a zone-naive value on +a defaulted insert; the read path repairs those to canonical form, so `find()` +stays uniform without a data migration. + + +--- + ### Boolean Types #### `boolean` @@ -374,9 +470,9 @@ is_active: ``` **Storage:** -- PostgreSQL: `BOOLEAN` +- SQL driver: `BOOLEAN` (SQLite stores `1`/`0`; the driver coerces it back to a + real JS boolean on read) - MongoDB: `Boolean` -- Redis: `1` or `0` **UI rendering:** ```html @@ -432,10 +528,12 @@ priority: required: true ``` -**Storage:** Stores `value` (not `label`) +**Storage:** Stores `value` (not `label`). Option `value`s must be lowercase +machine identifiers — the spec rejects `New`, `In Progress`, or `Closed_Won`. **Database mapping:** -- PostgreSQL: `VARCHAR` or `ENUM` +- SQL driver: `VARCHAR(255)` — the driver never emits a native `ENUM`, so adding + an option is a metadata change, not a schema migration - MongoDB: `String` **Use cases:** @@ -460,7 +558,8 @@ tags: ``` **Storage:** -- PostgreSQL: `TEXT[]` (array) or `JSONB` +- SQL driver: a `JSON` column holding the serialized array — not a native + `TEXT[]`, so the same DDL works on SQLite and MySQL - MongoDB: `[String]` **Example value:** `['customer', 'partner']` @@ -536,8 +635,19 @@ contacts: ``` **Database mapping:** -- PostgreSQL: `UUID` + Foreign Key constraint -- MongoDB: `ObjectId` or `String` +- SQL driver: `VARCHAR(255)` holding the related record id. Not a native `UUID` + column — record ids are opaque strings. A `multiple: true` lookup becomes a + `JSON` column instead. +- MongoDB: `String` + + +A relationship field authored with `reference:` gets **no database-level +`FOREIGN KEY` constraint**. The SQL driver's FK DDL is gated on a `reference_to` +property that the spec's `reference` never populates, and `master_detail` / +`tree` do not reach that branch at all. Referential integrity is enforced by the +**engine** instead: `deleteBehavior` is applied on delete, which is what produces +the `409 DELETE_RESTRICTED` above. + --- @@ -692,15 +802,24 @@ total_opportunity_value: - Aggregates a child object that references this object (via its `lookup`/`master_detail` field) - Set `relationshipField` only when the child has more than one reference back to this object -**Database implementation:** -- Materialized view (PostgreSQL) -- Aggregation pipeline (MongoDB) -- Background job recalculation +**Optional `filter`:** a `where`-style `FilterCondition` restricting *which* child +rows are aggregated, ANDed with the parent-FK match. This is what lets several +summaries roll the same child object into different totals: -**Performance:** -- Real-time: Recalculate on child write (slow) -- Batch: Update every N minutes (stale data) -- Hybrid: Increment/decrement on simple changes +```yaml +total_signups: + type: summary + summaryOperations: + object: engagement + function: count + filter: { type: signup } +``` + +**Implementation:** the summary is a **real numeric column** on the parent, +recomputed by the ObjectQL engine when a child row is inserted, updated, or +deleted. There is no materialized view, no MongoDB aggregation pipeline, and no +batch/hybrid recalculation mode — a child moving in or out of the `filter` +recomputes the parent on its next write like any other child update. --- @@ -717,10 +836,12 @@ case_number: **Example values:** `CASE-0001`, `CASE-0002`, ... **Format tokens:** -- `{0000}`: Zero-padded number -- `{YYYY}`: Year -- `{MM}`: Month -- `{DD}`: Day +- `{0000}`: Zero-padded counter +- `{YYYY}` / `{MM}` / `{DD}` / `{YYYYMMDD}`: date tokens (business timezone) +- `{field_name}`: interpolates another field's value + +The counter resets **per rendered prefix** — `AD{YYYYMMDD}{0000}` therefore +restarts at 1 each day. **Complex formats:** ```yaml @@ -730,9 +851,10 @@ invoice_number: # Generates: INV-2024-0001, INV-2024-0002, ... ``` -**Database implementation:** -- PostgreSQL: `SEQUENCE` -- MongoDB: Atomic counter collection +**Database implementation:** a `VARCHAR(255)` column, numbered from an atomic +counter row in the driver's own `_objectstack_sequences` table (bootstrapped from +the existing MAX on first use, and scoped per tenant when the object is +tenant-scoped). The driver does **not** create a native PostgreSQL `SEQUENCE`. --- @@ -748,7 +870,8 @@ metadata: ``` **Storage:** -- PostgreSQL: `JSONB` +- SQL driver: a `JSON` column (`json` on PostgreSQL/MySQL, `TEXT` on SQLite) — + the driver uses `json`, not `jsonb` - MongoDB: Native object **Query support:** @@ -783,7 +906,7 @@ There is no generic `array` field type. To store multiple values, use `tags` **Storage:** -- PostgreSQL: `TEXT[]` or `JSONB` +- SQL driver: a `JSON` column holding the serialized string array - MongoDB: `[String]` --- @@ -811,7 +934,7 @@ billing_address: ``` **Database mapping:** -- PostgreSQL: `JSONB` or composite type +- SQL driver: a `JSON` column (not a composite type) - MongoDB: Embedded document --- @@ -828,11 +951,15 @@ office_location: **Storage:** ```json { - "latitude": 37.7749, - "longitude": -122.4194 + "lat": 37.7749, + "lng": -122.4194 } ``` +`altitude` and `accuracy` (both in metres) are optional additional members. Note +the keys are `lat`/`lng` — the `{ latitude, longitude }` spelling was never +consumed by the runtime and has been retired from the value contract. + Proximity / radius ("near") search is **not** a built-in filter operator. ObjectQL's filter language exposes only `$eq`, `$ne`, `$gt`, `$gte`, `$lt`, @@ -844,8 +971,10 @@ geospatial indexes), which the SQL and in-memory drivers do not provide. **Database mapping:** -- PostgreSQL: `POINT` or `GEOGRAPHY` -- MongoDB: GeoJSON +- SQL driver: a `JSON` column — the driver does not emit a native `POINT` / + `GEOGRAPHY` column, which is the other half of why proximity search is not + available +- MongoDB: an embedded document --- @@ -859,27 +988,37 @@ avatar: multiple: false # set true to store an array of file references ``` -**Storage:** +**Stored value:** an opaque `sys_file` id string. The expanded read form is the +media metadata object, whose only required member is `url`: + ```json { - "filename": "profile.jpg", - "content_type": "image/jpeg", + "url": "https://cdn.example.com/files/abc123.jpg", + "name": "profile.jpg", "size": 1024000, - "url": "https://cdn.example.com/files/abc123.jpg" + "mimeType": "image/jpeg" } ``` -**Storage backends:** +`alt` and `duration` are the other optional members. The keys are `name` and +`mimeType` — not `filename` / `content_type`. + + +Deployments predating the file-as-reference migration may still hold the inline +metadata object (or a bare URL) as the *stored* value. The engine warns rather +than rejects until `os migrate files-to-references --apply` has run, so an +existing database keeps working while it is backfilled. + + +**Storage backends** (configured on the file-storage connector, not the field): - `local`: Server filesystem - `s3`: Amazon S3 - `azure_blob`: Azure Blob Storage - `gcs`: Google Cloud Storage -**Features:** -- Automatic upload handling -- Content-type validation -- Virus scanning (optional) -- CDN integration +…plus `dropbox`, `box`, `onedrive`, `google_drive`, `sharepoint`, `ftp`, and +`custom`. Upload-time processing — thumbnail generation, virus scanning — is +configured there too, under `contentProcessing`. --- @@ -904,21 +1043,37 @@ the uploaded file reference. ## Type Conversion Matrix -How types convert between databases: - -| ObjectQL Type | PostgreSQL | MongoDB | Redis | -|---------------|------------|---------|-------| -| `text` | `VARCHAR` / `TEXT` | `String` | `String` | -| `number` | `NUMERIC` | `Number` | `String` | -| `currency` | `JSONB` | `Object` | `String` (JSON) | -| `date` | `DATE` | `Date` | `String` (ISO) | -| `datetime` | `TIMESTAMP` | `Date` | `String` (ISO) | -| `boolean` | `BOOLEAN` | `Boolean` | `0` / `1` | -| `select` | `VARCHAR` / `ENUM` | `String` | `String` | -| `lookup` | `UUID` + FK | `ObjectId` | `String` | -| `formula` | `GENERATED` | Virtual | Computed | -| `json` | `JSONB` | `Object` | `String` (JSON) | -| `location` | `POINT` | `GeoJSON` | `String` (JSON) | +The column each type gets from the SQL driver, per dialect: + +| ObjectQL Type | PostgreSQL | MySQL | SQLite | +|---------------|------------|-------|--------| +| `text` | `TEXT` | `TEXT` | `TEXT` | +| `email` / `url` / `phone` | `VARCHAR(255)` | `VARCHAR(255)` | `VARCHAR(255)` | +| `number` / `currency` / `percent` | `REAL` | `FLOAT` | `REAL` | +| `date` | `DATE` | `DATE` | `TEXT` (`YYYY-MM-DD`) | +| `datetime` | `TIMESTAMPTZ` | `DATETIME(3)` | `TEXT` (canonical `…Z`) | +| `time` | `TIME` | `TIME(3)` | `TEXT` (`HH:MM:SS[.fff]`) | +| `boolean` / `toggle` | `BOOLEAN` | `BOOLEAN` | `INTEGER` `0`/`1` | +| `select` / `radio` | `VARCHAR(255)` | `VARCHAR(255)` | `VARCHAR(255)` | +| `multiselect` / `tags` | `JSON` | `JSON` | `TEXT` (JSON) | +| `lookup` / `master_detail` / `tree` | `VARCHAR(255)` | `VARCHAR(255)` | `VARCHAR(255)` | +| `summary` | `REAL` | `FLOAT` | `REAL` | +| `autonumber` | `VARCHAR(255)` | `VARCHAR(255)` | `VARCHAR(255)` | +| `formula` | *(no column — virtual)* | *(no column)* | *(no column)* | +| `json` / `location` / `address` | `JSON` | `JSON` | `TEXT` (JSON) | + +Any field flagged `multiple: true` becomes a `JSON` column regardless of its +type. Relationship columns are plain id strings with no database `FOREIGN KEY` +constraint (see `lookup` above). The MongoDB driver is schemaless — it issues no +DDL and stores the value it is given. + + +There is no Redis persistence backend. ObjectQL's data drivers are SQL +(PostgreSQL / MySQL / SQLite, plus a SQLite-WASM build of the same driver for the +browser), MongoDB, and in-memory. Redis appears in the platform only as an +optional *cluster-primitives* driver (pub/sub, locks, KV, counters), never as a +place records are stored. + ## Sensitive Data: Masking & Encryption From b8016db100612e74965464345422e4eeda30bf8e Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 30 Jul 2026 07:16:40 +0000 Subject: [PATCH 3/3] chore: trigger CI The docs commit landed on the branch without GitHub Actions firing (0 workflow runs for that SHA; only the Vercel status attached). Neither a close/reopen re-triggered it, so this empty commit forces a synchronize event to get real CI signal before merge. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01TPxNwPnjcn599ujXpU3ibJ