From bdcb41db99f0fb2edd93d002a10cdda690f7df68 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 11 Aug 2026 06:34:58 +0000 Subject: [PATCH] feat(driver-mongodb): lower dateGranularity groupBy to $dateToString bucket labels, and publish supports.queryDateGranularity (#7580) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `driver-mongodb` published no `queryDateGranularity`, so `engine.aggregate` bucketed every granularity in memory: correct answers, but the whole result set shipped to the client before the rollup. #7550 refused a bucketed node at the builder rather than implement or silently drop it, and sized the native lowering as a card of its own. This is that card. The bucket LABELS are the engine's own spellings, because the engine picks between the pushed-down and in-memory paths per query and a drill-down can cross that seam. All five declared granularities are advertised — `week` included, where driver-sql on SQLite cannot, because `$dateToString` has both halves of the ISO-8601 week date (`%G`/`%V`). No `$dateTrunc`: it answers a truncated DATE that still needs formatting, raises the server floor to MongoDB 5.0, and adds binSize/startOfWeek semantics this fleet cannot observe. `$dateToString` answers the label directly out of one operator, and fewer unobserved operators is the whole argument. All three ADR-0053 storage forms are served through one total expression: `$convert … onError/onNull: null` mirrors `bucketDateValue`'s totality, so a BSON Date, `YYYY-MM-DD` text, `HH:MM:SS` text, null, missing and unparseable junk all bucket the way the engine buckets them. The capability record and the builder's refusal read ONE constant, so an advertised granularity the builder would refuse cannot exist — the failure worse than advertising nothing, since the engine stops bucketing in memory on the strength of the bit. Parity is proven by running the SAME rows through the real `applyInMemoryAggregation` and through the emitted pipeline. driver-mongodb can depend on objectql where driver-sql cannot (objectql depends on no driver), so this is a devDependency rather than the hand-copied `bucketDateValue` the three SQL bucket suites carry, and the drift their `⚠️ Keep in sync` comments admit they cannot detect does not exist here. The strict in-process evaluator moved to `mongodb-pipeline-evaluator.testkit.ts` so both suites share one reader; only the date operators were added. It models the manual independently — its own ISO-8601 grammar, `%G`/`%V` from the standard — so the two sides CAN disagree and the comparison means something. ⚠️ Nothing here has met a real mongod (#5517: proxy 403 on the binary). The bound is written into the suite header, onto the published capability, and beside the lowering. Fixes #7580 Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_019QkW7hhVvhTxh4V9Wmnqdy --- .changeset/mongodb-native-date-granularity.md | 50 ++ packages/drivers/driver-mongodb/package.json | 1 + packages/drivers/driver-mongodb/src/index.ts | 6 +- .../mongodb-aggregation-translation.test.ts | 281 +++-------- .../driver-mongodb/src/mongodb-aggregation.ts | 268 +++++++++-- .../src/mongodb-date-bucket-parity.test.ts | 394 ++++++++++++++++ .../driver-mongodb/src/mongodb-driver.ts | 31 +- .../src/mongodb-pipeline-evaluator.testkit.ts | 445 ++++++++++++++++++ pnpm-lock.yaml | 3 + 9 files changed, 1220 insertions(+), 259 deletions(-) create mode 100644 .changeset/mongodb-native-date-granularity.md create mode 100644 packages/drivers/driver-mongodb/src/mongodb-date-bucket-parity.test.ts create mode 100644 packages/drivers/driver-mongodb/src/mongodb-pipeline-evaluator.testkit.ts diff --git a/.changeset/mongodb-native-date-granularity.md b/.changeset/mongodb-native-date-granularity.md new file mode 100644 index 0000000000..64342d603e --- /dev/null +++ b/.changeset/mongodb-native-date-granularity.md @@ -0,0 +1,50 @@ +--- +"@objectstack/driver-mongodb": patch +--- + +feat(driver-mongodb): bucket `dateGranularity` groupBy server-side, and publish `supports.queryDateGranularity` (#7580) + +`driver-mongodb` now lowers a `dateGranularity`-bearing `groupBy` into the +aggregation pipeline and advertises the capability, so `engine.aggregate` pushes +a bucketed aggregate down to MongoDB instead of fetching every matching row and +bucketing it in JS. + +**Answers do not change — where the work happens does.** `MongoDBDriver.supports` +published no `queryDateGranularity`, so the engine already bucketed every +granularity in memory and the results were correct. What was missing was the +index/server-side half: a year-over-year rollup shipped the whole result set to +the client first. #7550 refused a bucketed node at the builder rather than +implement or silently drop it; this replaces that refusal with the lowering it +described. + +**All five granularities `DateGranularity` declares are advertised** — `day`, +`week`, `month`, `quarter`, `year`. The bucket LABELS are the engine's own +spellings (`'2024'`, `'2024-Q1'`, `'2024-01'`, `'2024-01-15'`, ISO `'2025-W01'`), +because the engine picks between the pushed-down and in-memory paths per query +and a drill-down can cross that seam. `week: true` where `driver-sql` on SQLite +carries `week: false`: MongoDB's `$dateToString` has both halves of the ISO-8601 +week date (`%G`/`%V`), SQLite has neither. + +**All three ADR-0053 storage forms are served.** This driver stores `datetime` as +a BSON `Date` but `date` and `time` as timezone-naive TEXT, so the lowering reads +the instant through `$convert … onError/onNull: null` — total, exactly like the +in-memory `bucketDateValue`, which puts null, missing and unparseable values in +one empty bucket. A `Field.time` column is a wall clock and not an instant: both +paths agree it has no bucket, rather than one of them inventing a day. + +**Timezones are unchanged and stay engine-side.** `engine.aggregate` forces the +in-memory path for any non-UTC reference zone (ADR-0053 Phase 2 D2) and the AST +it hands a driver carries no `timezone`, so this bucketing is UTC by +construction. + +The #7550 refusal is kept for a granularity outside the advertised record — +`NOT_IMPLEMENTED` / 501 in the ADR-0112 envelope, now naming what *is* bucketed +here — and it reads the same constant the capability record publishes, so the +two cannot drift. + +⚠️ **Bound, stated because a green suite reads as more than it is.** Parity with +the engine's labels is proven through a strict in-process pipeline evaluator, not +against a live mongod: this environment cannot fetch a mongod binary (#5517). The +`$convert` / `$dateToString` / `$concat` / `$switch` semantics the lowering stands +on are documentation-derived. The bound is written into the suite header, onto +the published capability, and beside the lowering. diff --git a/packages/drivers/driver-mongodb/package.json b/packages/drivers/driver-mongodb/package.json index 130e96de2a..af2601339b 100644 --- a/packages/drivers/driver-mongodb/package.json +++ b/packages/drivers/driver-mongodb/package.json @@ -26,6 +26,7 @@ "nanoid": "^6.0.0" }, "devDependencies": { + "@objectstack/objectql": "workspace:*", "@types/node": "^26.1.2", "mongodb-memory-server": "^11.2.0", "typescript": "^6.0.3", diff --git a/packages/drivers/driver-mongodb/src/index.ts b/packages/drivers/driver-mongodb/src/index.ts index 06b9cc2890..c074e4f611 100644 --- a/packages/drivers/driver-mongodb/src/index.ts +++ b/packages/drivers/driver-mongodb/src/index.ts @@ -5,7 +5,11 @@ import { MongoDBDriver } from './mongodb-driver.js'; export { MongoDBDriver }; export type { MongoDBDriverConfig } from './mongodb-driver.js'; export { translateFilter } from './mongodb-filter.js'; -export { buildAggregationPipeline, postProcessAggregation } from './mongodb-aggregation.js'; +export { + buildAggregationPipeline, + postProcessAggregation, + MONGODB_DATE_GRANULARITIES, +} from './mongodb-aggregation.js'; export type { AggregationInput } from './mongodb-aggregation.js'; export { MongoDBMultiTenantUnsupportedError, diff --git a/packages/drivers/driver-mongodb/src/mongodb-aggregation-translation.test.ts b/packages/drivers/driver-mongodb/src/mongodb-aggregation-translation.test.ts index 65db00839a..89ebc1bca9 100644 --- a/packages/drivers/driver-mongodb/src/mongodb-aggregation-translation.test.ts +++ b/packages/drivers/driver-mongodb/src/mongodb-aggregation-translation.test.ts @@ -29,7 +29,7 @@ * possible at all — the same split as `mongodb-filter-logic-translation.test.ts` * and the shape #6814's closing conditions asked for. * - * ## Why there is a pipeline evaluator in here + * ## Why there is a pipeline evaluator * * The builder's output is a pipeline and the shared cases are stated as VALUES, * so something has to bridge the two. Pinning the emitted stages literally for @@ -46,11 +46,18 @@ * pre-fix lowerings is replayed through it and must FAIL the case it broke — so * "all green" cannot mean "the evaluator says yes to anything". * + * [#7580] The evaluator itself now lives in + * `mongodb-pipeline-evaluator.testkit.ts`, because the date-bucket parity suite + * needs the same strict reader and a second, laxer copy of it would be the + * easiest way to bless a broken lowering. It moved verbatim; only the date + * operators the bucket lowering emits were ADDED, so nothing this file asserts + * changed meaning. + * * What it deliberately does NOT answer: whether a real mongod agrees. `$cond` / - * `$ifNull` / `$addToSet` are modelled here from the documentation, not - * observed. That is the question the opt-in half would answer, and it is - * recorded as open rather than implied — this environment cannot fetch a mongod - * binary at all, and a suite nobody has executed is a claim, not a check. + * `$ifNull` / `$addToSet` are modelled from the documentation, not observed. + * That is the question the opt-in half would answer, and it is recorded as open + * rather than implied — this environment cannot fetch a mongod binary at all, + * and a suite nobody has executed is a claim, not a check. */ import { describe, it, expect } from 'vitest'; @@ -60,214 +67,17 @@ import { type AggregationCase, type GroupByNode, } from '@objectstack/spec/data'; -import { buildAggregationPipeline, postProcessAggregation, type AggregationInput } from './mongodb-aggregation.js'; +import { + buildAggregationPipeline, + postProcessAggregation, + MONGODB_DATE_GRANULARITIES, + type AggregationInput, +} from './mongodb-aggregation.js'; +import { runPipeline, type Doc } from './mongodb-pipeline-evaluator.testkit.js'; /** The alias every case's measure is projected under — never a fixture column. */ const MEASURE = 'measure'; -// ── A deliberately strict reader of the emitted pipeline ──────────────────── - -/** Thrown for any shape this evaluator does not model — never swallowed. */ -class UnsupportedShape extends Error {} - -/** A field path that resolved to nothing. MongoDB distinguishes it from `null`. */ -const MISSING = Symbol('missing'); - -type Doc = Record; - -/** Resolve a `'$a.b'` field path against a document, or {@link MISSING}. */ -function resolvePath(doc: Doc, ref: string): unknown { - if (!ref.startsWith('$')) throw new UnsupportedShape(`not a field path: ${ref}`); - let current: unknown = doc; - for (const part of ref.slice(1).split('.')) { - if (current === null || typeof current !== 'object' || Array.isArray(current)) return MISSING; - if (!Object.prototype.hasOwnProperty.call(current, part)) return MISSING; - current = (current as Doc)[part]; - } - return current; -} - -/** - * Evaluate an aggregation EXPRESSION against one document. - * - * Models only the operators this builder emits. `$$`-prefixed system variables - * are refused rather than guessed at — `$$ROOT` reaches the `array_agg` arm, and - * an evaluator that quietly resolved it would be asserting a semantics nobody - * checked. - */ -function evalExpr(doc: Doc, expr: unknown): unknown { - if (typeof expr === 'string' && expr.startsWith('$')) { - if (expr.startsWith('$$')) throw new UnsupportedShape(`system variable '${expr}' is not modelled`); - return resolvePath(doc, expr); - } - if (expr === null || typeof expr !== 'object') return expr; // literal - if (Array.isArray(expr)) return expr.map((e) => evalExpr(doc, e)); - - const keys = Object.keys(expr as Doc); - if (keys.length !== 1) throw new UnsupportedShape(`expression with ${keys.length} keys: ${keys.join(', ')}`); - const [op] = keys; - const arg = (expr as Doc)[op]; - - switch (op) { - case '$ifNull': { - if (!Array.isArray(arg) || arg.length !== 2) throw new UnsupportedShape('$ifNull takes [expr, replacement]'); - const value = evalExpr(doc, arg[0]); - return value === MISSING || value === null ? evalExpr(doc, arg[1]) : value; - } - case '$eq': { - if (!Array.isArray(arg) || arg.length !== 2) throw new UnsupportedShape('$eq takes two operands'); - // MISSING and null compare EQUAL under `$eq` in the aggregation language, - // which is why the `count` lowering routes through `$ifNull` first rather - // than relying on it. - const left = evalExpr(doc, arg[0]); - const right = evalExpr(doc, arg[1]); - const norm = (v: unknown) => (v === MISSING ? null : v); - return norm(left) === norm(right); - } - case '$cond': { - if (!Array.isArray(arg) || arg.length !== 3) throw new UnsupportedShape('$cond takes [if, then, else]'); - return evalExpr(doc, arg[0]) ? evalExpr(doc, arg[1]) : evalExpr(doc, arg[2]); - } - default: - throw new UnsupportedShape(`unsupported aggregation expression operator '${op}'`); - } -} - -/** One accumulator, folded over the documents of a single group. */ -function accumulate(rows: Doc[], acc: unknown): unknown { - if (acc === null || typeof acc !== 'object' || Array.isArray(acc)) { - throw new UnsupportedShape(`accumulator must be a one-key document, got ${JSON.stringify(acc)}`); - } - const keys = Object.keys(acc as Doc); - if (keys.length !== 1) throw new UnsupportedShape(`accumulator with ${keys.length} keys`); - const [op] = keys; - const arg = (acc as Doc)[op]; - const values = rows.map((row) => evalExpr(row, arg)); - /** MongoDB's arithmetic accumulators ignore missing and non-numeric values. */ - const numbers = values.filter((v): v is number => typeof v === 'number'); - - switch (op) { - case '$sum': - return numbers.reduce((a, b) => a + b, 0); - case '$avg': - return numbers.length === 0 ? null : numbers.reduce((a, b) => a + b, 0) / numbers.length; - case '$min': - return numbers.length === 0 ? null : Math.min(...numbers); - case '$max': - return numbers.length === 0 ? null : Math.max(...numbers); - case '$addToSet': { - // `$addToSet` skips a MISSING field and keeps an explicit `null` — the - // whole of #6814 lives in that second half. - const set: unknown[] = []; - for (const v of values) { - if (v === MISSING) continue; - if (!set.includes(v)) set.push(v); - } - return set; - } - case '$push': - return values.filter((v) => v !== MISSING); - default: - throw new UnsupportedShape(`unsupported accumulator '${op}'`); - } -} - -/** Execute an emitted pipeline over the fixture. Throws on any shape not modelled. */ -export function runPipeline(rows: readonly Doc[], pipeline: readonly Doc[]): Doc[] { - let docs: Doc[] = rows.map((row) => ({ ...row })); - - for (const stage of pipeline) { - const keys = Object.keys(stage); - if (keys.length !== 1) throw new UnsupportedShape(`pipeline stage with ${keys.length} keys`); - const [name] = keys; - const spec = stage[name]; - - switch (name) { - case '$group': { - const { _id: idSpec, ...accumulators } = spec as Doc; - const groups = new Map(); - for (const doc of docs) { - let id: unknown; - if (idSpec === null) { - id = null; - } else if (idSpec && typeof idSpec === 'object' && !Array.isArray(idSpec)) { - const key: Doc = {}; - for (const [outKey, ref] of Object.entries(idSpec as Doc)) { - const value = evalExpr(doc, ref); - key[outKey] = value === MISSING ? null : value; - } - id = key; - } else { - throw new UnsupportedShape(`$group._id must be null or a document, got ${JSON.stringify(idSpec)}`); - } - const bucket = JSON.stringify(id); - if (!groups.has(bucket)) groups.set(bucket, { id, rows: [] }); - groups.get(bucket)!.rows.push(doc); - } - docs = [...groups.values()].map(({ id, rows: groupRows }) => { - const out: Doc = { _id: id }; - for (const [alias, acc] of Object.entries(accumulators)) { - out[alias] = accumulate(groupRows, acc); - } - return out; - }); - break; - } - case '$project': { - docs = docs.map((doc) => { - const out: Doc = {}; - for (const [key, rule] of Object.entries(spec as Doc)) { - if (rule === 0 || rule === false) continue; - if (rule === 1 || rule === true) { - const value = resolvePath(doc, `$${key}`); - if (value !== MISSING) out[key] = value; - continue; - } - if (typeof rule === 'string') { - const value = resolvePath(doc, rule); - if (value !== MISSING) out[key] = value; - continue; - } - throw new UnsupportedShape(`unsupported $project rule for '${key}': ${JSON.stringify(rule)}`); - } - return out; - }); - break; - } - case '$match': - // Reachable only from `opts.where`, which no case in the shared set - // spells. Refused rather than approximated: this file's `$match` would - // be a second, weaker copy of the matcher - // `mongodb-filter-logic-translation.test.ts` already owns. - throw new UnsupportedShape('$match is not modelled here — see mongodb-filter-logic-translation.test.ts'); - case '$sort': { - const entries = Object.entries(spec as Doc); - docs = [...docs].sort((a, b) => { - for (const [field, dir] of entries) { - if (dir !== 1 && dir !== -1) throw new UnsupportedShape(`$sort direction ${String(dir)}`); - const av = a[field]; - const bv = b[field]; - if (av === bv) continue; - return ((av as never) < (bv as never) ? -1 : 1) * (dir as number); - } - return 0; - }); - break; - } - case '$skip': - docs = docs.slice(spec as number); - break; - case '$limit': - docs = docs.slice(0, spec as number); - break; - default: - throw new UnsupportedShape(`unsupported pipeline stage '${name}'`); - } - } - - return docs; -} - // ── Driving the shared case-set ───────────────────────────────────────────── /** How a case's single `groupBy` column is spelled on the wire. */ @@ -375,25 +185,56 @@ describe('the emitted pipeline', () => { describe('a groupBy entry with no lowering is REFUSED, not ignored', () => { const aggregations: AggregationInput[] = [{ function: 'count', alias: 'n' }]; - it('a dateGranularity node answers NOT_IMPLEMENTED / 501', () => { + /** + * [#7580] These two pins USED to assert that every declared granularity is + * refused. That was the true statement while this driver bucketed nothing; it + * is false now, and the honest update is to assert the NEW substance rather + * than to delete the coverage: + * + * - a granularity the driver ADVERTISES must LOWER (below, and in full in + * `mongodb-date-bucket-parity.test.ts`); + * - a granularity it does not advertise must still refuse, with the ADR-0112 + * envelope intact. + * + * Since the advertised record is now all five of `DateGranularity`, the second + * population is reachable only from outside the declared enum — a caller that + * hands this exported builder a granularity the spec does not name, which is + * exactly the caller the refusal was always written for. It is tested with one + * rather than deleted as unreachable, because "unreachable today" is how a + * refusal quietly becomes a `default:` that answers. + */ + it('an ADVERTISED granularity lowers instead of refusing', () => { + for (const g of ['day', 'week', 'month', 'quarter', 'year'] as const) { + expect(MONGODB_DATE_GRANULARITIES[g], `${g} must be advertised`).toBe(true); + const pipeline = buildAggregationPipeline({ + aggregations, + groupBy: [{ field: 'closed_at', dateGranularity: g }], + }); + expect(pipeline[0], g).toHaveProperty('$group'); + expect(JSON.stringify(pipeline), g).toContain('$dateToString'); + } + }); + + it('an UNADVERTISED granularity answers NOT_IMPLEMENTED / 501', () => { let thrown: (Error & { code?: string; status?: number }) | undefined; try { - buildAggregationPipeline({ aggregations, groupBy: [{ field: 'closed_at', dateGranularity: 'month' }] }); + // Outside `DateGranularity` — the only population left, and the one the + // refusal exists for: a direct caller going around the capability record. + buildAggregationPipeline({ + aggregations, + groupBy: [{ field: 'closed_at', dateGranularity: 'fortnight' } as unknown as GroupByNode], + }); } catch (err) { thrown = err as Error & { code?: string; status?: number }; } expect(thrown, 'a granularity this backend cannot bucket must not be silently dropped').toBeDefined(); expect(thrown!.code).toBe('NOT_IMPLEMENTED'); expect(thrown!.status).toBe(501); - expect(thrown!.message).toMatch(/Date bucketing by 'month' is not supported/); + expect(thrown!.message).toMatch(/Date bucketing by 'fortnight' is not supported/); expect(thrown!.message).toMatch(/queryDateGranularity/); - }); - - it('refuses every declared granularity, so none is half-supported', () => { - for (const g of ['day', 'week', 'month', 'quarter', 'year'] as const) { - expect(() => buildAggregationPipeline({ aggregations, groupBy: [{ field: 'closed_at', dateGranularity: g }] })) - .toThrow(new RegExp(`Date bucketing by '${g}'`)); - } + // The message names what IS bucketed, so a reader is told where the boundary + // is rather than only that they crossed it — the driver-sql wording. + expect(thrown!.message).toContain('Bucketed here: day, month, quarter, week, year (driver-mongodb)'); }); it('an entry that is neither half of the union answers INVALID_QUERY / 400', () => { @@ -414,7 +255,7 @@ describe('a groupBy entry with no lowering is REFUSED, not ignored', () => { it('refuses before a pipeline exists, so no stage is built from a half-read groupBy', () => { expect(() => buildAggregationPipeline({ aggregations, - groupBy: ['region', { field: 'closed_at', dateGranularity: 'year' }], + groupBy: ['region', { field: 'closed_at', dateGranularity: 'fortnight' } as unknown as GroupByNode], })).toThrow(/Date bucketing/); }); }); diff --git a/packages/drivers/driver-mongodb/src/mongodb-aggregation.ts b/packages/drivers/driver-mongodb/src/mongodb-aggregation.ts index 727ccfd253..0b3b50f36d 100644 --- a/packages/drivers/driver-mongodb/src/mongodb-aggregation.ts +++ b/packages/drivers/driver-mongodb/src/mongodb-aggregation.ts @@ -9,7 +9,7 @@ import type { Document } from 'mongodb'; import { StandardErrorCode } from '@objectstack/spec/api'; -import type { GroupByNode } from '@objectstack/spec/data'; +import type { DateGranularityValue, GroupByNode } from '@objectstack/spec/data'; import { translateFilter } from './mongodb-filter.js'; import type { TemporalFieldKindResolver } from './mongodb-temporal.js'; @@ -24,18 +24,70 @@ export interface AggregationInput { filter?: unknown; } +/** + * [#7580] The granularities this driver buckets NATIVELY, published verbatim as + * `MongoDBDriver.supports.queryDateGranularity`. + * + * ## One constant, two readers — which is what makes "declared = enforced" true + * + * The capability record the engine reads and the set {@link normalizeGroupBy} + * agrees to lower are THE SAME OBJECT. A driver that advertises a granularity + * its builder then refuses is strictly worse than one that advertises nothing: + * the engine stops bucketing in memory on the strength of the advertisement, so + * a query that used to answer starts throwing 501. Splitting the two into + * separate literals is how that happens, so they are not split. + * `mongodb-date-bucket-parity.test.ts` pins the identity so a future edit to one + * cannot silently miss the other. + * + * ## Why all five, when `driver-sql` on SQLite advertises four + * + * SQLite has no ISO-week format specifier, so `driver-sql` sets `week: false` + * and lets the engine bucket weeks in memory. MongoDB's `$dateToString` has both + * halves of the ISO-8601 week date — `%G` (ISO week-YEAR) and `%V` (ISO week + * number, zero-padded to 2) — which is exactly the label + * `bucketDateValue` computes by hand. The dialect difference is real, so the + * records differ. + * + * ⚠️ **Documentation-derived, not observed** (#5517). Every `$dateToString` + * format specifier and every `$convert`/`$concat`/`$switch` null rule this + * record stands on is read from the MongoDB manual; this fleet cannot fetch a + * mongod binary (proxy 403), so no assertion here has met a real server. See + * the bound stated at the top of `mongodb-date-bucket-parity.test.ts`. + */ +export const MONGODB_DATE_GRANULARITIES: Record = { + day: true, + week: true, + month: true, + quarter: true, + year: true, +}; + +/** The advertised granularities, in a stable order, for refusal messages. */ +const BUCKETED_HERE: string[] = Object.entries(MONGODB_DATE_GRANULARITIES) + .filter(([, on]) => on === true) + .map(([g]) => g) + .sort(); + /** * One `groupBy` entry after the declared union has been read: the field the - * `$group._id` keys on, and the name the group value is PROJECTED under. + * `$group._id` keys on, the name the group value is PROJECTED under, and the + * expression whose value defines the group. + * + * [#6850] `field` and `outKey` are separate because `GroupByNodeSchema.alias` + * renames the projection without moving the grouping — the rule #6401 converged + * the three SQL faces onto (`alias ?? field`), and the one + * `in-memory-aggregation.ts` has always applied. * - * [#6850] The two are separate because `GroupByNodeSchema.alias` renames the - * projection without moving the grouping — the rule #6401 converged the three - * SQL faces onto (`alias ?? field`), and the one `in-memory-aggregation.ts` has - * always applied. + * [#7580] `expr` is separate from `field` for the same class of reason: a + * date-bucketed node groups by a LABEL computed from the field, not by the + * field. Keeping it on the target rather than re-deriving it at `$group` time is + * what stops the `_id` and the `$project` that flattens it from disagreeing + * about which of the two a node meant. */ interface GroupByTarget { field: string; outKey: string; + expr: Document | string; } /** @@ -54,18 +106,163 @@ interface GroupByTarget { */ function normalizeGroupBy(nodes: readonly GroupByNode[]): GroupByTarget[] { return nodes.map((node) => { - if (typeof node === 'string') return { field: node, outKey: node }; + if (typeof node === 'string') return { field: node, outKey: node, expr: `$${node}` }; if (node && typeof node === 'object' && typeof node.field === 'string' && node.field !== '') { - // A DATE-BUCKETED node has no lowering here — refuse it rather than group - // by the raw instant, which would answer one bucket per distinct - // timestamp and look like a working query. - if (node.dateGranularity) refuseDateBucketedGroupBy(node.dateGranularity); - return { field: node.field, outKey: node.alias ?? node.field }; + const outKey = node.alias ?? node.field; + // [#7580] A DATE-BUCKETED node groups by the engine's bucket LABEL. A + // granularity outside {@link MONGODB_DATE_GRANULARITIES} is still refused + // rather than grouped by the raw instant, which would answer one bucket + // per distinct timestamp and look like a working query. + if (node.dateGranularity) { + if (MONGODB_DATE_GRANULARITIES[node.dateGranularity] !== true) { + refuseDateBucketedGroupBy(node.dateGranularity); + } + return { field: node.field, outKey, expr: buildDateBucketExpr(node.field, node.dateGranularity) }; + } + return { field: node.field, outKey, expr: `$${node.field}` }; } throw malformedGroupByError(node); }); } +/** + * [#7580] The instant a bucket expression reads, from whatever form the column + * holds it in. + * + * ## Why a conversion step exists at all (ADR-0053 storage forms) + * + * This driver does NOT store every temporal field as a BSON `Date`. Per + * `mongodb-temporal.ts`'s canon table: `datetime` is a BSON `Date`, but `date` + * is `YYYY-MM-DD` TEXT and `time` is `HH:MM:SS[.fff]` TEXT — both timezone-naive + * by ADR-0053 Phase 1, both deliberately NOT instants. A date operator applied + * straight to `$field` therefore meets a string on two of the three declared + * kinds, and MongoDB's date operators do not silently tolerate that. + * + * `$convert … onError: null, onNull: null` makes the read TOTAL, which is the + * property that matters, because the in-memory reference is total too: + * `bucketDateValue` returns `null` for a null, absent or unparseable instant and + * puts them all in one bucket (#3839). Matching that shape is not a nicety — + * a non-total expression would fail the WHOLE aggregation on one junk row, where + * the engine's fallback answers. + * + * Measured, per declared kind, against `bucketDateValue`'s `new Date(...)`: + * + * | Stored form | in-memory reference | this expression | + * |---|---|---| + * | BSON `Date` (`datetime`) | the instant | identity — `$convert` of a date is the date | + * | `'2024-01-15'` (`date`) | `new Date('2024-01-15')` = midnight **UTC** | `$dateFromString` semantics: midnight UTC | + * | `'14:30:00'` (`time`) | `Invalid Date` → `null` bucket | not a date → `onError` → `null` bucket | + * | epoch ms number | `new Date(ms)` | `$convert` reads a numeric as epoch ms | + * | `null` / missing | `null` bucket | `onNull` → `null` bucket | + * + * ⚠️ **The residual divergence, stated rather than papered over.** JS's date + * parser accepts legacy spellings that ISO-8601 does not (`'2024-01-15 10:00:00'` + * with a space, `'2024/01/15'`), so a column holding one of those buckets in + * memory and empties here. Those are not forms this driver WRITES — every write + * goes through `coerceTemporalValue`, which canonicalises to the table above — + * so the exposure is pre-#4047 legacy rows only, and it is the same exposure + * every SQL face already carries (`strftime` parses no more of them than + * `$dateFromString` does). It is recorded here because "advertised" has to mean + * something exact. + */ +function instantOf(field: string): Document { + return { $convert: { input: `$${field}`, to: 'date', onError: null, onNull: null } }; +} + +/** + * [#7580] The expression whose value IS the engine's bucket label for `field` at + * `granularity` — the `$group._id` half of native date bucketing. + * + * ## Labels, not instants — and therefore no `$dateTrunc` + * + * The contract is not "truncate the instant", it is "produce the string + * `bucketDateValue` produces", because the engine's in-memory fallback emits + * LABELS and `engine.aggregate` picks between the two paths per query on a + * capability bit. A drill-down that crosses that seam has to see the same + * spelling on both sides. `$dateTrunc` (the route the card sketched) answers a + * truncated DATE, which would still need formatting afterwards — so it buys a + * stage, raises the server floor to MongoDB 5.0, and adds `binSize` / + * `startOfWeek` semantics this fleet cannot observe. `$dateToString` alone + * answers the label directly, on a 3.6 floor, out of one operator. Fewer + * unobserved semantics is the whole argument: every operator here is one more + * documentation-derived claim (#5517), so the lowering that needs the fewest + * wins. + * + * ## The five labels, against `bucketDateValue` + * + * | Granularity | Reference | Emitted | + * |---|---|---| + * | `year` | `String(y)` → `'2024'` | `%Y` | + * | `month` | `` `${y}-${MM}` `` → `'2024-01'` | `%Y-%m` | + * | `day` | `` `${y}-${MM}-${DD}` `` → `'2024-01-15'` | `%Y-%m-%d` | + * | `week` | ISO-8601 week date → `'2025-W01'` | `%G-W%V` | + * | `quarter` | `` `${y}-Q${n}` `` → `'2024-Q1'` | `%Y` + `-Q` + a month switch | + * + * `%G`/`%V` are the ISO week-YEAR and the zero-padded ISO week number — the two + * quantities `bucketDateValue` computes by hand off the Thursday of the week, so + * `2024-12-30` labels `'2025-W01'` on both sides rather than `'2024-W01'`. + * + * ## Why `quarter` is spelled with a `$switch` over strings + * + * MongoDB has no quarter specifier, so the digit has to be derived. Deriving it + * arithmetically (`$ceil` of `$divide` of `$month`) would make the label depend + * on how `$toString` formats a *double* — `1` vs `1.0` is the difference between + * `'2024-Q1'` and a label nothing else in the repo produces, and it is precisely + * the kind of claim this environment cannot check. Comparing the zero-padded + * `%m` STRING instead (`'01' <= '03'` is lexicographic and exact for fixed-width + * two-digit numerals) keeps every value in the expression a literal string. + * + * Null propagation falls out of `$concat`, which the manual defines as returning + * `null` if ANY argument is null: on a null instant `%Y` is null, so the whole + * label is null and lands in the same empty bucket the reference uses. The + * `$switch` still evaluates on that path (`$lte: [null, '03']` is true under BSON + * sort order, so it answers `'1'`) — harmlessly, because `$concat` has already + * decided. No `$cond` guard is needed, and one fewer operator is one fewer + * unobserved claim. + * + * ⚠️ Known, shared bound: `%Y` is 4-digit zero-padded where `String(y)` is not, + * so a year before 1000 labels `'0999'` here and `'999'` in memory. Every SQL + * face has the identical property (`strftime('%Y')` pads too) and advertises + * `year` regardless; it is recorded, not silently inherited. + */ +function buildDateBucketExpr(field: string, granularity: DateGranularityValue): Document { + const fmt = (format: string): Document => ({ $dateToString: { format, date: instantOf(field) } }); + + switch (granularity) { + case 'year': + return fmt('%Y'); + case 'month': + return fmt('%Y-%m'); + case 'day': + return fmt('%Y-%m-%d'); + case 'week': + return fmt('%G-W%V'); + case 'quarter': + return { + $concat: [ + fmt('%Y'), + '-Q', + { + $switch: { + branches: [ + { case: { $lte: [fmt('%m'), '03'] }, then: '1' }, + { case: { $lte: [fmt('%m'), '06'] }, then: '2' }, + { case: { $lte: [fmt('%m'), '09'] }, then: '3' }, + ], + default: '4', + }, + }, + ], + }; + default: + // Unreachable through `DateGranularityValue`, and refused rather than + // defaulted: a granularity this switch does not know is a granularity with + // no label, and guessing one is the silent-answer failure this whole card + // exists to close. + return refuseDateBucketedGroupBy(granularity); + } +} + /** * [#6850] A `groupBy` entry asks for a date BUCKET — the twin of `driver-sql`'s * and `driver-turso`'s `refuseDateBucketedGroupBy`, first sentence for first @@ -74,32 +271,32 @@ function normalizeGroupBy(nodes: readonly GroupByNode[]): GroupByTarget[] { * bucket expression for it, so it is a capability gap in the backend rather than * a mistake in the query. * - * This driver buckets NOTHING natively, which is exactly what it publishes: - * `MongoDBDriver.supports` carries no `queryDateGranularity` key at all (see the - * comment on that field), so the engine buckets every granularity in memory and - * never pushes a bucketed item down here. The refusal therefore fires only for a - * caller that went around the capability bit and reached the builder directly, - * which is the caller this message is written for. - * - * A native lowering is buildable — MongoDB has `$dateTrunc` — and this refusal - * is not a verdict that it cannot be. It is not a one-liner either: the engine's - * fallback produces LABELS (`'2026-01'`, `'2026-Q1'`, ISO `'2026-W03'` — see - * `bucketDateValue` and `SqlDriver.buildDateBucketExpr`), so a pushdown here has - * to emit those strings, publish `supports.queryDateGranularity`, and be held to - * `date-bucket-parity.test.ts`. Until then, refusing is what keeps a declared - * key from being silently ignored. + * [#7580] The population it refuses has SHRUNK, and the shape has not. This + * driver now lowers every granularity in {@link MONGODB_DATE_GRANULARITIES} — + * all five `DateGranularity` declares — and publishes exactly that record as + * `MongoDBDriver.supports.queryDateGranularity`, so the refusal no longer fires + * for any spec-valid granularity. It is kept, and kept total, for the two + * callers that can still reach it: one that hands the builder a granularity + * string outside the declared enum (this module is exported, and `groupBy` + * arrives through an `any` cast on the driver's own `aggregate`), and any future + * edit that narrows the advertised record without narrowing the lowering. The + * refusal is the local half of "declared = enforced" — the capability record + * says what the engine may push down, this says what the builder will actually + * lower, and they read the same constant so they cannot disagree. + * + * The message names what IS bucketed here, so a reader is told where the + * boundary is rather than only that they crossed it — the `driver-sql` wording, + * first sentence for first sentence (#5907, #6212, ADR-0112). */ function refuseDateBucketedGroupBy(granularity: string): never { const err = new Error( `Date bucketing by '${granularity}' is not supported by this backend. ` - + `Bucketed here: none (driver-mongodb). ` + + `Bucketed here: ${BUCKETED_HERE.length > 0 ? BUCKETED_HERE.join(', ') : 'none'} (driver-mongodb). ` + `The query is spelled correctly and @objectstack/spec DateGranularity declares it — this is ` + `a capability gap in the backend, not a mistake in the query, which is why it answers ` + `NOT_IMPLEMENTED/501 rather than a 400. A driver publishes the granularities it buckets ` + `natively as \`supports.queryDateGranularity\`; the engine reads that record and buckets ` - + `in memory for every granularity absent from it, which is always correct (#6212). This ` - + `driver publishes none, so a bucketed groupBy reaches here only when a caller goes around ` - + `the engine (#6850).`, + + `in memory for every granularity absent from it, which is always correct (#6212).`, ) as Error & { code?: string; status?: number }; err.code = StandardErrorCode.enum.NOT_IMPLEMENTED; err.status = 501; @@ -178,10 +375,13 @@ export function buildAggregationPipeline(opts: { const groupAccumulators: Document = {}; // Build _id from groupBy fields. The _id key is the PROJECTED name and its - // value the FIELD path, so an alias renames the output column without - // moving the grouping — `alias ?? field`, the #6401 rule. - for (const { field, outKey } of groupTargets) { - groupId[outKey] = `$${field}`; + // value the grouping EXPRESSION — a bare field path for a plain node, the + // bucket label expression for a `dateGranularity` one (#7580) — so an alias + // renames the output column without moving the grouping (`alias ?? field`, + // the #6401 rule) and a bucketed node groups by its label rather than by the + // raw instant. + for (const { outKey, expr } of groupTargets) { + groupId[outKey] = expr; } // Build accumulators from aggregation descriptors diff --git a/packages/drivers/driver-mongodb/src/mongodb-date-bucket-parity.test.ts b/packages/drivers/driver-mongodb/src/mongodb-date-bucket-parity.test.ts new file mode 100644 index 0000000000..91c4cd6cf6 --- /dev/null +++ b/packages/drivers/driver-mongodb/src/mongodb-date-bucket-parity.test.ts @@ -0,0 +1,394 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * Date-bucket parity for `driver-mongodb` (#7580) — the contract that lets + * `MongoDBDriver.supports.queryDateGranularity` be published at all. + * + * ## The contract + * + * A driver that advertises `supports.queryDateGranularity[g]` tells + * `engine.aggregate` it may push `dateGranularity: g` down instead of fetching + * rows and bucketing them in JS (`engine.ts`, aggregate dispatch). The two are + * then one feature with two implementations, and the engine picks between them + * PER QUERY — so the same rows bucketed either way must carry the same LABELS, + * byte for byte. A dashboard can cross that seam mid-drill-down. + * + * "Same labels" includes the EMPTY bucket: a row with no instant must key the + * same way on both sides (#3839). It is the one label neither side computes — + * each just propagates its own idea of nothing — which is exactly why it drifted + * apart for so long without a test to notice. + * + * ## Why the reference is the REAL engine, not a copy + * + * `driver-sql`, `driver-turso` and `driver-sqlite-wasm` each hand-copy + * `bucketDateValue` into their bucket suites, because `driver-sql` cannot depend + * on `objectql`. Those three files say so themselves, and say what it costs: a + * copy that stops tracking its original leaves the copy and the driver agreeing + * with each other while both are wrong. The repo's answer is + * `checkDateBucketParity` (@objectstack/verify), which imports the real + * `applyInMemoryAggregation` — but it drives a LIVE driver through + * `create`/`find`/`aggregate`, so `driver-mongodb` cannot join it in this fleet. + * + * Measured rather than inherited: `objectql` depends on no driver, so + * `driver-mongodb → objectql` is acyclic, and as a **devDependency** it does not + * enter the published package. So this suite imports + * {@link applyInMemoryAggregation} and {@link bucketDateValue} themselves. The + * reference side here IS the engine's fallback — the drift the `⚠️ Keep in sync` + * comments admit they cannot detect does not exist for this driver. + * + * ## ⚠️ The bound this suite carries, in full (#5517) + * + * **Nothing here has met a real mongod.** This fleet cannot fetch a mongod + * binary (proxy 403 on the download), so the emitted pipeline is executed by the + * strict in-process evaluator in `mongodb-pipeline-evaluator.testkit.ts`, whose + * `$convert` / `$dateToString` / `$concat` / `$switch` / `$lte` semantics are + * modelled from the MongoDB manual, not observed. What this suite proves is that + * the LOWERING agrees with the engine under those documented semantics. What it + * cannot prove is that a real server implements them as documented. That bound + * is the same one `mongodb-aggregation-translation.test.ts` carries for + * `$cond`/`$ifNull`/`$addToSet` (#6850/#6814), it is stated on + * `MongoDBDriver.supports` where the capability is published, and it is why the + * evaluator models the documentation independently — its ISO-8601 parser and its + * `%G`/`%V` computation are written from the standard, never delegated to + * `new Date(...)` or transcribed from `bucketDateValue`, so the two sides CAN + * disagree and this comparison means something. + * + * Two guards against the failure mode a two-implementation comparison invites — + * both sides sharing one wrong idea: + * + * 1. the literal label pins below, which state the required spellings out loud + * (`'2025-W01'` for 2024-12-30, `'2024-Q3'` for 2024-07-01) rather than + * deriving them from either side; + * 2. the discrimination block at the bottom, which replays a plausible-but-wrong + * lowering (`%Y-%V`, `%G` dropped) through the evaluator and requires it to + * FAIL — so "all green" cannot mean "the evaluator says yes to anything". + */ + +import { describe, it, expect } from 'vitest'; +import { applyInMemoryAggregation, bucketDateValue } from '@objectstack/objectql'; +import type { DateGranularityValue } from '@objectstack/spec/data'; +import { MongoDBDriver } from './mongodb-driver.js'; +import { + buildAggregationPipeline, + MONGODB_DATE_GRANULARITIES, + type AggregationInput, +} from './mongodb-aggregation.js'; +import { runPipeline, type Doc } from './mongodb-pipeline-evaluator.testkit.js'; + +const GRANULARITIES: DateGranularityValue[] = ['day', 'week', 'month', 'quarter', 'year']; + +/** + * The empty bucket's label, chosen so no `String(value)` can produce it — the + * `checkDateBucketParity` device. A side spelling "empty" as the STRING + * `'(null)'`, or as `'null'` from stringifying a SQL NULL, lands under its own + * literal and reads as a different label than the other side's real `null`. + * Under a plain `String()` those two would compare EQUAL and this suite would + * bless the divergence. + */ +const EMPTY_LABEL = '‹empty bucket›'; + +/** + * The fixture rows, in the storage forms THIS DRIVER writes (ADR-0053, and the + * canon table in `mongodb-temporal.ts`) — which is the whole storage-form half of + * this card: + * + * - `at` — `Field.datetime`, stored as a BSON `Date`; + * - `on` — `Field.date`, stored as timezone-naive `YYYY-MM-DD` TEXT; + * - `tod` — `Field.time`, stored as timezone-naive `HH:MM:SS` TEXT. + * + * `at` and `on` name the SAME calendar day on every row, so a granularity that + * labels them differently is a storage-form leak even when each column is + * internally consistent — the cross-column pass `checkDateBucketParity` runs. + * + * The instants are `checkDateBucketParity`'s own, so this evidence lines up with + * what the SQL family is held to: they straddle year, quarter, month and ISO-week + * boundaries (2024-12-30 is 2025-W01) and midnight in both directions. Three + * rows are added that a live-driver fixture cannot easily carry: + * + * - `b8` — an explicit `null` in every temporal column (the empty bucket, #3839); + * - `b9` — the columns MISSING entirely, which MongoDB distinguishes from null + * and `$convert` routes through `onNull` rather than `onError`; + * - `b10` — unparseable junk, the row that proves the lowering is TOTAL. A + * non-total bucket expression fails the whole aggregation on this row where + * the engine's fallback answers. + */ +const ROWS: Doc[] = [ + { id: 'b1', at: new Date('2024-01-15T10:00:00.000Z'), on: '2024-01-15', tod: '10:00:00' }, + { id: 'b2', at: new Date('2024-06-30T23:59:59.000Z'), on: '2024-06-30', tod: '23:59:59' }, + { id: 'b3', at: new Date('2024-07-01T00:00:00.000Z'), on: '2024-07-01', tod: '00:00:00' }, + { id: 'b4', at: new Date('2024-12-30T12:00:00.000Z'), on: '2024-12-30', tod: '12:00:00' }, + { id: 'b5', at: new Date('2025-01-01T00:00:00.000Z'), on: '2025-01-01', tod: '00:00:00' }, + { id: 'b6', at: new Date('2025-05-19T09:00:00.000Z'), on: '2025-05-19', tod: '09:00:00' }, + { id: 'b7', at: new Date('2025-05-19T22:30:00.000Z'), on: '2025-05-19', tod: '22:30:00' }, + { id: 'b8', at: null, on: null, tod: null }, + { id: 'b9' }, + { id: 'b10', at: 'not-a-date', on: 'not-a-date', tod: 'not-a-date' }, +]; + +const COUNT: AggregationInput[] = [{ function: 'count', alias: 'n' }]; + +function astFor(field: string, granularity: DateGranularityValue) { + return { + groupBy: [{ field, dateGranularity: granularity }], + aggregations: [{ function: 'count' as const, alias: 'n' }], + }; +} + +/** `{label: count}` from aggregate rows keyed by `field`, empty bucket out of band. */ +function labelCounts(rows: any[], field: string): Record { + const out: Record = {}; + for (const row of rows ?? []) { + const v = row?.[field]; + const key = v == null ? EMPTY_LABEL : String(v); + out[key] = (out[key] ?? 0) + Number(row?.n ?? 0); + } + return out; +} + +/** What the DRIVER answers: the emitted pipeline, executed by the strict evaluator. */ +function pushedDown(field: string, granularity: DateGranularityValue): Record { + const pipeline = buildAggregationPipeline({ + aggregations: COUNT, + groupBy: [{ field, dateGranularity: granularity }], + }); + return labelCounts(runPipeline(ROWS, pipeline as Doc[]), field); +} + +/** What the ENGINE answers on the same rows, through its real in-memory fallback. */ +function inMemory(field: string, granularity: DateGranularityValue): Record { + return labelCounts(applyInMemoryAggregation(ROWS, astFor(field, granularity) as never), field); +} + +// ── The parity contract, per advertised cell ──────────────────────────────── + +describe('driver-mongodb date buckets match the engine, per advertised granularity × field kind', () => { + const FIELD_KINDS: Array<{ field: string; storage: string }> = [ + { field: 'at', storage: 'Field.datetime — BSON Date' }, + { field: 'on', storage: 'Field.date — YYYY-MM-DD text' }, + { field: 'tod', storage: 'Field.time — HH:MM:SS text' }, + ]; + + for (const { field, storage } of FIELD_KINDS) { + for (const granularity of GRANULARITIES) { + it(`${storage} @ ${granularity}`, () => { + expect( + MONGODB_DATE_GRANULARITIES[granularity], + 'this suite only covers ADVERTISED cells — an unadvertised one is bucketed in memory by definition', + ).toBe(true); + expect(pushedDown(field, granularity)).toEqual(inMemory(field, granularity)); + }); + } + } + + it('the datetime and date columns describe the same days, so they bucket identically', () => { + // The storage-form leak `checkDateBucketParity`'s cross-column pass exists + // for: each column can be internally consistent with its own reference while + // the two disagree, which is exactly the #3773 shape. + for (const granularity of GRANULARITIES) { + const byAt = pushedDown('at', granularity); + const byOn = pushedDown('on', granularity); + expect(Object.values(byAt).reduce((a, b) => a + b, 0), granularity).toBe(ROWS.length); + expect(byAt, granularity).toEqual(byOn); + } + }); + + it('a Field.time column has no instant, and BOTH paths say so rather than inventing one', () => { + // `'10:00:00'` is a wall clock, not an instant (#2004) — `new Date('10:00:00')` + // is an Invalid Date and `$convert` reports a conversion error. Every row + // therefore lands in the empty bucket on both sides. Asserted explicitly + // because "the two maps are equal" would also hold if both sides invented + // the same wrong day, and because it is the measurement that says a `time` + // field needs no separate refusal: the advertised cell is served, degenerately + // but faithfully. + for (const granularity of GRANULARITIES) { + expect(pushedDown('tod', granularity), granularity).toEqual({ [EMPTY_LABEL]: ROWS.length }); + } + }); + + it('null, MISSING and unparseable share ONE empty bucket, on both sides', () => { + // Three rows, one bucket — `bucketDateValue` deliberately does not tell them + // apart (#3839) and neither does the lowering: `onNull` takes the first two, + // `onError` the third, and all three answer `null`. + for (const granularity of GRANULARITIES) { + expect(pushedDown('at', granularity)[EMPTY_LABEL], granularity).toBe(3); + expect(inMemory('at', granularity)[EMPTY_LABEL], granularity).toBe(3); + } + }); +}); + +// ── The label spellings, stated rather than derived ───────────────────────── + +describe('the bucket LABELS are the engine spellings, literally', () => { + /** One instant through the whole lowering, as the single label it produces. */ + function labelOf(instant: Date, granularity: DateGranularityValue): string | null { + const pipeline = buildAggregationPipeline({ + aggregations: COUNT, + groupBy: [{ field: 'at', dateGranularity: granularity }], + }); + const out = runPipeline([{ at: instant, n: 1 }], pipeline as Doc[]); + expect(out).toHaveLength(1); + return (out[0].at ?? null) as string | null; + } + + const CASES: Array<[string, DateGranularityValue, string]> = [ + ['2024-01-15T10:00:00.000Z', 'year', '2024'], + ['2024-01-15T10:00:00.000Z', 'quarter', '2024-Q1'], + ['2024-01-15T10:00:00.000Z', 'month', '2024-01'], + ['2024-01-15T10:00:00.000Z', 'day', '2024-01-15'], + ['2024-01-15T10:00:00.000Z', 'week', '2024-W03'], + // Quarter boundaries, both sides of the same midnight. + ['2024-06-30T23:59:59.000Z', 'quarter', '2024-Q2'], + ['2024-07-01T00:00:00.000Z', 'quarter', '2024-Q3'], + ['2024-10-01T00:00:00.000Z', 'quarter', '2024-Q4'], + // The ISO week-YEAR is not the calendar year: 2024-12-30 is 2025-W01. This + // is the single most drift-prone label in the set, and the reason a `%Y-W%V` + // lowering (which would answer '2024-W01') is wrong in a way that only shows + // up one week a year. + ['2024-12-30T12:00:00.000Z', 'week', '2025-W01'], + ['2024-12-30T12:00:00.000Z', 'year', '2024'], + ['2024-12-30T12:00:00.000Z', 'month', '2024-12'], + ['2025-01-01T00:00:00.000Z', 'week', '2025-W01'], + // …and symmetrically, a January instant whose ISO week belongs to the year + // before: 2027-01-01 is a Friday, so it is 2026-W53. + ['2027-01-01T00:00:00.000Z', 'week', '2026-W53'], + ['2027-01-01T00:00:00.000Z', 'year', '2027'], + ]; + + for (const [iso, granularity, expected] of CASES) { + it(`${iso} @ ${granularity} → ${expected}`, () => { + expect(labelOf(new Date(iso), granularity)).toBe(expected); + // …and the engine agrees, which is what makes the literal a contract + // rather than a transcription of today's output. + expect(bucketDateValue(new Date(iso), granularity)).toBe(expected); + }); + } +}); + +// ── Declared = enforced ───────────────────────────────────────────────────── + +describe('the published capability record and the lowering are the same set', () => { + const driver = new MongoDBDriver({ url: 'mongodb://127.0.0.1:27017/parity-probe' }); + + it('MongoDBDriver.supports publishes the builder\'s own record, not a copy', () => { + // Identity, not equality. Two literals that happen to agree today are the + // mechanism by which a capability record drifts from the lowering it + // advertises — and that drift is the one failure worse than advertising + // nothing, because the engine STOPS bucketing in memory on the strength of + // the bit and a working query starts answering 501. + expect(driver.supports.queryDateGranularity).toBe(MONGODB_DATE_GRANULARITIES); + }); + + it('every advertised granularity actually lowers', () => { + for (const [granularity, advertised] of Object.entries(MONGODB_DATE_GRANULARITIES)) { + if (advertised !== true) continue; + expect( + () => + buildAggregationPipeline({ + aggregations: COUNT, + groupBy: [{ field: 'at', dateGranularity: granularity as DateGranularityValue }], + }), + granularity, + ).not.toThrow(); + } + }); + + it('advertises exactly the five granularities @objectstack/spec declares', () => { + // MongoDB has both halves of the ISO-8601 week date (`%G`/`%V`), which is why + // this record carries `week: true` where `driver-sql` on SQLite carries + // `week: false` — the dialects genuinely differ, and the records say so. + expect(MONGODB_DATE_GRANULARITIES).toEqual({ + day: true, week: true, month: true, quarter: true, year: true, + }); + expect(Object.keys(MONGODB_DATE_GRANULARITIES).sort()).toEqual([...GRANULARITIES].sort()); + }); +}); + +// ── Timezones stay engine-side ────────────────────────────────────────────── + +describe('the pushdown is UTC, which is the only thing it could be', () => { + it('the builder takes no timezone, and the engine never pushes a non-UTC bucket down', () => { + // ADR-0053 Phase 2 (D2), pinned engine-side by + // `engine-aggregate-timezone.test.ts`: `engine.aggregate` forces the + // in-memory path whenever a non-UTC reference zone meets a date bucket, and + // the AST it hands a driver carries no `timezone` key at all. So this + // lowering cannot be asked for a zoned bucket — measured here rather than + // assumed, by showing that the zoned label for a straddling instant is a + // DIFFERENT label, i.e. that a driver silently answering the UTC one would + // be answering the wrong question. + const straddling = new Date('2024-03-01T03:00:00.000Z'); // NY: 2024-02-29 + expect(bucketDateValue(straddling, 'day')).toBe('2024-03-01'); + expect(bucketDateValue(straddling, 'day', 'America/New_York')).toBe('2024-02-29'); + + const pipeline = buildAggregationPipeline({ + aggregations: COUNT, + groupBy: [{ field: 'at', dateGranularity: 'day' }], + }); + const out = runPipeline([{ at: straddling, n: 1 }], pipeline as Doc[]); + expect(out[0].at).toBe('2024-03-01'); + // Nothing in the emitted pipeline names a zone, so there is no seam through + // which one could be half-applied. + expect(JSON.stringify(pipeline)).not.toContain('timezone'); + }); +}); + +// ── The evaluator is not the thing being tested, so it gets tested ────────── + +describe('the evaluator discriminates on the date operators too', () => { + it('the plausible-but-wrong ISO week lowering (%Y instead of %G) FAILS the year-boundary row', () => { + // The bug this suite exists to make impossible: `%Y-W%V` looks right, agrees + // with `%G-W%V` on 51 weeks of the year, and mislabels the week that spans + // New Year — exactly the row `checkDateBucketParity`'s fixture carries. + const wrong = [ + { + $group: { + _id: { + at: { + $dateToString: { + format: '%Y-W%V', + date: { $convert: { input: '$at', to: 'date', onError: null, onNull: null } }, + }, + }, + }, + n: { $sum: 1 }, + }, + }, + { $project: { _id: 0, at: '$_id.at', n: 1 } }, + ]; + const out = runPipeline([{ at: new Date('2024-12-30T12:00:00.000Z') }], wrong as Doc[]); + expect(out[0].at).toBe('2024-W01'); + expect(out[0].at).not.toBe(bucketDateValue(new Date('2024-12-30T12:00:00.000Z'), 'week')); + }); + + it('refuses a date operator handed something that is not a date', () => { + // Without the `$convert` step a text-stored `Field.date` column reaches + // `$dateToString` as a STRING. A real mongod raises; an evaluator that + // quietly coerced would bless a lowering that fails in production on two of + // the three declared temporal kinds. + expect(() => + runPipeline([{ on: '2024-01-15' }], [ + { $group: { _id: { on: { $dateToString: { format: '%Y', date: '$on' } } }, n: { $sum: 1 } } }, + ] as Doc[]), + ).toThrow(/the lowering must convert first/); + }); + + it('parses ISO-8601 by its own grammar, so it CAN disagree with new Date()', () => { + // The property that makes every comparison above meaningful. JS accepts a + // space separator; ISO-8601 (and therefore `$dateFromString`) does not — so + // the residual divergence `instantOf` documents shows up as a difference + // here instead of being absorbed by a shared parser. + const legacy = '2024-01-15 10:00:00'; + expect(Number.isNaN(new Date(legacy).getTime())).toBe(false); // JS parses it + const pipeline = buildAggregationPipeline({ + aggregations: COUNT, + groupBy: [{ field: 'at', dateGranularity: 'day' }], + }); + expect(runPipeline([{ at: legacy, n: 1 }], pipeline as Doc[])[0].at ?? null).toBeNull(); + // Not a form this driver ever WRITES — `coerceTemporalValue` canonicalises a + // `datetime` to a BSON Date and a `date` to `YYYY-MM-DD` — so this is the + // pre-#4047 legacy-row exposure, recorded rather than papered over. (The + // engine's exact label for it depends on the host zone, since JS reads a + // zone-naive legacy string as LOCAL time; that it produces one at all is the + // divergence.) + expect(bucketDateValue(legacy, 'day')).not.toBeNull(); + }); +}); diff --git a/packages/drivers/driver-mongodb/src/mongodb-driver.ts b/packages/drivers/driver-mongodb/src/mongodb-driver.ts index c0c18f13ef..d939ed3f75 100644 --- a/packages/drivers/driver-mongodb/src/mongodb-driver.ts +++ b/packages/drivers/driver-mongodb/src/mongodb-driver.ts @@ -31,6 +31,7 @@ import { import { buildAggregationPipeline, postProcessAggregation, + MONGODB_DATE_GRANULARITIES, } from './mongodb-aggregation.js'; import { syncCollectionSchema, dropCollection } from './mongodb-schema.js'; import { @@ -86,13 +87,35 @@ export class MongoDBDriver implements IDataDriver { * Capability advertisement (#4634, ADR-0049): only the bits with an engine * reader survive. This driver batches its schema DDL round-trips * ({@link syncSchemasBatch}), so it opts in via the one bit the engine ANDs - * with method presence. It owns neither persistent autonumber sequences nor - * native date bucketing here, so `autonumber`/`queryDateGranularity` stay - * absent and the engine keeps its fallbacks. Everything the old 30-bit - * literal declared is expressed by the methods this class implements. + * with method presence. It owns no persistent autonumber sequences, so + * `autonumber` stays absent and the engine keeps that fallback. Everything the + * old 30-bit literal declared is expressed by the methods this class + * implements. + * + * [#7580] `queryDateGranularity` is now PUBLISHED, where it used to be + * deliberately absent. `buildAggregationPipeline` lowers a + * `dateGranularity`-bearing `groupBy` into `$dateToString` bucket labels + * server-side, so `engine.aggregate` may push a bucketed aggregate down here + * instead of fetching every row and bucketing it in JS. The record published + * is the builder's own `MONGODB_DATE_GRANULARITIES` — not a second literal + * that could drift from it — because an advertised granularity the builder + * then refuses turns a query the engine used to answer into a 501. + * + * What this bit does NOT claim: agreement with a real mongod. The lowering's + * `$dateToString` / `$convert` semantics are documentation-derived and held to + * the engine's labels by an in-process evaluator, because this fleet cannot + * fetch a mongod binary (#5517) — the bound is stated in full at the top of + * `mongodb-date-bucket-parity.test.ts`, and it is the reason that suite exists + * rather than an enrolment in `checkDateBucketParity`, which needs a server. + * + * Timezones stay engine-side: `engine.aggregate` forces the in-memory path for + * any non-UTC reference zone (ADR-0053 Phase 2 D2), and the AST it hands a + * driver carries no `timezone` at all, so this bucketing is UTC by + * construction — the only thing it could mean. */ public readonly supports = { batchSchemaSync: true, + queryDateGranularity: MONGODB_DATE_GRANULARITIES, }; private client: MongoClient; diff --git a/packages/drivers/driver-mongodb/src/mongodb-pipeline-evaluator.testkit.ts b/packages/drivers/driver-mongodb/src/mongodb-pipeline-evaluator.testkit.ts new file mode 100644 index 0000000000..d2363bed55 --- /dev/null +++ b/packages/drivers/driver-mongodb/src/mongodb-pipeline-evaluator.testkit.ts @@ -0,0 +1,445 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * A deliberately strict, in-process reader of the pipelines + * `buildAggregationPipeline` emits — the server-free half #5517 requires, in the + * one place both suites that need it can share it. + * + * ## Why it exists + * + * The builder's output is a PIPELINE and every contract it must satisfy is + * stated as VALUES: `AGGREGATION_CASES` names the numbers each group must + * produce, and `bucketDateValue` names the label each instant must carry. Pinning + * the emitted stages literally would pin today's spelling rather than the + * semantics — and it is the semantics that keep being wrong. Every defect + * #6850/#6814 found emitted a perfectly well-formed pipeline; the + * `"[object Object]"` `$group._id` was valid MongoDB that grouped by nothing. + * + * ## Why it is strict + * + * Every stage, accumulator and expression it does not model is a thrown + * {@link UnsupportedShape}, never a tolerated no-op. A stand-in more permissive + * than the real engine turns a suite into a green light for broken code. Its own + * discrimination is proved in `mongodb-aggregation-translation.test.ts`, where + * each pre-fix lowering is replayed through it and must FAIL the case it broke. + * + * ## ⚠️ What it deliberately does NOT answer + * + * Whether a real mongod agrees. Every operator below is modelled from the + * MongoDB manual, not observed: this fleet cannot fetch a mongod binary at all + * (proxy 403 on the download — #5517), and a suite nobody has executed against + * the real thing is a claim, not a check. The date operators added for #7580 + * (`$convert` → date, `$dateToString`, `$concat`, `$switch`, `$lte`) carry that + * bound exactly as the #6850/#6814 ones do. + * + * The single most important discipline that makes the bound survivable: this + * file models the DOCUMENTED semantics, never the behaviour the lowering + * happens to want. `$convert`'s string arm parses ISO-8601 with its own + * grammar rather than deferring to JS `new Date(...)` — otherwise the evaluator + * and the in-memory reference would agree by construction and the legacy-format + * divergence `mongodb-aggregation.ts#instantOf` documents would be invisible + * here. Likewise `%G`/`%V` are computed from the ISO-8601 week-date definition, + * not transcribed from `bucketDateValue`. + * + * `.testkit.ts` (not `.test.ts`) so it carries no `describe` of its own and can + * be imported by two suites without registering their tests twice — the + * `legacy-datetime-storage.testkit.ts` convention from `driver-sql`. + */ + +/** Thrown for any shape this evaluator does not model — never swallowed. */ +export class UnsupportedShape extends Error {} + +/** A field path that resolved to nothing. MongoDB distinguishes it from `null`. */ +export const MISSING = Symbol('missing'); + +export type Doc = Record; + +/** Resolve a `'$a.b'` field path against a document, or {@link MISSING}. */ +export function resolvePath(doc: Doc, ref: string): unknown { + if (!ref.startsWith('$')) throw new UnsupportedShape(`not a field path: ${ref}`); + let current: unknown = doc; + for (const part of ref.slice(1).split('.')) { + if (current === null || typeof current !== 'object' || Array.isArray(current)) return MISSING; + if (!Object.prototype.hasOwnProperty.call(current, part)) return MISSING; + current = (current as Doc)[part]; + } + return current; +} + +// ── Date semantics, modelled from the manual ──────────────────────────────── + +/** + * The ISO-8601 grammar `$dateFromString` accepts with no explicit `format` — + * which is also what `$convert … to: 'date'` uses for a string input. + * + * Written out rather than delegated to `new Date(s)` ON PURPOSE. JS's parser + * additionally accepts a pile of legacy spellings (`'2024-01-15 10:00:00'` with a + * space separator, `'2024/01/15'`, RFC-2822 dates); deferring to it here would + * make this evaluator accept exactly what the in-memory reference accepts, so the + * one storage-form divergence `instantOf` documents would compare EQUAL in every + * parity assertion instead of showing up. A model that cannot disagree with the + * thing it is checking is not a model. + */ +const ISO_8601 = + /^(\d{4})-(\d{2})-(\d{2})(?:T(\d{2}):(\d{2})(?::(\d{2})(?:\.(\d{1,3}))?)?(Z|[+-]\d{2}:?\d{2})?)?$/; + +function parseIso8601(s: string): Date | null { + const m = ISO_8601.exec(s); + if (!m) return null; + const [, y, mo, d, hh = '0', mi = '0', ss = '0', frac = '0', zone] = m; + const ms = Number(`${frac}00`.slice(0, 3)); + let epoch = Date.UTC(Number(y), Number(mo) - 1, Number(d), Number(hh), Number(mi), Number(ss), ms); + if (zone && zone !== 'Z') { + const sign = zone.startsWith('-') ? -1 : 1; + const [oh, om] = zone.slice(1).replace(':', '').match(/\d{2}/g)!.map(Number); + epoch -= sign * (oh * 60 + om) * 60_000; + } + const out = new Date(epoch); + // A syntactically well-formed but impossible date (`2024-02-31`) rolls over in + // `Date.UTC`; MongoDB rejects it, so reject it here too rather than silently + // labelling March. + if ( + out.getUTCFullYear() !== Number(y) + || out.getUTCMonth() !== Number(mo) - 1 + || out.getUTCDate() !== Number(d) + ) { + return null; + } + return out; +} + +/** A conversion the manual defines as an error, so the caller's `onError` answers. */ +const CONVERT_ERROR = Symbol('convert-error'); + +/** + * `$convert … to: 'date'`, per the manual's conversion table: a date converts to + * itself, a numeric is read as epoch milliseconds, a string is parsed as + * ISO-8601, and every other input is a conversion ERROR (which the caller's + * `onError` then answers). `null` and a missing field take `onNull`. + */ +function convertToDate(value: unknown): Date | typeof CONVERT_ERROR | null { + if (value === MISSING || value === null) return null; // → onNull + if (value instanceof Date) return Number.isNaN(value.getTime()) ? CONVERT_ERROR : value; + if (typeof value === 'number') { + if (!Number.isFinite(value)) return CONVERT_ERROR; + const d = new Date(value); + return Number.isNaN(d.getTime()) ? CONVERT_ERROR : d; + } + if (typeof value === 'string') return parseIso8601(value) ?? CONVERT_ERROR; + return CONVERT_ERROR; +} + +const pad = (n: number, w: number): string => String(Math.abs(n)).padStart(w, '0'); + +/** + * The ISO-8601 week date of an instant: its week-numbering YEAR (`%G`) and week + * number (`%V`). + * + * Straight from the definition — week 1 is the week containing the year's first + * Thursday, weeks run Monday..Sunday — rather than transcribed from + * `bucketDateValue`. Both implement the same standard, which is the point: they + * have to agree because ISO-8601 says so, not because one was copied. + */ +function isoWeekDate(d: Date): { year: number; week: number } { + // Move to the Thursday of this instant's week; the year that Thursday falls in + // IS the ISO week-numbering year, by definition. + const thursday = new Date(Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), d.getUTCDate())); + const isoWeekday = thursday.getUTCDay() === 0 ? 7 : thursday.getUTCDay(); // Mon=1..Sun=7 + thursday.setUTCDate(thursday.getUTCDate() + (4 - isoWeekday)); + const year = thursday.getUTCFullYear(); + const jan1 = Date.UTC(year, 0, 1); + const dayOfYear = Math.round((thursday.getTime() - jan1) / 86_400_000); // 0-based + return { year, week: Math.floor(dayOfYear / 7) + 1 }; +} + +/** `$dateToString`'s format specifiers, for the ones this builder emits. */ +function formatDate(d: Date, format: string): string { + let out = ''; + for (let i = 0; i < format.length; i += 1) { + if (format[i] !== '%') { + out += format[i]; + continue; + } + const spec = format[i + 1]; + i += 1; + switch (spec) { + case '%': out += '%'; break; + case 'Y': out += pad(d.getUTCFullYear(), 4); break; + case 'm': out += pad(d.getUTCMonth() + 1, 2); break; + case 'd': out += pad(d.getUTCDate(), 2); break; + case 'H': out += pad(d.getUTCHours(), 2); break; + case 'M': out += pad(d.getUTCMinutes(), 2); break; + case 'S': out += pad(d.getUTCSeconds(), 2); break; + case 'G': out += pad(isoWeekDate(d).year, 4); break; + case 'V': out += pad(isoWeekDate(d).week, 2); break; + default: + throw new UnsupportedShape(`unsupported $dateToString format specifier '%${String(spec)}'`); + } + } + return out; +} + +/** + * BSON canonical sort order, for the two types `$lte` meets here. Null sorts + * BELOW every string — which is why the `quarter` lowering can let its `$switch` + * run on a null instant without a guard: it answers a digit, and the surrounding + * `$concat` has already decided the whole label is null. + */ +function bsonLte(left: unknown, right: unknown): boolean { + const rank = (v: unknown): number => { + if (v === MISSING || v === null) return 0; + if (typeof v === 'number') return 1; + if (typeof v === 'string') return 2; + throw new UnsupportedShape(`$lte over an unmodelled BSON type: ${typeof v}`); + }; + const lr = rank(left); + const rr = rank(right); + if (lr !== rr) return lr < rr; + if (lr === 0) return true; // null <= null + return (left as string | number) <= (right as string | number); +} + +/** MongoDB's expression truthiness: `false`, `null`, `0` and missing are false. */ +function truthy(v: unknown): boolean { + return !(v === false || v === null || v === MISSING || v === 0 || v === undefined); +} + +// ── Expressions ───────────────────────────────────────────────────────────── + +/** + * Evaluate an aggregation EXPRESSION against one document. + * + * Models only the operators this builder emits. `$$`-prefixed system variables + * are refused rather than guessed at — `$$ROOT` reaches the `array_agg` arm, and + * an evaluator that quietly resolved it would be asserting a semantics nobody + * checked. + */ +export function evalExpr(doc: Doc, expr: unknown): unknown { + if (typeof expr === 'string' && expr.startsWith('$')) { + if (expr.startsWith('$$')) throw new UnsupportedShape(`system variable '${expr}' is not modelled`); + return resolvePath(doc, expr); + } + if (expr === null || typeof expr !== 'object') return expr; // literal + if (Array.isArray(expr)) return expr.map((e) => evalExpr(doc, e)); + + const keys = Object.keys(expr as Doc); + if (keys.length !== 1) throw new UnsupportedShape(`expression with ${keys.length} keys: ${keys.join(', ')}`); + const [op] = keys; + const arg = (expr as Doc)[op]; + + switch (op) { + case '$ifNull': { + if (!Array.isArray(arg) || arg.length !== 2) throw new UnsupportedShape('$ifNull takes [expr, replacement]'); + const value = evalExpr(doc, arg[0]); + return value === MISSING || value === null ? evalExpr(doc, arg[1]) : value; + } + case '$eq': { + if (!Array.isArray(arg) || arg.length !== 2) throw new UnsupportedShape('$eq takes two operands'); + // MISSING and null compare EQUAL under `$eq` in the aggregation language, + // which is why the `count` lowering routes through `$ifNull` first rather + // than relying on it. + const left = evalExpr(doc, arg[0]); + const right = evalExpr(doc, arg[1]); + const norm = (v: unknown) => (v === MISSING ? null : v); + return norm(left) === norm(right); + } + case '$lte': { + if (!Array.isArray(arg) || arg.length !== 2) throw new UnsupportedShape('$lte takes two operands'); + return bsonLte(evalExpr(doc, arg[0]), evalExpr(doc, arg[1])); + } + case '$cond': { + if (!Array.isArray(arg) || arg.length !== 3) throw new UnsupportedShape('$cond takes [if, then, else]'); + return truthy(evalExpr(doc, arg[0])) ? evalExpr(doc, arg[1]) : evalExpr(doc, arg[2]); + } + case '$switch': { + const spec = arg as { branches?: Array<{ case: unknown; then: unknown }>; default?: unknown }; + if (!spec || !Array.isArray(spec.branches)) throw new UnsupportedShape('$switch takes { branches, default? }'); + for (const branch of spec.branches) { + if (!branch || !('case' in branch) || !('then' in branch)) { + throw new UnsupportedShape('$switch branch takes { case, then }'); + } + if (truthy(evalExpr(doc, branch.case))) return evalExpr(doc, branch.then); + } + // The manual: no matching branch and no `default` is an ERROR, not null. + if (!('default' in spec)) throw new UnsupportedShape('$switch matched no branch and declares no default'); + return evalExpr(doc, spec.default); + } + case '$concat': { + if (!Array.isArray(arg)) throw new UnsupportedShape('$concat takes an array of operands'); + const parts = arg.map((a) => evalExpr(doc, a)); + // "If any argument resolves to null or refers to a field that is missing, + // $concat returns null" — the whole of this lowering's null propagation. + if (parts.some((p) => p === null || p === MISSING)) return null; + for (const p of parts) { + if (typeof p !== 'string') throw new UnsupportedShape(`$concat over a non-string operand: ${typeof p}`); + } + return parts.join(''); + } + case '$convert': { + const spec = arg as { input?: unknown; to?: unknown; onError?: unknown; onNull?: unknown }; + if (!spec || typeof spec !== 'object') throw new UnsupportedShape('$convert takes a document'); + if (spec.to !== 'date') throw new UnsupportedShape(`$convert to '${String(spec.to)}' is not modelled`); + const converted = convertToDate(evalExpr(doc, spec.input)); + if (converted === CONVERT_ERROR) { + if (!('onError' in spec)) throw new UnsupportedShape('$convert failed and declares no onError'); + return evalExpr(doc, spec.onError); + } + if (converted === null) { + if (!('onNull' in spec)) throw new UnsupportedShape('$convert saw null and declares no onNull'); + return evalExpr(doc, spec.onNull); + } + return converted; + } + case '$dateToString': { + const spec = arg as { format?: unknown; date?: unknown; onNull?: unknown }; + if (!spec || typeof spec.format !== 'string') throw new UnsupportedShape('$dateToString takes { format, date }'); + const value = evalExpr(doc, spec.date); + // "If the date is null or missing, $dateToString returns null" unless + // `onNull` says otherwise. + if (value === null || value === MISSING) return 'onNull' in spec ? evalExpr(doc, spec.onNull) : null; + if (!(value instanceof Date) || Number.isNaN(value.getTime())) { + // A real mongod raises here rather than answering; refusing keeps the + // evaluator from blessing a lowering that feeds it a non-date. + throw new UnsupportedShape('$dateToString received a non-date — the lowering must convert first'); + } + return formatDate(value, spec.format); + } + default: + throw new UnsupportedShape(`unsupported aggregation expression operator '${op}'`); + } +} + +/** One accumulator, folded over the documents of a single group. */ +export function accumulate(rows: Doc[], acc: unknown): unknown { + if (acc === null || typeof acc !== 'object' || Array.isArray(acc)) { + throw new UnsupportedShape(`accumulator must be a one-key document, got ${JSON.stringify(acc)}`); + } + const keys = Object.keys(acc as Doc); + if (keys.length !== 1) throw new UnsupportedShape(`accumulator with ${keys.length} keys`); + const [op] = keys; + const arg = (acc as Doc)[op]; + const values = rows.map((row) => evalExpr(row, arg)); + /** MongoDB's arithmetic accumulators ignore missing and non-numeric values. */ + const numbers = values.filter((v): v is number => typeof v === 'number'); + + switch (op) { + case '$sum': + return numbers.reduce((a, b) => a + b, 0); + case '$avg': + return numbers.length === 0 ? null : numbers.reduce((a, b) => a + b, 0) / numbers.length; + case '$min': + return numbers.length === 0 ? null : Math.min(...numbers); + case '$max': + return numbers.length === 0 ? null : Math.max(...numbers); + case '$addToSet': { + // `$addToSet` skips a MISSING field and keeps an explicit `null` — the + // whole of #6814 lives in that second half. + const set: unknown[] = []; + for (const v of values) { + if (v === MISSING) continue; + if (!set.includes(v)) set.push(v); + } + return set; + } + case '$push': + return values.filter((v) => v !== MISSING); + default: + throw new UnsupportedShape(`unsupported accumulator '${op}'`); + } +} + +/** Execute an emitted pipeline over the fixture. Throws on any shape not modelled. */ +export function runPipeline(rows: readonly Doc[], pipeline: readonly Doc[]): Doc[] { + let docs: Doc[] = rows.map((row) => ({ ...row })); + + for (const stage of pipeline) { + const keys = Object.keys(stage); + if (keys.length !== 1) throw new UnsupportedShape(`pipeline stage with ${keys.length} keys`); + const [name] = keys; + const spec = stage[name]; + + switch (name) { + case '$group': { + const { _id: idSpec, ...accumulators } = spec as Doc; + const groups = new Map(); + for (const doc of docs) { + let id: unknown; + if (idSpec === null) { + id = null; + } else if (idSpec && typeof idSpec === 'object' && !Array.isArray(idSpec)) { + const key: Doc = {}; + for (const [outKey, ref] of Object.entries(idSpec as Doc)) { + const value = evalExpr(doc, ref); + key[outKey] = value === MISSING ? null : value; + } + id = key; + } else { + throw new UnsupportedShape(`$group._id must be null or a document, got ${JSON.stringify(idSpec)}`); + } + const bucket = JSON.stringify(id); + if (!groups.has(bucket)) groups.set(bucket, { id, rows: [] }); + groups.get(bucket)!.rows.push(doc); + } + docs = [...groups.values()].map(({ id, rows: groupRows }) => { + const out: Doc = { _id: id }; + for (const [alias, acc] of Object.entries(accumulators)) { + out[alias] = accumulate(groupRows, acc); + } + return out; + }); + break; + } + case '$project': { + docs = docs.map((doc) => { + const out: Doc = {}; + for (const [key, rule] of Object.entries(spec as Doc)) { + if (rule === 0 || rule === false) continue; + if (rule === 1 || rule === true) { + const value = resolvePath(doc, `$${key}`); + if (value !== MISSING) out[key] = value; + continue; + } + if (typeof rule === 'string') { + const value = resolvePath(doc, rule); + if (value !== MISSING) out[key] = value; + continue; + } + throw new UnsupportedShape(`unsupported $project rule for '${key}': ${JSON.stringify(rule)}`); + } + return out; + }); + break; + } + case '$match': + // Reachable only from `opts.where`, which no case in the shared set + // spells. Refused rather than approximated: this file's `$match` would + // be a second, weaker copy of the matcher + // `mongodb-filter-logic-translation.test.ts` already owns. + throw new UnsupportedShape('$match is not modelled here — see mongodb-filter-logic-translation.test.ts'); + case '$sort': { + const entries = Object.entries(spec as Doc); + docs = [...docs].sort((a, b) => { + for (const [field, dir] of entries) { + if (dir !== 1 && dir !== -1) throw new UnsupportedShape(`$sort direction ${String(dir)}`); + const av = a[field]; + const bv = b[field]; + if (av === bv) continue; + return ((av as never) < (bv as never) ? -1 : 1) * (dir as number); + } + return 0; + }); + break; + } + case '$skip': + docs = docs.slice(spec as number); + break; + case '$limit': + docs = docs.slice(0, spec as number); + break; + default: + throw new UnsupportedShape(`unsupported pipeline stage '${name}'`); + } + } + + return docs; +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 90a4398409..b94e39704c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -886,6 +886,9 @@ importers: specifier: ^6.0.0 version: 6.0.0 devDependencies: + '@objectstack/objectql': + specifier: workspace:* + version: link:../../objectql '@types/node': specifier: ^26.1.2 version: 26.1.2