diff --git a/.changeset/hash-shadow-unique-key-utf8mb4.md b/.changeset/hash-shadow-unique-key-utf8mb4.md new file mode 100644 index 0000000000..79ad12fad3 --- /dev/null +++ b/.changeset/hash-shadow-unique-key-utf8mb4.md @@ -0,0 +1,25 @@ +--- +'@objectstack/driver-sql': minor +--- + +driver-sql (MySQL): carry an over-long UNIQUE index on a hash-shadow column + +On utf8mb4 InnoDB a key part holds at most 3072 bytes (768 characters), so a +full-value UNIQUE index over a longer column is inexpressible — an OAuth access +token that is a multi-KB JWT cannot be made keyable by any declared bound. +Measured on live MySQL 8.0.46, 7 of 44 exported platform objects failed +`syncSchema` outright and landed registered with their declared uniqueness +absent (Postgres 16.13: 0 of 44). + +Such a UNIQUE index is now carried by a driver-owned `__hash` column — a +`STORED GENERATED` `VARBINARY(32)` holding the full, untruncated SHA-256 of the +key values — with the unique index on that column. Uniqueness is still enforced +over the whole value: distinct values sharing a long prefix are both accepted +(the property that ruled out prefix-unique indexes), NULLs stay distinct, and a +composite tuple containing NULL conflicts with nothing. + +The shadow is created only *after* the server refuses the direct index, so the +dialect divergence is selected by the error code rather than by a dialect check: +Postgres and SQLite are byte-identical to before. Non-unique indexes are +deliberately left refused — an index over a digest accelerates no lookup the +planner can reach. diff --git a/packages/drivers/driver-sql/src/schema-drift.ts b/packages/drivers/driver-sql/src/schema-drift.ts index eb68d9ab90..3bf6c09c2e 100644 --- a/packages/drivers/driver-sql/src/schema-drift.ts +++ b/packages/drivers/driver-sql/src/schema-drift.ts @@ -327,6 +327,36 @@ export interface ManagedDriftEntry extends SchemaDiffEntry { /** Columns the driver creates unconditionally — never metadata fields. */ export const BUILTIN_COLUMNS = new Set(['id', 'created_at', 'updated_at']); +/** + * Suffix of a HASH-SHADOW column — the driver-owned column that carries a + * declared UNIQUE index MySQL cannot express over the values themselves + * (#11627). See `SqlDriver.createHashShadowUniqueIndex` for why it exists and + * what it stores. + */ +export const HASH_SHADOW_SUFFIX = '__hash'; + +/** + * Is this physical column a driver-owned hash shadow (#11627)? + * + * ⚠️ Load-bearing for the ORPHAN differ below, and the reason this predicate + * is exported rather than inlined. A hash-shadow column exists in the database + * and — by construction — in no metadata field, which is the exact shape the + * orphan pass reports as `unmapped_column` with a `drop_column` op. Dropping it + * would take the UNIQUE index it carries with it, silently returning the object + * to "registered but its declared uniqueness unenforced" — the very state + * #11374/#11627 exist to end, reached this time through the migration tool + * rather than through a refused DDL. + * + * Matched by SUFFIX rather than by a registry of known names, deliberately: the + * differ runs against a database whose metadata it is comparing to, and a + * shadow whose declared index has since been removed must still be recognised + * as driver-owned (it is then cleaned up by the index's own removal path, not + * by a blind column drop). + */ +export function isHashShadowColumn(name: string): boolean { + return name.endsWith(HASH_SHADOW_SUFFIX); +} + /** Minimal shape of an introspected physical column (see SqlDriver.introspectColumns). */ export interface PhysicalColumn { name: string; @@ -838,6 +868,10 @@ export function diffManagedTable(args: { // ── orphaned columns (physical column, no metadata field) ────────── for (const col of columns) { if (BUILTIN_COLUMNS.has(col.name)) continue; + // Driver-owned hash shadow (#11627), never a metadata field — see + // {@link isHashShadowColumn} for why dropping it as an orphan would + // silently disable the UNIQUE constraint it carries. + if (isHashShadowColumn(col.name)) continue; if (expectedColumns.has(col.name)) continue; out.push({ kind: 'unmapped_column', diff --git a/packages/drivers/driver-sql/src/sql-driver-11627-hash-shadow-key.test.ts b/packages/drivers/driver-sql/src/sql-driver-11627-hash-shadow-key.test.ts new file mode 100644 index 0000000000..6410300805 --- /dev/null +++ b/packages/drivers/driver-sql/src/sql-driver-11627-hash-shadow-key.test.ts @@ -0,0 +1,332 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * #11627 — the hash-shadow key: carrying a declared UNIQUE index MySQL cannot + * express over the values themselves. + * + * ## The defect + * + * On utf8mb4 InnoDB a key part holds at most 3072 bytes (768 characters), so a + * full-value UNIQUE index over a longer column is INEXPRESSIBLE. Measured on + * live MySQL 8.0.46 across every exported platform object: 7 of 44 failed + * `syncSchema` outright (6 `ER_BLOB_KEY_WITHOUT_LENGTH` + 1 `ER_TOO_LONG_KEY`); + * Postgres 16.13 took all 44. An OAuth access token may legitimately be a + * multi-KB JWT, so no declared bound can rescue those columns. + * + * ## Why a shadow and not a prefix index + * + * The maintainer's 2026-08-24 ruling on #11374 chose the hash route and + * rejected prefix-unique indexes, on measurement: `UNIQUE KEY (token(191))` + * enforces uniqueness over the PREFIX, so two genuinely distinct tokens that + * share their first 191 characters collide and the second is refused as + * `ER_DUP_ENTRY` — on `sys_session.token`, a valid sign-in refused as a + * duplicate. The prefix-collision test below is the executable form of that + * distinction: it is the assertion a prefix index would fail. + * + * ## What the live cell reads, and what it deliberately does NOT read + * + * Every physical claim here is read from `information_schema` in a SEPARATE + * query — never from the DDL this driver emitted. The emitted DDL is this + * change's own output; asserting on it would prove only that the driver said + * what it said. `SUB_PART IS NULL` is the load-bearing one: it is what + * distinguishes the shadow index from the rejected prefix index, which would + * report a sub-part. + * + * Opt-in, like every live cell in this package: + * + * OS_TEST_MYSQL_URL=mysql://root:root@127.0.0.1:3306/conformance \ + * pnpm --filter @objectstack/driver-sql test + */ + +import { describe, it, expect, afterEach } from 'vitest'; +import { createHash } from 'node:crypto'; +import { SqlDriver } from '../src/index.js'; +import { isHashShadowColumn, HASH_SHADOW_SUFFIX } from './schema-drift.js'; +import { MYSQL_CELL, PG_CELL, dialectCell, declareDialectCell } from './live-dialect-matrix.testkit.js'; + +/** An object with one bounded text field carrying a UNIQUE index over it. */ +const uniqueOn = (name: string, maxLength: number | undefined) => ({ + name, + fields: { v: { type: 'text', ...(maxLength === undefined ? {} : { maxLength }) } }, + indexes: [{ fields: ['v'], unique: true, name: `uniq_${name}_v` }], +}); + +// ── Dialect-free: the naming contract the differ and the driver share ─────── + +describe('hash-shadow column naming (#11627)', () => { + /** + * The differ and the driver must agree on what a shadow column IS, or the + * orphan pass proposes dropping the column that carries a live UNIQUE + * constraint. Two modules, one predicate — asserted here rather than trusted. + */ + it('is recognised as driver-owned by the drift differ', () => { + const shadow = (SqlDriver as any).hashShadowColumnFor('uniq_sys_oauth_access_token_token'); + expect(shadow).toBe(`uniq_sys_oauth_access_token_token${HASH_SHADOW_SUFFIX}`); + expect(isHashShadowColumn(shadow)).toBe(true); + expect(isHashShadowColumn('token')).toBe(false); + }); + + /** + * MySQL identifiers cap at 64 characters, and the shadow name is derived from + * an index name that is itself derived from table + columns. Truncation alone + * would alias two long index names sharing a prefix onto ONE column — a + * second constraint silently landing on the first one's shadow. The digest + * is what makes the overflow branch injective. + */ + it('stays inside MySQL 64-char identifiers without aliasing long names', () => { + const a = 'uniq_' + 'x'.repeat(70) + '_alpha'; + const b = 'uniq_' + 'x'.repeat(70) + '_beta'; + const sa = (SqlDriver as any).hashShadowColumnFor(a); + const sb = (SqlDriver as any).hashShadowColumnFor(b); + expect(sa.length).toBeLessThanOrEqual(64); + expect(sb.length).toBeLessThanOrEqual(64); + expect(sa).not.toBe(sb); + expect(isHashShadowColumn(sa)).toBe(true); + }); +}); + +// ── The dialects that never refuse must be untouched ──────────────────────── + +describe('dialects with no key-length ceiling are unchanged (#11627)', () => { + let driver: SqlDriver; + afterEach(async () => { await driver?.disconnect().catch(() => {}); }); + + /** + * ⛔ The negative half, and the reason "MySQL syncs now" is not this file's + * whole assertion. The shadow is selected BY THE SERVER'S ERROR CODE, not by + * a dialect getter, so a dialect that takes the direct index must keep it and + * gain no column. A change that applied the shadow everywhere would pass + * every positive assertion in this file and fail exactly here. + */ + it('SQLite keeps the direct index and grows no shadow column', async () => { + driver = new SqlDriver(dialectCell('sqlite').config()); + await driver.initObjects([uniqueOn('os11627_sqlite', 4096)]); + const info = await (driver as any).knex('os11627_sqlite').columnInfo(); + expect(Object.keys(info)).toContain('v'); + expect(Object.keys(info).filter((c) => isHashShadowColumn(c))).toEqual([]); + }); +}); + +// ── Live MySQL: the accept transition, the boundary, and the semantics ────── + +declareDialectCell(MYSQL_CELL, 'hash-shadow key (#11627)', (cell) => { + describe('hash-shadow key on live MySQL (#11627)', () => { + let driver: SqlDriver; + afterEach(async () => { await driver?.disconnect().catch(() => {}); }); + + /** Physical truth, read back from the catalog rather than from our DDL. */ + const catalog = async (table: string) => { + const knex = (driver as any).knex; + const cols = await knex + .select('COLUMN_NAME', 'DATA_TYPE', 'CHARACTER_MAXIMUM_LENGTH', 'EXTRA', 'GENERATION_EXPRESSION') + .from('information_schema.COLUMNS') + .where({ TABLE_SCHEMA: knex.client.database(), TABLE_NAME: table }); + const idx = await knex + .select('INDEX_NAME', 'NON_UNIQUE', 'COLUMN_NAME', 'SUB_PART') + .from('information_schema.STATISTICS') + .where({ TABLE_SCHEMA: knex.client.database(), TABLE_NAME: table }); + return { cols, idx }; + }; + + /** + * The accept transition: schema creation MySQL REFUSED before this change + * now succeeds, and the constraint is carried on a full-width digest. + */ + it('creates a UNIQUE index over a 1024-char column, on a varbinary(32) shadow', async () => { + driver = new SqlDriver(cell.config()); + await driver.initObjects([uniqueOn('os11627_wide', 1024)]); + + const { cols, idx } = await catalog('os11627_wide'); + const shadow = cols.find((c: any) => isHashShadowColumn(c.COLUMN_NAME)); + expect(shadow, 'a shadow column must exist').toBeTruthy(); + // The DIGEST WIDTH ACTUALLY STORED — the number the collision bound is + // computed over. 32 bytes is the FULL SHA-256, deliberately untruncated. + expect(shadow.DATA_TYPE).toBe('varbinary'); + expect(Number(shadow.CHARACTER_MAXIMUM_LENGTH)).toBe(32); + expect(String(shadow.EXTRA)).toContain('STORED GENERATED'); + expect(String(shadow.GENERATION_EXPRESSION).toLowerCase()).toContain('sha2'); + + // The source column keeps its declared shape: nothing was narrowed to + // make it keyable, which is the whole point of not using a bound here. + expect(String(cols.find((c: any) => c.COLUMN_NAME === 'v').DATA_TYPE)).toBe('text'); + + const carried = idx.filter((i: any) => isHashShadowColumn(i.COLUMN_NAME)); + expect(carried.length).toBe(1); + expect(Number(carried[0].NON_UNIQUE)).toBe(0); // genuinely UNIQUE + // ⛔ The control that separates this from the REJECTED route: a prefix + // index reports a SUB_PART. The shadow index keys a whole column. + expect(carried[0].SUB_PART).toBeNull(); + }); + + /** + * The boundary, both sides, read from the catalog: 768 characters is the + * last width MySQL keys directly (768 x 4 = 3072 bytes exactly), and 769 is + * the first that needs the shadow. Asserting only the wide case would pass + * for an implementation that shadowed EVERYTHING. + */ + it('switches to the shadow at exactly MAX_KEYABLE_VARCHAR_CHARS + 1', async () => { + driver = new SqlDriver(cell.config()); + await driver.initObjects([uniqueOn('os11627_at', 768), uniqueOn('os11627_over', 769)]); + + const at = await catalog('os11627_at'); + expect(String(at.cols.find((c: any) => c.COLUMN_NAME === 'v').DATA_TYPE)).toBe('varchar'); + expect(at.cols.filter((c: any) => isHashShadowColumn(c.COLUMN_NAME))).toEqual([]); + expect(at.idx.some((i: any) => i.COLUMN_NAME === 'v' && Number(i.NON_UNIQUE) === 0)).toBe(true); + + const over = await catalog('os11627_over'); + expect(String(over.cols.find((c: any) => c.COLUMN_NAME === 'v').DATA_TYPE)).toBe('text'); + expect(over.cols.filter((c: any) => isHashShadowColumn(c.COLUMN_NAME)).length).toBe(1); + }); + + /** + * ⛔ The assertion a PREFIX index fails. Two distinct values sharing their + * first 191 characters must BOTH be accepted — this is the measured + * behaviour that got prefix-unique rejected by the ruling. + */ + it('accepts distinct values that share a long prefix, and still rejects real duplicates', async () => { + driver = new SqlDriver(cell.config()); + await driver.initObjects([uniqueOn('os11627_sem', 1024)]); + const knex = (driver as any).knex; + + const shared = 'x'.repeat(191); + await knex('os11627_sem').insert({ id: 'a', v: `${shared}AAA` }); + await knex('os11627_sem').insert({ id: 'b', v: `${shared}BBB` }); + expect((await knex('os11627_sem').whereIn('id', ['a', 'b'])).length).toBe(2); + + // A multi-KB JWT — the value shape that made a bound indefensible. + await knex('os11627_sem').insert({ id: 'jwt', v: 'J'.repeat(4000) }); + + // …and the constraint is real: the same value twice is refused. + await expect(knex('os11627_sem').insert({ id: 'dup', v: `${shared}AAA` })).rejects.toThrow(); + }); + + /** + * NULL must stay DISTINCT, exactly as under a direct UNIQUE index. + * `SHA2(NULL)` is NULL, so the shadow is NULL and MySQL does not collide + * NULLs. Had the expression coalesced NULL to a string, every NULL row + * would hash identically and the second one would be refused — a silent + * tightening of the declared constraint. + */ + it('keeps NULLs distinct, as a direct UNIQUE index would', async () => { + driver = new SqlDriver(cell.config()); + await driver.initObjects([uniqueOn('os11627_null', 1024)]); + const knex = (driver as any).knex; + await knex('os11627_null').insert([{ id: 'n1', v: null }, { id: 'n2', v: null }, { id: 'n3', v: null }]); + expect((await knex('os11627_null').whereNull('v')).length).toBe(3); + }); + + /** + * A COMPOSITE unique hashes the tuple, and a tuple containing NULL must + * conflict with nothing — MySQL's own composite-UNIQUE semantics. `CONCAT` + * returning NULL for any NULL argument is what delivers that; `CONCAT_WS` + * would have skipped the NULL and made two different tuples collide. + */ + it('hashes a composite tuple, keeps any-NULL tuples non-conflicting, and stays injective', async () => { + driver = new SqlDriver(cell.config()); + const composite = { + name: 'os11627_comp', + fields: { a: { type: 'text', maxLength: 1024 }, b: { type: 'text', maxLength: 1024 } }, + indexes: [{ fields: ['a', 'b'], unique: true, name: 'uniq_os11627_comp_ab' }], + }; + await driver.initObjects([composite]); + const knex = (driver as any).knex; + + await knex('os11627_comp').insert([{ id: '1', a: 'x', b: 'y' }, { id: '2', a: 'x', b: 'yy' }]); + // Two rows with a NULL component must coexist. + await knex('os11627_comp').insert([{ id: 'n1', a: 'x', b: null }, { id: 'n2', a: 'x', b: null }]); + expect((await knex('os11627_comp').whereNull('b')).length).toBe(2); + // Separator injectivity: ('xy','') and ('x','y') are different tuples. + await knex('os11627_comp').insert([{ id: 's1', a: 'xy', b: '' }, { id: 's2', a: 'x', b: 'y2' }]); + // …and the composite constraint still bites. + await expect(knex('os11627_comp').insert({ id: 'dup', a: 'x', b: 'y' })).rejects.toThrow(); + }); + + /** + * The digest stored is the one this repo can independently recompute — the + * check that the constraint is over SHA-256 of the value and not over some + * server-side variant of it. + */ + it('stores the full SHA-256 of the value, byte for byte', async () => { + driver = new SqlDriver(cell.config()); + await driver.initObjects([uniqueOn('os11627_digest', 1024)]); + const knex = (driver as any).knex; + const value = 'token-' + 'z'.repeat(900); + await knex('os11627_digest').insert({ id: 'd', v: value }); + const shadowCol = (SqlDriver as any).hashShadowColumnFor('uniq_os11627_digest_v'); + const [row] = await knex('os11627_digest').select(shadowCol).where({ id: 'd' }); + const stored: Buffer = row[shadowCol]; + expect(stored.length).toBe(32); + expect(stored.toString('hex')).toBe(createHash('sha256').update(value).digest('hex')); + }); + + /** + * The clause-② half: once uniqueness is enforced over a DIGEST, MySQL's + * `ER_DUP_ENTRY` quotes raw binary and names the shadow index, so a real + * duplicate and a digest collision are indistinguishable from the error + * alone — measured: `Duplicate entry '\xA0\x02\x13...' for key + * 'proto.uniq_token'`. An operator reading that has been told nothing. + * + * The driver resolves it with one read on the failure path. This asserts + * the ORDINARY branch: a genuine duplicate is named in the DECLARED terms + * (the constraint and its source columns), not left as a digest. + */ + it('names the declared columns on a duplicate instead of MySQL binary digest', async () => { + driver = new SqlDriver(cell.config()); + await driver.initObjects([uniqueOn('os11627_dup', 1024)]); + await driver.create('os11627_dup', { v: 'T'.repeat(900) }); + await expect(driver.create('os11627_dup', { v: 'T'.repeat(900) })).rejects.toThrow( + /duplicate value for the UNIQUE constraint 'uniq_os11627_dup_v'.*\(v\)/s, + ); + }); + + /** + * ⛔ NON-UNIQUE indexes are deliberately NOT shadowed. An index over a + * digest serves no lookup the planner can reach for `WHERE col = ?`, so + * creating one would trade a loud refusal for a table that syncs, costs + * writes on every row, and accelerates nothing. This asserts the refusal is + * still a refusal — the scope limit is a decision, not an omission. + */ + it('leaves a non-unique unkeyable index refused rather than shadowing it', async () => { + driver = new SqlDriver(cell.config()); + const nonUnique = { + name: 'os11627_nonuniq', + fields: { v: { type: 'text', maxLength: 1024 } }, + indexes: [{ fields: ['v'], unique: false, name: 'idx_os11627_nonuniq_v' }], + }; + await expect(driver.initObjects([nonUnique])).rejects.toThrow( + /hash-shadow|cannot create index|BLOB\/TEXT/i, + ); + }); + }); +}); + +// ── Postgres is the control: it never refused, so nothing may change ──────── + +declareDialectCell(PG_CELL, 'hash-shadow key (#11627)', (cell) => { + describe('Postgres control (#11627)', () => { + let driver: SqlDriver; + afterEach(async () => { await driver?.disconnect().catch(() => {}); }); + + /** + * Postgres took all 44 platform objects before this change and must take + * them after, with its physical schema byte-identical — no shadow column, + * and the UNIQUE index still on the value itself. This is the assertion + * that catches a change which "fixed MySQL" by degrading everyone. + */ + it('keeps the direct UNIQUE index on the value and grows no shadow column', async () => { + driver = new SqlDriver(cell.config()); + await driver.initObjects([uniqueOn('os11627_pg', 1024)]); + const knex = (driver as any).knex; + const cols = await knex + .select('column_name') + .from('information_schema.columns') + .where({ table_name: 'os11627_pg' }); + expect(cols.map((c: any) => c.column_name).filter((c: string) => isHashShadowColumn(c))).toEqual([]); + const idx = await knex.raw( + `SELECT indexdef FROM pg_indexes WHERE tablename = 'os11627_pg'`, + ); + const defs = (idx.rows ?? []).map((r: any) => String(r.indexdef)).join('\n'); + expect(defs).toMatch(/UNIQUE INDEX .*uniq_os11627_pg_v.*\(v\)/i); + }); + }); +}); diff --git a/packages/drivers/driver-sql/src/sql-driver-keyed-text-mysql.test.ts b/packages/drivers/driver-sql/src/sql-driver-keyed-text-mysql.test.ts index d63aa91e57..c6fc4b2083 100644 --- a/packages/drivers/driver-sql/src/sql-driver-keyed-text-mysql.test.ts +++ b/packages/drivers/driver-sql/src/sql-driver-keyed-text-mysql.test.ts @@ -93,6 +93,34 @@ const tooWideObject = () => ({ indexes: [{ fields: ['token'], unique: true }], }); +/** + * The same two shapes with a NON-UNIQUE index, which is what still lands in the + * named refusal after #11627. + * + * ⚠️ Why this file grew these: the two objects above declare UNIQUE indexes, and + * #11627 made a UNIQUE index over an unkeyable column expressible — it is now + * carried on a hash-shadow column instead of being refused. That is a ruled + * behaviour change (maintainer, 2026-08-24 on #11374), so the assertions that + * pinned "unkeyable ⇒ refused" for those objects were pinning a branch that no + * longer exists for them, and were rewritten rather than deleted or silenced. + * The refusal itself is NOT gone — it is the disposition for a NON-UNIQUE + * unkeyable index, where a digest would serve no lookup — so the half of this + * file that proves the driver never weakens a constraint keeps a live subject. + */ +const UNBOUNDED_NONUNIQUE_TABLE = 'os11374_unbounded_nonuniq'; +const unboundedNonUniqueObject = () => ({ + name: UNBOUNDED_NONUNIQUE_TABLE, + fields: { token: { type: 'text' } }, + indexes: [{ fields: ['token'], unique: false }], +}); + +const TOO_WIDE_NONUNIQUE_TABLE = 'os11374_too_wide_nonuniq'; +const tooWideNonUniqueObject = () => ({ + name: TOO_WIDE_NONUNIQUE_TABLE, + fields: { token: { type: 'text', maxLength: 1024 } }, + indexes: [{ fields: ['token'], unique: false }], +}); + // ── The emission rule, on a dialect every runner has ──────────────────────── describe('keyed text columns take their declared maxLength (#11374)', () => { @@ -140,7 +168,8 @@ declareDialectCell(MYSQL_CELL, 'keyed text columns (#11374)', (cell) => { let driver: SqlDriver; afterEach(async () => { - for (const t of [BOUNDED_TABLE, UNBOUNDED_TABLE, TOO_WIDE_TABLE, 'os11374_prefix']) { + for (const t of [BOUNDED_TABLE, UNBOUNDED_TABLE, TOO_WIDE_TABLE, 'os11374_prefix', + UNBOUNDED_NONUNIQUE_TABLE, TOO_WIDE_NONUNIQUE_TABLE]) { await driver?.execute(`drop table if exists ${t}`).catch(() => {}); } await driver?.disconnect().catch(() => {}); @@ -177,30 +206,68 @@ declareDialectCell(MYSQL_CELL, 'keyed text columns (#11374)', (cell) => { expect(subPart ?? null).toBeNull(); }); - it('refuses an unkeyable column by name instead of weakening the constraint', async () => { + it('refuses a NON-UNIQUE unkeyable column by name instead of weakening it', async () => { driver = new SqlDriver(cell.config()); - await driver.execute(`drop table if exists ${UNBOUNDED_TABLE}`).catch(() => {}); + await driver.execute(`drop table if exists ${UNBOUNDED_NONUNIQUE_TABLE}`).catch(() => {}); // Loud, and specifically loud: the raw server error names a column in a // table that was just created successfully, which reads as an index - // quirk rather than an object whose declared uniqueness is now absent. - await expect(driver.initObjects([unboundedObject()])).rejects.toThrow( - /cannot create index 'uniq_os11374_unbounded_token'.*declares no `maxLength`/s, + // quirk rather than an object whose declared index is now absent. + await expect(driver.initObjects([unboundedNonUniqueObject()])).rejects.toThrow( + /cannot create index 'idx_os11374_unbounded_nonuniq_token'.*declares no `maxLength`/s, ); // ⛔ The negative half, and the point of the whole disposition: no index // was substituted. A prefix index here would have made `initObjects` - // resolve and left a constraint that means something else. - expect(await indexNames(driver, UNBOUNDED_TABLE)).toEqual([]); + // resolve and left an index that means something else. #11627 did NOT + // relax this — a hash shadow is offered only to a UNIQUE index, because + // a digest serves no lookup an ordinary index exists to accelerate. + expect(await indexNames(driver, UNBOUNDED_NONUNIQUE_TABLE)).toEqual([]); }); - it('gives a bound past the key ceiling the same named refusal', async () => { + it('gives a NON-UNIQUE bound past the key ceiling the same named refusal', async () => { driver = new SqlDriver(cell.config()); - await driver.execute(`drop table if exists ${TOO_WIDE_TABLE}`).catch(() => {}); - await expect(driver.initObjects([tooWideObject()])).rejects.toThrow( - /cannot create index 'uniq_os11374_too_wide_token'.*wider than 768 characters/s, + await driver.execute(`drop table if exists ${TOO_WIDE_NONUNIQUE_TABLE}`).catch(() => {}); + await expect(driver.initObjects([tooWideNonUniqueObject()])).rejects.toThrow( + /cannot create index 'idx_os11374_too_wide_nonuniq_token'.*wider than 768 characters/s, ); - expect(await indexNames(driver, TOO_WIDE_TABLE)).toEqual([]); + expect(await indexNames(driver, TOO_WIDE_NONUNIQUE_TABLE)).toEqual([]); + }); + + /** + * The UNIQUE half of the same two shapes, after #11627: expressible, and + * expressed WITHOUT the prefix index this file exists to rule out. + * + * This is the assertion that replaced the two refusal pins above for the + * unique case. It deliberately re-checks `sub_part`, the same discriminator + * the bounded case uses: the constraint moving onto a shadow column must + * not quietly become the prefix constraint the ruling rejected. + */ + it('carries the UNIQUE cases on a hash shadow, still never a prefix index', async () => { + driver = new SqlDriver(cell.config()); + await driver.execute(`drop table if exists ${UNBOUNDED_TABLE}`).catch(() => {}); + await driver.execute(`drop table if exists ${TOO_WIDE_TABLE}`).catch(() => {}); + + await driver.initObjects([unboundedObject(), tooWideObject()]); + + for (const [table, index] of [ + [UNBOUNDED_TABLE, 'uniq_os11374_unbounded_token'], + [TOO_WIDE_TABLE, 'uniq_os11374_too_wide_token'], + ] as const) { + expect(await indexNames(driver, table)).toContain(index); + const rows = (await rowsOf( + driver, + `select sub_part as SUB_PART, column_name as COLUMN_NAME, non_unique as NON_UNIQUE + from information_schema.statistics + where table_schema = database() and table_name = ? and index_name = ?`, + [table, index], + )) as any[]; + expect(rows.length).toBe(1); + // Whole key part, not a prefix — the rejected route reports a sub_part. + expect(rows[0].SUB_PART ?? null).toBeNull(); + expect(Number(rows[0].NON_UNIQUE)).toBe(0); + expect(String(rows[0].COLUMN_NAME)).toContain('__hash'); + } }); /** diff --git a/packages/drivers/driver-sql/src/sql-driver.ts b/packages/drivers/driver-sql/src/sql-driver.ts index cf4f8ed509..42a925fc83 100644 --- a/packages/drivers/driver-sql/src/sql-driver.ts +++ b/packages/drivers/driver-sql/src/sql-driver.ts @@ -5490,9 +5490,22 @@ export class SqlDriver implements IDataDriver { const result = await builder.insert(formatted).returning('*'); return this.formatOutput(object, result[0]); } catch (error) { - if (!mayRetry || attempt >= AUTONUMBER_COLLISION_RETRIES) throw error; + // #11627: on a table whose UNIQUE index is carried by a hash shadow, + // `ER_DUP_ENTRY` quotes a binary digest and names the shadow index, so + // neither the operator nor this driver can read WHICH value conflicted + // — and a genuine duplicate is indistinguishable from a (near- + // impossible) digest collision. Resolve it by reading the source + // columns back, on the failure path only, and re-throw the SAME error + // carrying whichever answer is true. Returns the error untouched when + // this is not a shadow conflict. + const shadowed = await this.decorateHashShadowDuplicate( + this.rotationWriteTarget(object) ?? object, + formatted, + error, + ); + if (!mayRetry || attempt >= AUTONUMBER_COLLISION_RETRIES) throw shadowed; const colliding = await this.collidingAutoNumberReservations(error, reservations, options); - if (colliding.length === 0) throw error; + if (colliding.length === 0) throw shadowed; for (const reservation of colliding) { await this.resyncSequenceToDataMax(reservation); // Clear only what collided, so the next pass regenerates exactly @@ -10928,6 +10941,48 @@ export class SqlDriver implements IDataDriver { // field-level fix; the boot still fails loudly, it just says why. const unkeyable = await this.explainUnkeyableTextColumn(tableName, name, columns, e); if (unkeyable) { + // #11627: a UNIQUE index MySQL cannot key directly is expressible + // after all — over a SHA-256 shadow of the same values. Attempted + // only AFTER the server has refused the direct index, deliberately: + // the same reasoning `explainRowSizeOverflow` records for the row + // budget. A pre-flight would have to reproduce MySQL's 3072-byte key + // arithmetic and, wrong in the strict direction, would move an object + // to a shadow key on a server that would have taken the real index — + // a physical schema decided by our arithmetic rather than by the + // server's. Reacting to the refusal cannot over-apply, and it is what + // keeps Postgres and SQLite (which never refuse) byte-identical to + // before: the dialect divergence is selected BY THE ERROR CODE, never + // by a dialect getter. + // + // ⛔ UNIQUE only. A non-unique index exists for an ACCESS PATH, and an + // index over a hash serves no lookup the planner can find on its own + // — `WHERE col = ?` cannot use it without rewriting the read side to + // filter on the digest too. Silently creating one would turn a loud + // refusal into a table that syncs, costs writes, and accelerates + // nothing. Those cases stay refused below, and stay tracked. + if (unique) { + try { + if (await this.createHashShadowUniqueIndex(tableName, name, columns)) { + existing.add(name); + continue; + } + } catch (shadowErr: any) { + const shadowMsg = String(shadowErr?.message ?? shadowErr); + if (/already exists|duplicate key name/i.test(shadowMsg)) { + existing.add(name); + continue; + } + // Fall through to the named refusal, which is still the honest + // outcome — but say that the shadow route was tried and why it + // did not land, so this does not read as never having been + // attempted. + (this.logger.error ?? this.logger.warn)( + `[sql-driver] hash-shadow UNIQUE index '${name}' on "${tableName}" could not be created ` + + `(#11627); falling back to the refusal below.`, + shadowMsg, + ); + } + } (this.logger.error ?? this.logger.warn)(unkeyable, msg); throw Object.assign(new Error(`${unkeyable} (server said: ${msg})`), { code: (e as { code?: string }).code, @@ -13438,6 +13493,269 @@ export class SqlDriver implements IDataDriver { ); } + /** + * Suffix of the driver-owned HASH-SHADOW column (#11627). Mirrored by + * `schema-drift.ts`'s {@link isHashShadowColumn}, which must recognise the + * same column as driver-owned rather than orphaned. + */ + protected static readonly HASH_SHADOW_SUFFIX = '__hash'; + + /** + * The shadow column that carries `indexName`, capped to MySQL's 64-character + * identifier limit. + * + * Derived from the INDEX name rather than from the column list, deliberately: + * one shadow serves one declared index, a composite index has no single + * column to name it after, and the index name is already the differ's + * identity for the constraint ({@link normalizeDeclaredIndex}) — so re-sync + * finds the existing index by the same name and skips, and the shadow cannot + * drift away from the index it belongs to. + * + * The overflow branch keeps a truncated prefix for readability and appends a + * digest of the FULL name, so two long index names that share a prefix still + * get different shadows. + */ + protected static hashShadowColumnFor(indexName: string): string { + const direct = `${indexName}${SqlDriver.HASH_SHADOW_SUFFIX}`; + if (direct.length <= 64) return direct; + const digest = createHash('sha256').update(indexName).digest('hex').slice(0, 8); + const keep = 64 - SqlDriver.HASH_SHADOW_SUFFIX.length - digest.length - 1; + return `${indexName.slice(0, keep)}_${digest}${SqlDriver.HASH_SHADOW_SUFFIX}`; + } + + /** + * Carry a declared UNIQUE index on a SHADOW column holding a SHA-256 of its + * key values, for the case MySQL cannot express directly (#11627). + * + * ## Why this exists + * + * On utf8mb4 InnoDB a key part may hold at most 3072 bytes — 768 characters + * — so a full-value UNIQUE index over a longer column is INEXPRESSIBLE, not + * merely expensive. An OAuth access token may legitimately be a multi-KB JWT. + * Before this, `syncSchema` refused such an object outright (measured: 7 of + * 44 platform objects on MySQL 8.0.46, Postgres 0/44), leaving it registered + * with its declared uniqueness absent. {@link explainUnkeyableTextColumn} + * names that refusal; this method removes the cause for the UNIQUE case. + * + * ## Why a GENERATED column, and not application-computed like the precedent + * + * `_objectstack_sequences.key_hash` ({@link createSequencesTable}) hashes in + * application code because it is a cross-dialect PRIMARY KEY the driver + * writes on every counter bump. This shadow is different in the one way that + * matters: it exists ONLY on the dialect that refused the direct index, and + * only to carry a constraint. A `STORED GENERATED` column lets the SERVER + * compute it, which buys three properties application hashing cannot: + * + * - **No write path changes at all.** Every INSERT and UPDATE — including + * ones this driver never sees, from `os migrate`, a DBA, or replication — + * maintains the shadow. An app-computed shadow is only as correct as the + * set of writers that remember it. + * - **Existing rows are hashed by the ALTER itself**, so there is no + * backfill step that could partially complete. + * - **The constraint cannot be bypassed**, because nothing can write a + * shadow that disagrees with its source columns. + * + * ## Semantics, measured on live MySQL 8.0.46 (utf8mb4 / InnoDB) + * + * `UNHEX(SHA2(v, 256))` is the FULL 256-bit digest stored as `VARBINARY(32)` + * — 32 bytes, two orders of magnitude inside the 3072-byte ceiling, and + * ⚠️ deliberately NOT truncated: a truncated digest would be the number the + * collision bound is computed over, and there is no reason to pay that. + * + * - **Distinct values that share a long prefix both insert.** This is the + * property that rules OUT the prefix-index alternative and the reason + * this route was chosen over it (maintainer ruling on #11374, + * 2026-08-24): measured, two distinct tokens sharing their first 191 + * characters are BOTH accepted here, where `UNIQUE KEY (token(191))` + * rejected the second as `ER_DUP_ENTRY` — a valid sign-in refused as a + * duplicate. + * - **A genuine duplicate is still rejected** (`ER_DUP_ENTRY`). + * - **NULL stays distinct.** `SHA2(NULL, 256)` is NULL, so a NULL-valued + * row has a NULL shadow and a UNIQUE index does not collide NULLs — + * matching what a direct UNIQUE over the column would have done. + * - **Composites use `CONCAT` with a `0x1f` separator**, and `CONCAT` + * returning NULL when ANY argument is NULL is exactly MySQL's + * composite-UNIQUE semantics (a tuple with any NULL conflicts with + * nothing). The separator keeps the encoding injective, so ('xy','') + * and ('x','y') do not alias. + * + * ## The collision bound, and what a collision would look like + * + * Uniqueness now holds over SHA-256 of the value rather than the value. With + * a full 256-bit digest and n rows in one index, the birthday bound is + * n^2 / 2^257. At n = 10^9 rows that is under 10^-59 — below any rate the + * storage layer itself is trusted at. ⚠️ But the failure MODE is what + * matters, not only its probability: a collision surfaces as `ER_DUP_ENTRY` + * on the shadow, which is the SAME error a real duplicate raises, and MySQL + * quotes the raw binary digest rather than the offending value. Left alone + * that is an operator seeing an unexplainable duplicate — a wrong answer, not + * a crash. {@link explainHashShadowDuplicate} exists so the driver can tell + * the two apart by reading the source columns back, and name whichever it is. + * + * Returns `true` when the shadow index now exists. + */ + protected async createHashShadowUniqueIndex( + tableName: string, + indexName: string, + columns: string[], + ): Promise { + if (!this.isMysql) return false; + const shadow = SqlDriver.hashShadowColumnFor(indexName); + const ref = (c: string) => `\`${c.replace(/`/g, '``')}\``; + // ONE argument needs no separator, and CONCAT of one value would only add + // a chance to get the encoding wrong. + const expr = + columns.length === 1 + ? ref(columns[0]!) + : `CONCAT(${columns.map((c) => ref(c)).join(', 0x1f, ')})`; + const sql = + `ALTER TABLE ${ref(tableName)} ` + + `ADD COLUMN ${ref(shadow)} VARBINARY(32) GENERATED ALWAYS AS (UNHEX(SHA2(${expr}, 256))) STORED, ` + + `ADD UNIQUE KEY ${ref(indexName)} (${ref(shadow)})`; + await this.knex.raw(sql); + this.logger.warn( + `[sql-driver] UNIQUE index '${indexName}' on "${tableName}" is carried by the hash-shadow column ` + + `"${shadow}" (SHA-256 of ${columns.join(', ')}), because MySQL cannot key ${columns.length > 1 ? 'this column set' : 'a column'} ` + + `longer than ${SqlDriver.MAX_KEYABLE_VARCHAR_CHARS} characters directly (#11627). The declared ` + + `constraint is enforced over the full value; only the physical key differs.`, + { tableName, indexName, columns, shadow }, + ); + return true; + } + + /** + * Tell a genuine uniqueness violation apart from a hash COLLISION on a + * shadow-carried UNIQUE index (#11627), and name which one happened. + * + * ⚠️ Why this is not optional. Once uniqueness is enforced over SHA-256 of + * the value, `ER_DUP_ENTRY` has two possible causes that MySQL reports + * IDENTICALLY — it quotes the raw binary digest and the index name, and the + * digest tells an operator nothing about which row conflicted. The likely + * cause is the ordinary one (a real duplicate); the astronomically unlikely + * one is a collision, and it would present as the platform refusing a write + * that is, in fact, unique. That is a user-visible WRONG ANSWER rather than a + * crash, so "vanishingly unlikely" is not on its own an adequate answer to + * it. + * + * The disambiguation is one read on the failure path only (the same trade + * {@link explainUnkeyableTextColumn} makes): re-select by the SOURCE columns. + * A row that matches them is a real duplicate. NO row matching them, with the + * shadow index nevertheless reporting a conflict, is a collision — and the + * message says so, with the values, so it is reportable rather than baffling. + * + * Returns `null` when this failure is not a shadow-index conflict at all, so + * the caller's existing handling is unchanged. + */ + protected async explainHashShadowDuplicate( + tableName: string, + values: Record, + cause: unknown, + ): Promise { + if (!this.isMysql) return null; + if ((cause as { code?: string } | undefined)?.code !== 'ER_DUP_ENTRY') return null; + const message = String((cause as { message?: string } | undefined)?.message ?? ''); + // MySQL names the key as `table.index` (8.0) or `index` (5.7) — take the + // last path segment either way. + const keyed = /for key '([^']+)'/.exec(message); + if (!keyed) return null; + const indexName = keyed[1]!.split('.').pop()!; + const shadow = SqlDriver.hashShadowColumnFor(indexName); + let shadowExists = false; + try { + shadowExists = await this.knex.schema.hasColumn(tableName, shadow); + } catch { + return null; + } + if (!shadowExists) return null; + const sources = await this.hashShadowSourceColumns(tableName, indexName); + if (sources.length === 0) return null; + // Only the source columns the failing write actually supplied; a partial + // update cannot be re-selected on columns it never mentioned. + if (!sources.every((c) => Object.prototype.hasOwnProperty.call(values, c))) return null; + let existing = 0; + try { + const rows = await this.knex(tableName) + .where(Object.fromEntries(sources.map((c) => [c, values[c]]))) + .limit(1); + existing = rows.length; + } catch { + return null; + } + if (existing > 0) { + // The ordinary case: a real duplicate. Say so in the declared terms + // rather than leaving MySQL's binary digest as the only explanation. + return ( + `[sql-driver] duplicate value for the UNIQUE constraint '${indexName}' on "${tableName}" ` + + `(${sources.join(', ')}). The constraint is physically carried by a hash-shadow column, so the ` + + `server's own message quotes a binary digest instead of the value (#11627).` + ); + } + return ( + `[sql-driver] HASH COLLISION on the shadow-carried UNIQUE index '${indexName}' on "${tableName}" ` + + `(${sources.join(', ')}): the write was rejected as a duplicate, but NO existing row carries these ` + + `values. Uniqueness on this index is enforced over a SHA-256 of them (#11627), so two different ` + + `values produced the same digest. This is expected at a rate near 10^-59 for a billion rows — if ` + + `you are reading this, please report it with the values above; the write itself is legitimate and ` + + `is being refused.` + ); + } + + /** + * The declared key columns a shadow-carried index hashes, read back from the + * registered metadata so the disambiguating select above filters on the same + * columns the shadow was generated from. + */ + /** + * Attach {@link explainHashShadowDuplicate}'s verdict to a failing write, or + * return the error untouched (#11627). + * + * ⚠️ Wired into {@link create} only. `update` issues its statement through + * three paths with no shared catch, and a duplicate on the columns this + * route covers — an OAuth token, an issuer/account pair, a metadata identity + * — arrives on INSERT in every flow this repo has. An UPDATE that collides + * still fails correctly; it just still reports MySQL's binary digest. That + * gap is deliberate and named rather than left to be discovered. + */ + protected async decorateHashShadowDuplicate( + tableName: string, + values: Record, + error: unknown, + ): Promise { + let explanation: string | null = null; + try { + explanation = await this.explainHashShadowDuplicate(tableName, values, error); + } catch { + // The disambiguation is a diagnostic. If it cannot be made, the original + // failure is still the failure — never mask it with this one. + return error; + } + if (!explanation) return error; + (this.logger.error ?? this.logger.warn)(explanation); + return Object.assign(new Error(`${explanation} (server said: ${String((error as { message?: string })?.message ?? error)})`), { + code: (error as { code?: string })?.code, + cause: error, + }); + } + + protected async hashShadowSourceColumns(tableName: string, indexName: string): Promise { + try { + const rows: Array<{ GENERATION_EXPRESSION?: string; generation_expression?: string }> = + await this.knex + .select('GENERATION_EXPRESSION') + .from('information_schema.COLUMNS') + .where({ + TABLE_SCHEMA: this.knex.client.database(), + TABLE_NAME: tableName, + COLUMN_NAME: SqlDriver.hashShadowColumnFor(indexName), + }); + const expr = String(rows[0]?.GENERATION_EXPRESSION ?? rows[0]?.generation_expression ?? ''); + // `unhex(sha2(`a`,256))` or `unhex(sha2(concat(`a`,0x1f,`b`),256))` + return [...expr.matchAll(/`((?:[^`]|``)+)`/g)].map((m) => m[1]!.replace(/``/g, '`')); + } catch { + return []; + } + } + /** * MySQL's per-ROW budget over the DECLARED byte widths of a table's columns * (#11565) — the number its own refusal quotes and the only part of that