From 4c5e91bacc6fd89c94f0d59f0b9e3c6ad0db15f9 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 24 Aug 2026 16:16:44 +0000 Subject: [PATCH 1/2] fix(driver-sql): name the declarations behind MySQL's row-size refusal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MySQL charges every bounded column's DECLARED byte width against a per-row budget, independently of the per-column varchar ceiling, and refuses the CREATE naming no column and no declaration — about a table its author described entirely in metadata. Translate ER_TOO_BIG_ROWSIZE at the initObjects create/alter call sites into a refusal that names every contributing field, its emitted varchar width and its byte cost at the schema's real bytes-per-character. The same failure, re-worded: a translator cannot over-refuse by construction, where a pre-flight reproducing the server's arithmetic can. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01VK8rFDtg8eREaxBGX99Csn --- .../sql-driver-11565-row-byte-budget.test.ts | 320 ++++++++++++++ packages/drivers/driver-sql/src/sql-driver.ts | 396 +++++++++++++++++- 2 files changed, 699 insertions(+), 17 deletions(-) create mode 100644 packages/drivers/driver-sql/src/sql-driver-11565-row-byte-budget.test.ts diff --git a/packages/drivers/driver-sql/src/sql-driver-11565-row-byte-budget.test.ts b/packages/drivers/driver-sql/src/sql-driver-11565-row-byte-budget.test.ts new file mode 100644 index 0000000000..c8122d8113 --- /dev/null +++ b/packages/drivers/driver-sql/src/sql-driver-11565-row-byte-budget.test.ts @@ -0,0 +1,320 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * #11565 — MySQL's per-ROW budget over DECLARED column widths, and the + * diagnostic that names the declarations that spent it. + * + * ## The defect + * + * MySQL charges every bounded column's DECLARED byte width against a per-row + * budget, independently of the per-column `varchar` ceiling. An object whose + * fields declare enough total width simply fails `CREATE TABLE` — and the + * server's refusal names **no column and no declaration**. It says "You have to + * change some columns to TEXT or BLOBs" about a table its author described + * entirely in metadata, and nothing maps that back to the `maxLength` values + * responsible. Sixteen fields at `maxLength: 1024` is not an exotic object. + * + * ## Why a translator and not a pre-flight + * + * A pre-flight that sums declared widths BEFORE issuing DDL has to reproduce + * the server's arithmetic, and wrong in the strict direction it refuses an + * object MySQL would have accepted — a contract change. A translator cannot + * over-refuse by construction: it speaks only after the server has refused. + * Sitting inside `initObjects`' own loop it still sees `obj.fields`, so it + * names every contributing field exactly as a pre-flight would. + * {@link SqlDriver.explainRowSizeOverflow} carries the four measurements that + * decided it; this file is the executable half. + * + * ## What each half is worth + * + * The dialect-free block runs everywhere, including Test Core. It pins the + * arithmetic, the agreement between the width mirror and `createColumn`'s own + * switch over every `FieldType` the spec declares, and — the half that stops + * "refuses" from passing for "refuses the RIGHT objects" — that an object past + * MySQL's budget still creates cleanly on a dialect that has no such budget. + * + * The MySQL block is the one that reds on the pre-fix tree. Opt-in: + * + * OS_TEST_MYSQL_URL=mysql://root:root@127.0.0.1:3306/conformance \ + * pnpm --filter @objectstack/driver-sql test + */ + +import { describe, it, expect, afterEach } from 'vitest'; +import { FieldType } from '@objectstack/spec/data'; +import { SqlDriver } from '../src/index.js'; +import { MYSQL_CELL, dialectCell, declareDialectCell } from './live-dialect-matrix.testkit.js'; + +/** An object of `count` string fields, each declaring the same `maxLength`. */ +const wideObject = (name: string, count: number, maxLength: number) => ({ + name, + fields: Object.fromEntries( + Array.from({ length: count }, (_, i) => [`f${i + 1}`, { type: 'string', maxLength }]), + ), +}); + +/** The same shape with NOTHING declared — `lookup` takes knex's varchar(255). */ +const undeclaredObject = (name: string, count: number) => ({ + name, + fields: Object.fromEntries( + Array.from({ length: count }, (_, i) => [`f${i + 1}`, { type: 'lookup' }]), + ), +}); + +// ── The arithmetic, and the mirror, on a dialect every runner has ─────────── + +describe('row byte budget — arithmetic and column mirror (#11565)', () => { + let driver: SqlDriver; + + afterEach(async () => { + await driver?.disconnect().catch(() => {}); + }); + + /** + * The length prefix moves at a BYTE-payload boundary, not a character one, so + * it moves with the charset — measured on MySQL 8.0.46 through the column + * counts a table can hold. A utf8mb4 `varchar(63)` payload is 252 bytes and + * 32 of them fit InnoDB's 8126-byte page limit (32 x 253 = 8096; at 254 bytes + * each, 32 would not fit), so that prefix is one byte. A `varchar(64)` + * payload is 256 bytes and behaves as the two-byte, off-page-eligible class. + * + * ⚠️ A payload of EXACTLY 255 is documented as the last one-byte width, and + * is deliberately not asserted here: such a column is never stored off-page, + * so the page limit binds at ~31 columns and neither limit can be made to + * discriminate 256 from 257 bytes. An unmeasurable claim does not get a pin. + */ + it('charges payload + varchar length prefix, and the prefix moves with the charset', () => { + const pack = (chars: number, bpc: number) => (SqlDriver as any).varcharPackLength(chars, bpc); + expect(pack(63, 4)).toBe(253); // utf8mb4: 252 + 1 + expect(pack(64, 4)).toBe(258); // utf8mb4: 256 + 2 + expect(pack(63, 1)).toBe(64); // latin1: 63 + 1 — the same width, a quarter the cost + expect(pack(300, 1)).toBe(302); // latin1: 300 + 2 + expect(pack(1024, 4)).toBe(4098); // the card's own row: 16 x 4098 = 65568 > 65535 + expect(pack(255, 4)).toBe(1022); // the DEFAULT width, which declares nothing + }); + + /** + * ⚠️ The mirror is a second reading of `createColumn`'s switch, so the risk it + * carries is drift. This pin removes the risk structurally rather than by + * review: one field of EVERY `FieldType` the spec declares, created for real, + * and the mirror's answer compared against the column that actually landed. A + * type added to the spec joins this pin without anyone remembering to. + */ + it('agrees with createColumn about every FieldType the spec declares', async () => { + const types = FieldType.options as readonly string[]; + expect(types.length).toBeGreaterThan(40); // the registry really was read + + const fields = Object.fromEntries(types.map((t) => [`f_${t}`, { type: t }])); + driver = new SqlDriver(dialectCell('sqlite').config()); + await driver.initObjects([{ name: 'os11565_every_type', fields }]); + const info: Record = await ( + driver as any + ).knex('os11565_every_type').columnInfo(); + + const mismatched: string[] = []; + for (const t of types) { + const column = `f_${t}`; + const mirrored = (driver as any).varcharColumnChars({ type: t }, undefined) as number | null; + const landed = info[column]; + const isVarchar = /varchar/i.test(String(landed?.type ?? '')); + const landedChars = isVarchar ? Number(landed?.maxLength) : null; + if (mirrored !== landedChars) { + mismatched.push( + `${t}: mirror says ${mirrored === null ? 'not a varchar' : `varchar(${mirrored})`}, ` + + `createColumn emitted ${landed === undefined ? 'no column' : String(landed.type)}`, + ); + } + } + expect(mismatched).toEqual([]); + }); + + /** + * ⛔ The negative half, and the reason "it refuses" is not the assertion this + * file makes: the budget is MySQL's, so an object past it must still create + * cleanly everywhere else. A pre-flight with a wrong constant would fail + * exactly here; a translator cannot, because it never runs. + */ + it('refuses nothing on a dialect with no row budget', async () => { + driver = new SqlDriver(dialectCell('sqlite').config()); + // 16 x maxLength 1024 — the shape live MySQL rejects, three lines down in + // the same repo. + await driver.initObjects([wideObject('os11565_wide_sqlite', 16, 1024)]); + const info: any = await (driver as any).knex('os11565_wide_sqlite').columnInfo(); + expect(Object.keys(info)).toContain('f16'); + expect(String(info.f16?.maxLength ?? '')).toBe('1024'); + }); + + /** The offender list is ordered by cost, and holds every varchar column. */ + it('profiles every varchar column, widest first', async () => { + driver = new SqlDriver(dialectCell('sqlite').config()); + const profile = (driver as any).rowWidthProfile( + { + narrow: { type: 'string', maxLength: 10 }, + widest: { type: 'string', maxLength: 4000 }, + middle: { type: 'string', maxLength: 1024 }, + unbounded_lookup: { type: 'lookup' }, + a_number: { type: 'number' }, + // Unkeyed text stays TEXT — it costs the budget a pointer, not a width. + body: { type: 'text', maxLength: 60000 }, + id: { type: 'string', maxLength: 9999 }, // built-in, never the author's + }, + new Map(), + 4, + ); + expect(profile.columns.map((c: any) => c.name)).toEqual([ + 'widest', + 'middle', + 'unbounded_lookup', + 'narrow', + ]); + expect(profile.columns[0]).toMatchObject({ chars: 4000, bytes: 16002 }); + expect(profile.columns[2]).toMatchObject({ chars: 255, bytes: 1022 }); + expect(profile.totalBytes).toBe(16002 + 4098 + 1022 + 41); + }); +}); + +// ── The half only a live MySQL can measure ────────────────────────────────── + +declareDialectCell(MYSQL_CELL, 'row byte budget (#11565)', (cell) => { + describe('row byte budget on live MySQL (#11565)', () => { + let driver: SqlDriver; + const TABLES = [ + 'os11565_ok', + 'os11565_over', + 'os11565_ok255', + 'os11565_over255', + 'os11565_grow', + 'os11565_narrow', + 'os11565_undeclared', + ]; + + afterEach(async () => { + for (const t of TABLES) await driver?.execute(`drop table if exists ${t}`).catch(() => {}); + await driver?.disconnect().catch(() => {}); + }); + + /** + * The measured boundaries below are utf8mb4's. On a latin1 schema the same + * declarations cost a quarter as much and every count here is wrong — so + * the cell asserts the multiplier rather than assuming it, the same way the + * matrix asserts its zone skew instead of hoping for it. + */ + it('runs on a 4-byte charset — the boundaries below are utf8mb4 numbers', async () => { + driver = new SqlDriver(cell.config()); + const seen = await (driver as any).schemaBytesPerChar(); + expect(seen).not.toBeNull(); + expect(seen.bytesPerChar).toBe(4); + }); + + /** + * Both sides of the boundary, in one test, because only the pair means + * anything: an implementation that refused every object would pass the + * second assertion alone. Measured through this driver (its built-in `id` + * varchar(255) is inside the budget, which is why the count is 15/16 here + * and 15/16 in raw SQL only by coincidence of the same width). + */ + it('creates 15 fields at maxLength 1024 and names all 16 when one more is added', async () => { + driver = new SqlDriver(cell.config()); + await driver.execute('drop table if exists os11565_ok'); + await driver.execute('drop table if exists os11565_over'); + + // ACCEPTS: unchanged behaviour, no diagnostic, a real table. + await driver.initObjects([wideObject('os11565_ok', 15, 1024)]); + const info: any = await (driver as any).knex('os11565_ok').columnInfo(); + expect(String(info.f15?.type)).toBe('varchar'); + expect(Number(info.f15?.maxLength)).toBe(1024); + + // REFUSES — with the fields the server would not name. + const failure = await driver + .initObjects([wideObject('os11565_over', 16, 1024)]) + .then(() => null) + .catch((e: any) => e); + expect(failure).toBeInstanceOf(Error); + const message = String(failure.message); + expect(message).toMatch(/cannot create table "os11565_over"/); + expect(message).toMatch(/65535-byte budget for one ROW/); + // Every contributing field, not merely "the table failed". + expect(message).toMatch(/Its 16 varchar column\(s\) take 65568 bytes/); + expect(message).toMatch(/"f1" varchar\(1024\) = 4098 bytes/); + expect(message).toMatch(/and 8 more/); + // The server's own sentence is kept, not replaced. + expect(message).toMatch(/server said: Row size too large/); + // Same failure, re-worded: the code survives for anything reading it. + expect(failure.code).toBe('ER_TOO_BIG_ROWSIZE'); + expect((failure.cause as any)?.code).toBe('ER_TOO_BIG_ROWSIZE'); + + // ⛔ And nothing was left behind: the object is not registered half-built. + const exists = await (driver as any).knex.schema.hasTable('os11565_over'); + expect(exists).toBe(false); + }); + + /** The card's second measured row, moved by the driver's own `id` column. */ + it('creates 63 fields at maxLength 255 and refuses 64', async () => { + driver = new SqlDriver(cell.config()); + await driver.execute('drop table if exists os11565_ok255'); + await driver.execute('drop table if exists os11565_over255'); + + await driver.initObjects([wideObject('os11565_ok255', 63, 255)]); + expect(await (driver as any).knex.schema.hasTable('os11565_ok255')).toBe(true); + + await expect(driver.initObjects([wideObject('os11565_over255', 64, 255)])).rejects.toThrow( + /cannot create table "os11565_over255".*Its 64 varchar column\(s\) take 65408 bytes/s, + ); + }); + + /** + * The path that is more likely than CREATE in a living app: a field added + * to an object that was already near the budget. The server refuses the ADD + * naming only the column being added, as if that one column were too wide — + * when the width is in fifteen columns nobody is touching. + */ + it('names the whole row when ALTER TABLE ADD COLUMN crosses the budget', async () => { + driver = new SqlDriver(cell.config()); + await driver.execute('drop table if exists os11565_grow'); + await driver.initObjects([wideObject('os11565_grow', 15, 1024)]); + + await expect(driver.initObjects([wideObject('os11565_grow', 16, 1024)])).rejects.toThrow( + /cannot add column\(s\) "f16" to "os11565_grow".*Its 16 varchar column\(s\)/s, + ); + }); + + /** + * The SECOND limit, which the card's threshold table does not reach and a + * 65535-byte pre-flight would have waved through: InnoDB's per-page limit, + * hit here by forty ordinary `maxLength: 63` fields — about a sixth of the + * 65535 budget. Reported with the number the SERVER quoted, not with 65535. + */ + it('reports InnoDB page-limit refusals with the page limit, not the row budget', async () => { + driver = new SqlDriver(cell.config()); + await driver.execute('drop table if exists os11565_narrow'); + + const failure = await driver + .initObjects([wideObject('os11565_narrow', 40, 63)]) + .then(() => null) + .catch((e: any) => e); + expect(failure).toBeInstanceOf(Error); + const message = String(failure.message); + expect(message).toMatch(/InnoDB's per-PAGE limit of 8126 bytes/); + expect(message).not.toMatch(/65535-byte budget for one ROW/); + expect(message).toMatch(/"f1" varchar\(63\) = 253 bytes/); + }); + + /** + * The shape a diagnostic reading only DECLARED bounds would have nothing to + * say about: sixty-four `lookup` fields, no `maxLength` anywhere, each + * silently taking knex's varchar(255). + */ + it('names the fields even when nothing declares a maxLength', async () => { + driver = new SqlDriver(cell.config()); + await driver.execute('drop table if exists os11565_undeclared'); + + const failure = await driver + .initObjects([undeclaredObject('os11565_undeclared', 64)]) + .then(() => null) + .catch((e: any) => e); + expect(failure).toBeInstanceOf(Error); + const message = String(failure.message); + expect(message).toMatch(/"f1" varchar\(255\) = 1022 bytes/); + expect(message).toMatch(/a field declaring NO `maxLength` still takes varchar\(255\)/i); + }); + }); +}); diff --git a/packages/drivers/driver-sql/src/sql-driver.ts b/packages/drivers/driver-sql/src/sql-driver.ts index 383ea31dd6..16db0e267a 100644 --- a/packages/drivers/driver-sql/src/sql-driver.ts +++ b/packages/drivers/driver-sql/src/sql-driver.ts @@ -3941,6 +3941,20 @@ export type SqlDriverConfig = Knex.Config & { * Implements the IDataDriver contract via Knex.js for optimal SQL * generation against PostgreSQL, MySQL, SQLite and other SQL databases. */ +/** + * One column's contribution to MySQL's per-ROW budget over DECLARED widths + * (#11565). + * + * `chars` is the `varchar(n)` width {@link SqlDriver.createColumn} emits for + * the field; `bytes` is what the server charges for it — the payload + * (`chars x bytesPerChar`) plus varchar's own length prefix. + */ +interface RowWidthContribution { + name: string; + chars: number; + bytes: number; +} + export class SqlDriver implements IDataDriver { // IDataDriver metadata public readonly name: string = 'com.objectstack.driver.sql'; @@ -8853,17 +8867,26 @@ export class SqlDriver implements IDataDriver { }); if (!exists) { - await this.knex.schema.createTable(tableName, (table) => { - table.string('id').primary(); - this.createAuditTimestampColumn(table, 'created_at'); - this.createAuditTimestampColumn(table, 'updated_at'); - if (obj.fields) { - for (const [name, field] of Object.entries(obj.fields)) { - if (builtinColumns.has(name)) continue; - this.createColumn(table, name, field, keyedColumns.get(name)); + try { + await this.knex.schema.createTable(tableName, (table) => { + table.string('id').primary(); + this.createAuditTimestampColumn(table, 'created_at'); + this.createAuditTimestampColumn(table, 'updated_at'); + if (obj.fields) { + for (const [name, field] of Object.entries(obj.fields)) { + if (builtinColumns.has(name)) continue; + this.createColumn(table, name, field, keyedColumns.get(name)); + } } - } - }); + }); + } catch (e: any) { + // #11565: MySQL charges every bounded column's DECLARED width against + // a per-ROW budget, and refuses the CREATE naming no column and no + // declaration — about a table its author described entirely in + // metadata. Re-throw the SAME failure carrying the field-level fix; + // anything that is not that refusal is re-thrown untouched. + await this.rethrowWithRowSizeExplanation(tableName, obj.fields, keyedColumns, null, e); + } this.tablesWithTimestamps.add(tableName); } else { const columnInfo = await this.knex(tableName).columnInfo(); @@ -8873,15 +8896,26 @@ export class SqlDriver implements IDataDriver { this.tablesWithTimestamps.add(tableName); } - await this.knex.schema.alterTable(tableName, (table) => { - if (obj.fields) { - for (const [name, field] of Object.entries(obj.fields)) { - if (!existingColumns.includes(name)) { - this.createColumn(table, name, field, keyedColumns.get(name)); + // #11565: the row budget is a property of the WHOLE row, so adding one + // ordinary column to a wide table is refused by the width of columns + // nobody is touching — with the same column-less server error. Named + // here so the ADD COLUMN path is not the one that keeps it. + const addedColumns = Object.keys(obj.fields ?? {}).filter( + (name) => !builtinColumns.has(name) && !existingColumns.includes(name), + ); + try { + await this.knex.schema.alterTable(tableName, (table) => { + if (obj.fields) { + for (const [name, field] of Object.entries(obj.fields)) { + if (!existingColumns.includes(name)) { + this.createColumn(table, name, field, keyedColumns.get(name)); + } } } - } - }); + }); + } catch (e: any) { + await this.rethrowWithRowSizeExplanation(tableName, obj.fields, keyedColumns, addedColumns, e); + } } // Materialize the table's index set: field-level `unique` (tenancy-aware @@ -13126,6 +13160,334 @@ export class SqlDriver implements IDataDriver { ); } + /** + * MySQL's per-ROW budget over the DECLARED byte widths of a table's columns + * (#11565) — the number its own refusal quotes and the only part of that + * refusal an operator can act on. + * + * Independent of the per-column `varchar` ceiling {@link MAX_VARCHAR_CHARS} + * bounds: every declaration can be individually legal and the table still be + * un-creatable. Measured on live MySQL 8.0.46 (utf8mb4 / InnoDB / DYNAMIC), + * through this driver rather than in raw SQL, so the built-in `id`, + * `created_at` and `updated_at` columns are inside the numbers: + * + * ``` + * 15 fields @ maxLength 1024 CREATE ok 63 fields @ maxLength 255 CREATE ok + * 16 fields @ maxLength 1024 ER_TOO_BIG_ROWSIZE 64 fields @ maxLength 255 ER_TOO_BIG_ROWSIZE + * ``` + * + * Postgres has no equivalent — it TOASTs — so nothing here is dialect-neutral + * behaviour that happens to be measured on MySQL. It is MySQL-only, and it is + * selected as MySQL-only **by the error code**, never by a dialect getter. + */ + protected static readonly MYSQL_ROW_BYTE_BUDGET = 65535; + + /** + * utf8mb4's bytes-per-character, used ONLY when the server cannot be asked + * (see {@link schemaBytesPerChar}). The real multiplier is a property of the + * schema, not of the metadata: measured on 8.0.46, `64 x varchar(1024)` is + * refused on a utf8mb4 database and CREATES on a latin1 one. That asymmetry + * is the whole reason this arithmetic is only ever run to EXPLAIN a refusal + * the server has already issued — see {@link explainRowSizeOverflow}. + */ + protected static readonly ASSUMED_BYTES_PER_CHAR = 4; + + /** + * What MySQL charges for one `varchar(chars)` column: the payload plus + * varchar's own length prefix, which is 1 byte while the payload fits in 255 + * and 2 bytes after. + * + * Bracketed by measurement rather than read off a doc page — the boundary is + * visible in the column counts a table can hold. On utf8mb4 a `varchar(63)` + * payload is 252 bytes and 32 such columns fit InnoDB's 8126-byte page limit + * (32 x 253 = 8096; at 254 bytes each they would not), so 252 takes ONE + * prefix byte; a `varchar(64)` payload of 256 bytes takes two. A payload of + * exactly 255 is the documented last one-byte width and is the one point + * neither limit can discriminate — a column that narrow is never stored + * off-page, so the page limit binds long before 256 vs 257 bytes could show. + * Off by one byte there would move a number in a message, never a verdict. + */ + protected static varcharPackLength(chars: number, bytesPerChar: number): number { + const payload = chars * bytesPerChar; + return payload + (payload < 256 ? 1 : 2); + } + + /** + * The `varchar(n)` width {@link SqlDriver.createColumn} emits for a field, or + * `null` when the column it emits is not a varchar at all (#11565). + * + * A READ-ONLY mirror of that switch, not a second decision: the two families + * that size a column from metadata delegate to the very helpers + * `createColumn` calls ({@link declaredVarcharLength}, + * {@link keyableTextLength}), and the remaining branches are grouped by what + * `createColumn` does with them, not restated per type. + * + * ⚠️ The branch that matters most for the row budget is the one that declares + * nothing: `lookup` / `user` / `auto_number` and the catch-all all spell + * `table.string(name)`, which is knex's `varchar(255)` — 1022 bytes on + * utf8mb4. An object with sixty lookups therefore reaches the budget having + * declared no `maxLength` anywhere, and a diagnostic that only looked at + * declared bounds would name nothing at all on the most reachable shape. + * + * The agreement between this mirror and `createColumn` is PINNED rather than + * asserted in prose: `sql-driver-11565-row-byte-budget.test.ts` builds one + * field of every `FieldType` the spec declares, creates the table, and + * compares this answer against `columnInfo()` column by column. A type added + * to the spec enters that pin automatically. + */ + protected varcharColumnChars(field: any, keyed?: { unique: boolean }): number | null { + // `multiple` is decided before the type switch in `createColumn` — a JSON + // column, whatever the element type would have been. + if (field?.multiple) return null; + const type = field?.type || 'string'; + switch (type) { + case 'string': + case 'email': + case 'url': + case 'phone': + case 'password': + return this.declaredVarcharLength(field); + case 'text': + case 'textarea': + case 'html': + case 'markdown': + return keyed ? this.keyableTextLength(field) : null; + // Virtual — `createColumn` returns without emitting anything. + case 'formula': + return null; + // The non-string primitives: INTEGER / REAL / BOOLEAN / DATE / DATETIME / + // TIME columns. None of them is sized from metadata and none is a varchar. + case 'integer': + case 'int': + case 'float': + case 'number': + case 'currency': + case 'percent': + case 'rating': + case 'slider': + case 'progress': + case 'summary': + case 'boolean': + case 'toggle': + case 'date': + case 'datetime': + case 'time': + return null; + default: + // `createColumn`'s catch-all, spelled the same way so the two cannot + // disagree about which types are JSON: everything else is + // `table.string(name)` at knex's default width. + return JSON_COLUMN_TYPES.has(type) ? null : SqlDriver.DEFAULT_STRING_VARCHAR_CHARS; + } + } + + /** + * Every varchar column this object's fields produce, widest first, with what + * each costs against the row budget at a given bytes-per-character (#11565). + * + * The built-in `id` / `created_at` / `updated_at` columns are excluded for the + * same reason `initObjects` skips them when iterating `obj.fields`: they are + * not authored, so naming them in a diagnostic addressed to the author points + * at something no declaration can change. They are NOT free — `id` alone is a + * `varchar(255)` — which is why the message says the sum sits on top of them + * rather than claiming to be the server's own total. + */ + protected rowWidthProfile( + fields: Record | undefined, + keyedColumns: ReadonlyMap, + bytesPerChar: number, + ): { columns: RowWidthContribution[]; totalBytes: number } { + const builtin = new Set(['id', ...AUDIT_TIMESTAMP_COLUMNS]); + const columns: RowWidthContribution[] = []; + for (const [name, field] of Object.entries(fields ?? {})) { + if (builtin.has(name)) continue; + const chars = this.varcharColumnChars(field, keyedColumns.get(name)); + if (chars === null) continue; + columns.push({ name, chars, bytes: SqlDriver.varcharPackLength(chars, bytesPerChar) }); + } + columns.sort((a, b) => b.bytes - a.bytes || a.name.localeCompare(b.name)); + return { columns, totalBytes: columns.reduce((sum, c) => sum + c.bytes, 0) }; + } + + /** + * The bytes-per-character the tables this driver creates actually get, read + * from the server. + * + * Knex emits no `CHARACTER SET` clause, so a created table takes the + * DATABASE's default charset — verified on 8.0.46: a table created with no + * clause in a `CHARACTER SET latin1` database comes back + * `TABLE_COLLATION = latin1_swedish_ci`. So `@@character_set_database` is the + * right question, and `information_schema.CHARACTER_SETS.MAXLEN` answers the + * multiplier for it (utf8mb4 → 4, utf8mb3 → 3, latin1/ascii → 1). + * + * One round-trip, and only ever on a path that is already throwing — the same + * trade {@link explainUnkeyableTextColumn} makes with its `columnInfo()` read. + * `null` when the read fails: a diagnostic must not turn into a second + * failure, and the advice holds with the assumed multiplier. + */ + protected async schemaBytesPerChar(): Promise<{ bytesPerChar: number; charset: string } | null> { + try { + const res: any = await this.knex.raw( + 'select cs.CHARACTER_SET_NAME as charset, cs.MAXLEN as maxlen ' + + 'from information_schema.CHARACTER_SETS cs ' + + 'where cs.CHARACTER_SET_NAME = @@character_set_database', + ); + const row = (Array.isArray(res) ? res[0]?.[0] : (res?.rows?.[0] ?? res?.[0])) ?? null; + const maxlen = Number(row?.maxlen ?? row?.MAXLEN); + const charset = String(row?.charset ?? row?.CHARSET ?? ''); + if (!Number.isInteger(maxlen) || maxlen <= 0 || !charset) return null; + return { bytesPerChar: maxlen, charset }; + } catch { + // Introspection is a nicety here; the advice below holds without it. + return null; + } + } + + /** + * Turn MySQL's `ER_TOO_BIG_ROWSIZE` into a message that names the columns + * whose declared widths spent the row budget (#11565). + * + * ## Why a post-hoc translator and NOT a pre-flight + * + * The card that raised this preferred a pre-flight — sum the declared widths + * before issuing DDL and refuse — on the grounds that a translator "only + * knows the table failed". That premise does not survive contact with the + * call site: this runs inside `initObjects`' own loop, where `obj.fields` and + * the resolved `keyedColumns` are in scope, so it names EVERY contributing + * field exactly as a pre-flight would. The advantage claimed for the + * pre-flight is not an advantage it has here. + * + * What the two shapes do not share is their failure direction, and the + * asymmetry is the whole argument: + * + * - A pre-flight has to REPRODUCE the server's arithmetic. Wrong in the lax + * direction it is merely useless — the server still refuses, with today's + * bad message. Wrong in the STRICT direction it refuses an object MySQL + * would have accepted, which is a contract change: the same metadata now + * means two different things depending on the driver's own model. + * - A translator cannot over-refuse BY CONSTRUCTION. It speaks only after + * the server has already refused, so no arithmetic error it makes can + * change which objects are accepted — the worst case is a less precise + * sentence attached to a refusal that was going to happen anyway. + * + * And the arithmetic is genuinely hard to hold, all four parts measured on + * 8.0.46 while choosing this shape: + * + * 1. **The charset multiplier is the schema's, not the metadata's.** + * `64 x varchar(1024)` is refused on utf8mb4 and CREATES on latin1. + * A pre-flight hard-coding 4 bytes/char over-refuses by 4x on a latin1 + * deployment. + * 2. **The length prefix moves** at a 255-byte payload boundary (see + * {@link varcharPackLength}). + * 3. **The null bitmap counts.** `163 x varchar(100)` is refused when the + * columns are nullable and CREATES when they are `NOT NULL` — same + * declared widths, different verdict. + * 4. **There is a SECOND, independent limit.** InnoDB's per-page limit + * (`Row size too large (> 8126)` on a 16K page) fires far below 65535 + * for many NARROW columns, because a column whose maximum is under ~256 + * bytes is never pushed off-page: measured, `varchar(63)` tops out at 32 + * columns and `varchar(64)` at 196. Through this driver, an object of 40 + * ordinary `maxLength: 63` fields is refused — a shape a 65535-byte + * pre-flight would have waved through, leaving the author with the + * unactionable server error the card was filed about. + * + * A pre-flight would also need a dialect predicate to know it is on MySQL. + * This needs none: `ER_TOO_BIG_ROWSIZE` is a MySQL code, so the diagnostic is + * MySQL-scoped by construction and costs nothing on every other dialect and + * on the success path. + * + * @param addedColumns the columns an `ALTER TABLE` was adding, or `null` for + * a `CREATE TABLE`. The budget is the whole ROW either way — a column added + * years later is refused by the width of columns nobody is touching — so + * the offender list is the same and only the opening clause differs. + */ + protected async explainRowSizeOverflow( + tableName: string, + fields: Record | undefined, + keyedColumns: ReadonlyMap, + addedColumns: string[] | null, + cause: unknown, + ): Promise { + if ((cause as { code?: string } | undefined)?.code !== 'ER_TOO_BIG_ROWSIZE') return null; + + const server = await this.schemaBytesPerChar(); + const bytesPerChar = server?.bytesPerChar ?? SqlDriver.ASSUMED_BYTES_PER_CHAR; + const charsetNote = server + ? `${bytesPerChar} bytes/character (${server.charset}, this schema's charset)` + : `${bytesPerChar} bytes/character (utf8mb4 assumed — the server's charset could not be read)`; + const { columns, totalBytes } = this.rowWidthProfile(fields, keyedColumns, bytesPerChar); + + // Two DIFFERENT limits answer with this one code, and they quote different + // numbers. Reading the server's own text keeps the message from asserting + // the 65535 one over a refusal that was really InnoDB's page limit. + const serverText = String((cause as { sqlMessage?: string; message?: string }).sqlMessage ?? ''); + const pageLimit = /row size too large \(>\s*(\d+)\)/i.exec(serverText)?.[1] ?? null; + const limitClause = pageLimit + ? `InnoDB's per-PAGE limit of ${pageLimit} bytes — which many NARROW bounded columns reach far below the ` + + `${SqlDriver.MYSQL_ROW_BYTE_BUDGET}-byte one, because a column narrower than about 256 bytes is never ` + + `stored off-page (measured on 8.0.46: 40 fields at \`maxLength: 63\` are refused)` + : `the server's ${SqlDriver.MYSQL_ROW_BYTE_BUDGET}-byte budget for one ROW`; + + const SHOWN = 8; + const listed = columns + .slice(0, SHOWN) + .map((c) => `"${c.name}" varchar(${c.chars}) = ${c.bytes} bytes`) + .join(', '); + const rest = columns.length > SHOWN ? `, and ${columns.length - SHOWN} more` : ''; + const inventory = + columns.length === 0 + ? `This object produces no varchar column at all, so the width is in the built-in columns or in a ` + + `type this diagnostic does not size — report it, because that is not a shape it was measured on.` + : `Its ${columns.length} varchar column(s) take ${totalBytes} bytes at ${charsetNote}, on top of the ` + + `built-in \`id\` (a varchar(${SqlDriver.DEFAULT_STRING_VARCHAR_CHARS})), \`created_at\` and ` + + `\`updated_at\` columns. Widest first: ${listed}${rest}.`; + + const opening = + addedColumns && addedColumns.length > 0 + ? `[sql-driver] cannot add column(s) ${addedColumns.map((c) => `"${c}"`).join(', ')} to "${tableName}"` + : `[sql-driver] cannot create table "${tableName}"`; + + return ( + `${opening} — MySQL refuses the whole row: every bounded column's DECLARED width is charged against ` + + `${limitClause}, whether or not a row ever holds that much. ${inventory} ` + + `Fix it on the declarations: lower \`maxLength\` on the widest field(s), or drop the bound from a ` + + `text-family field no index keys on so it is emitted as TEXT — an off-page TEXT column costs this ` + + `budget a small pointer rather than its full declared width. Note that a field declaring NO ` + + `\`maxLength\` still takes varchar(${SqlDriver.DEFAULT_STRING_VARCHAR_CHARS}) ` + + `(${SqlDriver.varcharPackLength(SqlDriver.DEFAULT_STRING_VARCHAR_CHARS, bytesPerChar)} bytes here), and ` + + `so do \`lookup\`, \`user\`, \`auto_number\` and the option types — an object can reach this limit ` + + `without declaring a bound anywhere. The server's own error names no column, which is the whole reason ` + + `this one does (#11565).` + ); + } + + /** + * Re-throw a schema-mutation failure carrying {@link explainRowSizeOverflow}'s + * explanation, or the original untouched when it is not a row-size refusal. + * + * The SAME failure, re-worded — the boot still fails, loudly, and no object is + * registered that MySQL declined to build. `sqlMessage` rather than `message` + * for the quoted server text: knex prefixes `message` with the entire DDL + * statement, which for the object that trips this is thousands of characters + * of `varchar(...)` and buries the one sentence the server actually said. + */ + private async rethrowWithRowSizeExplanation( + tableName: string, + fields: Record | undefined, + keyedColumns: ReadonlyMap, + addedColumns: string[] | null, + e: any, + ): Promise { + const overflow = await this.explainRowSizeOverflow(tableName, fields, keyedColumns, addedColumns, e); + if (!overflow) throw e; + const said = String(e?.sqlMessage ?? e?.message ?? e); + (this.logger.error ?? this.logger.warn)(overflow, said); + throw Object.assign(new Error(`${overflow} (server said: ${said})`), { + code: (e as { code?: string }).code, + cause: e, + }); + } + protected createColumn( table: Knex.CreateTableBuilder, name: string, From a483faedd593b13195f24ff2b8f673fb7584185c Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 24 Aug 2026 16:32:51 +0000 Subject: [PATCH 2/2] chore(changeset): MySQL row-size refusal names the declarations Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01VK8rFDtg8eREaxBGX99Csn --- .changeset/mysql-row-byte-budget-diagnostic.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/mysql-row-byte-budget-diagnostic.md diff --git a/.changeset/mysql-row-byte-budget-diagnostic.md b/.changeset/mysql-row-byte-budget-diagnostic.md new file mode 100644 index 0000000000..a50bf82ebb --- /dev/null +++ b/.changeset/mysql-row-byte-budget-diagnostic.md @@ -0,0 +1,5 @@ +--- +"@objectstack/driver-sql": patch +--- + +MySQL's row-size refusal now names the declarations that caused it (#11565). MySQL charges every bounded column's DECLARED byte width against a per-row budget, independently of the per-column `varchar` ceiling — measured on 8.0.46 through this driver, 15 fields at `maxLength: 1024` create and 16 are refused — and its own error names no column and no declaration, about a table its author described entirely in metadata. Schema sync now translates `ER_TOO_BIG_ROWSIZE` at both the `CREATE TABLE` and `ALTER TABLE ADD COLUMN` sites into the same failure re-worded: every varchar column the object produces, widest first, with its emitted width and its byte cost at the schema's real bytes-per-character (read from the server, not assumed), plus the fields that reach the budget while declaring nothing — `lookup`, `user`, `auto_number` and the option types all take `varchar(255)`. InnoDB's separate per-page limit answers with the same code and is reported with the number the server quoted rather than 65535. Deliberately a translator and not a pre-flight: it speaks only after the server has refused, so it cannot refuse an object MySQL would have accepted. Nothing is refused that was accepted before, and no other dialect is touched.