diff --git a/.changeset/mysql-row-byte-budget-diagnostic.md b/.changeset/mysql-row-byte-budget-diagnostic.md new file mode 100644 index 0000000000..a50bf82ebb --- /dev/null +++ b/.changeset/mysql-row-byte-budget-diagnostic.md @@ -0,0 +1,5 @@ +--- +"@objectstack/driver-sql": patch +--- + +MySQL's row-size refusal now names the declarations that caused it (#11565). MySQL charges every bounded column's DECLARED byte width against a per-row budget, independently of the per-column `varchar` ceiling — measured on 8.0.46 through this driver, 15 fields at `maxLength: 1024` create and 16 are refused — and its own error names no column and no declaration, about a table its author described entirely in metadata. Schema sync now translates `ER_TOO_BIG_ROWSIZE` at both the `CREATE TABLE` and `ALTER TABLE ADD COLUMN` sites into the same failure re-worded: every varchar column the object produces, widest first, with its emitted width and its byte cost at the schema's real bytes-per-character (read from the server, not assumed), plus the fields that reach the budget while declaring nothing — `lookup`, `user`, `auto_number` and the option types all take `varchar(255)`. InnoDB's separate per-page limit answers with the same code and is reported with the number the server quoted rather than 65535. Deliberately a translator and not a pre-flight: it speaks only after the server has refused, so it cannot refuse an object MySQL would have accepted. Nothing is refused that was accepted before, and no other dialect is touched. diff --git a/packages/drivers/driver-sql/src/sql-driver-11565-row-byte-budget.test.ts b/packages/drivers/driver-sql/src/sql-driver-11565-row-byte-budget.test.ts new file mode 100644 index 0000000000..c8122d8113 --- /dev/null +++ b/packages/drivers/driver-sql/src/sql-driver-11565-row-byte-budget.test.ts @@ -0,0 +1,320 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * #11565 — MySQL's per-ROW budget over DECLARED column widths, and the + * diagnostic that names the declarations that spent it. + * + * ## The defect + * + * MySQL charges every bounded column's DECLARED byte width against a per-row + * budget, independently of the per-column `varchar` ceiling. An object whose + * fields declare enough total width simply fails `CREATE TABLE` — and the + * server's refusal names **no column and no declaration**. It says "You have to + * change some columns to TEXT or BLOBs" about a table its author described + * entirely in metadata, and nothing maps that back to the `maxLength` values + * responsible. Sixteen fields at `maxLength: 1024` is not an exotic object. + * + * ## Why a translator and not a pre-flight + * + * A pre-flight that sums declared widths BEFORE issuing DDL has to reproduce + * the server's arithmetic, and wrong in the strict direction it refuses an + * object MySQL would have accepted — a contract change. A translator cannot + * over-refuse by construction: it speaks only after the server has refused. + * Sitting inside `initObjects`' own loop it still sees `obj.fields`, so it + * names every contributing field exactly as a pre-flight would. + * {@link SqlDriver.explainRowSizeOverflow} carries the four measurements that + * decided it; this file is the executable half. + * + * ## What each half is worth + * + * The dialect-free block runs everywhere, including Test Core. It pins the + * arithmetic, the agreement between the width mirror and `createColumn`'s own + * switch over every `FieldType` the spec declares, and — the half that stops + * "refuses" from passing for "refuses the RIGHT objects" — that an object past + * MySQL's budget still creates cleanly on a dialect that has no such budget. + * + * The MySQL block is the one that reds on the pre-fix tree. Opt-in: + * + * OS_TEST_MYSQL_URL=mysql://root:root@127.0.0.1:3306/conformance \ + * pnpm --filter @objectstack/driver-sql test + */ + +import { describe, it, expect, afterEach } from 'vitest'; +import { FieldType } from '@objectstack/spec/data'; +import { SqlDriver } from '../src/index.js'; +import { MYSQL_CELL, dialectCell, declareDialectCell } from './live-dialect-matrix.testkit.js'; + +/** An object of `count` string fields, each declaring the same `maxLength`. */ +const wideObject = (name: string, count: number, maxLength: number) => ({ + name, + fields: Object.fromEntries( + Array.from({ length: count }, (_, i) => [`f${i + 1}`, { type: 'string', maxLength }]), + ), +}); + +/** The same shape with NOTHING declared — `lookup` takes knex's varchar(255). */ +const undeclaredObject = (name: string, count: number) => ({ + name, + fields: Object.fromEntries( + Array.from({ length: count }, (_, i) => [`f${i + 1}`, { type: 'lookup' }]), + ), +}); + +// ── The arithmetic, and the mirror, on a dialect every runner has ─────────── + +describe('row byte budget — arithmetic and column mirror (#11565)', () => { + let driver: SqlDriver; + + afterEach(async () => { + await driver?.disconnect().catch(() => {}); + }); + + /** + * The length prefix moves at a BYTE-payload boundary, not a character one, so + * it moves with the charset — measured on MySQL 8.0.46 through the column + * counts a table can hold. A utf8mb4 `varchar(63)` payload is 252 bytes and + * 32 of them fit InnoDB's 8126-byte page limit (32 x 253 = 8096; at 254 bytes + * each, 32 would not fit), so that prefix is one byte. A `varchar(64)` + * payload is 256 bytes and behaves as the two-byte, off-page-eligible class. + * + * ⚠️ A payload of EXACTLY 255 is documented as the last one-byte width, and + * is deliberately not asserted here: such a column is never stored off-page, + * so the page limit binds at ~31 columns and neither limit can be made to + * discriminate 256 from 257 bytes. An unmeasurable claim does not get a pin. + */ + it('charges payload + varchar length prefix, and the prefix moves with the charset', () => { + const pack = (chars: number, bpc: number) => (SqlDriver as any).varcharPackLength(chars, bpc); + expect(pack(63, 4)).toBe(253); // utf8mb4: 252 + 1 + expect(pack(64, 4)).toBe(258); // utf8mb4: 256 + 2 + expect(pack(63, 1)).toBe(64); // latin1: 63 + 1 — the same width, a quarter the cost + expect(pack(300, 1)).toBe(302); // latin1: 300 + 2 + expect(pack(1024, 4)).toBe(4098); // the card's own row: 16 x 4098 = 65568 > 65535 + expect(pack(255, 4)).toBe(1022); // the DEFAULT width, which declares nothing + }); + + /** + * ⚠️ The mirror is a second reading of `createColumn`'s switch, so the risk it + * carries is drift. This pin removes the risk structurally rather than by + * review: one field of EVERY `FieldType` the spec declares, created for real, + * and the mirror's answer compared against the column that actually landed. A + * type added to the spec joins this pin without anyone remembering to. + */ + it('agrees with createColumn about every FieldType the spec declares', async () => { + const types = FieldType.options as readonly string[]; + expect(types.length).toBeGreaterThan(40); // the registry really was read + + const fields = Object.fromEntries(types.map((t) => [`f_${t}`, { type: t }])); + driver = new SqlDriver(dialectCell('sqlite').config()); + await driver.initObjects([{ name: 'os11565_every_type', fields }]); + const info: Record = await ( + driver as any + ).knex('os11565_every_type').columnInfo(); + + const mismatched: string[] = []; + for (const t of types) { + const column = `f_${t}`; + const mirrored = (driver as any).varcharColumnChars({ type: t }, undefined) as number | null; + const landed = info[column]; + const isVarchar = /varchar/i.test(String(landed?.type ?? '')); + const landedChars = isVarchar ? Number(landed?.maxLength) : null; + if (mirrored !== landedChars) { + mismatched.push( + `${t}: mirror says ${mirrored === null ? 'not a varchar' : `varchar(${mirrored})`}, ` + + `createColumn emitted ${landed === undefined ? 'no column' : String(landed.type)}`, + ); + } + } + expect(mismatched).toEqual([]); + }); + + /** + * ⛔ The negative half, and the reason "it refuses" is not the assertion this + * file makes: the budget is MySQL's, so an object past it must still create + * cleanly everywhere else. A pre-flight with a wrong constant would fail + * exactly here; a translator cannot, because it never runs. + */ + it('refuses nothing on a dialect with no row budget', async () => { + driver = new SqlDriver(dialectCell('sqlite').config()); + // 16 x maxLength 1024 — the shape live MySQL rejects, three lines down in + // the same repo. + await driver.initObjects([wideObject('os11565_wide_sqlite', 16, 1024)]); + const info: any = await (driver as any).knex('os11565_wide_sqlite').columnInfo(); + expect(Object.keys(info)).toContain('f16'); + expect(String(info.f16?.maxLength ?? '')).toBe('1024'); + }); + + /** The offender list is ordered by cost, and holds every varchar column. */ + it('profiles every varchar column, widest first', async () => { + driver = new SqlDriver(dialectCell('sqlite').config()); + const profile = (driver as any).rowWidthProfile( + { + narrow: { type: 'string', maxLength: 10 }, + widest: { type: 'string', maxLength: 4000 }, + middle: { type: 'string', maxLength: 1024 }, + unbounded_lookup: { type: 'lookup' }, + a_number: { type: 'number' }, + // Unkeyed text stays TEXT — it costs the budget a pointer, not a width. + body: { type: 'text', maxLength: 60000 }, + id: { type: 'string', maxLength: 9999 }, // built-in, never the author's + }, + new Map(), + 4, + ); + expect(profile.columns.map((c: any) => c.name)).toEqual([ + 'widest', + 'middle', + 'unbounded_lookup', + 'narrow', + ]); + expect(profile.columns[0]).toMatchObject({ chars: 4000, bytes: 16002 }); + expect(profile.columns[2]).toMatchObject({ chars: 255, bytes: 1022 }); + expect(profile.totalBytes).toBe(16002 + 4098 + 1022 + 41); + }); +}); + +// ── The half only a live MySQL can measure ────────────────────────────────── + +declareDialectCell(MYSQL_CELL, 'row byte budget (#11565)', (cell) => { + describe('row byte budget on live MySQL (#11565)', () => { + let driver: SqlDriver; + const TABLES = [ + 'os11565_ok', + 'os11565_over', + 'os11565_ok255', + 'os11565_over255', + 'os11565_grow', + 'os11565_narrow', + 'os11565_undeclared', + ]; + + afterEach(async () => { + for (const t of TABLES) await driver?.execute(`drop table if exists ${t}`).catch(() => {}); + await driver?.disconnect().catch(() => {}); + }); + + /** + * The measured boundaries below are utf8mb4's. On a latin1 schema the same + * declarations cost a quarter as much and every count here is wrong — so + * the cell asserts the multiplier rather than assuming it, the same way the + * matrix asserts its zone skew instead of hoping for it. + */ + it('runs on a 4-byte charset — the boundaries below are utf8mb4 numbers', async () => { + driver = new SqlDriver(cell.config()); + const seen = await (driver as any).schemaBytesPerChar(); + expect(seen).not.toBeNull(); + expect(seen.bytesPerChar).toBe(4); + }); + + /** + * Both sides of the boundary, in one test, because only the pair means + * anything: an implementation that refused every object would pass the + * second assertion alone. Measured through this driver (its built-in `id` + * varchar(255) is inside the budget, which is why the count is 15/16 here + * and 15/16 in raw SQL only by coincidence of the same width). + */ + it('creates 15 fields at maxLength 1024 and names all 16 when one more is added', async () => { + driver = new SqlDriver(cell.config()); + await driver.execute('drop table if exists os11565_ok'); + await driver.execute('drop table if exists os11565_over'); + + // ACCEPTS: unchanged behaviour, no diagnostic, a real table. + await driver.initObjects([wideObject('os11565_ok', 15, 1024)]); + const info: any = await (driver as any).knex('os11565_ok').columnInfo(); + expect(String(info.f15?.type)).toBe('varchar'); + expect(Number(info.f15?.maxLength)).toBe(1024); + + // REFUSES — with the fields the server would not name. + const failure = await driver + .initObjects([wideObject('os11565_over', 16, 1024)]) + .then(() => null) + .catch((e: any) => e); + expect(failure).toBeInstanceOf(Error); + const message = String(failure.message); + expect(message).toMatch(/cannot create table "os11565_over"/); + expect(message).toMatch(/65535-byte budget for one ROW/); + // Every contributing field, not merely "the table failed". + expect(message).toMatch(/Its 16 varchar column\(s\) take 65568 bytes/); + expect(message).toMatch(/"f1" varchar\(1024\) = 4098 bytes/); + expect(message).toMatch(/and 8 more/); + // The server's own sentence is kept, not replaced. + expect(message).toMatch(/server said: Row size too large/); + // Same failure, re-worded: the code survives for anything reading it. + expect(failure.code).toBe('ER_TOO_BIG_ROWSIZE'); + expect((failure.cause as any)?.code).toBe('ER_TOO_BIG_ROWSIZE'); + + // ⛔ And nothing was left behind: the object is not registered half-built. + const exists = await (driver as any).knex.schema.hasTable('os11565_over'); + expect(exists).toBe(false); + }); + + /** The card's second measured row, moved by the driver's own `id` column. */ + it('creates 63 fields at maxLength 255 and refuses 64', async () => { + driver = new SqlDriver(cell.config()); + await driver.execute('drop table if exists os11565_ok255'); + await driver.execute('drop table if exists os11565_over255'); + + await driver.initObjects([wideObject('os11565_ok255', 63, 255)]); + expect(await (driver as any).knex.schema.hasTable('os11565_ok255')).toBe(true); + + await expect(driver.initObjects([wideObject('os11565_over255', 64, 255)])).rejects.toThrow( + /cannot create table "os11565_over255".*Its 64 varchar column\(s\) take 65408 bytes/s, + ); + }); + + /** + * The path that is more likely than CREATE in a living app: a field added + * to an object that was already near the budget. The server refuses the ADD + * naming only the column being added, as if that one column were too wide — + * when the width is in fifteen columns nobody is touching. + */ + it('names the whole row when ALTER TABLE ADD COLUMN crosses the budget', async () => { + driver = new SqlDriver(cell.config()); + await driver.execute('drop table if exists os11565_grow'); + await driver.initObjects([wideObject('os11565_grow', 15, 1024)]); + + await expect(driver.initObjects([wideObject('os11565_grow', 16, 1024)])).rejects.toThrow( + /cannot add column\(s\) "f16" to "os11565_grow".*Its 16 varchar column\(s\)/s, + ); + }); + + /** + * The SECOND limit, which the card's threshold table does not reach and a + * 65535-byte pre-flight would have waved through: InnoDB's per-page limit, + * hit here by forty ordinary `maxLength: 63` fields — about a sixth of the + * 65535 budget. Reported with the number the SERVER quoted, not with 65535. + */ + it('reports InnoDB page-limit refusals with the page limit, not the row budget', async () => { + driver = new SqlDriver(cell.config()); + await driver.execute('drop table if exists os11565_narrow'); + + const failure = await driver + .initObjects([wideObject('os11565_narrow', 40, 63)]) + .then(() => null) + .catch((e: any) => e); + expect(failure).toBeInstanceOf(Error); + const message = String(failure.message); + expect(message).toMatch(/InnoDB's per-PAGE limit of 8126 bytes/); + expect(message).not.toMatch(/65535-byte budget for one ROW/); + expect(message).toMatch(/"f1" varchar\(63\) = 253 bytes/); + }); + + /** + * The shape a diagnostic reading only DECLARED bounds would have nothing to + * say about: sixty-four `lookup` fields, no `maxLength` anywhere, each + * silently taking knex's varchar(255). + */ + it('names the fields even when nothing declares a maxLength', async () => { + driver = new SqlDriver(cell.config()); + await driver.execute('drop table if exists os11565_undeclared'); + + const failure = await driver + .initObjects([undeclaredObject('os11565_undeclared', 64)]) + .then(() => null) + .catch((e: any) => e); + expect(failure).toBeInstanceOf(Error); + const message = String(failure.message); + expect(message).toMatch(/"f1" varchar\(255\) = 1022 bytes/); + expect(message).toMatch(/a field declaring NO `maxLength` still takes varchar\(255\)/i); + }); + }); +}); diff --git a/packages/drivers/driver-sql/src/sql-driver.ts b/packages/drivers/driver-sql/src/sql-driver.ts index 383ea31dd6..16db0e267a 100644 --- a/packages/drivers/driver-sql/src/sql-driver.ts +++ b/packages/drivers/driver-sql/src/sql-driver.ts @@ -3941,6 +3941,20 @@ export type SqlDriverConfig = Knex.Config & { * Implements the IDataDriver contract via Knex.js for optimal SQL * generation against PostgreSQL, MySQL, SQLite and other SQL databases. */ +/** + * One column's contribution to MySQL's per-ROW budget over DECLARED widths + * (#11565). + * + * `chars` is the `varchar(n)` width {@link SqlDriver.createColumn} emits for + * the field; `bytes` is what the server charges for it — the payload + * (`chars x bytesPerChar`) plus varchar's own length prefix. + */ +interface RowWidthContribution { + name: string; + chars: number; + bytes: number; +} + export class SqlDriver implements IDataDriver { // IDataDriver metadata public readonly name: string = 'com.objectstack.driver.sql'; @@ -8853,17 +8867,26 @@ export class SqlDriver implements IDataDriver { }); if (!exists) { - await this.knex.schema.createTable(tableName, (table) => { - table.string('id').primary(); - this.createAuditTimestampColumn(table, 'created_at'); - this.createAuditTimestampColumn(table, 'updated_at'); - if (obj.fields) { - for (const [name, field] of Object.entries(obj.fields)) { - if (builtinColumns.has(name)) continue; - this.createColumn(table, name, field, keyedColumns.get(name)); + try { + await this.knex.schema.createTable(tableName, (table) => { + table.string('id').primary(); + this.createAuditTimestampColumn(table, 'created_at'); + this.createAuditTimestampColumn(table, 'updated_at'); + if (obj.fields) { + for (const [name, field] of Object.entries(obj.fields)) { + if (builtinColumns.has(name)) continue; + this.createColumn(table, name, field, keyedColumns.get(name)); + } } - } - }); + }); + } catch (e: any) { + // #11565: MySQL charges every bounded column's DECLARED width against + // a per-ROW budget, and refuses the CREATE naming no column and no + // declaration — about a table its author described entirely in + // metadata. Re-throw the SAME failure carrying the field-level fix; + // anything that is not that refusal is re-thrown untouched. + await this.rethrowWithRowSizeExplanation(tableName, obj.fields, keyedColumns, null, e); + } this.tablesWithTimestamps.add(tableName); } else { const columnInfo = await this.knex(tableName).columnInfo(); @@ -8873,15 +8896,26 @@ export class SqlDriver implements IDataDriver { this.tablesWithTimestamps.add(tableName); } - await this.knex.schema.alterTable(tableName, (table) => { - if (obj.fields) { - for (const [name, field] of Object.entries(obj.fields)) { - if (!existingColumns.includes(name)) { - this.createColumn(table, name, field, keyedColumns.get(name)); + // #11565: the row budget is a property of the WHOLE row, so adding one + // ordinary column to a wide table is refused by the width of columns + // nobody is touching — with the same column-less server error. Named + // here so the ADD COLUMN path is not the one that keeps it. + const addedColumns = Object.keys(obj.fields ?? {}).filter( + (name) => !builtinColumns.has(name) && !existingColumns.includes(name), + ); + try { + await this.knex.schema.alterTable(tableName, (table) => { + if (obj.fields) { + for (const [name, field] of Object.entries(obj.fields)) { + if (!existingColumns.includes(name)) { + this.createColumn(table, name, field, keyedColumns.get(name)); + } } } - } - }); + }); + } catch (e: any) { + await this.rethrowWithRowSizeExplanation(tableName, obj.fields, keyedColumns, addedColumns, e); + } } // Materialize the table's index set: field-level `unique` (tenancy-aware @@ -13126,6 +13160,334 @@ export class SqlDriver implements IDataDriver { ); } + /** + * MySQL's per-ROW budget over the DECLARED byte widths of a table's columns + * (#11565) — the number its own refusal quotes and the only part of that + * refusal an operator can act on. + * + * Independent of the per-column `varchar` ceiling {@link MAX_VARCHAR_CHARS} + * bounds: every declaration can be individually legal and the table still be + * un-creatable. Measured on live MySQL 8.0.46 (utf8mb4 / InnoDB / DYNAMIC), + * through this driver rather than in raw SQL, so the built-in `id`, + * `created_at` and `updated_at` columns are inside the numbers: + * + * ``` + * 15 fields @ maxLength 1024 CREATE ok 63 fields @ maxLength 255 CREATE ok + * 16 fields @ maxLength 1024 ER_TOO_BIG_ROWSIZE 64 fields @ maxLength 255 ER_TOO_BIG_ROWSIZE + * ``` + * + * Postgres has no equivalent — it TOASTs — so nothing here is dialect-neutral + * behaviour that happens to be measured on MySQL. It is MySQL-only, and it is + * selected as MySQL-only **by the error code**, never by a dialect getter. + */ + protected static readonly MYSQL_ROW_BYTE_BUDGET = 65535; + + /** + * utf8mb4's bytes-per-character, used ONLY when the server cannot be asked + * (see {@link schemaBytesPerChar}). The real multiplier is a property of the + * schema, not of the metadata: measured on 8.0.46, `64 x varchar(1024)` is + * refused on a utf8mb4 database and CREATES on a latin1 one. That asymmetry + * is the whole reason this arithmetic is only ever run to EXPLAIN a refusal + * the server has already issued — see {@link explainRowSizeOverflow}. + */ + protected static readonly ASSUMED_BYTES_PER_CHAR = 4; + + /** + * What MySQL charges for one `varchar(chars)` column: the payload plus + * varchar's own length prefix, which is 1 byte while the payload fits in 255 + * and 2 bytes after. + * + * Bracketed by measurement rather than read off a doc page — the boundary is + * visible in the column counts a table can hold. On utf8mb4 a `varchar(63)` + * payload is 252 bytes and 32 such columns fit InnoDB's 8126-byte page limit + * (32 x 253 = 8096; at 254 bytes each they would not), so 252 takes ONE + * prefix byte; a `varchar(64)` payload of 256 bytes takes two. A payload of + * exactly 255 is the documented last one-byte width and is the one point + * neither limit can discriminate — a column that narrow is never stored + * off-page, so the page limit binds long before 256 vs 257 bytes could show. + * Off by one byte there would move a number in a message, never a verdict. + */ + protected static varcharPackLength(chars: number, bytesPerChar: number): number { + const payload = chars * bytesPerChar; + return payload + (payload < 256 ? 1 : 2); + } + + /** + * The `varchar(n)` width {@link SqlDriver.createColumn} emits for a field, or + * `null` when the column it emits is not a varchar at all (#11565). + * + * A READ-ONLY mirror of that switch, not a second decision: the two families + * that size a column from metadata delegate to the very helpers + * `createColumn` calls ({@link declaredVarcharLength}, + * {@link keyableTextLength}), and the remaining branches are grouped by what + * `createColumn` does with them, not restated per type. + * + * ⚠️ The branch that matters most for the row budget is the one that declares + * nothing: `lookup` / `user` / `auto_number` and the catch-all all spell + * `table.string(name)`, which is knex's `varchar(255)` — 1022 bytes on + * utf8mb4. An object with sixty lookups therefore reaches the budget having + * declared no `maxLength` anywhere, and a diagnostic that only looked at + * declared bounds would name nothing at all on the most reachable shape. + * + * The agreement between this mirror and `createColumn` is PINNED rather than + * asserted in prose: `sql-driver-11565-row-byte-budget.test.ts` builds one + * field of every `FieldType` the spec declares, creates the table, and + * compares this answer against `columnInfo()` column by column. A type added + * to the spec enters that pin automatically. + */ + protected varcharColumnChars(field: any, keyed?: { unique: boolean }): number | null { + // `multiple` is decided before the type switch in `createColumn` — a JSON + // column, whatever the element type would have been. + if (field?.multiple) return null; + const type = field?.type || 'string'; + switch (type) { + case 'string': + case 'email': + case 'url': + case 'phone': + case 'password': + return this.declaredVarcharLength(field); + case 'text': + case 'textarea': + case 'html': + case 'markdown': + return keyed ? this.keyableTextLength(field) : null; + // Virtual — `createColumn` returns without emitting anything. + case 'formula': + return null; + // The non-string primitives: INTEGER / REAL / BOOLEAN / DATE / DATETIME / + // TIME columns. None of them is sized from metadata and none is a varchar. + case 'integer': + case 'int': + case 'float': + case 'number': + case 'currency': + case 'percent': + case 'rating': + case 'slider': + case 'progress': + case 'summary': + case 'boolean': + case 'toggle': + case 'date': + case 'datetime': + case 'time': + return null; + default: + // `createColumn`'s catch-all, spelled the same way so the two cannot + // disagree about which types are JSON: everything else is + // `table.string(name)` at knex's default width. + return JSON_COLUMN_TYPES.has(type) ? null : SqlDriver.DEFAULT_STRING_VARCHAR_CHARS; + } + } + + /** + * Every varchar column this object's fields produce, widest first, with what + * each costs against the row budget at a given bytes-per-character (#11565). + * + * The built-in `id` / `created_at` / `updated_at` columns are excluded for the + * same reason `initObjects` skips them when iterating `obj.fields`: they are + * not authored, so naming them in a diagnostic addressed to the author points + * at something no declaration can change. They are NOT free — `id` alone is a + * `varchar(255)` — which is why the message says the sum sits on top of them + * rather than claiming to be the server's own total. + */ + protected rowWidthProfile( + fields: Record | undefined, + keyedColumns: ReadonlyMap, + bytesPerChar: number, + ): { columns: RowWidthContribution[]; totalBytes: number } { + const builtin = new Set(['id', ...AUDIT_TIMESTAMP_COLUMNS]); + const columns: RowWidthContribution[] = []; + for (const [name, field] of Object.entries(fields ?? {})) { + if (builtin.has(name)) continue; + const chars = this.varcharColumnChars(field, keyedColumns.get(name)); + if (chars === null) continue; + columns.push({ name, chars, bytes: SqlDriver.varcharPackLength(chars, bytesPerChar) }); + } + columns.sort((a, b) => b.bytes - a.bytes || a.name.localeCompare(b.name)); + return { columns, totalBytes: columns.reduce((sum, c) => sum + c.bytes, 0) }; + } + + /** + * The bytes-per-character the tables this driver creates actually get, read + * from the server. + * + * Knex emits no `CHARACTER SET` clause, so a created table takes the + * DATABASE's default charset — verified on 8.0.46: a table created with no + * clause in a `CHARACTER SET latin1` database comes back + * `TABLE_COLLATION = latin1_swedish_ci`. So `@@character_set_database` is the + * right question, and `information_schema.CHARACTER_SETS.MAXLEN` answers the + * multiplier for it (utf8mb4 → 4, utf8mb3 → 3, latin1/ascii → 1). + * + * One round-trip, and only ever on a path that is already throwing — the same + * trade {@link explainUnkeyableTextColumn} makes with its `columnInfo()` read. + * `null` when the read fails: a diagnostic must not turn into a second + * failure, and the advice holds with the assumed multiplier. + */ + protected async schemaBytesPerChar(): Promise<{ bytesPerChar: number; charset: string } | null> { + try { + const res: any = await this.knex.raw( + 'select cs.CHARACTER_SET_NAME as charset, cs.MAXLEN as maxlen ' + + 'from information_schema.CHARACTER_SETS cs ' + + 'where cs.CHARACTER_SET_NAME = @@character_set_database', + ); + const row = (Array.isArray(res) ? res[0]?.[0] : (res?.rows?.[0] ?? res?.[0])) ?? null; + const maxlen = Number(row?.maxlen ?? row?.MAXLEN); + const charset = String(row?.charset ?? row?.CHARSET ?? ''); + if (!Number.isInteger(maxlen) || maxlen <= 0 || !charset) return null; + return { bytesPerChar: maxlen, charset }; + } catch { + // Introspection is a nicety here; the advice below holds without it. + return null; + } + } + + /** + * Turn MySQL's `ER_TOO_BIG_ROWSIZE` into a message that names the columns + * whose declared widths spent the row budget (#11565). + * + * ## Why a post-hoc translator and NOT a pre-flight + * + * The card that raised this preferred a pre-flight — sum the declared widths + * before issuing DDL and refuse — on the grounds that a translator "only + * knows the table failed". That premise does not survive contact with the + * call site: this runs inside `initObjects`' own loop, where `obj.fields` and + * the resolved `keyedColumns` are in scope, so it names EVERY contributing + * field exactly as a pre-flight would. The advantage claimed for the + * pre-flight is not an advantage it has here. + * + * What the two shapes do not share is their failure direction, and the + * asymmetry is the whole argument: + * + * - A pre-flight has to REPRODUCE the server's arithmetic. Wrong in the lax + * direction it is merely useless — the server still refuses, with today's + * bad message. Wrong in the STRICT direction it refuses an object MySQL + * would have accepted, which is a contract change: the same metadata now + * means two different things depending on the driver's own model. + * - A translator cannot over-refuse BY CONSTRUCTION. It speaks only after + * the server has already refused, so no arithmetic error it makes can + * change which objects are accepted — the worst case is a less precise + * sentence attached to a refusal that was going to happen anyway. + * + * And the arithmetic is genuinely hard to hold, all four parts measured on + * 8.0.46 while choosing this shape: + * + * 1. **The charset multiplier is the schema's, not the metadata's.** + * `64 x varchar(1024)` is refused on utf8mb4 and CREATES on latin1. + * A pre-flight hard-coding 4 bytes/char over-refuses by 4x on a latin1 + * deployment. + * 2. **The length prefix moves** at a 255-byte payload boundary (see + * {@link varcharPackLength}). + * 3. **The null bitmap counts.** `163 x varchar(100)` is refused when the + * columns are nullable and CREATES when they are `NOT NULL` — same + * declared widths, different verdict. + * 4. **There is a SECOND, independent limit.** InnoDB's per-page limit + * (`Row size too large (> 8126)` on a 16K page) fires far below 65535 + * for many NARROW columns, because a column whose maximum is under ~256 + * bytes is never pushed off-page: measured, `varchar(63)` tops out at 32 + * columns and `varchar(64)` at 196. Through this driver, an object of 40 + * ordinary `maxLength: 63` fields is refused — a shape a 65535-byte + * pre-flight would have waved through, leaving the author with the + * unactionable server error the card was filed about. + * + * A pre-flight would also need a dialect predicate to know it is on MySQL. + * This needs none: `ER_TOO_BIG_ROWSIZE` is a MySQL code, so the diagnostic is + * MySQL-scoped by construction and costs nothing on every other dialect and + * on the success path. + * + * @param addedColumns the columns an `ALTER TABLE` was adding, or `null` for + * a `CREATE TABLE`. The budget is the whole ROW either way — a column added + * years later is refused by the width of columns nobody is touching — so + * the offender list is the same and only the opening clause differs. + */ + protected async explainRowSizeOverflow( + tableName: string, + fields: Record | undefined, + keyedColumns: ReadonlyMap, + addedColumns: string[] | null, + cause: unknown, + ): Promise { + if ((cause as { code?: string } | undefined)?.code !== 'ER_TOO_BIG_ROWSIZE') return null; + + const server = await this.schemaBytesPerChar(); + const bytesPerChar = server?.bytesPerChar ?? SqlDriver.ASSUMED_BYTES_PER_CHAR; + const charsetNote = server + ? `${bytesPerChar} bytes/character (${server.charset}, this schema's charset)` + : `${bytesPerChar} bytes/character (utf8mb4 assumed — the server's charset could not be read)`; + const { columns, totalBytes } = this.rowWidthProfile(fields, keyedColumns, bytesPerChar); + + // Two DIFFERENT limits answer with this one code, and they quote different + // numbers. Reading the server's own text keeps the message from asserting + // the 65535 one over a refusal that was really InnoDB's page limit. + const serverText = String((cause as { sqlMessage?: string; message?: string }).sqlMessage ?? ''); + const pageLimit = /row size too large \(>\s*(\d+)\)/i.exec(serverText)?.[1] ?? null; + const limitClause = pageLimit + ? `InnoDB's per-PAGE limit of ${pageLimit} bytes — which many NARROW bounded columns reach far below the ` + + `${SqlDriver.MYSQL_ROW_BYTE_BUDGET}-byte one, because a column narrower than about 256 bytes is never ` + + `stored off-page (measured on 8.0.46: 40 fields at \`maxLength: 63\` are refused)` + : `the server's ${SqlDriver.MYSQL_ROW_BYTE_BUDGET}-byte budget for one ROW`; + + const SHOWN = 8; + const listed = columns + .slice(0, SHOWN) + .map((c) => `"${c.name}" varchar(${c.chars}) = ${c.bytes} bytes`) + .join(', '); + const rest = columns.length > SHOWN ? `, and ${columns.length - SHOWN} more` : ''; + const inventory = + columns.length === 0 + ? `This object produces no varchar column at all, so the width is in the built-in columns or in a ` + + `type this diagnostic does not size — report it, because that is not a shape it was measured on.` + : `Its ${columns.length} varchar column(s) take ${totalBytes} bytes at ${charsetNote}, on top of the ` + + `built-in \`id\` (a varchar(${SqlDriver.DEFAULT_STRING_VARCHAR_CHARS})), \`created_at\` and ` + + `\`updated_at\` columns. Widest first: ${listed}${rest}.`; + + const opening = + addedColumns && addedColumns.length > 0 + ? `[sql-driver] cannot add column(s) ${addedColumns.map((c) => `"${c}"`).join(', ')} to "${tableName}"` + : `[sql-driver] cannot create table "${tableName}"`; + + return ( + `${opening} — MySQL refuses the whole row: every bounded column's DECLARED width is charged against ` + + `${limitClause}, whether or not a row ever holds that much. ${inventory} ` + + `Fix it on the declarations: lower \`maxLength\` on the widest field(s), or drop the bound from a ` + + `text-family field no index keys on so it is emitted as TEXT — an off-page TEXT column costs this ` + + `budget a small pointer rather than its full declared width. Note that a field declaring NO ` + + `\`maxLength\` still takes varchar(${SqlDriver.DEFAULT_STRING_VARCHAR_CHARS}) ` + + `(${SqlDriver.varcharPackLength(SqlDriver.DEFAULT_STRING_VARCHAR_CHARS, bytesPerChar)} bytes here), and ` + + `so do \`lookup\`, \`user\`, \`auto_number\` and the option types — an object can reach this limit ` + + `without declaring a bound anywhere. The server's own error names no column, which is the whole reason ` + + `this one does (#11565).` + ); + } + + /** + * Re-throw a schema-mutation failure carrying {@link explainRowSizeOverflow}'s + * explanation, or the original untouched when it is not a row-size refusal. + * + * The SAME failure, re-worded — the boot still fails, loudly, and no object is + * registered that MySQL declined to build. `sqlMessage` rather than `message` + * for the quoted server text: knex prefixes `message` with the entire DDL + * statement, which for the object that trips this is thousands of characters + * of `varchar(...)` and buries the one sentence the server actually said. + */ + private async rethrowWithRowSizeExplanation( + tableName: string, + fields: Record | undefined, + keyedColumns: ReadonlyMap, + addedColumns: string[] | null, + e: any, + ): Promise { + const overflow = await this.explainRowSizeOverflow(tableName, fields, keyedColumns, addedColumns, e); + if (!overflow) throw e; + const said = String(e?.sqlMessage ?? e?.message ?? e); + (this.logger.error ?? this.logger.warn)(overflow, said); + throw Object.assign(new Error(`${overflow} (server said: ${said})`), { + code: (e as { code?: string }).code, + cause: e, + }); + } + protected createColumn( table: Knex.CreateTableBuilder, name: string,