From 447ca49d8fcf39bcc073e712f838d595083bd44e Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 12 Aug 2026 08:54:33 +0000 Subject: [PATCH 1/2] fix(verify): make the RLS by-id-write class reachable and stop skips reading as passes The `verify --rls` runner reported 0 HOLES over a probe that could not, by construction, observe the class it claims to prove. - The probe persona was a bare `signUp()` member holding no object grants, so `checkObjectPermission` answered 403 before record scope was ever consulted. Measured on the stock showcase: 11 of 13 "consistent" verdicts were that object-gate 403; only 2 were a record-scope 404. - A `showcase_account` auto-record 400 cascaded into four further skips (8 of 23 objects skipped), and the summary line still read "0 HOLES". The probe is now the persona the class needs: object read+edit plus an owner-scoped SELECT-only narrowing, registered at boot so its policies are on the resolution path. That is deliberately the authoring shape that was the hole, so every object of every verified app is now a live regression guard for the by-id write-scope derivation. Reachability is MEASURED per object (a LIST 403 reports `probe-blocked`, never a pass), an unsatisfiable admin create falls back to adopting an existing row so one failure no longer cascades, and the report separates PROVEN objects from NOT-PROVEN ones in both the structured summary and the formatted output. --- packages/cli/src/commands/verify.ts | 39 +- packages/verify/src/index.ts | 18 +- packages/verify/src/rls.ts | 490 ++++++++++++++++++++-- scripts/check-verify-stand-in-erasure.mjs | 3 + 4 files changed, 513 insertions(+), 37 deletions(-) diff --git a/packages/cli/src/commands/verify.ts b/packages/cli/src/commands/verify.ts index ed20975d7e..c27f1965a4 100644 --- a/packages/cli/src/commands/verify.ts +++ b/packages/cli/src/commands/verify.ts @@ -10,8 +10,11 @@ import { formatReport, runRlsProofs, formatRlsReport, + provisionRlsProbePersona, + rlsProbeSecurity, type VerifyReport, type RlsReport, + type RlsProbeDescriptor, } from '@objectstack/verify'; import { loadConfig } from '../utils/config.js'; @@ -106,22 +109,50 @@ export default class Verify extends Command { // proving its authorization. let rls: RlsReport | undefined; if (flags.rls) { - const rlsStack = await bootStack(config, { multiTenant }); + // [#7685] The by-id-write class is only REACHABLE for a persona holding + // the object-level grants AND standing outside the record scope. A bare + // `signUp()` member holds no grants, so `checkObjectPermission` answers + // 403 before record scope is consulted and every probe was masked — 11 of + // 13 "consistent" showcase verdicts were that 403, an unfalsifiable green. + // `rlsProbeSecurity` registers the capability #7665's acceptance + // criterion 2 names (object read+edit, owner-scoped SELECT only) and + // carries the app's declared default profile through unchanged. + const rlsStack = await bootStack(config, { multiTenant, security: rlsProbeSecurity(config) }); try { const adminToken = await rlsStack.signIn(); - const memberToken = await rlsStack.signUp('verify-member@objectstack.test'); - rls = await runRlsProofs(rlsStack, adminToken, memberToken, config); + let probeToken: string; + let probe: RlsProbeDescriptor; + try { + const persona = await provisionRlsProbePersona(rlsStack, config); + probeToken = persona.token; + probe = { label: persona.email, grantedObjects: persona.grantedObjects }; + } catch (e) { + // Prefer failing to falling back (Route & surface ownership §3): a + // weaker persona still produces a report, so the degradation is + // recorded on the report itself AND counted as a hard failure below. + // Silently probing with an ungranted member is the exact false green + // this issue exists to remove. + probeToken = await rlsStack.signUp('verify-member@objectstack.test'); + probe = { + label: 'verify-member@objectstack.test', + degraded: `probe persona provisioning failed: ${(e as Error).message}`, + }; + } + rls = await runRlsProofs(rlsStack, adminToken, probeToken, config, { probe }); } finally { await rlsStack.stop(); } } // Failure contract: a "real" runtime break the app's author must see. + // A degraded RLS probe counts: the run reported verdicts it could not have + // established, which is worse than no verifier at all. const hardFailures = crud.summary.createFailed + crud.summary.readFailed + crud.summary.fidelityGaps + - (rls?.summary.holes ?? 0); + (rls?.summary.holes ?? 0) + + (rls?.probe.degraded ? 1 : 0); if (flags.json) { this.log(JSON.stringify({ app: crud.app, config: absolutePath, multiTenant, crud, rls, hardFailures }, null, 2)); diff --git a/packages/verify/src/index.ts b/packages/verify/src/index.ts index d8c379b602..f742f4ad77 100644 --- a/packages/verify/src/index.ts +++ b/packages/verify/src/index.ts @@ -16,8 +16,22 @@ export type { CrudCase, DerivedAssert, AssertKind, RelationalRef } from './deriv export { runCrudVerification, formatReport } from './verify.js'; export type { VerifyReport, ObjectVerifyResult } from './verify.js'; -export { runRlsProofs, formatRlsReport } from './rls.js'; -export type { RlsReport, RlsResult } from './rls.js'; +export { + runRlsProofs, + formatRlsReport, + provisionRlsProbePersona, + rlsProbePermissionSet, + rlsProbeSecurity, + RLS_PROBE_EMAIL, +} from './rls.js'; +export type { + RlsReport, + RlsResult, + RlsStatus, + RlsProbeDescriptor, + RlsProbePersona, + RlsProofOptions, +} from './rls.js'; // ADR-0060 — reusable conformance-ledger helper (static complement to the // runtime harness): classify every declarable property, fail closed on drift. diff --git a/packages/verify/src/rls.ts b/packages/verify/src/rls.ts index 57156176f8..17073124cb 100644 --- a/packages/verify/src/rls.ts +++ b/packages/verify/src/rls.ts @@ -10,29 +10,349 @@ // A user who CANNOT READ a record must not be able to WRITE it. // ("You can't mutate what you can't see.") // -// Derivation, per object: admin creates a record; a fresh member (no roles or -// grants) tries to read it, then tries to mutate it by id; we re-read as admin -// to see if the row actually changed. If the member couldn't see it yet changed -// it, that's the #1994 class of hole — regardless of the app's sharing config. +// Derivation, per object: admin creates a record; a probe persona tries to read +// it, then tries to mutate it by id; we re-read as admin to see if the row +// actually changed. If the probe couldn't see it yet changed it, that's the +// #1994 class of hole — regardless of the app's sharing config. +// +// ## [#7685] Two defects that made this runner's green mean nothing +// +// **1. The probe was masked by the OBJECT gate.** The persona used to be a bare +// `signUp()` member holding no object grants, so `checkObjectPermission` answered +// 403 *before* record scope was ever consulted — and the runner recorded that +// 403 as `rls-consistent`. Measured on the stock showcase: 11 of 13 "consistent" +// verdicts were `GET 403` (the object gate) and only 2 were `GET 404` (record +// scope). The by-id-write class was therefore STRUCTURALLY unreachable on those +// 11: no platform change could have flipped them to `rls-hole`, so their green +// was not evidence of anything. `provisionRlsProbePersona` below mints the +// persona the class needs — object read+edit, no record-scope grants — and the +// per-object LIST reachability probe MEASURES that the object gate is open +// rather than assuming it, reporting `probe-blocked` (never a pass) when it is +// not. +// +// **2. A skip read as a pass, and cascaded.** A `showcase_account` auto-record +// 400 skipped that object AND every object with a required relation to it — +// 8 of 23 objects skipped on a stock run — while the summary line reported +// "0 HOLES" as if the run were a clean bill of health. A skip is exactly where +// the privately-reported #7665 defect hid. Two changes: an unsatisfiable create +// now falls back to ADOPTING an existing row (seed data) so one failure no +// longer cascades into its dependents, and the report separates PROVEN objects +// from NOT-PROVEN ones (`member-visible` / `probe-blocked` / `skipped`) in both +// the structured summary and the formatted output. +// +// ⛔ What this runner still cannot reach, stated so its green is not over-read: +// an object whose narrowing is authored on a POSITION the probe does not hold +// (the showcase's `positions: ['contributor']` rules) is unnarrowed for this +// persona, so the probe reads every row and the object reports `member-visible` +// — honest, but not a proof. Reaching those needs a per-declared-position +// persona; tracked as a follow-on. /* eslint-disable @typescript-eslint/no-explicit-any */ +import { SecurityPlugin, securityDefaultPermissionSets, appSecurityPluginOptions } from '@objectstack/plugin-security'; +import type { PermissionSet } from '@objectstack/spec/security'; + import type { VerifyStack } from './harness.js'; import { deriveCrudCases, fillRelationalRefs } from './derive.js'; const PROBE_TYPES = new Set(['text', 'textarea', 'string']); const MUTATION = 'rls-mutated-by-B'; +/** Default identity of the object-granted probe persona (`provisionRlsProbePersona`). */ +export const RLS_PROBE_EMAIL = 'verify-rls-probe@objectstack.test'; +const RLS_PROBE_PASSWORD = 'Rls-Probe-Pass-123'; +const RLS_PROBE_PERMISSION_SET = 'verify_rls_probe'; + +export type RlsStatus = + /** PROVEN: the probe could not read the row and could not mutate it. */ + | 'rls-consistent' + /** PROVEN HOLE: the probe could not read the row yet mutated it by id. */ + | 'rls-hole' + /** NOT PROVEN: the probe CAN read the row, so there is no cross-owner scenario here. */ + | 'member-visible' + /** NOT PROVEN: the OBJECT gate refused the probe, so record scope was never consulted. */ + | 'probe-blocked' + /** NOT PROVEN: no probe target could be established at all. */ + | 'skipped'; + export interface RlsResult { object: string; - status: 'rls-consistent' | 'rls-hole' | 'member-visible' | 'skipped'; + status: RlsStatus; detail?: string; + /** + * How the probe target row was obtained. `adopted` means the derived admin + * create was rejected and an existing (seeded) row was used instead — the + * cascade-stopper. Absent when no target was established. + */ + target?: 'created' | 'adopted'; +} + +/** Describes the persona the probe ran as, so a run's REACH is legible. */ +export interface RlsProbeDescriptor { + /** Human label for the persona (an email, or a fixture description). */ + label: string; + /** How many objects the persona was granted object-level read+edit on. */ + grantedObjects?: number; + /** + * Set when the intended object-granted persona could NOT be provisioned and + * the run fell back to a weaker one. A degraded run proves strictly less than + * it appears to; callers surface it and fail rather than reporting success. + */ + degraded?: string; } export interface RlsReport { app: string; + /** Which persona drove the by-id probes. */ + probe: RlsProbeDescriptor; results: RlsResult[]; - summary: { objects: number; consistent: number; holes: number; memberVisible: number; skipped: number }; + /** + * Every object the run did NOT prove, with the reason — surfaced separately + * from `results` so "what did this run fail to look at" is one field, not a + * filter a consumer has to remember to apply. + */ + unproven: Array<{ object: string; status: RlsStatus; detail?: string }>; + summary: { + objects: number; + consistent: number; + holes: number; + memberVisible: number; + probeBlocked: number; + skipped: number; + /** Objects on which the by-id-write class was actually exercised. */ + proven: number; + /** Objects the run did not exercise (`memberVisible + probeBlocked + skipped`). */ + unproven: number; + }; +} + +export interface RlsProofOptions { + /** + * The persona `memberToken` belongs to. Reporting only — every reachability + * verdict below is MEASURED per object, never taken from this descriptor, + * because a declared persona that silently failed to receive its grants is + * precisely the state this runner must not report as a pass. + */ + probe?: RlsProbeDescriptor; +} + +/** The identity + token of an object-granted probe persona. */ +export interface RlsProbePersona { + token: string; + email: string; + userId: string; + /** Name of the minted `sys_permission_set` row. */ + permissionSet: string; + /** Objects the minted set grants read+edit on. */ + grantedObjects: number; +} + +function genId(prefix: string): string { + return `${prefix}_${Math.random().toString(36).slice(2, 10)}${Math.random().toString(36).slice(2, 6)}`; +} + +function rowsOf(payload: any): any[] { + if (Array.isArray(payload)) return payload; + const list = payload?.records ?? payload?.data ?? payload?.value; + return Array.isArray(list) ? list : []; +} + +/** + * The probe persona's capability, derived from the app's own metadata: object + * read+edit on every declared object, narrowed by an OWNER policy authored + * `operation: 'select'` only. + * + * Both halves are load-bearing, and neither is a dial to soften: + * + * - **read+edit** is what stops `checkObjectPermission` answering 403 first. + * Without it the record-level gate is never consulted and the runner's verdict + * is about the object gate, not about RLS (#7685). + * - **owner-scoped SELECT only** is what puts the persona OUTSIDE the record + * scope, which is the other half of #7665's acceptance criterion. It is also + * deliberately the exact AUTHORING SHAPE that was the hole: with no + * `update`-class predicate, a platform that does not derive the write scope + * from the caller's select narrowing lets the by-id PATCH through on a row the + * persona gets 404 on. So this set turns every object of every verified app + * into a live regression guard for that derivation — the runner answers + * `rls-hole` the day it stops holding. + * + * No `create`/`delete`, no positions, no system permissions: the persona owns + * nothing, so the admin-created probe row is outside its scope by construction + * rather than by fixture coincidence. + * + * ⚠️ The narrowing is VERIFIER-authored, so a `rls-consistent` verdict is a + * statement about the PLATFORM's by-id-write gate — not a statement that the + * app's own authored policies are right. The app's authorization config is what + * the per-app dogfood proofs cover. + */ +export function rlsProbePermissionSet(config: any): PermissionSet { + const objects: Record = {}; + const rowLevelSecurity: Array> = []; + for (const o of (config?.objects ?? []) as any[]) { + if (!o?.name) continue; + objects[o.name] = { allowRead: true, allowEdit: true }; + rowLevelSecurity.push({ + name: `${o.name}_rls_probe_scope`, + label: `RLS probe scope for ${o.name}`, + description: + 'Verifier-authored owner narrowing (select only) — puts the probe persona outside the ' + + 'scope of every record it did not create, so the by-id-write class is reachable (#7685).', + object: o.name, + operation: 'select', + using: 'created_by == current_user.id', + enabled: true, + }); + } + return { + name: RLS_PROBE_PERMISSION_SET, + label: 'Verify RLS Probe', + objects, + rowLevelSecurity, + } as unknown as PermissionSet; +} + +/** + * The `SecurityPlugin` an `objectstack verify --rls` boot needs: the platform + * defaults, the app's own declared default profile, **and** the probe capability + * above. + * + * The probe set must be REGISTERED at boot, not written as a bare + * `sys_permission_set` row: `PermissionEvaluator.resolvePermissionSets` resolves + * a name from metadata first, the bootstrap list second, and the DB row only as + * a last resort — and that last-resort loader hydrates `objects` / `fields` / + * `systemPermissions` / `tabPermissions` but NOT `rowLevelSecurity`. A row-only + * probe set would therefore grant the object bits and silently drop the + * narrowing, which is the one thing that makes the persona a probe at all. + * + * `appSecurityPluginOptions(config)` is carried through verbatim so the app's + * declared default profile still resolves exactly as it does under `bootStack`'s + * own default and under `objectstack serve` (#7001) — this plugin replaces that + * default WHOLE, so anything it forgets to carry is silently missing from the + * run. + */ +export function rlsProbeSecurity(config: any): SecurityPlugin { + return new SecurityPlugin({ + ...(appSecurityPluginOptions(config) ?? {}), + defaultPermissionSets: [...securityDefaultPermissionSets, rlsProbePermissionSet(config)], + }); +} + +/** + * Sign up the probe persona and GRANT it {@link rlsProbePermissionSet} — the + * capability the #1994 class actually needs (object read+edit, owner-scoped + * select, nothing else). + * + * Requires the stack to have been booted with {@link rlsProbeSecurity}, which is + * what puts the set's `rowLevelSecurity` on the resolution path. This function + * only writes the two RBAC link rows, and it writes them through the kernel's + * ObjectQL engine under a system context, the same way `bootstrapPlatformAdmin` + * seeds them: the persona is test SETUP, not the surface under test, so routing + * it through the data door would make the proof depend on whether this + * deployment happens to let an admin POST `sys_permission_set` — a second thing + * that can fail for reasons unrelated to RLS. + * + * Throws when the stack has no ObjectQL engine or the user cannot be resolved. + * Callers must NOT swallow that into a weaker persona silently: run degraded and + * say so (`RlsProbeDescriptor.degraded`), or fail. + */ +export async function provisionRlsProbePersona( + stack: VerifyStack, + config: any, + opts: { email?: string; password?: string } = {}, +): Promise { + const email = opts.email ?? RLS_PROBE_EMAIL; + const password = opts.password ?? RLS_PROBE_PASSWORD; + + await stack.signUp(email, password); + + const ql = await stack.kernel.getServiceAsync('objectql'); + if (!ql || typeof ql.find !== 'function' || typeof ql.insert !== 'function') { + throw new Error( + 'verify: cannot provision the RLS probe persona — no ObjectQL engine on this stack. ' + + 'The probe needs object-level read+edit grants, without which every by-id probe is ' + + 'masked by the object gate and the #1994 class is unreachable (#7685).', + ); + } + const sysCtx = { context: { isSystem: true } }; + + const users = rowsOf(await ql.find('sys_user', { where: { email }, limit: 1 }, sysCtx)); + const userId = users[0]?.id; + if (!userId) { + throw new Error(`verify: cannot provision the RLS probe persona — no sys_user row for ${email}.`); + } + + const probeSet = rlsProbePermissionSet(config); + + // The grant is keyed by ROW ID, and `resolveUserAuthzGrants` reads the row + // only to learn its NAME — the definition then resolves from the registered + // bootstrap list (`rlsProbeSecurity`), which is the copy that carries the + // narrowing. Find-or-create so it works whether or not this boot already + // seeded the registered set into the table. + const existing = rowsOf( + await ql.find('sys_permission_set', { where: { name: RLS_PROBE_PERMISSION_SET }, limit: 1 }, sysCtx), + ); + let permissionSetId = existing[0]?.id; + if (!permissionSetId) { + permissionSetId = genId('ps'); + await ql.insert( + 'sys_permission_set', + { + id: permissionSetId, + name: RLS_PROBE_PERMISSION_SET, + label: probeSet.label, + description: + 'Ephemeral persona minted by `objectstack verify --rls`: object-level read+edit on every ' + + 'declared object plus an owner-scoped SELECT narrowing, so a by-id refusal is attributable ' + + 'to the record gate rather than the object gate (#7685).', + object_permissions: JSON.stringify(probeSet.objects ?? {}), + field_permissions: '{}', + system_permissions: '[]', + row_level_security: JSON.stringify(probeSet.rowLevelSecurity ?? []), + tab_permissions: '{}', + active: true, + }, + sysCtx, + ); + } + await ql.insert( + 'sys_user_permission_set', + { + id: genId('ups'), + user_id: userId, + permission_set_id: permissionSetId, + organization_id: null, + granted_by: null, + }, + sysCtx, + ); + + // Re-sign-in so the probe's session is issued after the grant exists. The + // per-request resolver reads the grant tables live, so this is belt-and-braces + // — but a session minted before the row is exactly the shape that makes a + // provisioning bug look like an enforcement result. + const token = await stack.signIn(email, password); + + return { + token, + email, + userId: String(userId), + permissionSet: RLS_PROBE_PERMISSION_SET, + grantedObjects: Object.keys(probeSet.objects ?? {}).length, + }; +} + +/** The id of an existing row of `object`, read as admin, or null. */ +async function firstExistingId(stack: VerifyStack, adminToken: string, object: string): Promise { + const res = await stack.apiAs(adminToken, 'GET', `/data/${object}?$top=1`); + if (res.status !== 200) return null; + let payload: any; + try { + payload = await res.json(); + } catch { + return null; + } + const id = rowsOf(payload)[0]?.id; + return id ? String(id) : null; } export async function runRlsProofs( @@ -40,12 +360,13 @@ export async function runRlsProofs( adminToken: string, memberToken: string, config: any, + opts: RlsProofOptions = {}, ): Promise { const cases = deriveCrudCases(config); const results: RlsResult[] = []; - // Admin-created ids, threaded so a detail's required relation points at a real - // master (topological order created it first) — lets the #1994 invariant reach - // relationship-dense objects, not just the leaves. + // Admin-created (or adopted) ids, threaded so a detail's required relation + // points at a real master (topological order created it first) — lets the + // #1994 invariant reach relationship-dense objects, not just the leaves. const createdIds = new Map(); for (const c of cases) { @@ -56,21 +377,70 @@ export async function runRlsProofs( const probe = (c.asserts ?? []).find((a) => PROBE_TYPES.has(a.type)); if (!probe) { results.push({ object: c.object, status: 'skipped', detail: 'no plain-text probe field' }); continue; } - const { body, missing } = fillRelationalRefs(c, createdIds); - if (missing) { results.push({ object: c.object, status: 'skipped', detail: missing }); continue; } + let resolved = fillRelationalRefs(c, createdIds); + if (resolved.missing) { + // [#7685] Stop the CASCADE. An upstream object whose derived record the + // app's own validation rejects used to skip every dependent object too + // (one `showcase_account` 400 → four further skips). The dependency only + // needs SOME real master row, not one this run created — so adopt an + // existing (seeded) one before giving up. + let adopted = false; + for (const ref of c.relationalRefs ?? []) { + if (!ref.required || createdIds.has(ref.target)) continue; + const existing = await firstExistingId(stack, adminToken, ref.target); + if (existing) { createdIds.set(ref.target, existing); adopted = true; } + } + if (adopted) resolved = fillRelationalRefs(c, createdIds); + } + if (resolved.missing) { results.push({ object: c.object, status: 'skipped', detail: resolved.missing }); continue; } - // Admin (owner) creates the record. - const created = await stack.apiAs(adminToken, 'POST', `/data/${c.object}`, body); - if (created.status >= 300) { - results.push({ object: c.object, status: 'skipped', detail: `admin create failed (${created.status})` }); + // Admin (owner) creates the record — or, when the app's own validation + // rejects the derived body, adopt an existing row so the object is still + // probed instead of silently dropping out of the run. + let target: RlsResult['target'] = 'created'; + let id: string | null = null; + let createDetail = ''; + const created = await stack.apiAs(adminToken, 'POST', `/data/${c.object}`, resolved.body); + if (created.status < 300) { + const cj = (await created.json()) as any; + const createdId = cj?.id ?? cj?.record?.id; + if (createdId) id = String(createdId); + else createDetail = 'admin create returned no id'; + } else { + createDetail = `admin create failed (${created.status})`; + } + if (!id) { + const existing = await firstExistingId(stack, adminToken, c.object); + if (existing) { id = existing; target = 'adopted'; } + } + if (!id) { + results.push({ object: c.object, status: 'skipped', detail: `${createDetail}; no existing ${c.object} row to adopt` }); continue; } - const cj = (await created.json()) as any; - const id = cj?.id ?? cj?.record?.id; - if (!id) { results.push({ object: c.object, status: 'skipped', detail: 'no id from create' }); continue; } - createdIds.set(c.object, String(id)); + createdIds.set(c.object, id); + const via = target === 'adopted' ? ` [target adopted from existing rows — ${createDetail}]` : ''; - // Member B: can they SEE it? + // ── Reachability, MEASURED (#7685) ──────────────────────────────────────── + // Does the object-level gate let this persona through at all? A 403 on the + // plain LIST means `checkObjectPermission` refuses before record scope is + // consulted, so nothing below could ever observe the by-id-write class on + // this object — and recording that as `rls-consistent` (which is what this + // runner did for 11 of 13 "consistent" showcase objects) is a false pass. + const list = await stack.apiAs(memberToken, 'GET', `/data/${c.object}?$top=1`); + if (list.status === 403) { + results.push({ + object: c.object, + status: 'probe-blocked', + target, + detail: + `the probe persona holds no object-level READ grant on ${c.object} (LIST 403), so the ` + + 'object gate answers before record scope — the by-id-write class was NOT exercised here. ' + + `Not a pass.${via}`, + }); + continue; + } + + // Probe: can they SEE it? const bRead = await stack.apiAs(memberToken, 'GET', `/data/${c.object}/${id}`); let canRead = false; if (bRead.status === 200) { @@ -78,7 +448,7 @@ export async function runRlsProofs( canRead = !!rec && rec.id === id; } - // Member B: try to MUTATE it by id. + // Probe: try to MUTATE it by id. const bWrite = await stack.apiAs(memberToken, 'PATCH', `/data/${c.object}/${id}`, { [probe.field]: MUTATION }); // Ground truth: re-read as admin — did the row actually change? @@ -87,39 +457,97 @@ export async function runRlsProofs( const changed = afterVal === MUTATION; if (canRead) { - results.push({ object: c.object, status: 'member-visible', detail: 'member can read this object — not a cross-owner scenario (no RLS isolation, or read is granted)' }); + results.push({ + object: c.object, + status: 'member-visible', + target, + detail: + 'the probe can read this object — not a cross-owner scenario, so the by-id-write class is ' + + `not exercised here (no record-scope narrowing reaches this persona, or read is granted)${via}`, + }); } else if (changed) { results.push({ object: c.object, status: 'rls-hole', - detail: `member B cannot read it (GET ${bRead.status}) yet MUTATED it by id (PATCH ${bWrite.status}) — by-id write bypassed RLS (#1994 class)`, + target, + detail: `the probe cannot read it (GET ${bRead.status}) yet MUTATED it by id (PATCH ${bWrite.status}) — by-id write bypassed RLS (#1994 class)${via}`, }); } else { results.push({ object: c.object, status: 'rls-consistent', - detail: `member B cannot read (GET ${bRead.status}) and could not mutate (PATCH ${bWrite.status}, row unchanged)`, + target, + detail: `the probe cannot read (GET ${bRead.status}) and could not mutate (PATCH ${bWrite.status}, row unchanged)${via}`, }); } } + const count = (s: RlsStatus) => results.filter((r) => r.status === s).length; + const consistent = count('rls-consistent'); + const holes = count('rls-hole'); + const memberVisible = count('member-visible'); + const probeBlocked = count('probe-blocked'); + const skipped = count('skipped'); const summary = { objects: results.length, - consistent: results.filter((r) => r.status === 'rls-consistent').length, - holes: results.filter((r) => r.status === 'rls-hole').length, - memberVisible: results.filter((r) => r.status === 'member-visible').length, - skipped: results.filter((r) => r.status === 'skipped').length, + consistent, + holes, + memberVisible, + probeBlocked, + skipped, + proven: consistent + holes, + unproven: memberVisible + probeBlocked + skipped, + }; + const unproven = results + .filter((r) => r.status === 'member-visible' || r.status === 'probe-blocked' || r.status === 'skipped') + .map((r) => ({ object: r.object, status: r.status, detail: r.detail })); + + return { + app: config?.manifest?.id ?? 'app', + probe: opts.probe ?? { label: 'unspecified persona' }, + results, + unproven, + summary, }; - return { app: config?.manifest?.id ?? 'app', results, summary }; } export function formatRlsReport(report: RlsReport): string { const lines: string[] = [`\n=== objectstack verify (RLS / #1994) — ${report.app} ===`]; for (const r of report.results) { - const mark = r.status === 'rls-hole' ? '✗✗' : r.status === 'rls-consistent' ? '✓' : r.status === 'member-visible' ? '·' : '–'; + const mark = + r.status === 'rls-hole' ? '✗✗' + : r.status === 'rls-consistent' ? '✓' + : r.status === 'member-visible' ? '·' + : r.status === 'probe-blocked' ? '!' + : '–'; lines.push(` ${mark} ${r.object} [${r.status}] ${r.detail ?? ''}`); } const s = report.summary; - lines.push(` ── ${s.consistent} consistent, ${s.holes} HOLES, ${s.memberVisible} member-visible, ${s.skipped} skipped`); + const p = report.probe; + lines.push( + ` ── probe persona: ${p.label}${p.grantedObjects != null ? ` (object read+edit on ${p.grantedObjects} object(s))` : ''}`, + ); + if (p.degraded) { + lines.push(` ⛔ DEGRADED RUN — ${p.degraded}`); + lines.push(' Every verdict below proves LESS than it reads: without object-level grants the'); + lines.push(' object gate answers before record scope, so the by-id-write class is unreachable.'); + } + lines.push( + ` ── ${s.proven} PROVEN (${s.consistent} consistent, ${s.holes} HOLES) · ` + + `${s.unproven} NOT PROVEN (${s.memberVisible} member-visible, ${s.probeBlocked} probe-blocked, ${s.skipped} skipped)`, + ); + // A skip must never read as a pass (#7685). The old summary line reported + // "0 HOLES" over a run that had actually looked at 15 of 23 objects. + if (s.unproven > 0) { + lines.push( + ` ⚠ ${s.unproven} of ${s.objects} object(s) were NOT proven — this run is not a clean bill of health.`, + ); + } + if (s.objects > 0 && s.probeBlocked === s.objects) { + lines.push( + ' ⛔ EVERY object was probe-blocked — the persona holds no object grants at all, so this run', + ' established nothing. Check that the probe persona was provisioned.', + ); + } return lines.join('\n'); } diff --git a/scripts/check-verify-stand-in-erasure.mjs b/scripts/check-verify-stand-in-erasure.mjs index 81a39407b2..d5877bd85a 100644 --- a/scripts/check-verify-stand-in-erasure.mjs +++ b/scripts/check-verify-stand-in-erasure.mjs @@ -210,6 +210,9 @@ const NOT_A_STAND_IN = { 'takes the `VerifyStack` that `bootStack` in this same package returned — a concrete handle, ' + 'not a surface an out-of-tree implementer provides.', runRlsProofs: 'same `VerifyStack` handle as runCrudVerification.', + provisionRlsProbePersona: + 'same `VerifyStack` handle as runCrudVerification — it MINTS the RBAC rows for the #7685 ' + + 'object-granted RLS probe persona through that stack, so there is no second implementer.', formatReport: 'formats a `VerifyReport` this package produced; presentation, not conformance.', formatRlsReport: 'formats an `RlsReport` this package produced; presentation, not conformance.', fillRelationalRefs: From 952e7e7ee0ed968ee2abb6ac0b5d6f660be4bdd6 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 12 Aug 2026 08:59:41 +0000 Subject: [PATCH 2/2] test(qa): record the by-id-write and controlled-by-parent verdicts; cover the runner's new not-proven statuses MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ADR-0049's point is the RECORD telling the truth, so both matrix rows were re-decided on their own evidence rather than downgraded by assertion. `rls-by-id-write` stays `enforced`, but its enforcement citation was incomplete in a way that mattered: the named pre-image re-read is a no-op under select-only authoring, which was #7665. Ablating the write-scope derivation while leaving the pre-image re-read in place turns 16 of 20 probed showcase objects into `rls-hole`, so the row now names both halves. The shared proof file is NOT vacuous for this row — since #7665/PR #7792 it carries a dedicated select-only block whose member set grants full CRUD, so its refusals are the record gate. `controlled-by-parent` stays `enforced` on its own dedicated proof, whose fixture likewise grants full CRUD on both objects. Neither row was downgraded, so the mirroring showcase `permission-sets.ts` comment is accurate as shipped and is left alone. The runner's unit oracle gains the two statuses that stop a not-proven object reading as a pass (`probe-blocked`, and skip/adoption accounting). --- .changeset/verify-rls-probe-reachability.md | 66 +++++++++++ .../dogfood/test/authz-conformance.matrix.ts | 7 +- packages/qa/dogfood/test/rls-runner.test.ts | 110 +++++++++++++++++- 3 files changed, 175 insertions(+), 8 deletions(-) create mode 100644 .changeset/verify-rls-probe-reachability.md diff --git a/.changeset/verify-rls-probe-reachability.md b/.changeset/verify-rls-probe-reachability.md new file mode 100644 index 0000000000..73a02f8d26 --- /dev/null +++ b/.changeset/verify-rls-probe-reachability.md @@ -0,0 +1,66 @@ +--- +"@objectstack/verify": minor +"@objectstack/cli": minor +--- + +fix(verify): `--rls` reported 0 HOLES over a probe that could not reach the class it claims to prove (#7685) + +`objectstack verify --rls` exercises one invariant — **you cannot mutate what you +cannot see** (#1994). It reported `0 HOLES` on both example apps. Neither number +was evidence. + +**The probe was answered by the OBJECT gate, not by record scope.** The persona +was a bare `signUp()` member holding no object grants, so +`checkObjectPermission` refused with 403 before the row-level gate was ever +consulted — and the runner banked that 403 as `rls-consistent`. Measured on the +stock showcase: **11 of 13 "consistent" verdicts were the object-gate 403**, and +only 2 were a record-scope 404. On those 11 objects no platform regression could +have produced `rls-hole`, so their green was unfalsifiable by construction. + +**A skip read as a pass, and it cascaded.** One `showcase_account` auto-record +400 skipped that object and every object with a required relation to it — 8 of +23 objects skipped — while the summary line still said `0 HOLES`. + +## What changed + +- **The probe persona is now the one the class needs**: object read+edit on every + declared object, narrowed by an owner policy authored `operation: 'select'` + only, registered at boot (`rlsProbeSecurity`) so its policies are actually on + the resolution path. That is deliberately the authoring shape that WAS the hole + (#7665), so every object of every verified app is now a live regression guard + for the by-id write-scope derivation. +- **Reachability is measured, not assumed.** Before probing an object the runner + asks whether the persona can list it at all; a 403 is reported as + `probe-blocked` — a distinct status that is never a pass — instead of being + recorded as a consistent verdict. +- **An unsatisfiable create no longer cascades.** When the app's own validation + rejects the derived record, the runner adopts an existing row as the probe + target, so the object is probed and its dependents keep their master. +- **The report distinguishes PROVEN from NOT-PROVEN.** `RlsReport.summary` gains + `proven` / `unproven` / `probeBlocked`, `RlsReport.unproven` lists every object + the run did not exercise with its reason, `RlsReport.probe` names the persona, + and the formatted output prints an explicit "this run is not a clean bill of + health" line whenever anything went unproven. +- **A degraded probe fails the run.** If the persona cannot be provisioned the + report says so and `verify` exits non-zero, rather than quietly probing with an + ungranted member and reporting success. + +## Measured, before → after + +| | showcase | crm | +|:--|:--|:--| +| before | 13 consistent (11 object-masked), 0 holes, 2 member-visible, **8 skipped** | 4 consistent (all object-masked), 0 holes, **2 skipped** | +| after | **20 of 23 PROVEN**, 0 holes, 3 unproven | **6 of 6 PROVEN**, 0 holes, 0 unproven | + +The remaining 3 are honestly unprovable by this runner: one object has no +plain-text field to mutate, two are read-only federated objects. + +The new green is falsifiable: ablating the #7665 write-scope derivation in +`plugin-security` — while leaving the #1994 pre-image re-read fully in place — +turns 16 of the 20 proven showcase objects into `rls-hole` and exits 1. + +## No behaviour change outside the verifier + +This is tooling: no runtime, spec or enforcement path is touched. +`runRlsProofs`' existing four-argument call shape still works, and consumers that +only read the report gain fields rather than losing any. diff --git a/packages/qa/dogfood/test/authz-conformance.matrix.ts b/packages/qa/dogfood/test/authz-conformance.matrix.ts index 2c852ea507..04e547edb9 100644 --- a/packages/qa/dogfood/test/authz-conformance.matrix.ts +++ b/packages/qa/dogfood/test/authz-conformance.matrix.ts @@ -37,7 +37,9 @@ export const AUTHZ_CONFORMANCE: AuthzPrimitive[] = [ { id: 'rls-read', summary: 'RLS `using` read filter', state: 'enforced', enforcement: 'plugin-security/security-plugin.ts computeRlsFilter (AND-injected)', proof: 'rls-fixture.dogfood.test.ts' }, { id: 'rls-by-id-write', summary: 'by-id write enforcement (#1994)', state: 'enforced', - enforcement: 'plugin-security/security-plugin.ts pre-image re-read', proof: 'rls-fixture.dogfood.test.ts' }, + enforcement: 'plugin-security/security-plugin.ts step 2.7 pre-image re-read, composed with computeRlsFilter\'s write-scope DERIVATION (#7665): when no update/delete-class policy applies, the write class is compiled from the caller\'s SELECT narrowing, so the pre-image gate has a predicate to enforce', + proof: 'rls-fixture.dogfood.test.ts', + note: '[#7685] Re-verified as `enforced`, both halves of the enforcement named because ONLY BOTH hold it. The pre-image re-read alone is a no-op under select-only authoring — that was #7665, and it is why the site had to be re-cited here: ablating the derivation while leaving the pre-image re-read fully in place turns 16 of 20 probed showcase objects into `rls-hole`. The proof is NOT vacuous for this row despite sharing `rls-fixture.dogfood.test.ts` with `rls-read`: since #7665/PR #7792 that file carries a dedicated select-only block whose member set grants FULL CRUD on `rls_note` (so a refusal is the record gate, never the object gate) asserting the by-id PATCH is refused with the row unchanged, plus that an in-scope write still lands. Second, independent measurement: `objectstack verify --rls`, whose probe persona reaches this class since #7685 — 20/23 showcase and 6/6 crm objects PROVEN, 0 holes, and 16 holes under the same ablation.' }, { id: 'rls-write-check', summary: 'RLS `check` write post-image validation (ADR-0058 D4)', state: 'enforced', enforcement: 'plugin-security/security-plugin.ts step 3.6 — compileCelToFilter + matchesFilterCondition against the post-image (fail-closed)', note: 'Unit-proven in plugin-security/security-plugin.test.ts (RLS check enforcement); see ADR-0058 D7 ledger.' }, @@ -46,7 +48,8 @@ export const AUTHZ_CONFORMANCE: AuthzPrimitive[] = [ { id: 'owd-public-read', summary: 'OWD public_read (everyone reads, owner writes)', state: 'enforced', enforcement: 'plugin-sharing/sharing-service.ts (read model + canEdit)', proof: 'showcase-public-read-owd.dogfood.test.ts' }, { id: 'controlled-by-parent', summary: 'master-detail controlled_by_parent', state: 'enforced', - enforcement: 'plugin-security/security-plugin.ts computeControlledByParentFilter + assertControlledByParentWrite', proof: 'controlled-by-parent.dogfood.test.ts' }, + enforcement: 'plugin-security/security-plugin.ts computeControlledByParentFilter + assertControlledByParentWrite', proof: 'controlled-by-parent.dogfood.test.ts', + note: '[#7685] Re-verified as `enforced` on its OWN evidence, not by association with `rls-by-id-write`. The cited proof is DEDICATED and non-vacuous: `fixtures/cbp-fixture.ts` grants the member full CRUD on BOTH `cbp_account` and `cbp_note`, so every refusal it asserts is the derived record gate rather than the object gate, and the detail carries no authored RLS at all — access is derived from the owner-scoped master. It asserts the derived READ denial, the derived by-id WRITE denial with the row unchanged as ground truth, and that a note under a master the member owns stays readable AND writable (so the guard is not over-blocking). Second measurement: `verify --rls` probes `showcase_invoice_line` — a real `controlled_by_parent` detail — as `rls-consistent`, and it flips to `rls-hole` when the #7665 write-scope derivation its master depends on is ablated.' }, { id: 'multi-tenant', summary: 'organization isolation', state: 'enforced', enforcement: '@objectstack/organizations (enterprise) + Layer 0 tenant wall (plugin-security/tenant-layer.ts, AND-composed ahead of business RLS — ADR-0095 D1)', proof: 'rls-multitenant.dogfood.test.ts' }, { id: 'multi-tenant-write-postimage', summary: 'Layer 0 tenant post-image check on INSERT + UPDATE (#2937 / Finding 1 — a forged OR re-pointed organization_id cannot cross the tenant wall)', state: 'enforced', diff --git a/packages/qa/dogfood/test/rls-runner.test.ts b/packages/qa/dogfood/test/rls-runner.test.ts index 6be82af6a4..511f0718de 100644 --- a/packages/qa/dogfood/test/rls-runner.test.ts +++ b/packages/qa/dogfood/test/rls-runner.test.ts @@ -1,10 +1,25 @@ // Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. // // Unit proof that the RLS runner's #1994 classification is correct — driven by a -// scripted fake stack so we can exercise the three outcomes deterministically -// (a live owner-isolated fixture to exercise them end-to-end is the next step). +// scripted fake stack so we can exercise the outcomes deterministically. // // The invariant: a user who CANNOT READ a record must not be able to WRITE it. +// +// This is also the runner's DETECTOR-LIVENESS oracle. The live fixture +// (`rls-fixture.dogfood.test.ts`) can no longer plant the hole — #7665 closed +// the class platform-side, so its red block is now a green regression guard — +// and `objectstack verify --rls` reports 0 holes against the real apps. Only a +// scripted stack can still answer "the runner CAN say `rls-hole`", which is what +// keeps every green above from being a runner that lost the ability to fail. +// +// [#7685] The three classifications below are joined by the two the same issue +// added, and both exist because a NOT-PROVEN object used to be indistinguishable +// from a proven one: +// • `probe-blocked` — the OBJECT gate refused the persona, so record scope was +// never consulted. On the stock showcase that was 11 of 13 "consistent" +// verdicts, i.e. green that no platform change could have turned red. +// • target adoption — an admin create the app's own validation rejects no +// longer cascades its dependents out of the run. import { describe, it, expect } from 'vitest'; import { runRlsProofs } from '@objectstack/verify'; @@ -15,24 +30,47 @@ const CONFIG = { objects: [{ name: 'note', fields: { name: { type: 'text', required: true } } }], }; -/** A fake stack: admin always sees/owns; member behaviour is scripted per scenario. */ -function fakeStack(opts: { +interface FakeOpts { memberCanRead: boolean; memberWriteMutates: boolean; // does member's PATCH actually change the row? -}): VerifyStack { + /** The object-level gate refuses the member outright (403 before record scope). */ + objectGateDenies?: boolean; + /** Status the admin POST answers with (e.g. 400 — the app's own validation). */ + adminCreateStatus?: number; + /** Rows that already exist on the object, as seed data would. */ + seeded?: Array>; +} + +/** A fake stack: admin always sees/owns; member behaviour is scripted per scenario. */ +function fakeStack(opts: FakeOpts): VerifyStack { const store: Record = {}; + for (const row of opts.seeded ?? []) store[String(row.id)] = { ...row }; const json = (body: unknown, status = 200) => new Response(JSON.stringify(body), { status, headers: { 'Content-Type': 'application/json' } }); const apiAs: VerifyStack['apiAs'] = async (token, method, path, body) => { const isAdmin = token === 'admin'; - const [, , object, id] = path.split('/'); // /data// + // `/data/[?query]` (list) or `/data//` (by id) + const [, , objectSegment, id] = path.split('/'); + const object = String(objectSegment).split('?')[0]; + + // The object-level gate answers first, for every verb — exactly what made + // the grant-less persona's verdicts meaningless (#7685). + if (!isAdmin && opts.objectGateDenies) return json({ code: 'PERMISSION_DENIED' }, 403); + if (method === 'POST') { + if (opts.adminCreateStatus) return json({ error: 'VALIDATION_FAILED' }, opts.adminCreateStatus); const newId = 'rec1'; store[newId] = { id: newId, ...(body as object) }; return json({ object, id: newId, record: store[newId] }, 201); } if (method === 'GET') { + if (id === undefined) { + // LIST — the runner's reachability probe, and the admin-side source the + // cascade-stopper adopts a target from. + if (!isAdmin && !opts.memberCanRead) return json({ records: [] }); + return json({ records: Object.values(store) }); + } if (!isAdmin && !opts.memberCanRead) return json({ error: 'not found' }, 404); return json({ object, id, record: store[id] ?? null }); } @@ -78,3 +116,63 @@ describe('runRlsProofs #1994 classification', () => { expect(report.results[0].status).toBe('member-visible'); }); }); + +describe('[#7685] a NOT-PROVEN object never reads as a pass', () => { + it('reports probe-blocked — NOT rls-consistent — when the OBJECT gate refuses the persona', async () => { + // The exact shape of the old defect: the persona holds no object grants, so + // the 403 that comes back says nothing about record scope. The runner used + // to bank it as `rls-consistent`; a platform regression in the by-id write + // could not have flipped it, which is what made the green worthless. + const stack = fakeStack({ memberCanRead: false, memberWriteMutates: true, objectGateDenies: true }); + const report = await runRlsProofs(stack, 'admin', 'member', CONFIG); + expect(report.results[0].status).toBe('probe-blocked'); + expect(report.summary.consistent).toBe(0); + expect(report.summary.proven).toBe(0); + expect(report.summary.unproven).toBe(1); + expect(report.unproven.map((u) => u.object)).toEqual(['note']); + }); + + it('counts member-visible and skipped as UNPROVEN, separately from holes', async () => { + const stack = fakeStack({ memberCanRead: true, memberWriteMutates: false }); + const report = await runRlsProofs(stack, 'admin', 'member', CONFIG); + expect(report.summary).toMatchObject({ + objects: 1, consistent: 0, holes: 0, memberVisible: 1, probeBlocked: 0, skipped: 0, + proven: 0, unproven: 1, + }); + }); + + it('carries the probe descriptor through to the report so a run\'s REACH is legible', async () => { + const stack = fakeStack({ memberCanRead: false, memberWriteMutates: false }); + const report = await runRlsProofs(stack, 'admin', 'member', CONFIG, { + probe: { label: 'probe@test', grantedObjects: 1 }, + }); + expect(report.probe).toEqual({ label: 'probe@test', grantedObjects: 1 }); + }); +}); + +describe('[#7685] an unsatisfiable admin create does not cascade objects out of the run', () => { + it('adopts an existing row when the derived create is rejected, and still probes the object', async () => { + const stack = fakeStack({ + memberCanRead: false, + memberWriteMutates: true, + adminCreateStatus: 400, + seeded: [{ id: 'seed1', name: 'seeded' }], + }); + const report = await runRlsProofs(stack, 'admin', 'member', CONFIG); + expect(report.results[0].target).toBe('adopted'); + // The point of adopting: the object is PROVEN rather than skipped — and it + // is the hole here, which a skip would have hidden. + expect(report.results[0].status).toBe('rls-hole'); + expect(report.summary.skipped).toBe(0); + expect(report.summary.holes).toBe(1); + }); + + it('still skips — with the create failure in the reason — when there is nothing to adopt', async () => { + const stack = fakeStack({ memberCanRead: false, memberWriteMutates: false, adminCreateStatus: 400 }); + const report = await runRlsProofs(stack, 'admin', 'member', CONFIG); + expect(report.results[0].status).toBe('skipped'); + expect(report.results[0].detail).toContain('admin create failed (400)'); + expect(report.results[0].detail).toContain('no existing note row to adopt'); + expect(report.summary.proven).toBe(0); + }); +});