From 849b2acde83efd76f165d10e02c9bbb652c79b2b Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 13 Aug 2026 16:08:02 +0000 Subject: [PATCH] feat(service-cluster): carry an admitted node count through the multi-node gate (#8367) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `registerMultiNodeGate` consumed `{ allowMultiNode(): { allowed, reason } }` — a bare boolean verdict with no node count in the contract — so the only refusal a license could express was denying the whole cluster. The maintainer ruled on 2026-08-13 (recorded on objectstack-ai/cloud#1275) that a licensed `max_nodes` overflow must refuse the excess replicas, run up to the paid limit and warn loudly, explicitly NOT whole-cluster degrade. That verdict was previously inexpressible. A gate verdict may now carry `admitted`, and `checkMultiNodeAllowed(requested?)` forwards the caller's intended count and returns a normalized verdict carrying totalized `refused` / `capped`. `capped` marks only a PARTIAL refusal, so the licensed-overflow case cannot be conflated with the unlicensed one. Backward compatible: `requested` is an optional parameter and `admitted` an optional return field, so the existing zero-arg boolean-shaped provider that @objectstack/security-enterprise registers stays valid and reads as "no cap". The counts are ADVISORY at this seam and the code says so: the gate is consulted once per process at boot, where a replica has no membership view and no ordinal, so no replica can tell whether it is one of the admitted N. Binding enforcement needs an atomic slot claim, tracked separately. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01ARidKDYSCD56LaygrvDPnk --- .changeset/multinode-gate-admitted-count.md | 46 +++++ .../services/service-cluster/src/index.ts | 2 + .../src/multi-node-gate.test.ts | 155 ++++++++++++++- .../service-cluster/src/multi-node-gate.ts | 184 +++++++++++++++++- 4 files changed, 380 insertions(+), 7 deletions(-) create mode 100644 .changeset/multinode-gate-admitted-count.md diff --git a/.changeset/multinode-gate-admitted-count.md b/.changeset/multinode-gate-admitted-count.md new file mode 100644 index 0000000000..97be13d499 --- /dev/null +++ b/.changeset/multinode-gate-admitted-count.md @@ -0,0 +1,46 @@ +--- +"@objectstack/service-cluster": minor +--- + +feat(service-cluster): the multi-node gate can carry an admitted node count, so a license cap can refuse the excess replicas instead of the whole cluster (#8367) + +`registerMultiNodeGate` consumed `{ allowMultiNode(): { allowed, reason } }` — a +bare boolean verdict with **no node count in the contract**. The maintainer ruled +on 2026-08-13 (recorded on `objectstack-ai/cloud#1275`) that a licensed +`max_nodes` overflow must **refuse the excess replicas, run up to the paid limit, +and warn loudly** — explicitly *not* a whole-cluster degrade. Through a boolean +gate that verdict could not be stated at all: the only refusal a license could +express was `allowed: false`, which is precisely the whole-cluster degrade the +ruling rejects. + +A gate verdict may now carry `admitted` — how many nodes it admits — and +`checkMultiNodeAllowed(requested?)` forwards the caller's intended node count to +the gate and returns a normalized verdict: + +```ts +{ allowed: boolean; reason?: string; admitted?: number; refused: number; capped: boolean } +``` + +`refused` and `capped` are **totalized** (always present), so no consumer writes +`?? 0` over a third-party gate's output — the seam normalizes non-finite, +fractional and negative counts itself. `capped` marks only a **partial** refusal: +it stays `false` for an outright `allowed: false`, so the licensed-overflow case +and the unlicensed case cannot be conflated by a consumer. + +**Backward compatible.** `requested` is an optional parameter and `admitted` an +optional return field, so an existing zero-arg, boolean-shaped provider — the +shape `@objectstack/security-enterprise` registers today — remains valid and is +interpreted as "no count-based cap": it admits everything requested rather than +having a refusal invented for it. Existing zero-arg call sites are unaffected. + +**⚠️ The count is advisory at this seam — it is not yet enforcement.** The gate +is consulted once per process, at boot, by each replica independently, and at +that moment a replica has no cluster membership view (`nodeId` is random per +process; nothing tracks live nodes) and no ordinal — the only count available is +the operator-declared `OS_CLUSTER_REPLICAS`, identical in every replica. So every +replica computes the same verdict and none can tell whether *it* is one of the +admitted N or one of the excess. Binding enforcement additionally requires an +atomic slot claim against the shared cluster primitives this package already +ships (`ILock`/`ICounter`/`IKV`), which is tracked separately. Until then, +consumers should treat `refused > 0` as the trigger for the loud warning the +ruling requires, never as grounds to deny the cluster. diff --git a/packages/services/service-cluster/src/index.ts b/packages/services/service-cluster/src/index.ts index c8c9bd07e7..6de01c3936 100644 --- a/packages/services/service-cluster/src/index.ts +++ b/packages/services/service-cluster/src/index.ts @@ -72,4 +72,6 @@ export { checkMultiNodeAllowed, __resetMultiNodeGate, type MultiNodeGate, + type MultiNodeVerdict, + type ResolvedMultiNodeVerdict, } from './multi-node-gate.js'; diff --git a/packages/services/service-cluster/src/multi-node-gate.test.ts b/packages/services/service-cluster/src/multi-node-gate.test.ts index 7a9f7f1d71..b8f55b16bf 100644 --- a/packages/services/service-cluster/src/multi-node-gate.test.ts +++ b/packages/services/service-cluster/src/multi-node-gate.test.ts @@ -10,12 +10,18 @@ afterEach(() => __resetMultiNodeGate()); describe('multi-node gate', () => { it('allows when no gate is registered (open framework)', () => { - expect(checkMultiNodeAllowed()).toEqual({ allowed: true }); + expect(checkMultiNodeAllowed()).toEqual({ allowed: true, refused: 0, capped: false }); }); it('honors a denying gate with reason', () => { registerMultiNodeGate({ allowMultiNode: () => ({ allowed: false, reason: 'unlicensed' }) }); - expect(checkMultiNodeAllowed()).toEqual({ allowed: false, reason: 'unlicensed' }); + expect(checkMultiNodeAllowed()).toEqual({ + allowed: false, + reason: 'unlicensed', + admitted: 0, + refused: 0, + capped: false, + }); }); it('honors an allowing gate', () => { @@ -32,6 +38,149 @@ describe('multi-node gate', () => { it('reset restores open default', () => { registerMultiNodeGate({ allowMultiNode: () => ({ allowed: false }) }); __resetMultiNodeGate(); - expect(checkMultiNodeAllowed()).toEqual({ allowed: true }); + expect(checkMultiNodeAllowed()).toEqual({ allowed: true, refused: 0, capped: false }); + }); +}); + +// --------------------------------------------------------------------------- +// Count-carrying admission (#8367) +// +// The maintainer ruled (2026-08-13, recorded on cloud#1275) that a licensed +// `max_nodes` overflow refuses the EXCESS replicas and runs up to the paid +// limit -- explicitly NOT whole-cluster degrade. These pins fix the seam +// semantics that make such a verdict expressible. +// +// VACUITY NOTE: an assertion that only reads `allowed`/`reason`, or one whose +// requested count never exceeds the cap, passes verbatim against a completely +// UNWIDENED gate -- `refused` would be 0 either way. Every pin below therefore +// requests strictly more than the cap and asserts `refused`/`capped`, and the +// partial-cap pins assert `allowed === true` in the SAME expectation: a gate +// that denied the whole cluster would also produce `refused > 0`, so the +// `allowed` half is what distinguishes the ruled behaviour from the rejected +// one. +// --------------------------------------------------------------------------- +describe('multi-node gate: count-carrying admission', () => { + it('forwards the requested node count to the gate', () => { + const seen: Array = []; + registerMultiNodeGate({ + allowMultiNode: (requested) => { + seen.push(requested); + return { allowed: true }; + }, + }); + checkMultiNodeAllowed(5); + expect(seen).toEqual([5]); + }); + + it('admits up to the cap and refuses only the excess, staying allowed', () => { + registerMultiNodeGate({ + allowMultiNode: () => ({ allowed: true, reason: 'licensed', admitted: 3 }), + }); + // The ruled behaviour: run 3, refuse 2, do NOT deny the cluster. + expect(checkMultiNodeAllowed(5)).toEqual({ + allowed: true, + reason: 'licensed', + admitted: 3, + refused: 2, + capped: true, + }); + }); + + it('does not report a cap when the request fits inside it', () => { + registerMultiNodeGate({ + allowMultiNode: () => ({ allowed: true, admitted: 5 }), + }); + const verdict = checkMultiNodeAllowed(3); + expect(verdict).toEqual({ allowed: true, admitted: 3, refused: 0, capped: false }); + }); + + it('reports an outright denial as refusing everything, but never as a partial cap', () => { + registerMultiNodeGate({ + allowMultiNode: () => ({ allowed: false, reason: 'unlicensed' }), + }); + expect(checkMultiNodeAllowed(4)).toEqual({ + allowed: false, + reason: 'unlicensed', + admitted: 0, + refused: 4, + capped: false, + }); + }); + + it('treats an allowing gate that declares no count as uncapped', () => { + registerMultiNodeGate({ allowMultiNode: () => ({ allowed: true, reason: 'licensed' }) }); + expect(checkMultiNodeAllowed(9)).toEqual({ + allowed: true, + reason: 'licensed', + refused: 0, + capped: false, + }); + }); + + it('treats the open framework (no gate) as uncapped for any count', () => { + expect(checkMultiNodeAllowed(9)).toEqual({ allowed: true, refused: 0, capped: false }); + }); + + it('normalizes a degenerate admitted count from a third-party gate', () => { + // Contract-first: the seam normalizes, so no consumer writes `?? 0`. + registerMultiNodeGate({ allowMultiNode: () => ({ allowed: true, admitted: -2 }) }); + expect(checkMultiNodeAllowed(3)).toMatchObject({ admitted: 0, refused: 3 }); + + registerMultiNodeGate({ allowMultiNode: () => ({ allowed: true, admitted: 2.7 }) }); + expect(checkMultiNodeAllowed(4)).toMatchObject({ admitted: 2, refused: 2 }); + + registerMultiNodeGate({ allowMultiNode: () => ({ allowed: true, admitted: Number.NaN }) }); + expect(checkMultiNodeAllowed(4)).toMatchObject({ refused: 0, capped: false }); + }); + + it('ignores a meaningless requested count', () => { + registerMultiNodeGate({ allowMultiNode: () => ({ allowed: true, admitted: 2 }) }); + expect(checkMultiNodeAllowed(-1)).toMatchObject({ admitted: 2, refused: 0, capped: false }); + expect(checkMultiNodeAllowed(Number.NaN)).toMatchObject({ admitted: 2, refused: 0 }); + }); +}); + +// --------------------------------------------------------------------------- +// Backward compatibility with the published seam (#8367) +// +// `allowMultiNode` is consumed by @objectstack/security-enterprise in the cloud +// repo, which registers a ZERO-ARG arrow returning a bare `{ allowed, reason }` +// (cloud: apps/objectos-ee/objectstack.config.ts). Widening the seam must leave +// that provider working -- this block pins the exact shape it registers today, +// so a later signature change cannot strand the EE distribution silently. +// --------------------------------------------------------------------------- +describe('multi-node gate: existing boolean-shaped provider', () => { + /** Byte-for-byte the shape the EE distribution registers today. */ + const eeShapedProvider = { + allowMultiNode: () => ({ allowed: true, reason: 'offline license' }), + }; + + it('accepts a zero-arg boolean provider and honors its verdict uncounted', () => { + registerMultiNodeGate(eeShapedProvider); + expect(checkMultiNodeAllowed()).toMatchObject({ + allowed: true, + reason: 'offline license', + }); + }); + + it('keeps a zero-arg boolean provider correct under a COUNTED call', () => { + registerMultiNodeGate(eeShapedProvider); + // It declares no cap, so it must admit everything asked for -- the + // widening must not invent a refusal the provider never expressed. + expect(checkMultiNodeAllowed(12)).toEqual({ + allowed: true, + reason: 'offline license', + refused: 0, + capped: false, + }); + }); + + it('keeps a zero-arg DENYING boolean provider denying', () => { + registerMultiNodeGate({ + allowMultiNode: () => ({ allowed: false, reason: 'license does not include clustering' }), + }); + const verdict = checkMultiNodeAllowed(3); + expect(verdict.allowed).toBe(false); + expect(verdict.reason).toBe('license does not include clustering'); }); }); diff --git a/packages/services/service-cluster/src/multi-node-gate.ts b/packages/services/service-cluster/src/multi-node-gate.ts index 57e333e0a8..717677ef91 100644 --- a/packages/services/service-cluster/src/multi-node-gate.ts +++ b/packages/services/service-cluster/src/multi-node-gate.ts @@ -13,13 +13,132 @@ * rather than failing — multi-node is an add-on, not a precondition for the * runtime to serve. This is distinct from the split-brain guard, which throws * on an outright misconfiguration (memory driver declared multi-node). + * + * ## Two different questions (#8367) + * + * A gate answers one verdict that carries **two** separable facts: + * + * 1. *May this deployment run multi-node at all?* — `allowed`. A `false` here + * is the unlicensed case: there is no clustering entitlement, so the whole + * topology folds back to single-node. That has always been the behaviour + * and is unchanged. + * 2. *Given N replicas, how many are within the paid limit?* — `admitted`. + * This is the **licensed overflow** case, and it is a different question: + * the deployment IS entitled to cluster, it simply asked for more nodes + * than it paid for. + * + * Case 2 previously had no way to be expressed. The verdict was a bare boolean, + * so the only refusal a license could state was `allowed: false` — denying the + * entire cluster. The maintainer ruled on 2026-08-13 (recorded on cloud#1275) + * that a licensed `max_nodes` overflow must instead **refuse the excess + * replicas, run up to the paid limit, and warn loudly** — explicitly NOT a + * whole-cluster degrade. `admitted` is the seam that makes that verdict + * expressible. + * + * ## ⚠️ The count is ADVISORY at this seam — it is not yet enforcement + * + * A count-carrying verdict is **necessary but not sufficient** to make the + * ruling binding, and nothing in this module should be read as claiming + * otherwise. The gate is consulted **once per process, at boot, by each replica + * independently** (`os serve`, packages/cli/src/commands/serve.ts). At that + * moment a replica has: + * + * - no cluster membership view — nothing tracks which nodes are live; + * `nodeId` is generated randomly per process (see `cluster.ts`), and there + * is no join/leave registry to count; + * - no ordinal — the only count available is `OS_CLUSTER_REPLICAS`, an + * operator-*declared* desired count that is **identical in every replica** + * (see `split-brain-guard.ts`). + * + * So with a cap of 3 and 5 replicas booting, every replica computes the *same* + * verdict ("3 admitted") and none of them can know whether it is one of the + * admitted 3 or one of the excess 2. Acting on that verdict locally yields + * either "all 5 join" (nothing actually refused) or "all 5 refuse" (precisely + * the whole-cluster degrade the ruling rejects). + * + * Making "run N, refuse N+1" genuinely binding needs an **atomic slot claim** + * against the shared cluster primitives this package already ships (`ILock` / + * `ICounter` / `IKV` on a remote driver): each booting replica claims a slot, + * the (N+1)th claim fails, and *that* replica downgrades itself — plus slot + * release on shutdown and TTL expiry so a crashed replica does not leak its + * seat. That mechanism does not exist yet and is deliberately out of scope + * here; this module supplies the verdict it would consume. + * + * Until it lands, a consumer should treat `refused > 0` as the trigger for the + * **loud warning** the ruling requires, not as a licence to deny the cluster. + * + * ## Why not a per-node admission callback + * + * A `admitNode(nodeId): boolean` shape was considered and rejected on + * measurement: the registered gate is a **module singleton inside a single + * replica's process**, with no cross-process state. Each replica's provider + * would start its own private counter at zero and admit itself, so the shape + * would *look* like per-replica enforcement while enforcing nothing. A verdict + * that is honestly advisory is better than a callback that is silently vacuous. + */ + +/** + * The verdict a registered gate returns. + * + * Backward compatible by construction: `admitted` is optional, so a gate that + * returns a bare `{ allowed, reason }` — as the EE distribution does today — + * remains a valid provider and is interpreted as "no count-based cap". */ +export interface MultiNodeVerdict { + /** Whether a multi-node topology is authorized at all. */ + allowed: boolean; + /** Surfaced in logs. */ + reason?: string; + /** + * How many nodes this gate admits. **Omit** for no count-based cap (an + * allowing gate with no `admitted` admits everything requested). Values are + * normalized by {@link checkMultiNodeAllowed}: non-finite means uncapped, + * fractional is floored, negative clamps to 0. + */ + admitted?: number; +} + export interface MultiNodeGate { /** * Called before the runtime enables a remote-driver (multi-node) topology. * Return `allowed: false` to force single-node; `reason` is surfaced in logs. + * + * @param requested - How many nodes the caller intends to run, when it + * knows. **Optional**: an existing zero-arg implementation stays valid + * (a function of fewer parameters is assignable), which is what keeps + * `@objectstack/security-enterprise` working unchanged. Gates that + * enforce a cap return {@link MultiNodeVerdict.admitted}. */ - allowMultiNode(): { allowed: boolean; reason?: string }; + allowMultiNode(requested?: number): MultiNodeVerdict; +} + +/** + * A gate verdict normalized by the framework. Consumers read this shape and + * never have to write `?? 0` fallbacks over a third-party gate's output — + * normalization belongs at the seam, not in every consumer. + */ +export interface ResolvedMultiNodeVerdict { + /** Whether a multi-node topology is authorized at all. */ + allowed: boolean; + /** Surfaced in logs. */ + reason?: string; + /** + * How many of the requested nodes may run. **Absent** when the gate imposes + * no count cap. `0` when the gate denied outright. + */ + admitted?: number; + /** + * How many requested nodes exceed what the gate admits. `0` when uncapped, + * when the request fits, or when the caller declared no count. + */ + refused: number; + /** + * True only for a **partial** refusal — the licensed-overflow case, where + * the cluster runs at the cap and the excess is refused. Deliberately + * `false` for an outright denial (`allowed: false`), which is the separate + * unlicensed case, so a consumer cannot conflate the two. + */ + capped: boolean; } let registered: MultiNodeGate | undefined; @@ -32,12 +151,69 @@ export function registerMultiNodeGate(gate: MultiNodeGate): void { registered = gate; } +/** A positive, finite, whole node count — or `undefined` for "not declared". */ +function normalizeCount(value: number | undefined): number | undefined { + if (typeof value !== 'number' || !Number.isFinite(value) || value <= 0) return undefined; + return Math.floor(value); +} + /** * Resolve the multi-node decision. With no gate registered (open framework), - * multi-node is allowed. + * multi-node is allowed and uncapped. + * + * @param requested - How many nodes the caller intends to run, when it knows. + * Omit when no count is available; the verdict is then reported uncapped + * (`refused: 0`) because nothing was counted. Meaningless values (zero, + * negative, non-finite) are treated as "not declared". + * + * ⚠️ See the module doc: the returned counts are **advisory** — no replica can + * currently act on them alone to refuse itself. Use `refused > 0` to warn. */ -export function checkMultiNodeAllowed(): { allowed: boolean; reason?: string } { - return registered ? registered.allowMultiNode() : { allowed: true }; +export function checkMultiNodeAllowed(requested?: number): ResolvedMultiNodeVerdict { + const wanted = normalizeCount(requested); + + if (!registered) return { allowed: true, refused: 0, capped: false }; + + const verdict = registered.allowMultiNode(wanted); + + // Outright denial (unlicensed): everything asked for is refused, but this is + // NOT a partial cap — keep `capped` false so consumers can tell the ruled + // licensed-overflow case apart from the unlicensed one. + if (!verdict.allowed) { + return { + allowed: false, + ...(verdict.reason === undefined ? {} : { reason: verdict.reason }), + admitted: 0, + refused: wanted ?? 0, + capped: false, + }; + } + + const cap = + typeof verdict.admitted === 'number' && Number.isFinite(verdict.admitted) + ? Math.max(0, Math.floor(verdict.admitted)) + : undefined; + + // An allowing gate that declared no cap admits whatever was requested. + if (cap === undefined) { + return { + allowed: true, + ...(verdict.reason === undefined ? {} : { reason: verdict.reason }), + refused: 0, + capped: false, + }; + } + + const admitted = wanted === undefined ? cap : Math.min(cap, wanted); + const refused = wanted === undefined ? 0 : Math.max(0, wanted - cap); + + return { + allowed: true, + ...(verdict.reason === undefined ? {} : { reason: verdict.reason }), + admitted, + refused, + capped: refused > 0, + }; } /** Clear the registered gate. For tests. */