diff --git a/.changeset/multinode-gate-admitted-count.md b/.changeset/multinode-gate-admitted-count.md new file mode 100644 index 0000000000..97be13d499 --- /dev/null +++ b/.changeset/multinode-gate-admitted-count.md @@ -0,0 +1,46 @@ +--- +"@objectstack/service-cluster": minor +--- + +feat(service-cluster): the multi-node gate can carry an admitted node count, so a license cap can refuse the excess replicas instead of the whole cluster (#8367) + +`registerMultiNodeGate` consumed `{ allowMultiNode(): { allowed, reason } }` — a +bare boolean verdict with **no node count in the contract**. The maintainer ruled +on 2026-08-13 (recorded on `objectstack-ai/cloud#1275`) that a licensed +`max_nodes` overflow must **refuse the excess replicas, run up to the paid limit, +and warn loudly** — explicitly *not* a whole-cluster degrade. Through a boolean +gate that verdict could not be stated at all: the only refusal a license could +express was `allowed: false`, which is precisely the whole-cluster degrade the +ruling rejects. + +A gate verdict may now carry `admitted` — how many nodes it admits — and +`checkMultiNodeAllowed(requested?)` forwards the caller's intended node count to +the gate and returns a normalized verdict: + +```ts +{ allowed: boolean; reason?: string; admitted?: number; refused: number; capped: boolean } +``` + +`refused` and `capped` are **totalized** (always present), so no consumer writes +`?? 0` over a third-party gate's output — the seam normalizes non-finite, +fractional and negative counts itself. `capped` marks only a **partial** refusal: +it stays `false` for an outright `allowed: false`, so the licensed-overflow case +and the unlicensed case cannot be conflated by a consumer. + +**Backward compatible.** `requested` is an optional parameter and `admitted` an +optional return field, so an existing zero-arg, boolean-shaped provider — the +shape `@objectstack/security-enterprise` registers today — remains valid and is +interpreted as "no count-based cap": it admits everything requested rather than +having a refusal invented for it. Existing zero-arg call sites are unaffected. + +**⚠️ The count is advisory at this seam — it is not yet enforcement.** The gate +is consulted once per process, at boot, by each replica independently, and at +that moment a replica has no cluster membership view (`nodeId` is random per +process; nothing tracks live nodes) and no ordinal — the only count available is +the operator-declared `OS_CLUSTER_REPLICAS`, identical in every replica. So every +replica computes the same verdict and none can tell whether *it* is one of the +admitted N or one of the excess. Binding enforcement additionally requires an +atomic slot claim against the shared cluster primitives this package already +ships (`ILock`/`ICounter`/`IKV`), which is tracked separately. Until then, +consumers should treat `refused > 0` as the trigger for the loud warning the +ruling requires, never as grounds to deny the cluster. diff --git a/packages/services/service-cluster/src/index.ts b/packages/services/service-cluster/src/index.ts index c8c9bd07e7..6de01c3936 100644 --- a/packages/services/service-cluster/src/index.ts +++ b/packages/services/service-cluster/src/index.ts @@ -72,4 +72,6 @@ export { checkMultiNodeAllowed, __resetMultiNodeGate, type MultiNodeGate, + type MultiNodeVerdict, + type ResolvedMultiNodeVerdict, } from './multi-node-gate.js'; diff --git a/packages/services/service-cluster/src/multi-node-gate.test.ts b/packages/services/service-cluster/src/multi-node-gate.test.ts index 7a9f7f1d71..b8f55b16bf 100644 --- a/packages/services/service-cluster/src/multi-node-gate.test.ts +++ b/packages/services/service-cluster/src/multi-node-gate.test.ts @@ -10,12 +10,18 @@ afterEach(() => __resetMultiNodeGate()); describe('multi-node gate', () => { it('allows when no gate is registered (open framework)', () => { - expect(checkMultiNodeAllowed()).toEqual({ allowed: true }); + expect(checkMultiNodeAllowed()).toEqual({ allowed: true, refused: 0, capped: false }); }); it('honors a denying gate with reason', () => { registerMultiNodeGate({ allowMultiNode: () => ({ allowed: false, reason: 'unlicensed' }) }); - expect(checkMultiNodeAllowed()).toEqual({ allowed: false, reason: 'unlicensed' }); + expect(checkMultiNodeAllowed()).toEqual({ + allowed: false, + reason: 'unlicensed', + admitted: 0, + refused: 0, + capped: false, + }); }); it('honors an allowing gate', () => { @@ -32,6 +38,149 @@ describe('multi-node gate', () => { it('reset restores open default', () => { registerMultiNodeGate({ allowMultiNode: () => ({ allowed: false }) }); __resetMultiNodeGate(); - expect(checkMultiNodeAllowed()).toEqual({ allowed: true }); + expect(checkMultiNodeAllowed()).toEqual({ allowed: true, refused: 0, capped: false }); + }); +}); + +// --------------------------------------------------------------------------- +// Count-carrying admission (#8367) +// +// The maintainer ruled (2026-08-13, recorded on cloud#1275) that a licensed +// `max_nodes` overflow refuses the EXCESS replicas and runs up to the paid +// limit -- explicitly NOT whole-cluster degrade. These pins fix the seam +// semantics that make such a verdict expressible. +// +// VACUITY NOTE: an assertion that only reads `allowed`/`reason`, or one whose +// requested count never exceeds the cap, passes verbatim against a completely +// UNWIDENED gate -- `refused` would be 0 either way. Every pin below therefore +// requests strictly more than the cap and asserts `refused`/`capped`, and the +// partial-cap pins assert `allowed === true` in the SAME expectation: a gate +// that denied the whole cluster would also produce `refused > 0`, so the +// `allowed` half is what distinguishes the ruled behaviour from the rejected +// one. +// --------------------------------------------------------------------------- +describe('multi-node gate: count-carrying admission', () => { + it('forwards the requested node count to the gate', () => { + const seen: Array = []; + registerMultiNodeGate({ + allowMultiNode: (requested) => { + seen.push(requested); + return { allowed: true }; + }, + }); + checkMultiNodeAllowed(5); + expect(seen).toEqual([5]); + }); + + it('admits up to the cap and refuses only the excess, staying allowed', () => { + registerMultiNodeGate({ + allowMultiNode: () => ({ allowed: true, reason: 'licensed', admitted: 3 }), + }); + // The ruled behaviour: run 3, refuse 2, do NOT deny the cluster. + expect(checkMultiNodeAllowed(5)).toEqual({ + allowed: true, + reason: 'licensed', + admitted: 3, + refused: 2, + capped: true, + }); + }); + + it('does not report a cap when the request fits inside it', () => { + registerMultiNodeGate({ + allowMultiNode: () => ({ allowed: true, admitted: 5 }), + }); + const verdict = checkMultiNodeAllowed(3); + expect(verdict).toEqual({ allowed: true, admitted: 3, refused: 0, capped: false }); + }); + + it('reports an outright denial as refusing everything, but never as a partial cap', () => { + registerMultiNodeGate({ + allowMultiNode: () => ({ allowed: false, reason: 'unlicensed' }), + }); + expect(checkMultiNodeAllowed(4)).toEqual({ + allowed: false, + reason: 'unlicensed', + admitted: 0, + refused: 4, + capped: false, + }); + }); + + it('treats an allowing gate that declares no count as uncapped', () => { + registerMultiNodeGate({ allowMultiNode: () => ({ allowed: true, reason: 'licensed' }) }); + expect(checkMultiNodeAllowed(9)).toEqual({ + allowed: true, + reason: 'licensed', + refused: 0, + capped: false, + }); + }); + + it('treats the open framework (no gate) as uncapped for any count', () => { + expect(checkMultiNodeAllowed(9)).toEqual({ allowed: true, refused: 0, capped: false }); + }); + + it('normalizes a degenerate admitted count from a third-party gate', () => { + // Contract-first: the seam normalizes, so no consumer writes `?? 0`. + registerMultiNodeGate({ allowMultiNode: () => ({ allowed: true, admitted: -2 }) }); + expect(checkMultiNodeAllowed(3)).toMatchObject({ admitted: 0, refused: 3 }); + + registerMultiNodeGate({ allowMultiNode: () => ({ allowed: true, admitted: 2.7 }) }); + expect(checkMultiNodeAllowed(4)).toMatchObject({ admitted: 2, refused: 2 }); + + registerMultiNodeGate({ allowMultiNode: () => ({ allowed: true, admitted: Number.NaN }) }); + expect(checkMultiNodeAllowed(4)).toMatchObject({ refused: 0, capped: false }); + }); + + it('ignores a meaningless requested count', () => { + registerMultiNodeGate({ allowMultiNode: () => ({ allowed: true, admitted: 2 }) }); + expect(checkMultiNodeAllowed(-1)).toMatchObject({ admitted: 2, refused: 0, capped: false }); + expect(checkMultiNodeAllowed(Number.NaN)).toMatchObject({ admitted: 2, refused: 0 }); + }); +}); + +// --------------------------------------------------------------------------- +// Backward compatibility with the published seam (#8367) +// +// `allowMultiNode` is consumed by @objectstack/security-enterprise in the cloud +// repo, which registers a ZERO-ARG arrow returning a bare `{ allowed, reason }` +// (cloud: apps/objectos-ee/objectstack.config.ts). Widening the seam must leave +// that provider working -- this block pins the exact shape it registers today, +// so a later signature change cannot strand the EE distribution silently. +// --------------------------------------------------------------------------- +describe('multi-node gate: existing boolean-shaped provider', () => { + /** Byte-for-byte the shape the EE distribution registers today. */ + const eeShapedProvider = { + allowMultiNode: () => ({ allowed: true, reason: 'offline license' }), + }; + + it('accepts a zero-arg boolean provider and honors its verdict uncounted', () => { + registerMultiNodeGate(eeShapedProvider); + expect(checkMultiNodeAllowed()).toMatchObject({ + allowed: true, + reason: 'offline license', + }); + }); + + it('keeps a zero-arg boolean provider correct under a COUNTED call', () => { + registerMultiNodeGate(eeShapedProvider); + // It declares no cap, so it must admit everything asked for -- the + // widening must not invent a refusal the provider never expressed. + expect(checkMultiNodeAllowed(12)).toEqual({ + allowed: true, + reason: 'offline license', + refused: 0, + capped: false, + }); + }); + + it('keeps a zero-arg DENYING boolean provider denying', () => { + registerMultiNodeGate({ + allowMultiNode: () => ({ allowed: false, reason: 'license does not include clustering' }), + }); + const verdict = checkMultiNodeAllowed(3); + expect(verdict.allowed).toBe(false); + expect(verdict.reason).toBe('license does not include clustering'); }); }); diff --git a/packages/services/service-cluster/src/multi-node-gate.ts b/packages/services/service-cluster/src/multi-node-gate.ts index 57e333e0a8..717677ef91 100644 --- a/packages/services/service-cluster/src/multi-node-gate.ts +++ b/packages/services/service-cluster/src/multi-node-gate.ts @@ -13,13 +13,132 @@ * rather than failing — multi-node is an add-on, not a precondition for the * runtime to serve. This is distinct from the split-brain guard, which throws * on an outright misconfiguration (memory driver declared multi-node). + * + * ## Two different questions (#8367) + * + * A gate answers one verdict that carries **two** separable facts: + * + * 1. *May this deployment run multi-node at all?* — `allowed`. A `false` here + * is the unlicensed case: there is no clustering entitlement, so the whole + * topology folds back to single-node. That has always been the behaviour + * and is unchanged. + * 2. *Given N replicas, how many are within the paid limit?* — `admitted`. + * This is the **licensed overflow** case, and it is a different question: + * the deployment IS entitled to cluster, it simply asked for more nodes + * than it paid for. + * + * Case 2 previously had no way to be expressed. The verdict was a bare boolean, + * so the only refusal a license could state was `allowed: false` — denying the + * entire cluster. The maintainer ruled on 2026-08-13 (recorded on cloud#1275) + * that a licensed `max_nodes` overflow must instead **refuse the excess + * replicas, run up to the paid limit, and warn loudly** — explicitly NOT a + * whole-cluster degrade. `admitted` is the seam that makes that verdict + * expressible. + * + * ## ⚠️ The count is ADVISORY at this seam — it is not yet enforcement + * + * A count-carrying verdict is **necessary but not sufficient** to make the + * ruling binding, and nothing in this module should be read as claiming + * otherwise. The gate is consulted **once per process, at boot, by each replica + * independently** (`os serve`, packages/cli/src/commands/serve.ts). At that + * moment a replica has: + * + * - no cluster membership view — nothing tracks which nodes are live; + * `nodeId` is generated randomly per process (see `cluster.ts`), and there + * is no join/leave registry to count; + * - no ordinal — the only count available is `OS_CLUSTER_REPLICAS`, an + * operator-*declared* desired count that is **identical in every replica** + * (see `split-brain-guard.ts`). + * + * So with a cap of 3 and 5 replicas booting, every replica computes the *same* + * verdict ("3 admitted") and none of them can know whether it is one of the + * admitted 3 or one of the excess 2. Acting on that verdict locally yields + * either "all 5 join" (nothing actually refused) or "all 5 refuse" (precisely + * the whole-cluster degrade the ruling rejects). + * + * Making "run N, refuse N+1" genuinely binding needs an **atomic slot claim** + * against the shared cluster primitives this package already ships (`ILock` / + * `ICounter` / `IKV` on a remote driver): each booting replica claims a slot, + * the (N+1)th claim fails, and *that* replica downgrades itself — plus slot + * release on shutdown and TTL expiry so a crashed replica does not leak its + * seat. That mechanism does not exist yet and is deliberately out of scope + * here; this module supplies the verdict it would consume. + * + * Until it lands, a consumer should treat `refused > 0` as the trigger for the + * **loud warning** the ruling requires, not as a licence to deny the cluster. + * + * ## Why not a per-node admission callback + * + * A `admitNode(nodeId): boolean` shape was considered and rejected on + * measurement: the registered gate is a **module singleton inside a single + * replica's process**, with no cross-process state. Each replica's provider + * would start its own private counter at zero and admit itself, so the shape + * would *look* like per-replica enforcement while enforcing nothing. A verdict + * that is honestly advisory is better than a callback that is silently vacuous. + */ + +/** + * The verdict a registered gate returns. + * + * Backward compatible by construction: `admitted` is optional, so a gate that + * returns a bare `{ allowed, reason }` — as the EE distribution does today — + * remains a valid provider and is interpreted as "no count-based cap". */ +export interface MultiNodeVerdict { + /** Whether a multi-node topology is authorized at all. */ + allowed: boolean; + /** Surfaced in logs. */ + reason?: string; + /** + * How many nodes this gate admits. **Omit** for no count-based cap (an + * allowing gate with no `admitted` admits everything requested). Values are + * normalized by {@link checkMultiNodeAllowed}: non-finite means uncapped, + * fractional is floored, negative clamps to 0. + */ + admitted?: number; +} + export interface MultiNodeGate { /** * Called before the runtime enables a remote-driver (multi-node) topology. * Return `allowed: false` to force single-node; `reason` is surfaced in logs. + * + * @param requested - How many nodes the caller intends to run, when it + * knows. **Optional**: an existing zero-arg implementation stays valid + * (a function of fewer parameters is assignable), which is what keeps + * `@objectstack/security-enterprise` working unchanged. Gates that + * enforce a cap return {@link MultiNodeVerdict.admitted}. */ - allowMultiNode(): { allowed: boolean; reason?: string }; + allowMultiNode(requested?: number): MultiNodeVerdict; +} + +/** + * A gate verdict normalized by the framework. Consumers read this shape and + * never have to write `?? 0` fallbacks over a third-party gate's output — + * normalization belongs at the seam, not in every consumer. + */ +export interface ResolvedMultiNodeVerdict { + /** Whether a multi-node topology is authorized at all. */ + allowed: boolean; + /** Surfaced in logs. */ + reason?: string; + /** + * How many of the requested nodes may run. **Absent** when the gate imposes + * no count cap. `0` when the gate denied outright. + */ + admitted?: number; + /** + * How many requested nodes exceed what the gate admits. `0` when uncapped, + * when the request fits, or when the caller declared no count. + */ + refused: number; + /** + * True only for a **partial** refusal — the licensed-overflow case, where + * the cluster runs at the cap and the excess is refused. Deliberately + * `false` for an outright denial (`allowed: false`), which is the separate + * unlicensed case, so a consumer cannot conflate the two. + */ + capped: boolean; } let registered: MultiNodeGate | undefined; @@ -32,12 +151,69 @@ export function registerMultiNodeGate(gate: MultiNodeGate): void { registered = gate; } +/** A positive, finite, whole node count — or `undefined` for "not declared". */ +function normalizeCount(value: number | undefined): number | undefined { + if (typeof value !== 'number' || !Number.isFinite(value) || value <= 0) return undefined; + return Math.floor(value); +} + /** * Resolve the multi-node decision. With no gate registered (open framework), - * multi-node is allowed. + * multi-node is allowed and uncapped. + * + * @param requested - How many nodes the caller intends to run, when it knows. + * Omit when no count is available; the verdict is then reported uncapped + * (`refused: 0`) because nothing was counted. Meaningless values (zero, + * negative, non-finite) are treated as "not declared". + * + * ⚠️ See the module doc: the returned counts are **advisory** — no replica can + * currently act on them alone to refuse itself. Use `refused > 0` to warn. */ -export function checkMultiNodeAllowed(): { allowed: boolean; reason?: string } { - return registered ? registered.allowMultiNode() : { allowed: true }; +export function checkMultiNodeAllowed(requested?: number): ResolvedMultiNodeVerdict { + const wanted = normalizeCount(requested); + + if (!registered) return { allowed: true, refused: 0, capped: false }; + + const verdict = registered.allowMultiNode(wanted); + + // Outright denial (unlicensed): everything asked for is refused, but this is + // NOT a partial cap — keep `capped` false so consumers can tell the ruled + // licensed-overflow case apart from the unlicensed one. + if (!verdict.allowed) { + return { + allowed: false, + ...(verdict.reason === undefined ? {} : { reason: verdict.reason }), + admitted: 0, + refused: wanted ?? 0, + capped: false, + }; + } + + const cap = + typeof verdict.admitted === 'number' && Number.isFinite(verdict.admitted) + ? Math.max(0, Math.floor(verdict.admitted)) + : undefined; + + // An allowing gate that declared no cap admits whatever was requested. + if (cap === undefined) { + return { + allowed: true, + ...(verdict.reason === undefined ? {} : { reason: verdict.reason }), + refused: 0, + capped: false, + }; + } + + const admitted = wanted === undefined ? cap : Math.min(cap, wanted); + const refused = wanted === undefined ? 0 : Math.max(0, wanted - cap); + + return { + allowed: true, + ...(verdict.reason === undefined ? {} : { reason: verdict.reason }), + admitted, + refused, + capped: refused > 0, + }; } /** Clear the registered gate. For tests. */