diff --git a/.changeset/lucky-donkeys-vanish.md b/.changeset/lucky-donkeys-vanish.md new file mode 100644 index 0000000000..bc91a5e8c5 --- /dev/null +++ b/.changeset/lucky-donkeys-vanish.md @@ -0,0 +1,35 @@ +--- +"@objectstack/observability": minor +"@objectstack/cli": minor +--- + +Surface a licensed `max_nodes` oversell to operators as telemetry + +`os serve` already warned loudly at boot when `OS_CLUSTER_REPLICAS` declared more +nodes than the licence gate admits, but that warning existed only in one +process's startup output: an operator who scaled past their cap three weeks ago +had no way to ask the question today, and no way to alert on it. The same +advisory verdict is now also published through the deployment's configured +metrics backend, so it reaches the place operators already look. + +Three names join `SEMCONV` in `@objectstack/observability`, emitted once per boot +by `os serve` when a remote cluster driver is configured, each labelled with the +gate's own verdict vocabulary (`admitted` / `capped` / `refused`): + +- `cluster_declared_nodes` (gauge) — the replica count the operator **declared**; +- `cluster_admitted_nodes` (gauge) — how many of them the licence **admits**; +- `cluster_node_cap_verdicts_total` (counter) — one increment per process boot + that consulted the gate, so an alert stays writable after a one-shot gauge has + aged out of a push-based backend. + +**Visibility only — the cap remains advisory and nothing is refused.** The gate is +consulted once per process at boot, every replica computes the same verdict, and +none can know whether it is one of the admitted ones, so all of them still join. +The names say so on purpose: this process has no cluster membership view at all, +so a series called `cluster_nodes` or `cluster_active_nodes` would be a false +statement dressed as telemetry. Nothing here counts peers, and no accept/reject +behaviour changed. + +Absence is meaningful rather than an instrumentation gap: a single-node +deployment never consults the gate and emits nothing, and an emission also needs +a metrics backend configured via `OS_OBS_EXPORTER`. diff --git a/packages/cli/src/commands/serve-multi-node-cap-advisory.pin.test.ts b/packages/cli/src/commands/serve-multi-node-cap-advisory.pin.test.ts index 7f2883cb5e..dd9bdf3c5f 100644 --- a/packages/cli/src/commands/serve-multi-node-cap-advisory.pin.test.ts +++ b/packages/cli/src/commands/serve-multi-node-cap-advisory.pin.test.ts @@ -214,3 +214,82 @@ describe('the shape assertions ignore a comment that quotes the old call (#10514 expect(maskComments(regressed)).not.toMatch(/checkMultiNodeAllowed\(\s*[^)\s]/); }); }); + +/** + * THE SECOND PIN: serve's declared-count normalization still matches the gate's + * own, byte for byte modulo comments and whitespace. + * + * The telemetry reading (#12667) has to report the count the operator DECLARED, + * and the resolved verdict cannot give it back: `admitted` is `min(cap, + * wanted)`, so `{admitted: 3, refused: 0}` is produced BOTH by "declared 3 under + * a cap of 5" and by "declared nothing under a cap of 3". The declaration is + * only knowable from `OS_CLUSTER_REPLICAS`, so `serve.ts` normalizes that value + * itself — and a normalization that disagrees with the gate's would publish a + * declaration the gate never saw (a `0` or a `2.7` the gate had already thrown + * away as "not declared"). + * + * Both sides are read from the file that OWNS each, for the same reason the + * shape pin above is: an expected rule re-typed here would just relocate the + * divergence into this file, where it would be equally silent. + */ + +/** + * The brace-matched body of a top-level `function (…) … { … }`, with + * comments blanked and whitespace collapsed, so two implementations can be + * compared on what they DO. + * + * The first `{` after the declaration is taken as the body opener — true for + * both functions compared below (neither has an object type or a destructured + * parameter in its signature); a signature that grows one would need the scan + * to skip the parameter list first, and would fail loudly here rather than + * quietly compare the wrong span. + */ +function functionBody(source: string, name: string): string { + const masked = maskComments(source); + const at = masked.indexOf(`function ${name}(`); + expect(at, `function ${name} not found — did it move or get renamed?`).toBeGreaterThan(-1); + + const open = masked.indexOf('{', at); + expect(open, `function ${name} has no body brace`).toBeGreaterThan(-1); + + let depth = 1; + let i = open + 1; + for (; i < masked.length && depth > 0; i++) { + if (masked[i] === '{') depth++; + else if (masked[i] === '}') depth--; + } + expect(depth, `function ${name} body is unbalanced`).toBe(0); + + return masked.slice(open + 1, i - 1).replace(/\s+/g, ' ').trim(); +} + +describe('os serve ↔ multi-node gate: the declared-count rule', () => { + it("serve's `normalizeDeclaredNodeCount` still mirrors the gate's `normalizeCount`", () => { + const producer = functionBody(GATE_SOURCE, 'normalizeCount'); + const consumer = functionBody(SERVE_SOURCE, 'normalizeDeclaredNodeCount'); + + // Guard the extractor: two empty bodies would agree vacuously. + expect(producer).toContain('Number.isFinite'); + expect(producer).toContain('Math.floor'); + expect(producer.length).toBeGreaterThan(40); + + expect( + consumer, + 'packages/services/service-cluster/src/multi-node-gate.ts changed how it decides ' + + 'whether a requested node count counts as DECLARED. serve.ts mirrors that rule by ' + + 'hand (no static dependency) so its operator telemetry reports the same declaration ' + + 'the gate saw — update `normalizeDeclaredNodeCount` in ' + + 'packages/cli/src/commands/serve.ts to match.', + ).toEqual(producer); + }); + + it('the telemetry reading is fed from the DECLARED env var, not from a count of anything', () => { + // The whole card turns on this: `OS_CLUSTER_REPLICAS` is what the operator + // wrote, identical in every replica. There is no membership count to read + // instead, and a future edit that reached for one would be publishing a + // number this process cannot know. + expect(MASKED_SERVE_SOURCE).toMatch( + /describeMultiNodeCapTelemetry\(\s*verdict\s*,\s*Number\(process\.env\.OS_CLUSTER_REPLICAS\)\s*\)/, + ); + }); +}); diff --git a/packages/cli/src/commands/serve-multi-node-cap-telemetry.test.ts b/packages/cli/src/commands/serve-multi-node-cap-telemetry.test.ts new file mode 100644 index 0000000000..bb4552eca9 --- /dev/null +++ b/packages/cli/src/commands/serve-multi-node-cap-telemetry.test.ts @@ -0,0 +1,219 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * What an OPERATOR sees when `os serve` publishes the multi-node licence + * reading as telemetry (#12667 — maintainer ruling 2026-08-27, verbatim + * 「其他接受」, adopting option C on the `max_nodes` decision: make a licensed + * oversell VISIBLE; the atomic slot-claim enforcement mechanism is deliberately + * not built). + * + * These assertions are about the published SURFACE — the exact series, values + * and labels a dashboard receives — not about a value having been computed. + * The distinction matters here more than usual: the deliverable of this card is + * a reading that is honest about what the process can and cannot know, so a + * test that only checked "a number came out" would pass over every regression + * worth catching. + * + * ⚠️ The three facts that make this visibility and not enforcement, re-measured + * on the tree this landed against: + * + * - the gate is consulted ONCE PER PROCESS at boot (`serve.ts`); + * - there is NO cluster membership view — `generateNodeId` is random per + * process and there is no join/leave registry (`cluster.ts`); + * - `OS_CLUSTER_REPLICAS` is an operator-DECLARED count, identical in every + * replica (`split-brain-guard.ts`). + * + * So nothing in the process knows how many peers exist, and the surface must + * not read as though it does. The "honest naming" block at the bottom pins + * that, because it is the regression most likely to arrive later as a helpful + * wording change. + */ + +import { describe, it, expect } from 'vitest'; +import { + describeMultiNodeCapTelemetry, + type MultiNodeCapMetric, + type MultiNodeGateVerdict, +} from './serve.js'; + +/** + * The four verdicts the producer can hand this consumer, spelled the way + * `checkMultiNodeAllowed` builds them (`multi-node-gate.ts`). Same fixtures as + * `serve-multi-node-cap-advisory.test.ts`, deliberately: the two reaches of one + * advisory must be tested against one set of inputs, or they can drift into + * telling an operator two different stories about the same boot. + */ +const VERDICTS = { + /** No gate registered, or an allowing gate that declared no cap. */ + uncapped: { allowed: true, refused: 0, capped: false }, + /** A cap exists and the declared topology fits inside it. */ + withinCap: { allowed: true, admitted: 3, refused: 0, capped: false }, + /** The licensed-overflow case: 5 declared, 3 paid for. */ + overflow: { allowed: true, admitted: 3, refused: 2, capped: true }, + /** Unlicensed: the whole cluster is denied. `capped` stays false by design. */ + denied: { allowed: false, reason: 'no clustering entitlement', admitted: 0, refused: 5, capped: false }, +} satisfies Record; + +/** `NaN` is what `Number(process.env.OS_CLUSTER_REPLICAS)` yields when unset. */ +const NOT_DECLARED = Number(undefined); + +describe('describeMultiNodeCapTelemetry — the series an operator receives', () => { + it('a licensed overflow publishes the declared count, the admitted count and the boot event', () => { + expect(describeMultiNodeCapTelemetry(VERDICTS.overflow, 5)).toEqual([ + { name: 'cluster_node_cap_verdicts_total', kind: 'counter', value: 1, labels: { verdict: 'capped' } }, + { name: 'cluster_declared_nodes', kind: 'gauge', value: 5, labels: { verdict: 'capped' } }, + { name: 'cluster_admitted_nodes', kind: 'gauge', value: 3, labels: { verdict: 'capped' } }, + ]); + }); + + it('a topology that fits reads `admitted`, with declared and admitted agreeing', () => { + expect(describeMultiNodeCapTelemetry(VERDICTS.withinCap, 3)).toEqual([ + { name: 'cluster_node_cap_verdicts_total', kind: 'counter', value: 1, labels: { verdict: 'admitted' } }, + { name: 'cluster_declared_nodes', kind: 'gauge', value: 3, labels: { verdict: 'admitted' } }, + { name: 'cluster_admitted_nodes', kind: 'gauge', value: 3, labels: { verdict: 'admitted' } }, + ]); + }); + + it('an outright denial reads `refused`, NOT `capped` — the two are different facts', () => { + const samples = describeMultiNodeCapTelemetry(VERDICTS.denied, 5); + expect(samples.every((s) => s.labels.verdict === 'refused')).toBe(true); + expect(samples).toEqual([ + { name: 'cluster_node_cap_verdicts_total', kind: 'counter', value: 1, labels: { verdict: 'refused' } }, + { name: 'cluster_declared_nodes', kind: 'gauge', value: 5, labels: { verdict: 'refused' } }, + { name: 'cluster_admitted_nodes', kind: 'gauge', value: 0, labels: { verdict: 'refused' } }, + ]); + }); + + it('an uncapped gate publishes NO admitted series — a number there would invent a limit', () => { + const samples = describeMultiNodeCapTelemetry(VERDICTS.uncapped, 4); + expect(samples.map((s) => s.name)).toEqual([ + 'cluster_node_cap_verdicts_total', + 'cluster_declared_nodes', + ]); + // Specifically NOT `cluster_admitted_nodes 0`, which would read as "your + // licence admits zero nodes" on a deployment with no cap at all. + expect(samples.find((s) => s.name === 'cluster_admitted_nodes')).toBeUndefined(); + }); + + it('publishes NO declared series when nothing was declared — `0` would be a declaration of zero', () => { + const samples = describeMultiNodeCapTelemetry(VERDICTS.withinCap, NOT_DECLARED); + expect(samples.map((s) => s.name)).toEqual([ + 'cluster_node_cap_verdicts_total', + 'cluster_admitted_nodes', + ]); + expect(samples.find((s) => s.name === 'cluster_declared_nodes')).toBeUndefined(); + }); + + it('the boot event is emitted for every verdict, so a silent series is a CONFIGURATION answer', () => { + // An operator reading a dashboard has to be able to tell "gate consulted, + // everything fine" from "nothing here is instrumented". The counter is + // present in all four cases; absence therefore means the gate was never + // consulted (single-node boot) or no metrics backend is configured. + for (const verdict of Object.values(VERDICTS)) { + const counters = describeMultiNodeCapTelemetry(verdict, 5) + .filter((s) => s.kind === 'counter'); + expect(counters).toHaveLength(1); + expect(counters[0]!.name).toBe('cluster_node_cap_verdicts_total'); + expect(counters[0]!.value).toBe(1); + } + }); +}); + +describe('the declared count is normalized exactly as the gate normalizes its own input', () => { + // `checkMultiNodeAllowed` treats meaningless values (unset, zero, negative, + // non-finite) as "not declared" and floors a fractional one. The reading has + // to agree, or the surface reports a declaration the gate never saw. + const cases: Array<[number, number | undefined]> = [ + [NOT_DECLARED, undefined], + [0, undefined], + [-1, undefined], + [Number.POSITIVE_INFINITY, undefined], + [2.7, 2], + [3, 3], + ]; + + for (const [input, expected] of cases) { + it(`OS_CLUSTER_REPLICAS=${String(input)} → ${expected === undefined ? 'no declared series' : `declared ${expected}`}`, () => { + const declared = describeMultiNodeCapTelemetry(VERDICTS.uncapped, input) + .find((s) => s.name === 'cluster_declared_nodes'); + expect(declared?.value).toBe(expected); + }); + } +}); + +describe('⚠️ the surface never claims observed membership', () => { + /** + * THE assertion this card exists to protect. Every fact the process holds at + * this moment is a DECLARATION or a LICENCE verdict; it has no membership + * view whatsoever. A later "helpful" rename — `cluster_nodes`, + * `cluster_active_nodes`, a label `state="running"` — would turn a true + * reading into a false one while every other test here stayed green, because + * the numbers would not change at all. Only the words would. + */ + // + // ⚠️ The segment anchors are `[^a-z0-9]`, NOT `\b`. This regex was first + // written with `\b` and the vacuity proof at the bottom caught it + // immediately: `_` is a WORD character, so `\bactive\b` does not match + // inside `cluster_active_nodes` — the exact rename this guard exists to + // reject would have sailed through while all three sweeps below reported + // green. Metric names are snake_case, so the separator has to be treated as + // a boundary explicitly. + const MEMBERSHIP_CLAIMS = + /(?:^|[^a-z0-9])(?:running|active|live|alive|online|healthy|current|observed|actual|joined|members?|membership|peers?|connected|up)(?:[^a-z0-9]|$)/i; + + const ALL_SAMPLES = Object.values(VERDICTS).flatMap((v) => [ + ...describeMultiNodeCapTelemetry(v, 5), + ...describeMultiNodeCapTelemetry(v, NOT_DECLARED), + ]); + + it('guards itself: the sweep actually has samples to inspect', () => { + expect(ALL_SAMPLES.length).toBeGreaterThan(8); + }); + + it('no metric NAME claims an observed count', () => { + for (const sample of ALL_SAMPLES) { + expect( + sample.name, + `"${sample.name}" reads as a count of what is RUNNING. This process has no ` + + 'cluster membership view — nodeId is random per process and there is no ' + + 'join/leave registry — so such a series would be false. Name it for what ' + + 'is known: what the operator DECLARED, and what the licence ADMITS.', + ).not.toMatch(MEMBERSHIP_CLAIMS); + } + }); + + it('no LABEL name or value claims an observed count', () => { + for (const sample of ALL_SAMPLES) { + for (const [key, value] of Object.entries(sample.labels)) { + expect(key).not.toMatch(MEMBERSHIP_CLAIMS); + expect(value).not.toMatch(MEMBERSHIP_CLAIMS); + } + } + }); + + it('the names that ARE published say declared / admitted, and use the gate\'s own vocabulary', () => { + const names = new Set(ALL_SAMPLES.map((s) => s.name)); + expect(names).toEqual(new Set([ + 'cluster_node_cap_verdicts_total', + 'cluster_declared_nodes', + 'cluster_admitted_nodes', + ])); + + // The verdict label is the vocabulary #8367 / PR #8503 landed — not a + // second one invented for the display. + const words = new Set(ALL_SAMPLES.map((s) => s.labels.verdict)); + expect(words).toEqual(new Set(['admitted', 'capped', 'refused'])); + }); + + it('vacuity proof: the sweep DOES reject a membership-flavoured rename', () => { + // Without this, a regex that silently stopped matching would leave the + // three tests above green over exactly the rename they exist to catch. + expect('cluster_active_nodes').toMatch(MEMBERSHIP_CLAIMS); + expect('cluster_nodes_running').toMatch(MEMBERSHIP_CLAIMS); + expect('cluster_live_members').toMatch(MEMBERSHIP_CLAIMS); + expect('members').toMatch(MEMBERSHIP_CLAIMS); + // ...and does not reject the honest ones. + expect('cluster_declared_nodes').not.toMatch(MEMBERSHIP_CLAIMS); + expect('cluster_admitted_nodes').not.toMatch(MEMBERSHIP_CLAIMS); + }); +}); diff --git a/packages/cli/src/commands/serve.ts b/packages/cli/src/commands/serve.ts index 3aef836359..4b51811ed7 100644 --- a/packages/cli/src/commands/serve.ts +++ b/packages/cli/src/commands/serve.ts @@ -91,6 +91,17 @@ import { type ConsoleShaDrift, } from '../utils/console.js'; import dotenvFlow from 'dotenv-flow'; +// Metric NAMES and the metrics-service name, from the package that owns both. +// `buildServeObservability()` below reaches the same package through a dynamic +// `import()` with a "not installed — silently skip" catch; this STATIC import +// adds no new failure mode on top of it, and the measurement is written down +// so the next reader does not have to re-derive it: `@objectstack/runtime` is +// imported statically a few lines above, its index re-exports +// `./observability/index.js`, and that module statically imports +// `@objectstack/observability` for `OBSERVABILITY_METRICS_SERVICE`. So loading +// this command already requires the package to resolve; the dynamic form there +// is about the exporter CLASSES, not about reachability. +import { SEMCONV, OBSERVABILITY_METRICS_SERVICE, type MetricsRegistry } from '@objectstack/observability'; // --------------------------------------------------------------------------- // Observability bootstrap for `objectstack serve` @@ -2525,6 +2536,13 @@ export default class Serve extends Command { // only the in-memory driver — remote drivers (e.g. redis) come from the EE // distribution; if absent we fall back to the in-memory cluster. let clusterConfig: { driver: string; url?: string } | undefined; + // The gate's verdict, held for the operator-facing telemetry emitted near + // the end of boot (#12667). The gate is consulted exactly once per + // process and its answer is otherwise consumed on the spot by the boot + // warning, so a second consult later would be a second question, not a + // second reading of the same one. `undefined` means the gate was never + // consulted — a single-node boot — and nothing is emitted. + let multiNodeVerdict: MultiNodeGateVerdict | undefined; const __clusterDriver = process.env.OS_CLUSTER_DRIVER?.trim(); if (__clusterDriver && __clusterDriver !== 'memory') { // Multi-node authorization gate (open mechanism): a distribution (e.g. @@ -2561,6 +2579,10 @@ export default class Serve extends Command { // declared" — normalization lives at the seam on purpose, so there is // deliberately no `?? 0` or pre-parse here. const __gate = checkMultiNodeAllowed(Number(process.env.OS_CLUSTER_REPLICAS)); + // Held for BOTH branches below: an outright denial is as much an + // operator-facing reading as a licensed overflow, and only one of the + // two currently reaches a log line an operator is likely to still have. + multiNodeVerdict = __gate; if (!__gate.allowed) { console.warn( `[cluster] multi-node not authorized (${__gate.reason ?? 'denied'}) — ` + @@ -4474,6 +4496,21 @@ export default class Serve extends Command { if (Array.isArray(s) && s.length > 0) seedSummary = s; } catch { /* no seeds ran — nothing to show */ } + // ── Multi-node licence reading → telemetry (#12667) ──────────── + // The advisory the gate produced at boot, published where a deployment's + // metrics pipeline already looks. Emitted HERE, after every plugin has + // been registered, so a host that mounts its own + // `ObservabilityServicePlugin` is covered as well as serve's own + // auto-wired one — the auto-wire block runs long before the config + // plugins, and resolving there would have reached only half the hosts. + // + // ⛔ Visibility only. This publishes the SAME advisory verdict the boot + // warning renders as prose; it refuses nothing, counts no peers, and must + // never be reworded into a claim that it does. See + // `describeMultiNodeCapTelemetry` for why a real membership count is not + // available to this process at all. + if (multiNodeVerdict) emitMultiNodeCapTelemetry(kernel, multiNodeVerdict); + // ── Clean startup summary ────────────────────────────────────── // #8978 — the Config:/Artifact: row must name what actually booted, // never `relativeConfig` unconditionally (see resolveBannerConfigRow). @@ -5668,6 +5705,152 @@ export function formatMultiNodeCapAdvisory(verdict: MultiNodeGateVerdict): strin ); } +/** + * The gate verdict as one word, from the vocabulary the gate itself ships + * (`admitted` / `refused` / `capped`) — deliberately not a second vocabulary. + * + * - `refused` — `allowed: false`, the unlicensed case: the whole topology is + * refused and `os serve` downgrades to single-node. + * - `capped` — the licensed-overflow case: entitled to cluster, declared more + * nodes than the licence admits. **Advisory**: every declared replica still + * joins. + * - `admitted` — everything declared fits, or no cap was expressed at all. + */ +export type MultiNodeCapVerdictWord = 'admitted' | 'capped' | 'refused'; + +/** One metric observation the operator surface publishes. */ +export interface MultiNodeCapMetric { + /** Canonical name from `SEMCONV` — never a literal written here. */ + name: string; + kind: 'gauge' | 'counter'; + value: number; + labels: { verdict: MultiNodeCapVerdictWord }; +} + +/** + * A positive, finite, whole node count — or `undefined` for "not declared". + * + * ⚠️ A deliberate MIRROR of `normalizeCount` in + * `@objectstack/service-cluster`'s `multi-node-gate.ts`, kept BYTE-IDENTICAL in + * its body so `serve-multi-node-cap-advisory.pin.test.ts` can compare the two + * bodies derived from their own files rather than against a rule re-typed in a + * test. The mirror exists because the resolved verdict does not carry the + * declared count back: `admitted` is `min(cap, wanted)`, so `{admitted: 3, + * refused: 0}` is produced both by "declared 3 under a cap of 5" and by + * "declared nothing under a cap of 3". The declaration is only knowable from + * the env var the operator wrote, which is what this reads. + */ +function normalizeDeclaredNodeCount(value: number | undefined): number | undefined { + if (typeof value !== 'number' || !Number.isFinite(value) || value <= 0) return undefined; + return Math.floor(value); +} + +/** + * The operator-facing TELEMETRY reading of a multi-node gate verdict — the + * second reach of the same advisory the boot warning above renders as prose + * (#12667, ruled C on #8501, maintainer 2026-08-27 「其他接受」). + * + * ⚠️ **Visibility, not enforcement, and the shape says so.** Nothing here + * refuses anything: the gate is consulted once per process at boot, every + * replica computes the same verdict, and none can know whether it is one of the + * admitted ones — so all of them join. The two facts that make a real count + * impossible are unchanged: there is no cluster membership view (`nodeId` is + * random per process, no join/leave registry) and `OS_CLUSTER_REPLICAS` is an + * operator-DECLARED count identical in every replica. Hence + * `cluster_declared_nodes` and `cluster_admitted_nodes`: "declared vs what the + * licence admits" is an honest sentence, "N nodes running" is not, and would + * be a false statement dressed as telemetry. + * + * Why prose alone was not enough reach: the boot warning is one `console.warn` + * in one process's startup output. An operator who scaled past their cap three + * weeks ago has no way to ask the question today, and no way to alert on it. + * These observations put the same reading where the deployment's existing + * metrics pipeline already looks. + * + * Deliberate omissions, each of which would be a false number rather than a + * missing one: + * + * - no `cluster_declared_nodes` when nothing was declared — `0` would read + * as a declaration of zero replicas; + * - no `cluster_admitted_nodes` when the gate expressed no count cap + * (`admitted` absent) — any number there invents a limit nobody stated; + * - no series at all when the gate was never consulted (single-node boot). + * + * Pure and exported so the test suite can assert **what an operator sees** for + * each verdict rather than that some value was computed. + * + * @param verdict - the verdict `checkMultiNodeAllowed` returned at boot. + * @param declaredReplicas - `Number(process.env.OS_CLUSTER_REPLICAS)`, raw: + * normalization is this function's job, exactly as it is the gate's at its + * own seam. `NaN` (the variable is unset) means "not declared". + */ +export function describeMultiNodeCapTelemetry( + verdict: MultiNodeGateVerdict, + declaredReplicas: number, +): MultiNodeCapMetric[] { + const verdictWord: MultiNodeCapVerdictWord = + !verdict.allowed ? 'refused' : verdict.capped ? 'capped' : 'admitted'; + const labels = { verdict: verdictWord }; + + // The boot EVENT first: it is the one series that survives a push-based + // exporter's staleness window, so it is what an alert can be written against. + const metrics: MultiNodeCapMetric[] = [ + { name: SEMCONV.clusterNodeCapVerdictsTotal, kind: 'counter', value: 1, labels }, + ]; + + const declared = normalizeDeclaredNodeCount(declaredReplicas); + if (declared !== undefined) { + metrics.push({ name: SEMCONV.clusterDeclaredNodes, kind: 'gauge', value: declared, labels }); + } + if (typeof verdict.admitted === 'number') { + metrics.push({ name: SEMCONV.clusterAdmittedNodes, kind: 'gauge', value: verdict.admitted, labels }); + } + return metrics; +} + +/** + * Publish {@link describeMultiNodeCapTelemetry}'s observations into whatever + * metrics backend the deployment configured. + * + * Best-effort by contract: a metric call site must never throw, and an + * unconfigured deployment must not be worse off than before this existed — with + * no backend registered the boot warning remains the only reading, which is + * precisely the state this card set out to improve on rather than replace. + * + * ⚠️ `kernel.getService` THROWS on a miss (see `packages/core/src/kernel.ts`), + * so the lookup is wrapped rather than null-checked. Resolving through the + * kernel — not through the block `serve` built itself — is what lets a host + * that mounts its OWN `ObservabilityServicePlugin` receive these too. + */ +function emitMultiNodeCapTelemetry( + kernel: { getService?: (name: string) => T } | undefined, + verdict: MultiNodeGateVerdict, +): void { + // Typed with the slot's contract, never erased to `any` (#4127/#4251, + // `check:slot-lookup`): the erasure would also have switched off the argument + // check the two calls below depend on — see their note. + let metrics: MetricsRegistry | undefined; + try { + metrics = kernel?.getService?.(OBSERVABILITY_METRICS_SERVICE); + } catch { + return; // no observability backend configured — boot warning stands alone + } + if (!metrics) return; + try { + for (const sample of describeMultiNodeCapTelemetry(verdict, Number(process.env.OS_CLUSTER_REPLICAS))) { + // The two signatures differ in argument ORDER (`MetricsRegistry`): + // counter(name, labels, value) vs gauge(name, value, labels). Written out + // separately so the swap is visible — and, because the lookup above is + // typed, a swap is now a compile error rather than a green build emitting + // a label map where a number belongs. + if (sample.kind === 'counter') metrics.counter(sample.name, sample.labels, sample.value); + else metrics.gauge(sample.name, sample.value, sample.labels); + } + } catch { + // Per the metrics contract: never throw from a call site. + } +} + /** * Best-effort driver introspection. * diff --git a/packages/observability/src/semconv.ts b/packages/observability/src/semconv.ts index 31e15edae4..6d11794d40 100644 --- a/packages/observability/src/semconv.ts +++ b/packages/observability/src/semconv.ts @@ -111,6 +111,55 @@ export const SEMCONV = { registryLookupDurationMs: 'registry_lookup_duration_ms', /** Counter, labels: `source` (`r2`|`http`|`local`), `result` (`hit`|`miss`|`error`). */ registrySourceFetchesTotal: 'registry_source_fetches_total', + + // ── Cluster licensing — emitted by `@objectstack/cli`'s `os serve` at boot ── + // + // ⚠️ Every name in this group is a reading of what the operator DECLARED + // and what the licence gate SAID about that declaration. **None of them is + // a membership reading**, and none may be renamed into one. At the moment + // `os serve` consults the multi-node gate, the process has no cluster + // membership view at all: `nodeId` is generated randomly per process, there + // is no join/leave registry, and the only count in existence is + // `OS_CLUSTER_REPLICAS` — an operator-declared desired count that is + // identical in every replica. A series called `cluster_nodes` or + // `cluster_active_nodes` fed from here would be a false statement dressed + // as telemetry, so the honesty lives in the NAMES: `declared` is what the + // operator wrote, `admitted` is what the gate says fits the licence. + // + // The cap these describe is ADVISORY and is not enforced: no replica can + // act on the verdict alone to refuse itself, so a `capped` verdict means + // every declared replica still joins. Read a `capped` series as "this + // deployment is configured beyond what it paid for", never as "replicas + // were turned away". + // + // ⚠️ Absence is meaningful and is NOT an instrumentation gap: the gate is + // consulted only when `OS_CLUSTER_DRIVER` names a remote driver, so a + // single-node deployment emits nothing here at all. An emission also needs + // a configured metrics backend (`OS_OBS_EXPORTER`); with none, the boot log + // line remains the only reading. + /** + * Gauge, labels: `verdict` (`admitted`|`capped`|`refused`). The replica + * count the operator DECLARED via `OS_CLUSTER_REPLICAS` — not a count of + * anything observed. Absent when nothing was declared; deliberately never + * emitted as `0`, which would read as a declaration of zero replicas. + */ + clusterDeclaredNodes: 'cluster_declared_nodes', + /** + * Gauge, labels: `verdict` (`admitted`|`capped`|`refused`). How many of the + * declared nodes the licence gate admits. Absent when the gate imposes no + * count cap at all — emitting a number there would invent a limit that was + * never expressed. + */ + clusterAdmittedNodes: 'cluster_admitted_nodes', + /** + * Counter, labels: `verdict` (`admitted`|`capped`|`refused`). One + * increment per PROCESS BOOT that consulted the gate — a boot-event count, + * never a node count. It exists because the two gauges above are written + * once per boot and a push-based exporter lets a one-shot gauge age out of + * the backend; `increase(cluster_node_cap_verdicts_total{verdict="capped"}[1h]) > 0` + * stays alertable after the gauges have gone stale. + */ + clusterNodeCapVerdictsTotal: 'cluster_node_cap_verdicts_total', } as const; /** diff --git a/packages/services/service-cluster/src/multi-node-gate.ts b/packages/services/service-cluster/src/multi-node-gate.ts index 717677ef91..7acbe92137 100644 --- a/packages/services/service-cluster/src/multi-node-gate.ts +++ b/packages/services/service-cluster/src/multi-node-gate.ts @@ -67,6 +67,25 @@ * Until it lands, a consumer should treat `refused > 0` as the trigger for the * **loud warning** the ruling requires, not as a licence to deny the cluster. * + * ## What a consumer may do with the verdict TODAY (visibility, not enforcement) + * + * Maintainer ruling 2026-08-27, verbatim 「其他接受」 (adopting option C): a + * licensed `max_nodes` oversell is made **visible to operators**, and the + * atomic slot claim above is deliberately not built. `os serve` is the sole + * runtime consumer and gives the verdict two reaches, both read-only: + * + * 1. a loud boot warning (`formatMultiNodeCapAdvisory`), and + * 2. operator telemetry (`describeMultiNodeCapTelemetry`) — the metric family + * `cluster_declared_nodes` / `cluster_admitted_nodes` / + * `cluster_node_cap_verdicts_total` in `@objectstack/observability`'s + * `SEMCONV`. + * + * ⛔ Both are worded — and NAMED — around what the operator *declared* and what + * the licence *admits*, never around what is running. Nothing in this process + * knows the latter (that is the whole point of the section above), so a + * consumer that renames a reading into `cluster_nodes`, `active`, `live` or + * `members` is publishing a false statement, not clarifying a clumsy one. + * * ## Why not a per-node admission callback * * A `admitNode(nodeId): boolean` shape was considered and rejected on