From 7d942a75e75998b8fde88324980757532dbc76c4 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 27 Aug 2026 13:02:19 +0000 Subject: [PATCH 1/2] wip(cli): multi-node licence oversell telemetry --- .../serve-multi-node-cap-advisory.pin.test.ts | 79 +++++++ .../serve-multi-node-cap-telemetry.test.ts | 210 ++++++++++++++++++ packages/cli/src/commands/serve.ts | 176 +++++++++++++++ packages/observability/src/semconv.ts | 49 ++++ .../service-cluster/src/multi-node-gate.ts | 19 ++ 5 files changed, 533 insertions(+) create mode 100644 packages/cli/src/commands/serve-multi-node-cap-telemetry.test.ts diff --git a/packages/cli/src/commands/serve-multi-node-cap-advisory.pin.test.ts b/packages/cli/src/commands/serve-multi-node-cap-advisory.pin.test.ts index 7f2883cb5e..dd9bdf3c5f 100644 --- a/packages/cli/src/commands/serve-multi-node-cap-advisory.pin.test.ts +++ b/packages/cli/src/commands/serve-multi-node-cap-advisory.pin.test.ts @@ -214,3 +214,82 @@ describe('the shape assertions ignore a comment that quotes the old call (#10514 expect(maskComments(regressed)).not.toMatch(/checkMultiNodeAllowed\(\s*[^)\s]/); }); }); + +/** + * THE SECOND PIN: serve's declared-count normalization still matches the gate's + * own, byte for byte modulo comments and whitespace. + * + * The telemetry reading (#12667) has to report the count the operator DECLARED, + * and the resolved verdict cannot give it back: `admitted` is `min(cap, + * wanted)`, so `{admitted: 3, refused: 0}` is produced BOTH by "declared 3 under + * a cap of 5" and by "declared nothing under a cap of 3". The declaration is + * only knowable from `OS_CLUSTER_REPLICAS`, so `serve.ts` normalizes that value + * itself — and a normalization that disagrees with the gate's would publish a + * declaration the gate never saw (a `0` or a `2.7` the gate had already thrown + * away as "not declared"). + * + * Both sides are read from the file that OWNS each, for the same reason the + * shape pin above is: an expected rule re-typed here would just relocate the + * divergence into this file, where it would be equally silent. + */ + +/** + * The brace-matched body of a top-level `function (…) … { … }`, with + * comments blanked and whitespace collapsed, so two implementations can be + * compared on what they DO. + * + * The first `{` after the declaration is taken as the body opener — true for + * both functions compared below (neither has an object type or a destructured + * parameter in its signature); a signature that grows one would need the scan + * to skip the parameter list first, and would fail loudly here rather than + * quietly compare the wrong span. + */ +function functionBody(source: string, name: string): string { + const masked = maskComments(source); + const at = masked.indexOf(`function ${name}(`); + expect(at, `function ${name} not found — did it move or get renamed?`).toBeGreaterThan(-1); + + const open = masked.indexOf('{', at); + expect(open, `function ${name} has no body brace`).toBeGreaterThan(-1); + + let depth = 1; + let i = open + 1; + for (; i < masked.length && depth > 0; i++) { + if (masked[i] === '{') depth++; + else if (masked[i] === '}') depth--; + } + expect(depth, `function ${name} body is unbalanced`).toBe(0); + + return masked.slice(open + 1, i - 1).replace(/\s+/g, ' ').trim(); +} + +describe('os serve ↔ multi-node gate: the declared-count rule', () => { + it("serve's `normalizeDeclaredNodeCount` still mirrors the gate's `normalizeCount`", () => { + const producer = functionBody(GATE_SOURCE, 'normalizeCount'); + const consumer = functionBody(SERVE_SOURCE, 'normalizeDeclaredNodeCount'); + + // Guard the extractor: two empty bodies would agree vacuously. + expect(producer).toContain('Number.isFinite'); + expect(producer).toContain('Math.floor'); + expect(producer.length).toBeGreaterThan(40); + + expect( + consumer, + 'packages/services/service-cluster/src/multi-node-gate.ts changed how it decides ' + + 'whether a requested node count counts as DECLARED. serve.ts mirrors that rule by ' + + 'hand (no static dependency) so its operator telemetry reports the same declaration ' + + 'the gate saw — update `normalizeDeclaredNodeCount` in ' + + 'packages/cli/src/commands/serve.ts to match.', + ).toEqual(producer); + }); + + it('the telemetry reading is fed from the DECLARED env var, not from a count of anything', () => { + // The whole card turns on this: `OS_CLUSTER_REPLICAS` is what the operator + // wrote, identical in every replica. There is no membership count to read + // instead, and a future edit that reached for one would be publishing a + // number this process cannot know. + expect(MASKED_SERVE_SOURCE).toMatch( + /describeMultiNodeCapTelemetry\(\s*verdict\s*,\s*Number\(process\.env\.OS_CLUSTER_REPLICAS\)\s*\)/, + ); + }); +}); diff --git a/packages/cli/src/commands/serve-multi-node-cap-telemetry.test.ts b/packages/cli/src/commands/serve-multi-node-cap-telemetry.test.ts new file mode 100644 index 0000000000..c690241e85 --- /dev/null +++ b/packages/cli/src/commands/serve-multi-node-cap-telemetry.test.ts @@ -0,0 +1,210 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * What an OPERATOR sees when `os serve` publishes the multi-node licence + * reading as telemetry (#12667 — maintainer ruling 2026-08-27, verbatim + * 「其他接受」, adopting option C on the `max_nodes` decision: make a licensed + * oversell VISIBLE; the atomic slot-claim enforcement mechanism is deliberately + * not built). + * + * These assertions are about the published SURFACE — the exact series, values + * and labels a dashboard receives — not about a value having been computed. + * The distinction matters here more than usual: the deliverable of this card is + * a reading that is honest about what the process can and cannot know, so a + * test that only checked "a number came out" would pass over every regression + * worth catching. + * + * ⚠️ The three facts that make this visibility and not enforcement, re-measured + * on the tree this landed against: + * + * - the gate is consulted ONCE PER PROCESS at boot (`serve.ts`); + * - there is NO cluster membership view — `generateNodeId` is random per + * process and there is no join/leave registry (`cluster.ts`); + * - `OS_CLUSTER_REPLICAS` is an operator-DECLARED count, identical in every + * replica (`split-brain-guard.ts`). + * + * So nothing in the process knows how many peers exist, and the surface must + * not read as though it does. The "honest naming" block at the bottom pins + * that, because it is the regression most likely to arrive later as a helpful + * wording change. + */ + +import { describe, it, expect } from 'vitest'; +import { + describeMultiNodeCapTelemetry, + type MultiNodeCapMetric, + type MultiNodeGateVerdict, +} from './serve.js'; + +/** + * The four verdicts the producer can hand this consumer, spelled the way + * `checkMultiNodeAllowed` builds them (`multi-node-gate.ts`). Same fixtures as + * `serve-multi-node-cap-advisory.test.ts`, deliberately: the two reaches of one + * advisory must be tested against one set of inputs, or they can drift into + * telling an operator two different stories about the same boot. + */ +const VERDICTS = { + /** No gate registered, or an allowing gate that declared no cap. */ + uncapped: { allowed: true, refused: 0, capped: false }, + /** A cap exists and the declared topology fits inside it. */ + withinCap: { allowed: true, admitted: 3, refused: 0, capped: false }, + /** The licensed-overflow case: 5 declared, 3 paid for. */ + overflow: { allowed: true, admitted: 3, refused: 2, capped: true }, + /** Unlicensed: the whole cluster is denied. `capped` stays false by design. */ + denied: { allowed: false, reason: 'no clustering entitlement', admitted: 0, refused: 5, capped: false }, +} satisfies Record; + +/** `NaN` is what `Number(process.env.OS_CLUSTER_REPLICAS)` yields when unset. */ +const NOT_DECLARED = Number(undefined); + +describe('describeMultiNodeCapTelemetry — the series an operator receives', () => { + it('a licensed overflow publishes the declared count, the admitted count and the boot event', () => { + expect(describeMultiNodeCapTelemetry(VERDICTS.overflow, 5)).toEqual([ + { name: 'cluster_node_cap_verdicts_total', kind: 'counter', value: 1, labels: { verdict: 'capped' } }, + { name: 'cluster_declared_nodes', kind: 'gauge', value: 5, labels: { verdict: 'capped' } }, + { name: 'cluster_admitted_nodes', kind: 'gauge', value: 3, labels: { verdict: 'capped' } }, + ]); + }); + + it('a topology that fits reads `admitted`, with declared and admitted agreeing', () => { + expect(describeMultiNodeCapTelemetry(VERDICTS.withinCap, 3)).toEqual([ + { name: 'cluster_node_cap_verdicts_total', kind: 'counter', value: 1, labels: { verdict: 'admitted' } }, + { name: 'cluster_declared_nodes', kind: 'gauge', value: 3, labels: { verdict: 'admitted' } }, + { name: 'cluster_admitted_nodes', kind: 'gauge', value: 3, labels: { verdict: 'admitted' } }, + ]); + }); + + it('an outright denial reads `refused`, NOT `capped` — the two are different facts', () => { + const samples = describeMultiNodeCapTelemetry(VERDICTS.denied, 5); + expect(samples.every((s) => s.labels.verdict === 'refused')).toBe(true); + expect(samples).toEqual([ + { name: 'cluster_node_cap_verdicts_total', kind: 'counter', value: 1, labels: { verdict: 'refused' } }, + { name: 'cluster_declared_nodes', kind: 'gauge', value: 5, labels: { verdict: 'refused' } }, + { name: 'cluster_admitted_nodes', kind: 'gauge', value: 0, labels: { verdict: 'refused' } }, + ]); + }); + + it('an uncapped gate publishes NO admitted series — a number there would invent a limit', () => { + const samples = describeMultiNodeCapTelemetry(VERDICTS.uncapped, 4); + expect(samples.map((s) => s.name)).toEqual([ + 'cluster_node_cap_verdicts_total', + 'cluster_declared_nodes', + ]); + // Specifically NOT `cluster_admitted_nodes 0`, which would read as "your + // licence admits zero nodes" on a deployment with no cap at all. + expect(samples.find((s) => s.name === 'cluster_admitted_nodes')).toBeUndefined(); + }); + + it('publishes NO declared series when nothing was declared — `0` would be a declaration of zero', () => { + const samples = describeMultiNodeCapTelemetry(VERDICTS.withinCap, NOT_DECLARED); + expect(samples.map((s) => s.name)).toEqual([ + 'cluster_node_cap_verdicts_total', + 'cluster_admitted_nodes', + ]); + expect(samples.find((s) => s.name === 'cluster_declared_nodes')).toBeUndefined(); + }); + + it('the boot event is emitted for every verdict, so a silent series is a CONFIGURATION answer', () => { + // An operator reading a dashboard has to be able to tell "gate consulted, + // everything fine" from "nothing here is instrumented". The counter is + // present in all four cases; absence therefore means the gate was never + // consulted (single-node boot) or no metrics backend is configured. + for (const verdict of Object.values(VERDICTS)) { + const counters = describeMultiNodeCapTelemetry(verdict, 5) + .filter((s) => s.kind === 'counter'); + expect(counters).toHaveLength(1); + expect(counters[0]!.name).toBe('cluster_node_cap_verdicts_total'); + expect(counters[0]!.value).toBe(1); + } + }); +}); + +describe('the declared count is normalized exactly as the gate normalizes its own input', () => { + // `checkMultiNodeAllowed` treats meaningless values (unset, zero, negative, + // non-finite) as "not declared" and floors a fractional one. The reading has + // to agree, or the surface reports a declaration the gate never saw. + const cases: Array<[number, number | undefined]> = [ + [NOT_DECLARED, undefined], + [0, undefined], + [-1, undefined], + [Number.POSITIVE_INFINITY, undefined], + [2.7, 2], + [3, 3], + ]; + + for (const [input, expected] of cases) { + it(`OS_CLUSTER_REPLICAS=${String(input)} → ${expected === undefined ? 'no declared series' : `declared ${expected}`}`, () => { + const declared = describeMultiNodeCapTelemetry(VERDICTS.uncapped, input) + .find((s) => s.name === 'cluster_declared_nodes'); + expect(declared?.value).toBe(expected); + }); + } +}); + +describe('⚠️ the surface never claims observed membership', () => { + /** + * THE assertion this card exists to protect. Every fact the process holds at + * this moment is a DECLARATION or a LICENCE verdict; it has no membership + * view whatsoever. A later "helpful" rename — `cluster_nodes`, + * `cluster_active_nodes`, a label `state="running"` — would turn a true + * reading into a false one while every other test here stayed green, because + * the numbers would not change at all. Only the words would. + */ + const MEMBERSHIP_CLAIMS = + /\b(running|active|live|alive|online|healthy|current|observed|actual|joined|members?|membership|peers?|connected|up)\b/i; + + const ALL_SAMPLES = Object.values(VERDICTS).flatMap((v) => [ + ...describeMultiNodeCapTelemetry(v, 5), + ...describeMultiNodeCapTelemetry(v, NOT_DECLARED), + ]); + + it('guards itself: the sweep actually has samples to inspect', () => { + expect(ALL_SAMPLES.length).toBeGreaterThan(8); + }); + + it('no metric NAME claims an observed count', () => { + for (const sample of ALL_SAMPLES) { + expect( + sample.name, + `"${sample.name}" reads as a count of what is RUNNING. This process has no ` + + 'cluster membership view — nodeId is random per process and there is no ' + + 'join/leave registry — so such a series would be false. Name it for what ' + + 'is known: what the operator DECLARED, and what the licence ADMITS.', + ).not.toMatch(MEMBERSHIP_CLAIMS); + } + }); + + it('no LABEL name or value claims an observed count', () => { + for (const sample of ALL_SAMPLES) { + for (const [key, value] of Object.entries(sample.labels)) { + expect(key).not.toMatch(MEMBERSHIP_CLAIMS); + expect(value).not.toMatch(MEMBERSHIP_CLAIMS); + } + } + }); + + it('the names that ARE published say declared / admitted, and use the gate\'s own vocabulary', () => { + const names = new Set(ALL_SAMPLES.map((s) => s.name)); + expect(names).toEqual(new Set([ + 'cluster_node_cap_verdicts_total', + 'cluster_declared_nodes', + 'cluster_admitted_nodes', + ])); + + // The verdict label is the vocabulary #8367 / PR #8503 landed — not a + // second one invented for the display. + const words = new Set(ALL_SAMPLES.map((s) => s.labels.verdict)); + expect(words).toEqual(new Set(['admitted', 'capped', 'refused'])); + }); + + it('vacuity proof: the sweep DOES reject a membership-flavoured rename', () => { + // Without this, a regex that silently stopped matching would leave the + // three tests above green over exactly the rename they exist to catch. + expect('cluster_active_nodes').toMatch(MEMBERSHIP_CLAIMS); + expect('cluster_nodes_running').toMatch(MEMBERSHIP_CLAIMS); + expect('members').toMatch(MEMBERSHIP_CLAIMS); + // ...and does not reject the honest ones. + expect('cluster_declared_nodes').not.toMatch(MEMBERSHIP_CLAIMS); + expect('cluster_admitted_nodes').not.toMatch(MEMBERSHIP_CLAIMS); + }); +}); diff --git a/packages/cli/src/commands/serve.ts b/packages/cli/src/commands/serve.ts index 3aef836359..6b92daf7b6 100644 --- a/packages/cli/src/commands/serve.ts +++ b/packages/cli/src/commands/serve.ts @@ -91,6 +91,17 @@ import { type ConsoleShaDrift, } from '../utils/console.js'; import dotenvFlow from 'dotenv-flow'; +// Metric NAMES and the metrics-service name, from the package that owns both. +// `buildServeObservability()` below reaches the same package through a dynamic +// `import()` with a "not installed — silently skip" catch; this STATIC import +// adds no new failure mode on top of it, and the measurement is written down +// so the next reader does not have to re-derive it: `@objectstack/runtime` is +// imported statically a few lines above, its index re-exports +// `./observability/index.js`, and that module statically imports +// `@objectstack/observability` for `OBSERVABILITY_METRICS_SERVICE`. So loading +// this command already requires the package to resolve; the dynamic form there +// is about the exporter CLASSES, not about reachability. +import { SEMCONV, OBSERVABILITY_METRICS_SERVICE } from '@objectstack/observability'; // --------------------------------------------------------------------------- // Observability bootstrap for `objectstack serve` @@ -2525,6 +2536,13 @@ export default class Serve extends Command { // only the in-memory driver — remote drivers (e.g. redis) come from the EE // distribution; if absent we fall back to the in-memory cluster. let clusterConfig: { driver: string; url?: string } | undefined; + // The gate's verdict, held for the operator-facing telemetry emitted near + // the end of boot (#12667). The gate is consulted exactly once per + // process and its answer is otherwise consumed on the spot by the boot + // warning, so a second consult later would be a second question, not a + // second reading of the same one. `undefined` means the gate was never + // consulted — a single-node boot — and nothing is emitted. + let multiNodeVerdict: MultiNodeGateVerdict | undefined; const __clusterDriver = process.env.OS_CLUSTER_DRIVER?.trim(); if (__clusterDriver && __clusterDriver !== 'memory') { // Multi-node authorization gate (open mechanism): a distribution (e.g. @@ -2561,6 +2579,10 @@ export default class Serve extends Command { // declared" — normalization lives at the seam on purpose, so there is // deliberately no `?? 0` or pre-parse here. const __gate = checkMultiNodeAllowed(Number(process.env.OS_CLUSTER_REPLICAS)); + // Held for BOTH branches below: an outright denial is as much an + // operator-facing reading as a licensed overflow, and only one of the + // two currently reaches a log line an operator is likely to still have. + multiNodeVerdict = __gate; if (!__gate.allowed) { console.warn( `[cluster] multi-node not authorized (${__gate.reason ?? 'denied'}) — ` + @@ -4474,6 +4496,21 @@ export default class Serve extends Command { if (Array.isArray(s) && s.length > 0) seedSummary = s; } catch { /* no seeds ran — nothing to show */ } + // ── Multi-node licence reading → telemetry (#12667) ──────────── + // The advisory the gate produced at boot, published where a deployment's + // metrics pipeline already looks. Emitted HERE, after every plugin has + // been registered, so a host that mounts its own + // `ObservabilityServicePlugin` is covered as well as serve's own + // auto-wired one — the auto-wire block runs long before the config + // plugins, and resolving there would have reached only half the hosts. + // + // ⛔ Visibility only. This publishes the SAME advisory verdict the boot + // warning renders as prose; it refuses nothing, counts no peers, and must + // never be reworded into a claim that it does. See + // `describeMultiNodeCapTelemetry` for why a real membership count is not + // available to this process at all. + if (multiNodeVerdict) emitMultiNodeCapTelemetry(kernel, multiNodeVerdict); + // ── Clean startup summary ────────────────────────────────────── // #8978 — the Config:/Artifact: row must name what actually booted, // never `relativeConfig` unconditionally (see resolveBannerConfigRow). @@ -5668,6 +5705,145 @@ export function formatMultiNodeCapAdvisory(verdict: MultiNodeGateVerdict): strin ); } +/** + * The gate verdict as one word, from the vocabulary the gate itself ships + * (`admitted` / `refused` / `capped`) — deliberately not a second vocabulary. + * + * - `refused` — `allowed: false`, the unlicensed case: the whole topology is + * refused and `os serve` downgrades to single-node. + * - `capped` — the licensed-overflow case: entitled to cluster, declared more + * nodes than the licence admits. **Advisory**: every declared replica still + * joins. + * - `admitted` — everything declared fits, or no cap was expressed at all. + */ +export type MultiNodeCapVerdictWord = 'admitted' | 'capped' | 'refused'; + +/** One metric observation the operator surface publishes. */ +export interface MultiNodeCapMetric { + /** Canonical name from `SEMCONV` — never a literal written here. */ + name: string; + kind: 'gauge' | 'counter'; + value: number; + labels: { verdict: MultiNodeCapVerdictWord }; +} + +/** + * A positive, finite, whole node count — or `undefined` for "not declared". + * + * ⚠️ A deliberate MIRROR of `normalizeCount` in + * `@objectstack/service-cluster`'s `multi-node-gate.ts`, kept BYTE-IDENTICAL in + * its body so `serve-multi-node-cap-advisory.pin.test.ts` can compare the two + * bodies derived from their own files rather than against a rule re-typed in a + * test. The mirror exists because the resolved verdict does not carry the + * declared count back: `admitted` is `min(cap, wanted)`, so `{admitted: 3, + * refused: 0}` is produced both by "declared 3 under a cap of 5" and by + * "declared nothing under a cap of 3". The declaration is only knowable from + * the env var the operator wrote, which is what this reads. + */ +function normalizeDeclaredNodeCount(value: number | undefined): number | undefined { + if (typeof value !== 'number' || !Number.isFinite(value) || value <= 0) return undefined; + return Math.floor(value); +} + +/** + * The operator-facing TELEMETRY reading of a multi-node gate verdict — the + * second reach of the same advisory the boot warning above renders as prose + * (#12667, ruled C on #8501, maintainer 2026-08-27 「其他接受」). + * + * ⚠️ **Visibility, not enforcement, and the shape says so.** Nothing here + * refuses anything: the gate is consulted once per process at boot, every + * replica computes the same verdict, and none can know whether it is one of the + * admitted ones — so all of them join. The two facts that make a real count + * impossible are unchanged: there is no cluster membership view (`nodeId` is + * random per process, no join/leave registry) and `OS_CLUSTER_REPLICAS` is an + * operator-DECLARED count identical in every replica. Hence + * `cluster_declared_nodes` and `cluster_admitted_nodes`: "declared vs what the + * licence admits" is an honest sentence, "N nodes running" is not, and would + * be a false statement dressed as telemetry. + * + * Why prose alone was not enough reach: the boot warning is one `console.warn` + * in one process's startup output. An operator who scaled past their cap three + * weeks ago has no way to ask the question today, and no way to alert on it. + * These observations put the same reading where the deployment's existing + * metrics pipeline already looks. + * + * Deliberate omissions, each of which would be a false number rather than a + * missing one: + * + * - no `cluster_declared_nodes` when nothing was declared — `0` would read + * as a declaration of zero replicas; + * - no `cluster_admitted_nodes` when the gate expressed no count cap + * (`admitted` absent) — any number there invents a limit nobody stated; + * - no series at all when the gate was never consulted (single-node boot). + * + * Pure and exported so the test suite can assert **what an operator sees** for + * each verdict rather than that some value was computed. + * + * @param verdict - the verdict `checkMultiNodeAllowed` returned at boot. + * @param declaredReplicas - `Number(process.env.OS_CLUSTER_REPLICAS)`, raw: + * normalization is this function's job, exactly as it is the gate's at its + * own seam. `NaN` (the variable is unset) means "not declared". + */ +export function describeMultiNodeCapTelemetry( + verdict: MultiNodeGateVerdict, + declaredReplicas: number, +): MultiNodeCapMetric[] { + const verdictWord: MultiNodeCapVerdictWord = + !verdict.allowed ? 'refused' : verdict.capped ? 'capped' : 'admitted'; + const labels = { verdict: verdictWord }; + + // The boot EVENT first: it is the one series that survives a push-based + // exporter's staleness window, so it is what an alert can be written against. + const metrics: MultiNodeCapMetric[] = [ + { name: SEMCONV.clusterNodeCapVerdictsTotal, kind: 'counter', value: 1, labels }, + ]; + + const declared = normalizeDeclaredNodeCount(declaredReplicas); + if (declared !== undefined) { + metrics.push({ name: SEMCONV.clusterDeclaredNodes, kind: 'gauge', value: declared, labels }); + } + if (typeof verdict.admitted === 'number') { + metrics.push({ name: SEMCONV.clusterAdmittedNodes, kind: 'gauge', value: verdict.admitted, labels }); + } + return metrics; +} + +/** + * Publish {@link describeMultiNodeCapTelemetry}'s observations into whatever + * metrics backend the deployment configured. + * + * Best-effort by contract: a metric call site must never throw, and an + * unconfigured deployment must not be worse off than before this existed — with + * no backend registered the boot warning remains the only reading, which is + * precisely the state this card set out to improve on rather than replace. + * + * ⚠️ `kernel.getService` THROWS on a miss (see `packages/core/src/kernel.ts`), + * so the lookup is wrapped rather than null-checked. Resolving through the + * kernel — not through the block `serve` built itself — is what lets a host + * that mounts its OWN `ObservabilityServicePlugin` receive these too. + */ +function emitMultiNodeCapTelemetry(kernel: any, verdict: MultiNodeGateVerdict): void { + let metrics: any; + try { + metrics = kernel?.getService?.(OBSERVABILITY_METRICS_SERVICE); + } catch { + return; // no observability backend configured — boot warning stands alone + } + if (!metrics) return; + try { + for (const sample of describeMultiNodeCapTelemetry(verdict, Number(process.env.OS_CLUSTER_REPLICAS))) { + // The two signatures differ in argument ORDER (`MetricsRegistry` in + // @objectstack/observability): counter(name, labels, value) vs + // gauge(name, value, labels). Swapping them type-checks under an `any` + // registry and emits garbage, so they are written out separately. + if (sample.kind === 'counter') metrics.counter?.(sample.name, sample.labels, sample.value); + else metrics.gauge?.(sample.name, sample.value, sample.labels); + } + } catch { + // Per the metrics contract: never throw from a call site. + } +} + /** * Best-effort driver introspection. * diff --git a/packages/observability/src/semconv.ts b/packages/observability/src/semconv.ts index 31e15edae4..6d11794d40 100644 --- a/packages/observability/src/semconv.ts +++ b/packages/observability/src/semconv.ts @@ -111,6 +111,55 @@ export const SEMCONV = { registryLookupDurationMs: 'registry_lookup_duration_ms', /** Counter, labels: `source` (`r2`|`http`|`local`), `result` (`hit`|`miss`|`error`). */ registrySourceFetchesTotal: 'registry_source_fetches_total', + + // ── Cluster licensing — emitted by `@objectstack/cli`'s `os serve` at boot ── + // + // ⚠️ Every name in this group is a reading of what the operator DECLARED + // and what the licence gate SAID about that declaration. **None of them is + // a membership reading**, and none may be renamed into one. At the moment + // `os serve` consults the multi-node gate, the process has no cluster + // membership view at all: `nodeId` is generated randomly per process, there + // is no join/leave registry, and the only count in existence is + // `OS_CLUSTER_REPLICAS` — an operator-declared desired count that is + // identical in every replica. A series called `cluster_nodes` or + // `cluster_active_nodes` fed from here would be a false statement dressed + // as telemetry, so the honesty lives in the NAMES: `declared` is what the + // operator wrote, `admitted` is what the gate says fits the licence. + // + // The cap these describe is ADVISORY and is not enforced: no replica can + // act on the verdict alone to refuse itself, so a `capped` verdict means + // every declared replica still joins. Read a `capped` series as "this + // deployment is configured beyond what it paid for", never as "replicas + // were turned away". + // + // ⚠️ Absence is meaningful and is NOT an instrumentation gap: the gate is + // consulted only when `OS_CLUSTER_DRIVER` names a remote driver, so a + // single-node deployment emits nothing here at all. An emission also needs + // a configured metrics backend (`OS_OBS_EXPORTER`); with none, the boot log + // line remains the only reading. + /** + * Gauge, labels: `verdict` (`admitted`|`capped`|`refused`). The replica + * count the operator DECLARED via `OS_CLUSTER_REPLICAS` — not a count of + * anything observed. Absent when nothing was declared; deliberately never + * emitted as `0`, which would read as a declaration of zero replicas. + */ + clusterDeclaredNodes: 'cluster_declared_nodes', + /** + * Gauge, labels: `verdict` (`admitted`|`capped`|`refused`). How many of the + * declared nodes the licence gate admits. Absent when the gate imposes no + * count cap at all — emitting a number there would invent a limit that was + * never expressed. + */ + clusterAdmittedNodes: 'cluster_admitted_nodes', + /** + * Counter, labels: `verdict` (`admitted`|`capped`|`refused`). One + * increment per PROCESS BOOT that consulted the gate — a boot-event count, + * never a node count. It exists because the two gauges above are written + * once per boot and a push-based exporter lets a one-shot gauge age out of + * the backend; `increase(cluster_node_cap_verdicts_total{verdict="capped"}[1h]) > 0` + * stays alertable after the gauges have gone stale. + */ + clusterNodeCapVerdictsTotal: 'cluster_node_cap_verdicts_total', } as const; /** diff --git a/packages/services/service-cluster/src/multi-node-gate.ts b/packages/services/service-cluster/src/multi-node-gate.ts index 717677ef91..7acbe92137 100644 --- a/packages/services/service-cluster/src/multi-node-gate.ts +++ b/packages/services/service-cluster/src/multi-node-gate.ts @@ -67,6 +67,25 @@ * Until it lands, a consumer should treat `refused > 0` as the trigger for the * **loud warning** the ruling requires, not as a licence to deny the cluster. * + * ## What a consumer may do with the verdict TODAY (visibility, not enforcement) + * + * Maintainer ruling 2026-08-27, verbatim 「其他接受」 (adopting option C): a + * licensed `max_nodes` oversell is made **visible to operators**, and the + * atomic slot claim above is deliberately not built. `os serve` is the sole + * runtime consumer and gives the verdict two reaches, both read-only: + * + * 1. a loud boot warning (`formatMultiNodeCapAdvisory`), and + * 2. operator telemetry (`describeMultiNodeCapTelemetry`) — the metric family + * `cluster_declared_nodes` / `cluster_admitted_nodes` / + * `cluster_node_cap_verdicts_total` in `@objectstack/observability`'s + * `SEMCONV`. + * + * ⛔ Both are worded — and NAMED — around what the operator *declared* and what + * the licence *admits*, never around what is running. Nothing in this process + * knows the latter (that is the whole point of the section above), so a + * consumer that renames a reading into `cluster_nodes`, `active`, `live` or + * `members` is publishing a false statement, not clarifying a clumsy one. + * * ## Why not a per-node admission callback * * A `admitNode(nodeId): boolean` shape was considered and rejected on From b7d45784cebf6fb25d7ff66c4921b999555e9474 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 27 Aug 2026 13:34:52 +0000 Subject: [PATCH 2/2] feat(cli,observability): publish the licensed max_nodes oversell as operator telemetry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extends the reach of the boot-time cap advisory: the same advisory verdict is now also emitted through the deployment's configured metrics backend, so an operator can ask the question and alert on it long after the boot log scrolled away. Visibility only. The gate stays advisory, nothing is refused, and the surface is named around what the operator DECLARED and what the licence ADMITS — this process has no cluster membership view at all. --- .changeset/lucky-donkeys-vanish.md | 35 +++++++++++++++++++ .../serve-multi-node-cap-telemetry.test.ts | 11 +++++- packages/cli/src/commands/serve.ts | 27 ++++++++------ 3 files changed, 62 insertions(+), 11 deletions(-) create mode 100644 .changeset/lucky-donkeys-vanish.md diff --git a/.changeset/lucky-donkeys-vanish.md b/.changeset/lucky-donkeys-vanish.md new file mode 100644 index 0000000000..bc91a5e8c5 --- /dev/null +++ b/.changeset/lucky-donkeys-vanish.md @@ -0,0 +1,35 @@ +--- +"@objectstack/observability": minor +"@objectstack/cli": minor +--- + +Surface a licensed `max_nodes` oversell to operators as telemetry + +`os serve` already warned loudly at boot when `OS_CLUSTER_REPLICAS` declared more +nodes than the licence gate admits, but that warning existed only in one +process's startup output: an operator who scaled past their cap three weeks ago +had no way to ask the question today, and no way to alert on it. The same +advisory verdict is now also published through the deployment's configured +metrics backend, so it reaches the place operators already look. + +Three names join `SEMCONV` in `@objectstack/observability`, emitted once per boot +by `os serve` when a remote cluster driver is configured, each labelled with the +gate's own verdict vocabulary (`admitted` / `capped` / `refused`): + +- `cluster_declared_nodes` (gauge) — the replica count the operator **declared**; +- `cluster_admitted_nodes` (gauge) — how many of them the licence **admits**; +- `cluster_node_cap_verdicts_total` (counter) — one increment per process boot + that consulted the gate, so an alert stays writable after a one-shot gauge has + aged out of a push-based backend. + +**Visibility only — the cap remains advisory and nothing is refused.** The gate is +consulted once per process at boot, every replica computes the same verdict, and +none can know whether it is one of the admitted ones, so all of them still join. +The names say so on purpose: this process has no cluster membership view at all, +so a series called `cluster_nodes` or `cluster_active_nodes` would be a false +statement dressed as telemetry. Nothing here counts peers, and no accept/reject +behaviour changed. + +Absence is meaningful rather than an instrumentation gap: a single-node +deployment never consults the gate and emits nothing, and an emission also needs +a metrics backend configured via `OS_OBS_EXPORTER`. diff --git a/packages/cli/src/commands/serve-multi-node-cap-telemetry.test.ts b/packages/cli/src/commands/serve-multi-node-cap-telemetry.test.ts index c690241e85..bb4552eca9 100644 --- a/packages/cli/src/commands/serve-multi-node-cap-telemetry.test.ts +++ b/packages/cli/src/commands/serve-multi-node-cap-telemetry.test.ts @@ -150,8 +150,16 @@ describe('⚠️ the surface never claims observed membership', () => { * reading into a false one while every other test here stayed green, because * the numbers would not change at all. Only the words would. */ + // + // ⚠️ The segment anchors are `[^a-z0-9]`, NOT `\b`. This regex was first + // written with `\b` and the vacuity proof at the bottom caught it + // immediately: `_` is a WORD character, so `\bactive\b` does not match + // inside `cluster_active_nodes` — the exact rename this guard exists to + // reject would have sailed through while all three sweeps below reported + // green. Metric names are snake_case, so the separator has to be treated as + // a boundary explicitly. const MEMBERSHIP_CLAIMS = - /\b(running|active|live|alive|online|healthy|current|observed|actual|joined|members?|membership|peers?|connected|up)\b/i; + /(?:^|[^a-z0-9])(?:running|active|live|alive|online|healthy|current|observed|actual|joined|members?|membership|peers?|connected|up)(?:[^a-z0-9]|$)/i; const ALL_SAMPLES = Object.values(VERDICTS).flatMap((v) => [ ...describeMultiNodeCapTelemetry(v, 5), @@ -202,6 +210,7 @@ describe('⚠️ the surface never claims observed membership', () => { // three tests above green over exactly the rename they exist to catch. expect('cluster_active_nodes').toMatch(MEMBERSHIP_CLAIMS); expect('cluster_nodes_running').toMatch(MEMBERSHIP_CLAIMS); + expect('cluster_live_members').toMatch(MEMBERSHIP_CLAIMS); expect('members').toMatch(MEMBERSHIP_CLAIMS); // ...and does not reject the honest ones. expect('cluster_declared_nodes').not.toMatch(MEMBERSHIP_CLAIMS); diff --git a/packages/cli/src/commands/serve.ts b/packages/cli/src/commands/serve.ts index 6b92daf7b6..4b51811ed7 100644 --- a/packages/cli/src/commands/serve.ts +++ b/packages/cli/src/commands/serve.ts @@ -101,7 +101,7 @@ import dotenvFlow from 'dotenv-flow'; // `@objectstack/observability` for `OBSERVABILITY_METRICS_SERVICE`. So loading // this command already requires the package to resolve; the dynamic form there // is about the exporter CLASSES, not about reachability. -import { SEMCONV, OBSERVABILITY_METRICS_SERVICE } from '@objectstack/observability'; +import { SEMCONV, OBSERVABILITY_METRICS_SERVICE, type MetricsRegistry } from '@objectstack/observability'; // --------------------------------------------------------------------------- // Observability bootstrap for `objectstack serve` @@ -5822,22 +5822,29 @@ export function describeMultiNodeCapTelemetry( * kernel — not through the block `serve` built itself — is what lets a host * that mounts its OWN `ObservabilityServicePlugin` receive these too. */ -function emitMultiNodeCapTelemetry(kernel: any, verdict: MultiNodeGateVerdict): void { - let metrics: any; +function emitMultiNodeCapTelemetry( + kernel: { getService?: (name: string) => T } | undefined, + verdict: MultiNodeGateVerdict, +): void { + // Typed with the slot's contract, never erased to `any` (#4127/#4251, + // `check:slot-lookup`): the erasure would also have switched off the argument + // check the two calls below depend on — see their note. + let metrics: MetricsRegistry | undefined; try { - metrics = kernel?.getService?.(OBSERVABILITY_METRICS_SERVICE); + metrics = kernel?.getService?.(OBSERVABILITY_METRICS_SERVICE); } catch { return; // no observability backend configured — boot warning stands alone } if (!metrics) return; try { for (const sample of describeMultiNodeCapTelemetry(verdict, Number(process.env.OS_CLUSTER_REPLICAS))) { - // The two signatures differ in argument ORDER (`MetricsRegistry` in - // @objectstack/observability): counter(name, labels, value) vs - // gauge(name, value, labels). Swapping them type-checks under an `any` - // registry and emits garbage, so they are written out separately. - if (sample.kind === 'counter') metrics.counter?.(sample.name, sample.labels, sample.value); - else metrics.gauge?.(sample.name, sample.value, sample.labels); + // The two signatures differ in argument ORDER (`MetricsRegistry`): + // counter(name, labels, value) vs gauge(name, value, labels). Written out + // separately so the swap is visible — and, because the lookup above is + // typed, a swap is now a compile error rather than a green build emitting + // a label map where a number belongs. + if (sample.kind === 'counter') metrics.counter(sample.name, sample.labels, sample.value); + else metrics.gauge(sample.name, sample.value, sample.labels); } } catch { // Per the metrics contract: never throw from a call site.