diff --git a/packages/services/service-cluster/src/cluster-service-plugin.ts b/packages/services/service-cluster/src/cluster-service-plugin.ts index f92456d805..01714239c9 100644 --- a/packages/services/service-cluster/src/cluster-service-plugin.ts +++ b/packages/services/service-cluster/src/cluster-service-plugin.ts @@ -4,6 +4,7 @@ import type { Plugin, PluginContext } from '@objectstack/core'; import type { IClusterService } from '@objectstack/spec/contracts'; import type { ClusterCapabilityConfigInput } from '@objectstack/spec/kernel'; import { defineCluster } from './cluster.js'; +import { assertClusterDriverSafeForTopology } from './split-brain-guard.js'; /** * Options for `ClusterServicePlugin`. @@ -59,6 +60,11 @@ export class ClusterServicePlugin implements Plugin { this.cluster = defineCluster(this.options.config ?? {}); this.owned = true; } + // Split-brain guard (ADR-0010): fail fast if a multi-node topology is + // declared but the resolved driver is in-process. Runs before + // registration so a misconfiguration never half-wires the service. + assertClusterDriverSafeForTopology(this.cluster.driver); + ctx.registerService('cluster', this.cluster); ctx.logger.info( `ClusterServicePlugin: registered "${this.cluster.driver}" driver (node=${this.cluster.nodeId})`, diff --git a/packages/services/service-cluster/src/index.ts b/packages/services/service-cluster/src/index.ts index 17df79bef9..52a2df92bf 100644 --- a/packages/services/service-cluster/src/index.ts +++ b/packages/services/service-cluster/src/index.ts @@ -39,6 +39,12 @@ export { type ClusterServicePluginOptions, } from './cluster-service-plugin.js'; +export { + assertClusterDriverSafeForTopology, + declaresMultiNode, + type SplitBrainGuardEnv, +} from './split-brain-guard.js'; + export { MetadataClusterBridgePlugin } from './metadata-cluster-bridge-plugin.js'; // Re-export contracts for convenience. diff --git a/packages/services/service-cluster/src/split-brain-guard.test.ts b/packages/services/service-cluster/src/split-brain-guard.test.ts new file mode 100644 index 0000000000..aaf6ea3385 --- /dev/null +++ b/packages/services/service-cluster/src/split-brain-guard.test.ts @@ -0,0 +1,61 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +import { describe, it, expect } from 'vitest'; +import { + declaresMultiNode, + assertClusterDriverSafeForTopology, +} from './split-brain-guard.js'; + +describe('declaresMultiNode', () => { + it('false when nothing declared', () => { + expect(declaresMultiNode({})).toBe(false); + }); + it('true for OS_EXPECT_MULTI_NODE=true (case-insensitive, trimmed)', () => { + expect(declaresMultiNode({ OS_EXPECT_MULTI_NODE: 'true' })).toBe(true); + expect(declaresMultiNode({ OS_EXPECT_MULTI_NODE: ' TRUE ' })).toBe(true); + }); + it('false for OS_EXPECT_MULTI_NODE=false / other', () => { + expect(declaresMultiNode({ OS_EXPECT_MULTI_NODE: 'false' })).toBe(false); + expect(declaresMultiNode({ OS_EXPECT_MULTI_NODE: '1' })).toBe(false); + }); + it('replicas: >1 true, <=1 false, non-numeric false', () => { + expect(declaresMultiNode({ OS_CLUSTER_REPLICAS: '3' })).toBe(true); + expect(declaresMultiNode({ OS_CLUSTER_REPLICAS: '1' })).toBe(false); + expect(declaresMultiNode({ OS_CLUSTER_REPLICAS: '0' })).toBe(false); + expect(declaresMultiNode({ OS_CLUSTER_REPLICAS: 'abc' })).toBe(false); + }); +}); + +describe('assertClusterDriverSafeForTopology', () => { + it('no throw: single-node + memory (the common case)', () => { + expect(() => assertClusterDriverSafeForTopology('memory', {})).not.toThrow(); + }); + it('no throw: multi-node + remote driver', () => { + expect(() => + assertClusterDriverSafeForTopology('redis', { OS_EXPECT_MULTI_NODE: 'true' }), + ).not.toThrow(); + }); + it('THROWS: multi-node (OS_EXPECT_MULTI_NODE) + memory', () => { + expect(() => + assertClusterDriverSafeForTopology('memory', { OS_EXPECT_MULTI_NODE: 'true' }), + ).toThrow(/split-brain/); + }); + it('THROWS: multi-node (replicas>1) + memory, cites ADR-0010', () => { + expect(() => + assertClusterDriverSafeForTopology('memory', { OS_CLUSTER_REPLICAS: '2' }), + ).toThrow(/ADR-0010/); + }); + it('escape hatch: multi-node + memory + OS_ALLOW_MEMORY_CLUSTER_MULTINODE=true', () => { + expect(() => + assertClusterDriverSafeForTopology('memory', { + OS_EXPECT_MULTI_NODE: 'true', + OS_ALLOW_MEMORY_CLUSTER_MULTINODE: 'true', + }), + ).not.toThrow(); + }); + it('does not guard unknown custom drivers (author responsibility)', () => { + expect(() => + assertClusterDriverSafeForTopology('mycustom', { OS_EXPECT_MULTI_NODE: 'true' }), + ).not.toThrow(); + }); +}); diff --git a/packages/services/service-cluster/src/split-brain-guard.ts b/packages/services/service-cluster/src/split-brain-guard.ts new file mode 100644 index 0000000000..8011a9ebf5 --- /dev/null +++ b/packages/services/service-cluster/src/split-brain-guard.ts @@ -0,0 +1,78 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * Split-brain guard (ADR-0010, path A). + * + * The `memory` cluster driver keeps PubSub/Lock/KV/Counter state **per + * process**. On a single node that is correct; across multiple replicas + * each process holds its own state, so: + * - `ILock` is acquired locally by every replica -> no mutual exclusion; + * - `ICounter`/`IKV` versions diverge per replica; + * - `IPubSub` does not fan out across processes. + * + * This fails silently -- single-node tests and dev pass, only production + * multi-replica corrupts. The guard turns that silent corruption into a + * loud startup error when the operator has *declared* a multi-node + * topology yet wired an in-process driver. + * + * Detection is deliberately conservative (path A): it keys off an + * explicit operator declaration, not active peer discovery (that is the + * optional path B in ADR-0010). It cannot know at boot whether the + * cluster primitives are actually used cross-node, so it offers an + * escape hatch for the rare "replicas declared but primitives unused" + * case. + */ + +/** In-process drivers whose state does not coordinate across replicas. */ +const IN_PROCESS_DRIVERS = new Set(['memory']); + +/** Environment inputs the guard reads. */ +export interface SplitBrainGuardEnv { + /** `'true'` -> operator declares a multi-node deployment. */ + OS_EXPECT_MULTI_NODE?: string; + /** Replica count; `> 1` -> multi-node. */ + OS_CLUSTER_REPLICAS?: string; + /** `'true'` -> bypass the guard (replicas declared but primitives unused cross-node). */ + OS_ALLOW_MEMORY_CLUSTER_MULTINODE?: string; +} + +function isTrue(v: string | undefined): boolean { + return String(v).trim().toLowerCase() === 'true'; +} + +/** + * True when the operator has declared a multi-node topology via + * `OS_EXPECT_MULTI_NODE=true` or `OS_CLUSTER_REPLICAS` greater than 1. + */ +export function declaresMultiNode(env: SplitBrainGuardEnv = process.env): boolean { + if (isTrue(env.OS_EXPECT_MULTI_NODE)) return true; + const replicas = Number(env.OS_CLUSTER_REPLICAS); + return Number.isFinite(replicas) && replicas > 1; +} + +/** + * Throw if a multi-node topology is declared while the resolved cluster + * `driver` is in-process (`memory`), unless explicitly allowed via + * `OS_ALLOW_MEMORY_CLUSTER_MULTINODE=true`. + * + * Call this at startup *before* registering the cluster service so a + * misconfiguration fails fast. + */ +export function assertClusterDriverSafeForTopology( + driver: string, + env: SplitBrainGuardEnv = process.env, +): void { + if (!declaresMultiNode(env)) return; + if (!IN_PROCESS_DRIVERS.has(driver)) return; + if (isTrue(env.OS_ALLOW_MEMORY_CLUSTER_MULTINODE)) return; + + throw new Error( + `ClusterServicePlugin: multi-node deployment declared ` + + `(OS_EXPECT_MULTI_NODE / OS_CLUSTER_REPLICAS>1) but the cluster driver is ` + + `in-process "${driver}" -- its locks/counters/pub-sub do not coordinate across ` + + `processes, so multiple replicas silently split-brain. Configure a remote cluster ` + + `driver (e.g. @objectstack/service-cluster-redis) or a DB-backed driver. To override ` + + `(replicas declared but cluster primitives unused cross-node), set ` + + `OS_ALLOW_MEMORY_CLUSTER_MULTINODE=true. See cloud ADR-0010.`, + ); +}