From a081337cdc391c07b778e8f5f23d06d74457f4b1 Mon Sep 17 00:00:00 2001 From: djgrant <1670902+djgrant@users.noreply.github.com> Date: Wed, 22 Jul 2026 10:01:37 +0100 Subject: [PATCH 01/34] Add durable Yieldstar reconciliation foundation Introduce the durable reconciliation modules under durable/, with the Yieldstar runtime confined to a single adapter seam. Adds @notation/utils and moves error-matcher classification next to ErrorMatcher in @notation/resource as findErrorMatcher. The legacy reconciler remains in place until the runtime cutover. --- packages/reconciler/package.json | 7 +- .../reconciler/src/durable/coordination.ts | 55 ++ packages/reconciler/src/durable/deploy.ts | 37 ++ packages/reconciler/src/durable/destroy.ts | 46 ++ packages/reconciler/src/durable/emit.ts | 67 +++ packages/reconciler/src/durable/index.ts | 18 + packages/reconciler/src/durable/operations.ts | 495 ++++++++++++++++ .../reconciler/src/durable/state-backend.ts | 137 +++++ packages/reconciler/src/durable/stores.ts | 67 +++ packages/reconciler/src/durable/types.ts | 37 ++ packages/reconciler/src/durable/yieldstar.ts | 8 + packages/reconciler/src/events.ts | 53 ++ packages/reconciler/src/planner.ts | 81 +++ packages/reconciler/src/resource-registry.ts | 4 +- .../test/durable-reconciliation.test.ts | 531 ++++++++++++++++++ packages/reconciler/test/planner.test.ts | 36 ++ pnpm-lock.yaml | 65 ++- pnpm-workspace.yaml | 4 + 18 files changed, 1722 insertions(+), 26 deletions(-) create mode 100644 packages/reconciler/src/durable/coordination.ts create mode 100644 packages/reconciler/src/durable/deploy.ts create mode 100644 packages/reconciler/src/durable/destroy.ts create mode 100644 packages/reconciler/src/durable/emit.ts create mode 100644 packages/reconciler/src/durable/index.ts create mode 100644 packages/reconciler/src/durable/operations.ts create mode 100644 packages/reconciler/src/durable/state-backend.ts create mode 100644 packages/reconciler/src/durable/stores.ts create mode 100644 packages/reconciler/src/durable/types.ts create mode 100644 packages/reconciler/src/durable/yieldstar.ts create mode 100644 packages/reconciler/src/events.ts create mode 100644 packages/reconciler/src/planner.ts create mode 100644 packages/reconciler/test/durable-reconciliation.test.ts create mode 100644 packages/reconciler/test/planner.test.ts diff --git a/packages/reconciler/package.json b/packages/reconciler/package.json index 39ed2f0..3429d0d 100644 --- a/packages/reconciler/package.json +++ b/packages/reconciler/package.json @@ -14,7 +14,12 @@ "dependencies": { "@notation/resource": "workspace:*", "@notation/state": "workspace:*", + "@yieldstar/core": "0.5.0", "deep-object-diff": "^1.1.9", - "yieldstar": "^0.4.6" + "yieldstar": "0.5.0" + }, + "devDependencies": { + "@yieldstar/sqlite-runtime": "0.5.0", + "pino": "^9.9.0" } } diff --git a/packages/reconciler/src/durable/coordination.ts b/packages/reconciler/src/durable/coordination.ts new file mode 100644 index 0000000..7dac2d2 --- /dev/null +++ b/packages/reconciler/src/durable/coordination.ts @@ -0,0 +1,55 @@ +import type { ReconcilerEventEmitter } from "../events"; +import { emitOnce } from "./emit"; +import { deploymentCoordinationStore, type CoordinationState } from "./stores"; +import type { DurableStep, WorkflowStore } from "./yieldstar"; + +type CoordinationOptions = { + deploymentId: string; + executionId: string; + emit?: ReconcilerEventEmitter; +}; + +/** + * Prevents concurrent executions from mutating the same deployment. Names + * the holder so an operator can resume it after a crash. + */ +export async function* acquireDeploymentCoordination( + step: DurableStep, + opts: CoordinationOptions, +): AsyncGenerator, any> { + const coordination = yield* step.store(deploymentCoordinationStore, { + id: opts.deploymentId, + initial: { holder: null }, + }); + + const snapshot = yield* coordination.get("notation:coordination:inspect"); + const holder = snapshot.state.holder; + if (holder !== null && holder !== opts.executionId) { + yield* emitOnce(step, "notation:coordination:waiting", opts.emit, () => ({ + level: "warn", + event: "reconciler.coordination.waiting", + deploymentId: opts.deploymentId, + executionId: opts.executionId, + holderExecutionId: holder, + })); + } + + yield* coordination.take( + "notation:coordination:acquire", + (state) => state.holder === null || state.holder === opts.executionId, + (draft) => { + draft.holder = opts.executionId; + }, + ); + + return coordination; +} + +export function releaseDeploymentCoordination( + coordination: WorkflowStore, + executionId: string, +) { + return coordination.update("notation:coordination:release", (draft) => { + if (draft.holder === executionId) draft.holder = null; + }); +} diff --git a/packages/reconciler/src/durable/deploy.ts b/packages/reconciler/src/durable/deploy.ts new file mode 100644 index 0000000..3b93520 --- /dev/null +++ b/packages/reconciler/src/durable/deploy.ts @@ -0,0 +1,37 @@ +import { buildResourceDepthLevels } from "../dependency-graph"; +import { + acquireDeploymentCoordination, + releaseDeploymentCoordination, +} from "./coordination"; +import { reconcileResource, sweepOrphans } from "./operations"; +import type { DurableDeployOptions } from "./types"; +import type { DurableStep } from "./yieldstar"; + +export async function* deploy( + step: DurableStep, + opts: DurableDeployOptions, +): AsyncGenerator { + // Phase 1: take exclusive hold of the deployment. + const coordination = yield* acquireDeploymentCoordination(step, opts); + + try { + // Phase 2: reconcile in dependency order, so a resource only runs once + // its dependencies have converged. + for (const level of buildResourceDepthLevels(opts.resources)) { + for (const resource of level) { + yield* reconcileResource(step, resource, opts); + } + } + + // Phase 3: delete resources that are in state but no longer declared. + yield* sweepOrphans(step, opts, { + workflow: "deploy", + listKey: "notation:orphans:list", + warningKey: (nodeId) => `notation:orphan:${nodeId}:warning`, + deleteSuffix: "orphan", + }); + } finally { + // Phase 4: release the hold, even on error. + yield* releaseDeploymentCoordination(coordination, opts.executionId); + } +} diff --git a/packages/reconciler/src/durable/destroy.ts b/packages/reconciler/src/durable/destroy.ts new file mode 100644 index 0000000..8f01316 --- /dev/null +++ b/packages/reconciler/src/durable/destroy.ts @@ -0,0 +1,46 @@ +import { buildResourceDepthLevels } from "../dependency-graph"; +import { + acquireDeploymentCoordination, + releaseDeploymentCoordination, +} from "./coordination"; +import { deleteResource, sweepOrphans } from "./operations"; +import type { DurableDestroyOptions } from "./types"; +import type { DurableStep } from "./yieldstar"; + +/** Durably destroys persisted resources in reverse dependency order. */ +export async function* destroy( + step: DurableStep, + opts: DurableDestroyOptions, +): AsyncGenerator { + // Phase 1: take exclusive hold of the deployment. + const coordination = yield* acquireDeploymentCoordination(step, opts); + + try { + // Phase 2: delete in reverse dependency order, so dependents are gone + // before the resources they depend on. Resources with no persisted state + // were never created (or are already deleted) and are skipped. + const levels = buildResourceDepthLevels(opts.resources); + for (let index = levels.length - 1; index >= 0; index -= 1) { + for (const resource of levels[index]!) { + const stateNode = yield* step.run( + `notation:destroy:${resource.id}:state:lookup`, + () => opts.state.get(resource.id), + ); + if (!stateNode) continue; + resource.setOutput(stateNode.output); + yield* deleteResource(step, resource, opts, "destroy"); + } + } + + // Phase 3: delete resources that are in state but no longer declared. + yield* sweepOrphans(step, opts, { + workflow: "destroy", + listKey: "notation:destroy:orphans:list", + warningKey: (nodeId) => `notation:destroy:orphan:${nodeId}:warning`, + deleteSuffix: "destroy-orphan", + }); + } finally { + // Phase 4: release the hold, even on error. + yield* releaseDeploymentCoordination(coordination, opts.executionId); + } +} diff --git a/packages/reconciler/src/durable/emit.ts b/packages/reconciler/src/durable/emit.ts new file mode 100644 index 0000000..7be77b1 --- /dev/null +++ b/packages/reconciler/src/durable/emit.ts @@ -0,0 +1,67 @@ +import type { + OperationLifecycleEvent, + OperationLifecycleStatus, + OperationName, + ReconcilerEventEmitter, +} from "../events"; +import type { DurableStep } from "./yieldstar"; + +/** + * Emits an event inside a durable step so a replayed workflow does not + * re-emit events it already sent. + */ +export function emitOnce( + step: DurableStep, + key: string, + emit: ReconcilerEventEmitter | undefined, + event: () => Parameters[0], +) { + return step.run(key, async () => { + await emit?.(event()); + }); +} + +export function emitLifecycle( + step: DurableStep, + key: string, + emit: ReconcilerEventEmitter | undefined, + operation: OperationName, + status: OperationLifecycleStatus, + resource: LifecycleResource, + extra: { reason?: string; error?: unknown } = {}, +) { + return emitOnce(step, key, emit, () => + createLifecycleEvent(operation, status, resource, extra), + ); +} + +type LifecycleResource = { + id: string; + type: OperationLifecycleEvent["resourceType"]; +}; + +export function createLifecycleEvent( + operation: OperationName, + status: OperationLifecycleStatus, + resource: LifecycleResource, + extra: { reason?: string; error?: unknown } = {}, +): OperationLifecycleEvent { + const error = extra.error; + const details = + error === undefined + ? {} + : error instanceof Error + ? { errorName: error.name, errorMessage: error.message } + : { errorName: "UnknownError", errorMessage: String(error) }; + + return { + level: status === "error" ? "error" : "info", + event: "reconciler.operation.lifecycle", + operation, + status, + resourceId: resource.id, + resourceType: resource.type, + ...(extra.reason ? { reason: extra.reason } : {}), + ...details, + }; +} diff --git a/packages/reconciler/src/durable/index.ts b/packages/reconciler/src/durable/index.ts new file mode 100644 index 0000000..e42707f --- /dev/null +++ b/packages/reconciler/src/durable/index.ts @@ -0,0 +1,18 @@ +export { deploy } from "./deploy"; +export { destroy } from "./destroy"; +export { DurableStateBackend } from "./state-backend"; +export { + deploymentCoordinationStore, + resourceStateStore, + type CoordinationState, + type StoredResourceState, +} from "./stores"; +export { + DEFAULT_READ_POLL_OPTIONS, + DEFAULT_RETRY_OPTIONS, + type DurableDeployOptions, + type DurableDestroyOptions, + type DurableOperationOptions, + type PollOptions, +} from "./types"; +export type { DurableStep } from "./yieldstar"; diff --git a/packages/reconciler/src/durable/operations.ts b/packages/reconciler/src/durable/operations.ts new file mode 100644 index 0000000..6bca1d2 --- /dev/null +++ b/packages/reconciler/src/durable/operations.ts @@ -0,0 +1,495 @@ +import { + findErrorMatcher, + type BaseResource, + type ResourceType, +} from "@notation/resource"; +import { RevConflict } from "@notation/state"; +import { + createMissingResourceRegistryMatchWarningEvent, + createResourceRegistryFromResources, + resolveResourceClass, +} from "../resource-registry"; +import { decideAction, type ResourceAction } from "../plan"; +import { emitLifecycle, emitOnce } from "./emit"; +import type { DurableStateBackend } from "./state-backend"; +import { + resourceStateStore, + toStateNode, + type StoredResourceState, +} from "./stores"; +import { + DEFAULT_READ_POLL_OPTIONS, + DEFAULT_RETRY_OPTIONS, + type DurableDeployOptions, + type DurableOperationOptions, + type PollOptions, +} from "./types"; +import { + RetryableError, + type DurableStep, + type WorkflowStore, +} from "./yieldstar"; + +export async function* reconcileResource( + step: DurableStep, + resource: BaseResource, + opts: DurableDeployOptions, +): AsyncGenerator { + const prefix = `notation:resource:${resource.id}`; + + // Hydrate the resource from persisted state. The snapshot is kept so later + // writes can be conditional on the exact instance identity and version that + // was read here. + let stateNode = yield* step.run(`${prefix}:state:lookup`, () => + opts.state.get(resource.id), + ); + let stateStore: WorkflowStore | undefined; + let snapshot: + Awaited> | undefined; + if (stateNode) { + stateStore = yield* openResourceState(step, opts.state, resource.id); + snapshot = yield* stateStore.get(`${prefix}:state:get`); + stateNode = toStateNode(snapshot); + } + if (stateNode) resource.setOutput(stateNode.output); + + // Decide the operation from desired params vs persisted state. + const params = yield* step.run(`${prefix}:params`, () => + resource.getParams(), + ); + let action: ResourceAction = decideAction({ + resource, + stateNode: stateNode ?? undefined, + params, + }); + + // A noop is only trusted after the remote is read back: the provider may + // have drifted from persisted state, which upgrades the decision. + if (action.decision === "noop" && (opts.driftDetection ?? true)) { + const remote = yield* readRemote( + step, + resource, + opts, + `${prefix}:drift-read`, + ); + action = decideAction({ + resource, + stateNode: stateNode ?? undefined, + params, + driftRead: remote, + }); + } + + if (action.decision === "drift-update") { + const diff = action.patch; + yield* emitOnce(step, `${prefix}:drift-detected`, opts.emit, () => ({ + level: "info", + event: "reconciler.drift.detected", + resourceId: resource.id, + resourceType: resource.type, + diff, + })); + } + + yield* emitOnce(step, `${prefix}:decision`, opts.emit, () => ({ + level: "info", + event: "reconciler.deploy.decision", + resourceId: resource.id, + resourceType: resource.type, + decision: action.decision, + })); + + if (action.decision === "noop") return; + const operation = + action.decision === "create" || action.decision === "drift-recreate" + ? "create" + : "update"; + const patch = "patch" in action ? action.patch : {}; + yield* emitLifecycle( + step, + `${prefix}:${operation}:start`, + opts.emit, + operation, + "start", + resource, + ); + if (opts.dryRun) { + yield* emitLifecycle( + step, + `${prefix}:${operation}:dry-run`, + opts.emit, + operation, + "dry-run", + resource, + ); + return; + } + + try { + // Execute the provider call. Each call runs in its own durable step, so a + // replayed workflow never repeats a completed provider mutation. + if (operation === "create") { + const primaryKey = yield* runProviderCall( + step, + `${prefix}:create`, + () => resource.create(params), + resource, + opts.retryOptions, + ); + resource.setOutput(params); + if (primaryKey) resource.setOutput({ ...primaryKey, ...resource.output }); + } else { + if (!resource.update) { + yield* emitLifecycle( + step, + `${prefix}:update:skip`, + opts.emit, + "update", + "skip", + resource, + { reason: "update-not-implemented" }, + ); + return; + } + yield* runProviderCall( + step, + `${prefix}:update`, + () => + resource.update!( + resource.key, + patch, + params, + resource.toState(resource.output), + ), + resource, + opts.retryOptions, + ); + resource.setOutput({ ...resource.key, ...params }); + } + + // Read back the remote so persisted output reflects provider-assigned + // values, then persist conditionally against the snapshot read above. + const read = yield* readRemote( + step, + resource, + opts, + `${prefix}:read-after-write`, + ); + if (read.status === "found") + resource.setOutput({ ...resource.output, ...read.output }); + + const nextState: StoredResourceState = { + id: resource.id, + groupId: resource.groupId, + groupType: resource.groupType, + type: resource.type, + lastOperation: operation, + lastOperationAt: new Date().toISOString(), + config: resource.config, + params: resource.toState(params), + output: resource.toState(resource.output), + }; + + if (!stateStore || !snapshot) { + yield* step.store(resourceStateStore, { + id: opts.state.storeId(resource.id), + initial: nextState, + }); + } else { + const result = yield* stateStore.updateFrom( + `${prefix}:state:persist`, + snapshot, + () => nextState, + ); + if (!result.updated) + throw new RevConflict( + resource.id, + stateNode?.rev ?? 0, + result.actualVersion + 1, + ); + } + + yield* emitLifecycle( + step, + `${prefix}:${operation}:success`, + opts.emit, + operation, + "success", + resource, + ); + } catch (error) { + yield* emitLifecycle( + step, + `${prefix}:${operation}:error`, + opts.emit, + operation, + "error", + resource, + { error }, + ); + throw error; + } +} + +export async function* deleteResource( + step: DurableStep, + resource: BaseResource, + opts: DurableOperationOptions, + suffix: string, +): AsyncGenerator { + const prefix = `notation:${suffix}:${resource.id}`; + + // Hydrate output from persisted state; the delete call needs the primary + // key and the state removal must be conditional on this exact snapshot. + const stateStore = yield* openResourceState(step, opts.state, resource.id); + const snapshot = yield* stateStore.get(`${prefix}:state:get`); + const stateNode = toStateNode(snapshot); + resource.setOutput(stateNode.output); + + yield* emitLifecycle( + step, + `${prefix}:delete:start`, + opts.emit, + "delete", + "start", + resource, + ); + + if (opts.dryRun) { + yield* emitLifecycle( + step, + `${prefix}:delete:dry-run`, + opts.emit, + "delete", + "dry-run", + resource, + ); + return; + } + + try { + // An already-deleted remote is success, not failure: the goal state is + // absence, so a declared not-found error downgrades to a skip. + try { + yield* runProviderCall( + step, + `${prefix}:delete`, + () => resource.delete(resource.key, resource.toState(resource.output)), + resource, + opts.retryOptions, + ); + } catch (error) { + if (!findErrorMatcher(error, resource.notFoundOnError)) throw error; + yield* emitLifecycle( + step, + `${prefix}:delete:not-found`, + opts.emit, + "delete", + "skip", + resource, + { reason: "resource-not-found" }, + ); + } + + // State is removed only after the provider delete completes, and only if + // the store still matches the snapshot read before deleting. + const deleted = yield* stateStore.deleteFrom( + `${prefix}:state:delete`, + snapshot, + ); + if (!deleted.deleted) + throw new RevConflict(resource.id, stateNode.rev, undefined); + yield* emitLifecycle( + step, + `${prefix}:delete:success`, + opts.emit, + "delete", + "success", + resource, + ); + } catch (error) { + yield* emitLifecycle( + step, + `${prefix}:delete:error`, + opts.emit, + "delete", + "error", + resource, + { error }, + ); + throw error; + } +} + +export async function* readRemote( + step: DurableStep, + resource: BaseResource, + opts: DurableOperationOptions, + key: string, +): AsyncGenerator< + any, + | { status: "found"; output: Record } + | { status: "not-found" }, + any +> { + if (!resource.read) { + yield* emitLifecycle( + step, + `${key}:skip`, + opts.emit, + "read", + "skip", + resource, + { + reason: "read-not-implemented", + }, + ); + return { + status: "found", + output: { ...(await resource.getParams()), ...resource.output }, + }; + } + + yield* emitLifecycle( + step, + `${key}:start`, + opts.emit, + "read", + "start", + resource, + ); + try { + const output = yield* step.run(key, async () => { + const value = await resource.read!(resource.key); + // An unsettled read (a declared condition not yet met) re-polls + // durably instead of returning a half-provisioned remote. + const unsettled = (resource.retryReadOnCondition ?? []) + .filter(Boolean) + .find((condition) => { + const actual = value[condition!.key]; + return condition!.value === undefined + ? !actual + : actual !== condition!.value; + }); + if (unsettled) { + throw new RetryableError(unsettled.reason, { + ...(opts.readPollOptions ?? DEFAULT_READ_POLL_OPTIONS), + }); + } + return value; + }); + yield* emitLifecycle( + step, + `${key}:success`, + opts.emit, + "read", + "success", + resource, + ); + return { status: "found", output }; + } catch (error) { + if (findErrorMatcher(error, resource.notFoundOnError)) { + yield* emitLifecycle( + step, + `${key}:not-found`, + opts.emit, + "read", + "skip", + resource, + { reason: "resource-not-found" }, + ); + return { status: "not-found" }; + } + yield* emitLifecycle( + step, + `${key}:error`, + opts.emit, + "read", + "error", + resource, + { + error, + }, + ); + throw error; + } +} + +/** + * Runs a provider call in a durable step, converting errors the resource has + * declared retryable into durable retries. + */ +export function runProviderCall( + step: DurableStep, + key: string, + call: () => T | Promise, + resource: BaseResource, + retryOptions?: PollOptions, +) { + return step.run(key, async () => { + try { + return await call(); + } catch (error) { + const matcher = findErrorMatcher(error, resource.retryLaterOnError); + if (matcher) { + throw new RetryableError(matcher.reason, { + ...(retryOptions ?? DEFAULT_RETRY_OPTIONS), + }); + } + throw error; + } + }); +} + +/** + * Deletes persisted resources that are no longer in the desired set. A state + * node whose type has no registry entry is left in place and surfaced as a + * warning, because deleting it would need a resource class we cannot resolve. + */ +export async function* sweepOrphans( + step: DurableStep, + opts: DurableOperationOptions, + params: { + workflow: "deploy" | "destroy"; + listKey: string; + warningKey: (nodeId: string) => string; + deleteSuffix: string; + }, +): AsyncGenerator { + const resourceById = new Map( + opts.resources.map((resource) => [resource.id, resource]), + ); + const persisted = yield* step.run(params.listKey, () => opts.state.values()); + const registry = + opts.registry ?? createResourceRegistryFromResources(opts.resources); + + for (const node of persisted) { + if (resourceById.has(node.id)) continue; + + const Resource = resolveResourceClass(registry, node.type as ResourceType); + if (!Resource) { + yield* emitOnce(step, params.warningKey(node.id), opts.emit, () => + createMissingResourceRegistryMatchWarningEvent({ + workflow: params.workflow, + resourceId: node.id, + resourceType: node.type as ResourceType, + }), + ); + continue; + } + + const resource = new Resource({ id: node.id, config: node.config }); + resource.setOutput(node.output); + yield* deleteResource(step, resource, opts, params.deleteSuffix); + } +} + +export function openResourceState( + step: DurableStep, + state: DurableStateBackend, + resourceId: string, +) { + return step.store(resourceStateStore, { + id: state.storeId(resourceId), + }); +} diff --git a/packages/reconciler/src/durable/state-backend.ts b/packages/reconciler/src/durable/state-backend.ts new file mode 100644 index 0000000..e15d498 --- /dev/null +++ b/packages/reconciler/src/durable/state-backend.ts @@ -0,0 +1,137 @@ +import { RevConflict, type StateNode } from "@notation/state"; +import { + resourceStateStore, + toStateNode, + withoutRev, + type StoredResourceState, +} from "./stores"; +import type { StoreClient } from "./yieldstar"; + +export class DurableStateBackend { + readonly #client: StoreClient; + readonly #deploymentId: string; + readonly #prefix: string; + + constructor(client: StoreClient, deploymentId: string) { + this.#client = client; + this.#deploymentId = deploymentId; + // Keep deployment prefixes disjoint so orphan cleanup cannot delete + // another deployment's stores. + this.#prefix = `${encodeURIComponent(deploymentId)}:`; + } + + storeId(resourceId: string) { + return `${this.#prefix}${resourceId}`; + } + + async get(id: string): Promise { + const snapshot = await this.#tryGetSnapshot(this.storeId(id)); + return snapshot ? toStateNode(snapshot) : undefined; + } + + async #tryGetSnapshot( + storeId: string, + ): Promise< + | { state: StoredResourceState; instanceId: string; version: number } + | undefined + > { + try { + return await this.#client.getStore({ + definition: resourceStateStore, + id: storeId, + }); + } catch (error) { + const ids = await this.#client.listStores(resourceStateStore); + if (!ids.includes(storeId)) return undefined; + throw error; + } + } + + async has(id: string): Promise { + return (await this.get(id)) !== undefined; + } + + async update( + id: string, + expectedRev: number, + patch: Partial, + ): Promise<{ rev: number }> { + const storeId = this.storeId(id); + const snapshot = await this.#tryGetSnapshot(storeId); + if (!snapshot) { + if (expectedRev !== 0) throw new RevConflict(id, expectedRev, undefined); + const initial = { ...patch, id } as StoredResourceState; + const created = await this.#client.getOrCreateStore({ + definition: resourceStateStore, + id: storeId, + initial, + }); + return { rev: created.version + 1 }; + } + + const actualRev = snapshot.version + 1; + if (actualRev !== expectedRev) + throw new RevConflict(id, expectedRev, actualRev); + const result = await this.#client.updateStoreFrom({ + definition: resourceStateStore, + id: storeId, + snapshot, + updater: (draft) => { + Object.assign(draft, withoutRev(patch)); + }, + }); + if (!result.updated) throw new RevConflict(id, expectedRev, undefined); + return { rev: result.version + 1 }; + } + + async delete(id: string, expectedRev: number): Promise { + const storeId = this.storeId(id); + const snapshot = await this.#tryGetSnapshot(storeId); + if (!snapshot) { + if (expectedRev !== 0) throw new RevConflict(id, expectedRev, undefined); + return; + } + const actualRev = snapshot.version + 1; + if (actualRev !== expectedRev) + throw new RevConflict(id, expectedRev, actualRev); + const result = await this.#client.deleteStoreFrom({ + definition: resourceStateStore, + id: storeId, + snapshot, + }); + if (!result.deleted) throw new RevConflict(id, expectedRev, undefined); + } + + async values(): Promise { + const ids = await this.#client.listStores(resourceStateStore); + const snapshots = await Promise.all( + ids + .filter((id) => id.startsWith(this.#prefix)) + .map((id) => this.#tryGetSnapshot(id)), + ); + return snapshots + .filter((snapshot) => snapshot !== undefined) + .map(toStateNode); + } + + snapshot(id: string) { + return this.#client.getStore({ + definition: resourceStateStore, + id: this.storeId(id), + }); + } + + async clear(): Promise { + const ids = await this.#client.listStores(resourceStateStore); + await Promise.all( + ids + .filter((id) => id.startsWith(this.#prefix)) + .map((id) => + this.#client.deleteStore({ + definition: resourceStateStore, + id, + }), + ), + ); + } +} diff --git a/packages/reconciler/src/durable/stores.ts b/packages/reconciler/src/durable/stores.ts new file mode 100644 index 0000000..fba2cb3 --- /dev/null +++ b/packages/reconciler/src/durable/stores.ts @@ -0,0 +1,67 @@ +import type { StateNode } from "@notation/state"; +import { defineStore, type StandardSchemaV1 } from "./yieldstar"; + +export type StoredResourceState = Omit; +export type CoordinationState = { holder: string | null }; + +const storedResourceStateSchema = plainObjectSchema( + "Stored resource state", + (value) => + typeof value.id === "string" && + typeof value.type === "string" && + isPlainObject(value.config) && + isPlainObject(value.params) && + isPlainObject(value.output), +); +const coordinationStateSchema = plainObjectSchema( + "Deployment coordination state", + (value) => + "holder" in value && + (value.holder === null || typeof value.holder === "string"), +); + +export const resourceStateStore = defineStore( + "notation/resource-state", + storedResourceStateSchema, +); + +export const deploymentCoordinationStore = defineStore( + "notation/deployment-coordination", + coordinationStateSchema, +); + +export function toStateNode(snapshot: { + state: StoredResourceState; + version: number; +}): StateNode { + return { ...snapshot.state, rev: snapshot.version + 1 } as StateNode; +} + +export function withoutRev( + patch: Partial, +): Partial { + const { rev: _rev, ...stored } = patch; + return stored; +} + +function plainObjectSchema>( + label: string, + refine: (value: Record) => boolean, +): StandardSchemaV1 { + return { + "~standard": { + version: 1, + vendor: "notation", + validate(value) { + if (!isPlainObject(value) || !refine(value)) { + return { issues: [{ message: `${label} is invalid` }] }; + } + return { value: value as T }; + }, + }, + }; +} + +function isPlainObject(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} diff --git a/packages/reconciler/src/durable/types.ts b/packages/reconciler/src/durable/types.ts new file mode 100644 index 0000000..c139348 --- /dev/null +++ b/packages/reconciler/src/durable/types.ts @@ -0,0 +1,37 @@ +import type { BaseResource } from "@notation/resource"; +import type { ReconcilerEventEmitter } from "../events"; +import type { ResourceRegistry } from "../resource-registry"; +import type { DurableStateBackend } from "./state-backend"; + +export type PollOptions = { + maxAttempts: number; + retryInterval: number; +}; + +export const DEFAULT_RETRY_OPTIONS: PollOptions = { + maxAttempts: 10, + retryInterval: 1_000, +}; + +export const DEFAULT_READ_POLL_OPTIONS: PollOptions = { + maxAttempts: 30, + retryInterval: 1_000, +}; + +export type DurableOperationOptions = { + deploymentId: string; + executionId: string; + resources: BaseResource[]; + state: DurableStateBackend; + registry?: ResourceRegistry; + dryRun?: boolean; + emit?: ReconcilerEventEmitter; + retryOptions?: PollOptions; + readPollOptions?: PollOptions; +}; + +export type DurableDeployOptions = DurableOperationOptions & { + driftDetection?: boolean; +}; + +export type DurableDestroyOptions = DurableOperationOptions; diff --git a/packages/reconciler/src/durable/yieldstar.ts b/packages/reconciler/src/durable/yieldstar.ts new file mode 100644 index 0000000..13ea2dd --- /dev/null +++ b/packages/reconciler/src/durable/yieldstar.ts @@ -0,0 +1,8 @@ +import type { WorkflowFn } from "yieldstar"; + +export { RetryableError, defineStore } from "yieldstar"; +export type { WorkflowStore } from "yieldstar"; +export type { StandardSchemaV1, StoreClient } from "@yieldstar/core"; + +/** The durable step primitive the runtime hands to workflow functions. */ +export type DurableStep = Parameters>[0]; diff --git a/packages/reconciler/src/events.ts b/packages/reconciler/src/events.ts new file mode 100644 index 0000000..2cad300 --- /dev/null +++ b/packages/reconciler/src/events.ts @@ -0,0 +1,53 @@ +import type { ResourceType } from "@notation/resource"; + +export type OperationName = "create" | "read" | "update" | "delete"; + +export type OperationLifecycleStatus = + "start" | "success" | "error" | "skip" | "dry-run"; + +export type OperationLifecycleEvent = { + level: "info" | "error"; + event: "reconciler.operation.lifecycle"; + operation: OperationName; + status: OperationLifecycleStatus; + resourceId: string; + resourceType: ResourceType; + reason?: string; + errorName?: string; + errorMessage?: string; +}; + +export type ReconcilerDeployEvent = { + level: "info"; + event: "reconciler.deploy.decision"; + resourceId: string; + resourceType: string; + decision: "create" | "update" | "drift-update" | "drift-recreate" | "noop"; +}; + +export type ReconcilerDriftDetectedEvent = { + level: "info"; + event: "reconciler.drift.detected"; + resourceId: string; + resourceType: string; + diff: Record; +}; + +export type CoordinationWaitingEvent = { + level: "warn"; + event: "reconciler.coordination.waiting"; + deploymentId: string; + executionId: string; + holderExecutionId: string; +}; + +export type ReconcilerEvent = + | OperationLifecycleEvent + | CoordinationWaitingEvent + | ReconcilerDeployEvent + | ReconcilerDriftDetectedEvent + | import("./resource-registry").MissingResourceRegistryMatchWarningEvent; + +export type ReconcilerEventEmitter = ( + event: ReconcilerEvent, +) => void | Promise; diff --git a/packages/reconciler/src/planner.ts b/packages/reconciler/src/planner.ts new file mode 100644 index 0000000..6e90b56 --- /dev/null +++ b/packages/reconciler/src/planner.ts @@ -0,0 +1,81 @@ +import type { BaseResource } from "@notation/resource"; +import type { StateBackend } from "@notation/state"; +import { buildResourceDepthLevels } from "./dependency-graph"; +import { + decideAction, + getDependencyIds, + resolvePlanParams, + type Plan, + type PlanNode, +} from "./plan"; + +export type CreatePlanOptions = { + resources: BaseResource[]; + state: StateBackend; + driftDetection?: boolean; +}; + +export async function createPlan({ + resources, + state, + driftDetection = true, +}: CreatePlanOptions): Promise { + const resourceById = new Map( + resources.map((resource) => [resource.id, resource]), + ); + const nodes: PlanNode[] = []; + + for (const level of buildResourceDepthLevels(resources)) { + for (const resource of level) { + const stateNode = await state.get(resource.id); + if (stateNode) resource.setOutput(stateNode.output); + const params = await resolvePlanParams(resource); + let action = decideAction({ resource, stateNode, params }); + + if (action.decision === "noop" && driftDetection && resource.read) { + try { + const output = await resource.read(resource.key); + action = decideAction({ + resource, + stateNode, + params, + driftRead: { status: "found", output }, + }); + } catch (error) { + const notFound = resource.notFoundOnError?.some( + (matcher) => matcher.name === (error as Error)?.name, + ); + if (!notFound) throw error; + action = decideAction({ + resource, + stateNode, + params, + driftRead: { status: "not-found" }, + }); + } + } + + nodes.push({ + id: resource.id, + type: resource.type, + decision: action.decision, + ...("diff" in action ? { diff: action.diff } : {}), + params, + dependsOn: getDependencyIds(resource), + }); + } + } + + for (const stateNode of await state.values()) { + if (resourceById.has(stateNode.id)) continue; + nodes.push({ + id: stateNode.id, + type: stateNode.type, + decision: "delete-orphan", + params: stateNode.params, + dependsOn: [], + }); + } + + return { createdAt: new Date().toISOString(), nodes }; +} diff --git a/packages/reconciler/src/resource-registry.ts b/packages/reconciler/src/resource-registry.ts index 916c0ab..6916caa 100644 --- a/packages/reconciler/src/resource-registry.ts +++ b/packages/reconciler/src/resource-registry.ts @@ -6,7 +6,7 @@ export type MissingResourceRegistryMatchWarningEvent = { level: "warn"; event: "reconciler.orphan-deletion.skipped"; reason: "resource-type-not-registered"; - workflow: "deploy" | "refresh"; + workflow: "deploy" | "refresh" | "destroy"; resourceId: string; resourceType: ResourceType; }; @@ -46,7 +46,7 @@ export function resolveResourceClass( } export function createMissingResourceRegistryMatchWarningEvent(opts: { - workflow: "deploy" | "refresh"; + workflow: "deploy" | "refresh" | "destroy"; resourceId: string; resourceType: ResourceType; }): MissingResourceRegistryMatchWarningEvent { diff --git a/packages/reconciler/test/durable-reconciliation.test.ts b/packages/reconciler/test/durable-reconciliation.test.ts new file mode 100644 index 0000000..1d55152 --- /dev/null +++ b/packages/reconciler/test/durable-reconciliation.test.ts @@ -0,0 +1,531 @@ +import { + WorkflowRunner, + type HeapClient, + type WorkflowEvent, +} from "@yieldstar/core"; +import { + SqliteHeapClient, + SqliteStoreClient, + createSqliteDb, +} from "@yieldstar/sqlite-runtime/node"; +import { resource, type BaseResource } from "@notation/resource"; +import pino from "pino"; +import { createWorkflowRouter, workflow } from "yieldstar"; +import { describe, expect, it, vi } from "vitest"; +import * as durable from "../src/durable"; +import type { ReconcilerEvent } from "../src/events"; +import { + createResourceRegistry, + type ResourceRegistry, +} from "../src/resource-registry"; + +const logger = pino({ level: "silent" }); + +describe("durable execution and replay", () => { + it("waits durably for a retryable provider and persists after success", async () => { + let attempts = 0; + const PendingResource = resource({ type: "test/durable/pending" }) + .defineSchema({}) + .defineOperations({ + create: async () => { + attempts += 1; + if (attempts === 1) { + const error = new Error("provider is pending"); + error.name = "ProviderPending"; + throw error; + } + }, + delete: async () => undefined, + retryLaterOnError: [ + { name: "ProviderPending", reason: "provider is pending" }, + ], + }); + const runtime = createRuntime( + [new PendingResource({ id: "pending" })], + "durable-wait", + { retryOptions: { maxAttempts: 3, retryInterval: 1 } }, + ); + + await runtime.run("wait-execution"); + expect(attempts).toBe(1); + expect(runtime.scheduler.events).toHaveLength(1); + + await runtime.run("wait-execution"); + expect(attempts).toBe(2); + expect(await runtime.state.get("pending")).toMatchObject({ + id: "pending", + lastOperation: "create", + rev: 1, + }); + runtime.close(); + }); + + it("resumes after a crash without repeating a completed create", async () => { + const create = vi.fn(async () => undefined); + const TestResource = resource({ type: "test/durable/resume" }) + .defineSchema({}) + .defineOperations({ create, delete: async () => undefined }); + const runtime = createRuntime( + [new TestResource({ id: "resume" })], + "crash-resume", + { crashAfterStep: "notation:resource:resume:create" }, + ); + + await expect(runtime.run("resume-execution")).rejects.toThrow( + "simulated process crash", + ); + expect(create).toHaveBeenCalledOnce(); + expect(await runtime.state.get("resume")).toBeUndefined(); + + await runtime.run("resume-execution"); + expect(create).toHaveBeenCalledOnce(); + expect(await runtime.state.get("resume")).toMatchObject({ rev: 1 }); + runtime.close(); + }); + + it("resumes durable destroy after a crash without repeating delete", async () => { + const remove = vi.fn(async () => undefined); + const TestResource = resource({ type: "test/durable/destroy-resume" }) + .defineSchema({}) + .defineOperations({ create: async () => undefined, delete: remove }); + const runtime = createRuntime( + [new TestResource({ id: "destroyed" })], + "destroy-crash-resume", + { crashAfterStep: "notation:destroy:destroyed:delete" }, + ); + + await runtime.run("deploy-before-destroy"); + await expect(runtime.destroy("destroy-execution")).rejects.toThrow( + "simulated process crash", + ); + expect(remove).toHaveBeenCalledOnce(); + expect(await runtime.state.get("destroyed")).toBeDefined(); + + await runtime.destroy("destroy-execution"); + expect(remove).toHaveBeenCalledOnce(); + expect(await runtime.state.get("destroyed")).toBeUndefined(); + runtime.close(); + }); + + it("waits durably for a retryable delete before removing state", async () => { + let attempts = 0; + const PendingDelete = resource({ type: "test/durable/pending-delete" }) + .defineSchema({}) + .defineOperations({ + create: async () => undefined, + delete: async () => { + attempts += 1; + if (attempts === 1) { + const error = new Error("delete is pending"); + error.name = "DeletePending"; + throw error; + } + }, + retryLaterOnError: [ + { name: "DeletePending", reason: "delete is pending" }, + ], + }); + const runtime = createRuntime( + [new PendingDelete({ id: "pending-delete" })], + "durable-destroy-wait", + { retryOptions: { maxAttempts: 3, retryInterval: 1 } }, + ); + + await runtime.run("deploy-before-wait"); + await runtime.destroy("destroy-wait"); + expect(attempts).toBe(1); + expect(await runtime.state.get("pending-delete")).toBeDefined(); + + await runtime.destroy("destroy-wait"); + expect(attempts).toBe(2); + expect(await runtime.state.get("pending-delete")).toBeUndefined(); + runtime.close(); + }); +}); + +describe("dependency ordering", () => { + it("destroys dependents before their dependencies", async () => { + const order: string[] = []; + const Dependency = resource({ type: "test/durable/dependency" }) + .defineSchema({}) + .defineOperations({ + create: async () => undefined, + delete: async () => void order.push("dependency"), + }); + const Dependent = resource({ type: "test/durable/dependent" }) + .defineSchema({}) + .defineOperations({ + create: async () => undefined, + delete: async () => void order.push("dependent"), + }); + const dependency = new Dependency({ id: "dependency" }); + const dependent = new Dependent({ + id: "dependent", + dependencies: { dependency }, + }); + const runtime = createRuntime([dependency, dependent], "destroy-order"); + + await runtime.run("deploy-before-ordered-destroy"); + await runtime.destroy("ordered-destroy"); + + expect(order).toEqual(["dependent", "dependency"]); + runtime.close(); + }); +}); + +describe("conditional state persistence", () => { + it("uses store identity and version for conditional update and delete", async () => { + const runtime = createRuntime([], "conditional-state"); + await runtime.state.update("resource", 0, statePatch("resource")); + const originalSnapshot = await runtime.state.snapshot("resource"); + + const first = runtime.state.update("resource", 1, { + output: { winner: "first" }, + }); + const second = runtime.state.update("resource", 1, { + output: { winner: "second" }, + }); + const results = await Promise.allSettled([first, second]); + + expect( + results.filter((result) => result.status === "fulfilled"), + ).toHaveLength(1); + expect( + results.filter((result) => result.status === "rejected"), + ).toHaveLength(1); + await expect(runtime.state.delete("resource", 1)).rejects.toMatchObject({ + name: "RevConflict", + }); + expect(await runtime.state.get("resource")).toMatchObject({ rev: 2 }); + + await runtime.state.clear(); + await runtime.state.update("resource", 0, statePatch("resource")); + const staleDelete = await runtime.storeClient.deleteStoreFrom({ + definition: durable.resourceStateStore, + id: runtime.state.storeId("resource"), + snapshot: originalSnapshot, + }); + expect(staleDelete).toMatchObject({ + deleted: false, + reason: "conflict", + }); + expect(await runtime.state.get("resource")).toMatchObject({ rev: 1 }); + runtime.close(); + }); +}); + +describe("deployment coordination", () => { + it("serializes concurrent deployments through durable store waiting", async () => { + let unblockCreate!: () => void; + const blocked = new Promise((resolve) => { + unblockCreate = resolve; + }); + let started!: () => void; + const createStarted = new Promise((resolve) => { + started = resolve; + }); + const create = vi.fn(async () => { + started(); + await blocked; + }); + const TestResource = resource({ type: "test/durable/concurrent" }) + .defineSchema({}) + .defineOperations({ create, delete: async () => undefined }); + const runtime = createRuntime( + [new TestResource({ id: "shared" })], + "concurrent", + ); + + const first = runtime.run("deployment-a"); + await createStarted; + await runtime.run("deployment-b"); + expect(create).toHaveBeenCalledOnce(); + + unblockCreate(); + await first; + const wake = runtime.scheduler.events.find( + (event) => event.executionId === "deployment-b", + ); + expect(wake).toBeDefined(); + await runtime.runner.run(wake!, logger); + + expect(create).toHaveBeenCalledOnce(); + expect(await runtime.state.values()).toHaveLength(1); + runtime.close(); + }); + + it("emits a coordination waiting event when another execution holds the deployment", async () => { + let unblockCreate!: () => void; + const blocked = new Promise((resolve) => { + unblockCreate = resolve; + }); + let started!: () => void; + const createStarted = new Promise((resolve) => { + started = resolve; + }); + const TestResource = resource({ type: "test/durable/coordination" }) + .defineSchema({}) + .defineOperations({ + create: async () => { + started(); + await blocked; + }, + delete: async () => undefined, + }); + const events: ReconcilerEvent[] = []; + const runtime = createRuntime( + [new TestResource({ id: "held" })], + "coordination-waiting", + { emit: (event) => void events.push(event) }, + ); + + const first = runtime.run("holder-execution"); + await createStarted; + await runtime.run("waiter-execution"); + + expect( + events.find((event) => event.event === "reconciler.coordination.waiting"), + ).toMatchObject({ + level: "warn", + deploymentId: "coordination-waiting", + executionId: "waiter-execution", + holderExecutionId: "holder-execution", + }); + + unblockCreate(); + await first; + runtime.close(); + }); +}); + +describe("deployment scoping", () => { + it("scopes store listing to the exact deployment despite prefix-like IDs", async () => { + const database = createSqliteDb({ path: ":memory:" }); + const storeClient = new SqliteStoreClient({ + db: database, + schedulerClient: new TestScheduler(), + }); + const app = new durable.DurableStateBackend(storeClient, "app"); + const appBlue = new durable.DurableStateBackend(storeClient, "app:blue"); + + await app.update("site", 0, statePatch("site")); + await appBlue.update("site", 0, statePatch("site")); + + expect(await app.values()).toHaveLength(1); + expect(await appBlue.values()).toHaveLength(1); + + await app.clear(); + expect(await app.values()).toHaveLength(0); + expect(await appBlue.values()).toHaveLength(1); + expect(await appBlue.get("site")).toBeDefined(); + database.close(); + }); +}); + +describe("orphan deletion", () => { + it("deletes orphaned resources through the registry on a later deployment", async () => { + const deleteSpy = vi.fn(async () => undefined); + const OrphanResource = resource({ type: "test/durable/orphan" }) + .defineSchema({}) + .defineOperations({ create: async () => undefined, delete: deleteSpy }); + const resources: BaseResource[] = [new OrphanResource({ id: "orphan" })]; + const runtime = createRuntime(resources, "orphan-deletion", { + registry: createResourceRegistry([OrphanResource]), + }); + + await runtime.run("deploy-1"); + expect(await runtime.state.values()).toHaveLength(1); + + resources.length = 0; + await runtime.run("deploy-2"); + + expect(deleteSpy).toHaveBeenCalledOnce(); + expect(await runtime.state.values()).toHaveLength(0); + expect(await runtime.state.get("orphan")).toBeUndefined(); + runtime.close(); + }); +}); + +describe("drift detection and repair", () => { + it("emits drift detection and repairs remote drift with update", async () => { + let remote = { name: "expected" }; + const updateSpy = vi.fn(async () => { + remote = { name: "expected" }; + }); + const DriftResource = resource({ type: "test/durable/drift" }) + .defineSchema({ + name: { + presence: "required", + propertyType: "param", + valueType: "string" as any, + }, + }) + .defineOperations({ + create: async () => remote, + read: async () => remote, + update: updateSpy, + delete: async () => undefined, + }); + const events: ReconcilerEvent[] = []; + const runtime = createRuntime( + [new DriftResource({ id: "drifted", config: { name: "expected" } })], + "drift-repair", + { driftDetection: true, emit: (event) => void events.push(event) }, + ); + + await runtime.run("deploy-1"); + remote = { name: "drifted" }; + await runtime.run("deploy-2"); + + expect(updateSpy).toHaveBeenCalledOnce(); + expect( + events.find((event) => event.event === "reconciler.drift.detected"), + ).toMatchObject({ resourceId: "drifted", diff: { name: "expected" } }); + expect( + events.filter( + (event) => + event.event === "reconciler.deploy.decision" && + event.decision === "drift-update", + ), + ).toHaveLength(1); + runtime.close(); + }); +}); + +function createRuntime( + resources: BaseResource[], + deploymentId: string, + options: { + retryOptions?: { maxAttempts: number; retryInterval: number }; + crashAfterStep?: string; + registry?: ResourceRegistry; + driftDetection?: boolean; + emit?: (event: ReconcilerEvent) => void; + } = {}, +) { + const database = createSqliteDb({ path: ":memory:" }); + const scheduler = new TestScheduler(); + const sqliteHeap = new SqliteHeapClient(database); + const heap = options.crashAfterStep + ? new CrashAfterWriteHeap(sqliteHeap, options.crashAfterStep) + : sqliteHeap; + const storeClient = new SqliteStoreClient({ + db: database, + schedulerClient: scheduler, + }); + const state = new durable.DurableStateBackend(storeClient, deploymentId); + const deploy = workflow(async function* (step, event) { + yield* durable.deploy(step, { + deploymentId, + executionId: event.executionId, + resources, + state, + registry: options.registry, + driftDetection: options.driftDetection ?? false, + emit: options.emit, + retryOptions: options.retryOptions, + }); + }); + const destroy = workflow(async function* (step, event) { + yield* durable.destroy(step, { + deploymentId, + executionId: event.executionId, + resources, + state, + registry: options.registry, + emit: options.emit, + retryOptions: options.retryOptions, + }); + }); + const router = createWorkflowRouter({ deploy, destroy }); + const runner = new WorkflowRunner({ + router, + heapClient: heap, + storeClient, + schedulerClient: scheduler, + logger, + }); + + return { + runner, + scheduler, + state, + storeClient, + run(executionId: string) { + return runner.run( + { + workflowId: "deploy", + executionId, + params: {}, + context: new Map(), + }, + logger, + ); + }, + destroy(executionId: string) { + return runner.run( + { + workflowId: "destroy", + executionId, + params: {}, + context: new Map(), + }, + logger, + ); + }, + close() { + database.close(); + }, + }; +} + +class TestScheduler { + readonly events: WorkflowEvent[] = []; + + async requestWakeUp(event: WorkflowEvent) { + this.events.push(event); + } +} + +class CrashAfterWriteHeap implements HeapClient { + #crashed = false; + + constructor( + private readonly inner: HeapClient, + private readonly crashAfterStep: string, + ) {} + + readStep(params: { executionId: string; stepKey: string }) { + return this.inner.readStep(params); + } + + async writeStep(params: { + executionId: string; + stepKey: string; + stepAttempt: number; + stepDone: boolean; + stepResponseJson: string; + }) { + await this.inner.writeStep(params); + if ( + !this.#crashed && + params.stepKey === this.crashAfterStep && + params.stepDone + ) { + this.#crashed = true; + throw new Error("simulated process crash"); + } + } +} + +function statePatch(id: string) { + return { + id, + type: "test/durable/state", + config: {}, + params: {}, + output: {}, + lastOperation: "create" as const, + lastOperationAt: new Date().toISOString(), + }; +} diff --git a/packages/reconciler/test/planner.test.ts b/packages/reconciler/test/planner.test.ts new file mode 100644 index 0000000..a215e82 --- /dev/null +++ b/packages/reconciler/test/planner.test.ts @@ -0,0 +1,36 @@ +import { resource } from "@notation/resource"; +import { MemoryStateBackend } from "@notation/state"; +import { describe, expect, it } from "vitest"; +import { createPlan } from "../src/planner"; + +describe("createPlan", () => { + it("plans desired creates and persisted orphans without mutation execution", async () => { + const TestResource = resource({ type: "test/planner/resource" }) + .defineSchema({}) + .defineOperations({ + create: async () => undefined, + delete: async () => undefined, + }); + const state = new MemoryStateBackend(); + await state.update("orphan", 0, { + id: "orphan", + type: TestResource.type, + config: {}, + params: {}, + output: {}, + lastOperation: "create", + lastOperationAt: "2026-07-22T00:00:00.000Z", + }); + + const plan = await createPlan({ + resources: [new TestResource({ id: "desired" })], + state, + driftDetection: false, + }); + + expect(plan.nodes).toEqual([ + expect.objectContaining({ id: "desired", decision: "create" }), + expect.objectContaining({ id: "orphan", decision: "delete-orphan" }), + ]); + }); +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 82d8d14..82b4901 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -345,12 +345,25 @@ importers: '@notation/state': specifier: workspace:* version: link:../state + '@notation/utils': + specifier: workspace:* + version: link:../utils + '@yieldstar/core': + specifier: 0.5.0 + version: 0.5.0 deep-object-diff: specifier: ^1.1.9 version: 1.1.9 yieldstar: - specifier: ^0.4.6 - version: 0.4.6 + specifier: 0.5.0 + version: 0.5.0 + devDependencies: + '@yieldstar/sqlite-runtime': + specifier: 0.5.0 + version: 0.5.0 + pino: + specifier: ^9.9.0 + version: 9.14.0 packages/resource: {} @@ -1540,8 +1553,12 @@ packages: '@vitest/utils@4.1.10': resolution: {integrity: sha512-fy9am/HWxbaGt/Sawrp90vt6Y6jQwf1RX77cz3uwoJwJVMli/e1IEwRPnMNJ7vKfPTwo0diXifkpPvwH9v7nGA==} - '@yieldstar/core@0.4.6': - resolution: {integrity: sha512-6aJQ2NwA07YKdQpiyxdg+SCKwnHQsuxAGGW6uKZYydTS10ljL3uTE6mKdrt/22ehOtbXzjRa/xnECAWh3GM7pw==} + '@yieldstar/core@0.5.0': + resolution: {integrity: sha512-KaN1+AVg54W9G4VXHNmCRixV0MY123714YZgaA9EZx7mDgB0vfFA6T71rUnldGH86QEkocCwI4GCZzZ+0G4fyQ==} + + '@yieldstar/sqlite-runtime@0.5.0': + resolution: {integrity: sha512-zdG4kvOzEJFrTW52SY+WtdUuSvKhXs438H2/+TQotwi3fltLqva9EMgRJ/l9mcvC7nqAq1i0g2v+TZWjO/py4A==} + engines: {node: '>=22.6'} abstract-logging@2.0.1: resolution: {integrity: sha512-2BjRTZxTPvheOvGbBslFSYOUkr+SjPtOnrLP33f+VIWLzezQpZcqVg7ja3L4dBXmzzgwT+a029jRx5PCi3JuiA==} @@ -2228,9 +2245,6 @@ packages: pino-abstract-transport@3.0.0: resolution: {integrity: sha512-wlfUczU+n7Hy/Ha5j9a/gZNy7We5+cXp8YL+X+PG8S0KXxw7n/JXA3c46Y0zQznIJ83URJiwy7Lh56WLokNuxg==} - pino-std-serializers@7.0.0: - resolution: {integrity: sha512-e906FRY0+tV27iq4juKzSYPbUj2do2X2JX4EzSca1631EB2QJQUqGbDuERal7LCtOpxl6x3+nvo9NPZcmjkiFA==} - pino-std-serializers@7.1.0: resolution: {integrity: sha512-BndPH67/JxGExRgiX1dX0w1FvZck5Wa4aal9198SrRhZjH3GxKQUKIBnYJTdj2HDN3UQAS06HlfcSbQj2OHmaw==} @@ -2421,9 +2435,6 @@ packages: resolution: {integrity: sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==} engines: {node: '>=8'} - sonic-boom@4.2.0: - resolution: {integrity: sha512-INb7TM37/mAcsGmc9hyyI6+QR3rR1zVRu36B0NeGXKnOOLiZOfER5SA+N7X7k3yUYRzLWafduTDvJAfDswwEww==} - sonic-boom@4.2.1: resolution: {integrity: sha512-w6AxtubXa2wTXAUsZMMWERrsIRAdrK0Sc+FUytWvYAhBJLyuI4llrMIC1DtlNSdI99EI86KZum2MMq3EAZlF9Q==} @@ -2600,6 +2611,10 @@ packages: util-deprecate@1.0.2: resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} + uuid@11.1.1: + resolution: {integrity: sha512-vIYxrBCC/N/K+Js3qSN88go7kIfNPssr/hHCesKCQNAjmgvYS2oqr69kIufEG+O4+PfezOH4EbIeHCfFov8ZgQ==} + hasBin: true + vite@8.1.3: resolution: {integrity: sha512-Ds+gBRbj0lwRO2Y5hwnUBdxSwlAve9LeRyU4sNnAr0ewW0gWF0n5bgXgUzbgZ49MV9BVUAQUFYVcDUcilUExMA==} engines: {node: ^20.19.0 || >=22.12.0} @@ -2701,8 +2716,8 @@ packages: yallist@3.1.1: resolution: {integrity: sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==} - yieldstar@0.4.6: - resolution: {integrity: sha512-toVxX0hHx+AlD5S3MT7voOW/Jw7AX5Mqejp28BJErSHGdZVcuKyVIfzYknbfnRnmBxAeZd3wfVnk7JQJQzZUZQ==} + yieldstar@0.5.0: + resolution: {integrity: sha512-MKuo2uaHYdy+1u0O0Q3Po3eVLGEmiNACsbaQ9aqOD4miZyRT63fXc/mhk0V4n8fNtoMN1BORD9QiXq5ZuOBtkQ==} yoctocolors@2.1.1: resolution: {integrity: sha512-GQHQqAopRhwU8Kt1DDM8NjibDXHC8eoh1erhGAJPEyveY9qqVeXvVikNKrDz69sHowPMorbPUrH/mx8c50eiBQ==} @@ -3835,7 +3850,15 @@ snapshots: convert-source-map: 2.0.0 tinyrainbow: 3.1.0 - '@yieldstar/core@0.4.6': {} + '@yieldstar/core@0.5.0': + dependencies: + '@standard-schema/spec': 1.1.0 + + '@yieldstar/sqlite-runtime@0.5.0': + dependencies: + '@yieldstar/core': 0.5.0 + pino: 9.14.0 + uuid: 11.1.1 abstract-logging@2.0.1: {} @@ -4461,8 +4484,6 @@ snapshots: dependencies: split2: 4.2.0 - pino-std-serializers@7.0.0: {} - pino-std-serializers@7.1.0: {} pino@10.3.1: @@ -4485,12 +4506,12 @@ snapshots: atomic-sleep: 1.0.0 on-exit-leak-free: 2.1.2 pino-abstract-transport: 2.0.0 - pino-std-serializers: 7.0.0 + pino-std-serializers: 7.1.0 process-warning: 5.0.0 quick-format-unescaped: 4.0.4 real-require: 0.2.0 safe-stable-stringify: 2.5.0 - sonic-boom: 4.2.0 + sonic-boom: 4.2.1 thread-stream: 3.1.0 pirates@4.0.7: {} @@ -4661,10 +4682,6 @@ snapshots: slash@3.0.0: {} - sonic-boom@4.2.0: - dependencies: - atomic-sleep: 1.0.0 - sonic-boom@4.2.1: dependencies: atomic-sleep: 1.0.0 @@ -4816,6 +4833,8 @@ snapshots: util-deprecate@1.0.2: {} + uuid@11.1.1: {} + vite@8.1.3(@types/node@22.13.4)(esbuild@0.28.1)(jiti@2.7.0): dependencies: lightningcss: 1.32.0 @@ -4869,9 +4888,9 @@ snapshots: yallist@3.1.1: {} - yieldstar@0.4.6: + yieldstar@0.5.0: dependencies: - '@yieldstar/core': 0.4.6 + '@yieldstar/core': 0.5.0 nanoid: 5.1.16 pino: 9.14.0 serialize-error: 11.0.3 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 8d0e4c7..d1d8a41 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -3,6 +3,10 @@ packages: - "packages/*" allowBuilds: esbuild: true +minimumReleaseAgeExclude: + - "@yieldstar/core@0.5.0" + - "@yieldstar/sqlite-runtime@0.5.0" + - yieldstar@0.5.0 overrides: lodash-es@<4.18.1: ^4.18.1 js-yaml@3: 3.15.0 From 351384eb144fd44f5842b018995a3b02e2733f99 Mon Sep 17 00:00:00 2001 From: djgrant <1670902+djgrant@users.noreply.github.com> Date: Wed, 22 Jul 2026 15:22:35 +0100 Subject: [PATCH 02/34] Harden durable reconciliation semantics --- .../reconciler/src/durable/coordination.ts | 4 +- packages/reconciler/src/durable/emit.ts | 9 +-- packages/reconciler/src/durable/operations.ts | 27 +++++-- .../reconciler/src/durable/state-backend.ts | 52 ++++++++++---- packages/reconciler/src/durable/stores.ts | 10 ++- packages/reconciler/src/planner.ts | 6 +- .../test/durable-reconciliation.test.ts | 72 ++++++++++++++++++- packages/reconciler/test/planner.test.ts | 38 ++++++++++ 8 files changed, 182 insertions(+), 36 deletions(-) diff --git a/packages/reconciler/src/durable/coordination.ts b/packages/reconciler/src/durable/coordination.ts index 7dac2d2..2fb2579 100644 --- a/packages/reconciler/src/durable/coordination.ts +++ b/packages/reconciler/src/durable/coordination.ts @@ -1,5 +1,5 @@ import type { ReconcilerEventEmitter } from "../events"; -import { emitOnce } from "./emit"; +import { emitEvent } from "./emit"; import { deploymentCoordinationStore, type CoordinationState } from "./stores"; import type { DurableStep, WorkflowStore } from "./yieldstar"; @@ -25,7 +25,7 @@ export async function* acquireDeploymentCoordination( const snapshot = yield* coordination.get("notation:coordination:inspect"); const holder = snapshot.state.holder; if (holder !== null && holder !== opts.executionId) { - yield* emitOnce(step, "notation:coordination:waiting", opts.emit, () => ({ + yield* emitEvent(step, "notation:coordination:waiting", opts.emit, () => ({ level: "warn", event: "reconciler.coordination.waiting", deploymentId: opts.deploymentId, diff --git a/packages/reconciler/src/durable/emit.ts b/packages/reconciler/src/durable/emit.ts index 7be77b1..6cc14a9 100644 --- a/packages/reconciler/src/durable/emit.ts +++ b/packages/reconciler/src/durable/emit.ts @@ -6,11 +6,8 @@ import type { } from "../events"; import type { DurableStep } from "./yieldstar"; -/** - * Emits an event inside a durable step so a replayed workflow does not - * re-emit events it already sent. - */ -export function emitOnce( +/** Checkpoints delivery after the emitter returns. Emitters must tolerate a duplicate if the process crashes before that checkpoint. */ +export function emitEvent( step: DurableStep, key: string, emit: ReconcilerEventEmitter | undefined, @@ -30,7 +27,7 @@ export function emitLifecycle( resource: LifecycleResource, extra: { reason?: string; error?: unknown } = {}, ) { - return emitOnce(step, key, emit, () => + return emitEvent(step, key, emit, () => createLifecycleEvent(operation, status, resource, extra), ); } diff --git a/packages/reconciler/src/durable/operations.ts b/packages/reconciler/src/durable/operations.ts index 6bca1d2..c9bf94c 100644 --- a/packages/reconciler/src/durable/operations.ts +++ b/packages/reconciler/src/durable/operations.ts @@ -10,7 +10,7 @@ import { resolveResourceClass, } from "../resource-registry"; import { decideAction, type ResourceAction } from "../plan"; -import { emitLifecycle, emitOnce } from "./emit"; +import { emitEvent, emitLifecycle } from "./emit"; import type { DurableStateBackend } from "./state-backend"; import { resourceStateStore, @@ -82,7 +82,7 @@ export async function* reconcileResource( if (action.decision === "drift-update") { const diff = action.patch; - yield* emitOnce(step, `${prefix}:drift-detected`, opts.emit, () => ({ + yield* emitEvent(step, `${prefix}:drift-detected`, opts.emit, () => ({ level: "info", event: "reconciler.drift.detected", resourceId: resource.id, @@ -91,7 +91,7 @@ export async function* reconcileResource( })); } - yield* emitOnce(step, `${prefix}:decision`, opts.emit, () => ({ + yield* emitEvent(step, `${prefix}:decision`, opts.emit, () => ({ level: "info", event: "reconciler.deploy.decision", resourceId: resource.id, @@ -126,8 +126,8 @@ export async function* reconcileResource( } try { - // Execute the provider call. Each call runs in its own durable step, so a - // replayed workflow never repeats a completed provider mutation. + // Checkpoint successful provider calls. Provider mutations must be + // idempotent because a crash before the checkpoint can repeat them. if (operation === "create") { const primaryKey = yield* runProviderCall( step, @@ -174,6 +174,7 @@ export async function* reconcileResource( resource, opts, `${prefix}:read-after-write`, + { retryNotFound: true }, ); if (read.status === "found") resource.setOutput({ ...resource.output, ...read.output }); @@ -326,6 +327,7 @@ export async function* readRemote( resource: BaseResource, opts: DurableOperationOptions, key: string, + behaviour: { retryNotFound?: boolean } = {}, ): AsyncGenerator< any, | { status: "found"; output: Record } @@ -360,7 +362,18 @@ export async function* readRemote( ); try { const output = yield* step.run(key, async () => { - const value = await resource.read!(resource.key); + let value: Record; + try { + value = await resource.read!(resource.key); + } catch (error) { + const notFound = findErrorMatcher(error, resource.notFoundOnError); + if (behaviour.retryNotFound && notFound) { + throw new RetryableError(notFound.reason, { + ...(opts.readPollOptions ?? DEFAULT_READ_POLL_OPTIONS), + }); + } + throw error; + } // An unsettled read (a declared condition not yet met) re-polls // durably instead of returning a half-provisioned remote. const unsettled = (resource.retryReadOnCondition ?? []) @@ -468,7 +481,7 @@ export async function* sweepOrphans( const Resource = resolveResourceClass(registry, node.type as ResourceType); if (!Resource) { - yield* emitOnce(step, params.warningKey(node.id), opts.emit, () => + yield* emitEvent(step, params.warningKey(node.id), opts.emit, () => createMissingResourceRegistryMatchWarningEvent({ workflow: params.workflow, resourceId: node.id, diff --git a/packages/reconciler/src/durable/state-backend.ts b/packages/reconciler/src/durable/state-backend.ts index e15d498..b3703da 100644 --- a/packages/reconciler/src/durable/state-backend.ts +++ b/packages/reconciler/src/durable/state-backend.ts @@ -1,5 +1,7 @@ import { RevConflict, type StateNode } from "@notation/state"; +import { randomUUID } from "node:crypto"; import { + RESOURCE_CREATION_TOKEN, resourceStateStore, toStateNode, withoutRev, @@ -9,12 +11,10 @@ import type { StoreClient } from "./yieldstar"; export class DurableStateBackend { readonly #client: StoreClient; - readonly #deploymentId: string; readonly #prefix: string; constructor(client: StoreClient, deploymentId: string) { this.#client = client; - this.#deploymentId = deploymentId; // Keep deployment prefixes disjoint so orphan cleanup cannot delete // another deployment's stores. this.#prefix = `${encodeURIComponent(deploymentId)}:`; @@ -60,12 +60,20 @@ export class DurableStateBackend { const snapshot = await this.#tryGetSnapshot(storeId); if (!snapshot) { if (expectedRev !== 0) throw new RevConflict(id, expectedRev, undefined); - const initial = { ...patch, id } as StoredResourceState; + const creationToken = randomUUID(); + const initial = { + ...patch, + id, + [RESOURCE_CREATION_TOKEN]: creationToken, + } as StoredResourceState; const created = await this.#client.getOrCreateStore({ definition: resourceStateStore, id: storeId, initial, }); + if (created.state[RESOURCE_CREATION_TOKEN] !== creationToken) { + throw new RevConflict(id, expectedRev, created.version + 1); + } return { rev: created.version + 1 }; } @@ -80,7 +88,8 @@ export class DurableStateBackend { Object.assign(draft, withoutRev(patch)); }, }); - if (!result.updated) throw new RevConflict(id, expectedRev, undefined); + if (!result.updated) + throw new RevConflict(id, expectedRev, result.actualVersion + 1); return { rev: result.version + 1 }; } @@ -99,7 +108,12 @@ export class DurableStateBackend { id: storeId, snapshot, }); - if (!result.deleted) throw new RevConflict(id, expectedRev, undefined); + if (!result.deleted) + throw new RevConflict( + id, + expectedRev, + result.reason === "conflict" ? result.actualVersion + 1 : undefined, + ); } async values(): Promise { @@ -123,15 +137,27 @@ export class DurableStateBackend { async clear(): Promise { const ids = await this.#client.listStores(resourceStateStore); + const scopedIds = ids.filter((id) => id.startsWith(this.#prefix)); + const snapshots = await Promise.all( + scopedIds.map((id) => this.#tryGetSnapshot(id)), + ); await Promise.all( - ids - .filter((id) => id.startsWith(this.#prefix)) - .map((id) => - this.#client.deleteStore({ - definition: resourceStateStore, - id, - }), - ), + scopedIds.map(async (id, index) => { + const snapshot = snapshots[index]; + if (!snapshot) return; + const result = await this.#client.deleteStoreFrom({ + definition: resourceStateStore, + id, + snapshot, + }); + if (!result.deleted && result.reason === "conflict") { + throw new RevConflict( + id.slice(this.#prefix.length), + snapshot.version + 1, + result.actualVersion + 1, + ); + } + }), ); } } diff --git a/packages/reconciler/src/durable/stores.ts b/packages/reconciler/src/durable/stores.ts index fba2cb3..e2a1920 100644 --- a/packages/reconciler/src/durable/stores.ts +++ b/packages/reconciler/src/durable/stores.ts @@ -1,7 +1,11 @@ import type { StateNode } from "@notation/state"; import { defineStore, type StandardSchemaV1 } from "./yieldstar"; -export type StoredResourceState = Omit; +export const RESOURCE_CREATION_TOKEN = "$notationCreateToken"; + +export type StoredResourceState = Omit & { + [RESOURCE_CREATION_TOKEN]?: string; +}; export type CoordinationState = { holder: string | null }; const storedResourceStateSchema = plainObjectSchema( @@ -34,7 +38,9 @@ export function toStateNode(snapshot: { state: StoredResourceState; version: number; }): StateNode { - return { ...snapshot.state, rev: snapshot.version + 1 } as StateNode; + const { [RESOURCE_CREATION_TOKEN]: _creationToken, ...state } = + snapshot.state; + return { ...state, rev: snapshot.version + 1 } as StateNode; } export function withoutRev( diff --git a/packages/reconciler/src/planner.ts b/packages/reconciler/src/planner.ts index 6e90b56..1826703 100644 --- a/packages/reconciler/src/planner.ts +++ b/packages/reconciler/src/planner.ts @@ -1,4 +1,4 @@ -import type { BaseResource } from "@notation/resource"; +import { findErrorMatcher, type BaseResource } from "@notation/resource"; import type { StateBackend } from "@notation/state"; import { buildResourceDepthLevels } from "./dependency-graph"; import { @@ -42,9 +42,7 @@ export async function createPlan({ driftRead: { status: "found", output }, }); } catch (error) { - const notFound = resource.notFoundOnError?.some( - (matcher) => matcher.name === (error as Error)?.name, - ); + const notFound = findErrorMatcher(error, resource.notFoundOnError); if (!notFound) throw error; action = decideAction({ resource, diff --git a/packages/reconciler/test/durable-reconciliation.test.ts b/packages/reconciler/test/durable-reconciliation.test.ts index 1d55152..e6e44cc 100644 --- a/packages/reconciler/test/durable-reconciliation.test.ts +++ b/packages/reconciler/test/durable-reconciliation.test.ts @@ -60,7 +60,7 @@ describe("durable execution and replay", () => { runtime.close(); }); - it("resumes after a crash without repeating a completed create", async () => { + it("resumes after a crash following the create checkpoint", async () => { const create = vi.fn(async () => undefined); const TestResource = resource({ type: "test/durable/resume" }) .defineSchema({}) @@ -83,7 +83,7 @@ describe("durable execution and replay", () => { runtime.close(); }); - it("resumes durable destroy after a crash without repeating delete", async () => { + it("resumes after a crash following the delete checkpoint", async () => { const remove = vi.fn(async () => undefined); const TestResource = resource({ type: "test/durable/destroy-resume" }) .defineSchema({}) @@ -141,6 +141,46 @@ describe("durable execution and replay", () => { expect(await runtime.state.get("pending-delete")).toBeUndefined(); runtime.close(); }); + + it("retries a post-write not-found before persisting state", async () => { + let reads = 0; + const EventuallyReadable = resource({ + type: "test/durable/eventually-readable", + }) + .defineSchema({}) + .defineOperations({ + create: async () => undefined, + read: async () => { + reads += 1; + if (reads === 1) { + const error = new Error("not visible yet"); + error.name = "NotFound"; + throw error; + } + return {}; + }, + delete: async () => undefined, + notFoundOnError: [ + { name: "NotFound", reason: "resource is not visible yet" }, + ], + }); + const runtime = createRuntime( + [new EventuallyReadable({ id: "eventually-readable" })], + "post-write-read", + { readPollOptions: { maxAttempts: 3, retryInterval: 1 } }, + ); + + await runtime.run("post-write-read-execution"); + expect(reads).toBe(1); + expect(await runtime.state.get("eventually-readable")).toBeUndefined(); + + await runtime.run("post-write-read-execution"); + expect(reads).toBe(2); + expect(await runtime.state.get("eventually-readable")).toMatchObject({ + rev: 1, + }); + runtime.close(); + }); }); describe("dependency ordering", () => { @@ -174,6 +214,31 @@ describe("dependency ordering", () => { }); describe("conditional state persistence", () => { + it("allows only one concurrent create-if-absent", async () => { + const runtime = createRuntime([], "conditional-create"); + const first = runtime.state.update("resource", 0, { + ...statePatch("resource"), + output: { winner: "first" }, + }); + const second = runtime.state.update("resource", 0, { + ...statePatch("resource"), + output: { winner: "second" }, + }); + + const results = await Promise.allSettled([first, second]); + + expect( + results.filter((result) => result.status === "fulfilled"), + ).toHaveLength(1); + expect( + results.filter((result) => result.status === "rejected"), + ).toHaveLength(1); + const state = await runtime.state.get("resource"); + expect(state).toMatchObject({ rev: 1 }); + expect(state).not.toHaveProperty("$notationCreateToken"); + runtime.close(); + }); + it("uses store identity and version for conditional update and delete", async () => { const runtime = createRuntime([], "conditional-state"); await runtime.state.update("resource", 0, statePatch("resource")); @@ -397,6 +462,7 @@ function createRuntime( deploymentId: string, options: { retryOptions?: { maxAttempts: number; retryInterval: number }; + readPollOptions?: { maxAttempts: number; retryInterval: number }; crashAfterStep?: string; registry?: ResourceRegistry; driftDetection?: boolean; @@ -424,6 +490,7 @@ function createRuntime( driftDetection: options.driftDetection ?? false, emit: options.emit, retryOptions: options.retryOptions, + readPollOptions: options.readPollOptions, }); }); const destroy = workflow(async function* (step, event) { @@ -435,6 +502,7 @@ function createRuntime( registry: options.registry, emit: options.emit, retryOptions: options.retryOptions, + readPollOptions: options.readPollOptions, }); }); const router = createWorkflowRouter({ deploy, destroy }); diff --git a/packages/reconciler/test/planner.test.ts b/packages/reconciler/test/planner.test.ts index a215e82..f0011e5 100644 --- a/packages/reconciler/test/planner.test.ts +++ b/packages/reconciler/test/planner.test.ts @@ -33,4 +33,42 @@ describe("createPlan", () => { expect.objectContaining({ id: "orphan", decision: "delete-orphan" }), ]); }); + + it("honours the message constraint in not-found matchers", async () => { + const TestResource = resource({ type: "test/planner/not-found" }) + .defineSchema({}) + .defineOperations({ + create: async () => undefined, + read: async () => { + const error = new Error("access denied"); + error.name = "ProviderError"; + throw error; + }, + delete: async () => undefined, + notFoundOnError: [ + { + name: "ProviderError", + message: "not found", + reason: "resource does not exist", + }, + ], + }); + const state = new MemoryStateBackend(); + await state.update("existing", 0, { + id: "existing", + type: TestResource.type, + config: {}, + params: {}, + output: {}, + lastOperation: "create", + lastOperationAt: "2026-07-22T00:00:00.000Z", + }); + + await expect( + createPlan({ + resources: [new TestResource({ id: "existing" })], + state, + }), + ).rejects.toThrow("access denied"); + }); }); From 65d7badb78f7d9aa21c82c9d65d6032a246422c4 Mon Sep 17 00:00:00 2001 From: djgrant <1670902+djgrant@users.noreply.github.com> Date: Wed, 22 Jul 2026 15:33:13 +0100 Subject: [PATCH 03/34] Protect durable state revisions --- packages/reconciler/src/durable/state-backend.ts | 2 +- packages/reconciler/test/durable-reconciliation.test.ts | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/reconciler/src/durable/state-backend.ts b/packages/reconciler/src/durable/state-backend.ts index b3703da..88fa35c 100644 --- a/packages/reconciler/src/durable/state-backend.ts +++ b/packages/reconciler/src/durable/state-backend.ts @@ -62,7 +62,7 @@ export class DurableStateBackend { if (expectedRev !== 0) throw new RevConflict(id, expectedRev, undefined); const creationToken = randomUUID(); const initial = { - ...patch, + ...withoutRev(patch), id, [RESOURCE_CREATION_TOKEN]: creationToken, } as StoredResourceState; diff --git a/packages/reconciler/test/durable-reconciliation.test.ts b/packages/reconciler/test/durable-reconciliation.test.ts index e6e44cc..78ea78c 100644 --- a/packages/reconciler/test/durable-reconciliation.test.ts +++ b/packages/reconciler/test/durable-reconciliation.test.ts @@ -218,10 +218,12 @@ describe("conditional state persistence", () => { const runtime = createRuntime([], "conditional-create"); const first = runtime.state.update("resource", 0, { ...statePatch("resource"), + rev: 41, output: { winner: "first" }, }); const second = runtime.state.update("resource", 0, { ...statePatch("resource"), + rev: 42, output: { winner: "second" }, }); From 73546cf73349b27583b353365782b8a442f02763 Mon Sep 17 00:00:00 2001 From: djgrant <1670902+djgrant@users.noreply.github.com> Date: Fri, 24 Jul 2026 14:10:14 +0100 Subject: [PATCH 04/34] Replace durable store schemas with Valibot --- packages/reconciler/package.json | 1 + packages/reconciler/src/durable/stores.ts | 72 +++++++------------- packages/reconciler/src/durable/yieldstar.ts | 2 +- pnpm-lock.yaml | 18 ++++- 4 files changed, 40 insertions(+), 53 deletions(-) diff --git a/packages/reconciler/package.json b/packages/reconciler/package.json index 3429d0d..d1ec1db 100644 --- a/packages/reconciler/package.json +++ b/packages/reconciler/package.json @@ -16,6 +16,7 @@ "@notation/state": "workspace:*", "@yieldstar/core": "0.5.0", "deep-object-diff": "^1.1.9", + "valibot": "^1.4.2", "yieldstar": "0.5.0" }, "devDependencies": { diff --git a/packages/reconciler/src/durable/stores.ts b/packages/reconciler/src/durable/stores.ts index e2a1920..d25f3ab 100644 --- a/packages/reconciler/src/durable/stores.ts +++ b/packages/reconciler/src/durable/stores.ts @@ -1,46 +1,42 @@ import type { StateNode } from "@notation/state"; -import { defineStore, type StandardSchemaV1 } from "./yieldstar"; +import * as v from "valibot"; +import { defineStore } from "./yieldstar"; export const RESOURCE_CREATION_TOKEN = "$notationCreateToken"; -export type StoredResourceState = Omit & { - [RESOURCE_CREATION_TOKEN]?: string; -}; -export type CoordinationState = { holder: string | null }; - -const storedResourceStateSchema = plainObjectSchema( - "Stored resource state", - (value) => - typeof value.id === "string" && - typeof value.type === "string" && - isPlainObject(value.config) && - isPlainObject(value.params) && - isPlainObject(value.output), -); -const coordinationStateSchema = plainObjectSchema( - "Deployment coordination state", - (value) => - "holder" in value && - (value.holder === null || typeof value.holder === "string"), -); - export const resourceStateStore = defineStore( - "notation/resource-state", - storedResourceStateSchema, + "resource-state", + v.looseObject({ + id: v.string(), + type: v.string(), + config: v.record(v.string(), v.unknown()), + params: v.record(v.string(), v.unknown()), + output: v.record(v.string(), v.unknown()), + lastOperation: v.picklist(["drift", "create", "update", "delete"]), + lastOperationAt: v.string(), + [RESOURCE_CREATION_TOKEN]: v.optional(v.string()), + }), ); export const deploymentCoordinationStore = defineStore( - "notation/deployment-coordination", - coordinationStateSchema, + "deployment-coordination", + v.object({ holder: v.nullable(v.string()) }), ); +export type StoredResourceState = v.InferOutput< + typeof resourceStateStore.schema +>; +export type CoordinationState = v.InferOutput< + typeof deploymentCoordinationStore.schema +>; + export function toStateNode(snapshot: { state: StoredResourceState; version: number; }): StateNode { const { [RESOURCE_CREATION_TOKEN]: _creationToken, ...state } = snapshot.state; - return { ...state, rev: snapshot.version + 1 } as StateNode; + return { ...state, rev: snapshot.version + 1 }; } export function withoutRev( @@ -49,25 +45,3 @@ export function withoutRev( const { rev: _rev, ...stored } = patch; return stored; } - -function plainObjectSchema>( - label: string, - refine: (value: Record) => boolean, -): StandardSchemaV1 { - return { - "~standard": { - version: 1, - vendor: "notation", - validate(value) { - if (!isPlainObject(value) || !refine(value)) { - return { issues: [{ message: `${label} is invalid` }] }; - } - return { value: value as T }; - }, - }, - }; -} - -function isPlainObject(value: unknown): value is Record { - return typeof value === "object" && value !== null && !Array.isArray(value); -} diff --git a/packages/reconciler/src/durable/yieldstar.ts b/packages/reconciler/src/durable/yieldstar.ts index 13ea2dd..4fb26db 100644 --- a/packages/reconciler/src/durable/yieldstar.ts +++ b/packages/reconciler/src/durable/yieldstar.ts @@ -2,7 +2,7 @@ import type { WorkflowFn } from "yieldstar"; export { RetryableError, defineStore } from "yieldstar"; export type { WorkflowStore } from "yieldstar"; -export type { StandardSchemaV1, StoreClient } from "@yieldstar/core"; +export type { StoreClient } from "@yieldstar/core"; /** The durable step primitive the runtime hands to workflow functions. */ export type DurableStep = Parameters>[0]; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 82b4901..fd80fe6 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -345,15 +345,15 @@ importers: '@notation/state': specifier: workspace:* version: link:../state - '@notation/utils': - specifier: workspace:* - version: link:../utils '@yieldstar/core': specifier: 0.5.0 version: 0.5.0 deep-object-diff: specifier: ^1.1.9 version: 1.1.9 + valibot: + specifier: ^1.4.2 + version: 1.4.2(typescript@6.0.3) yieldstar: specifier: 0.5.0 version: 0.5.0 @@ -2615,6 +2615,14 @@ packages: resolution: {integrity: sha512-vIYxrBCC/N/K+Js3qSN88go7kIfNPssr/hHCesKCQNAjmgvYS2oqr69kIufEG+O4+PfezOH4EbIeHCfFov8ZgQ==} hasBin: true + valibot@1.4.2: + resolution: {integrity: sha512-gjdCvJ6d3RyHAneqxMYMW9QMCwYMb3jpOO0IyHZV1bnRHFBHrX3VkIILt5XYR0WhwHiH7Mty8ovuPZ/O3gamrg==} + peerDependencies: + typescript: '>=5' + peerDependenciesMeta: + typescript: + optional: true + vite@8.1.3: resolution: {integrity: sha512-Ds+gBRbj0lwRO2Y5hwnUBdxSwlAve9LeRyU4sNnAr0ewW0gWF0n5bgXgUzbgZ49MV9BVUAQUFYVcDUcilUExMA==} engines: {node: ^20.19.0 || >=22.12.0} @@ -4835,6 +4843,10 @@ snapshots: uuid@11.1.1: {} + valibot@1.4.2(typescript@6.0.3): + optionalDependencies: + typescript: 6.0.3 + vite@8.1.3(@types/node@22.13.4)(esbuild@0.28.1)(jiti@2.7.0): dependencies: lightningcss: 1.32.0 From fcb06914c546f3d23febf8aac0e83b7d51ae37ab Mon Sep 17 00:00:00 2001 From: djgrant <1670902+djgrant@users.noreply.github.com> Date: Mon, 27 Jul 2026 14:47:28 +0100 Subject: [PATCH 05/34] Adapt the durable reconciler to undefined reads The durable planner and operations now treat an undefined read as absence, and a tagged ResourceNotReadyError as a wait: durable operations retry it, while planning reports an indeterminate node carrying the message. --- packages/cli/src/plan.ts | 1 + packages/reconciler/src/durable/operations.ts | 127 +++++++----------- packages/reconciler/src/planner.ts | 35 +++-- .../test/durable-reconciliation.test.ts | 29 ++-- packages/reconciler/test/planner.test.ts | 80 +++++++++-- 5 files changed, 145 insertions(+), 127 deletions(-) diff --git a/packages/cli/src/plan.ts b/packages/cli/src/plan.ts index ef52ec8..787eaf0 100644 --- a/packages/cli/src/plan.ts +++ b/packages/cli/src/plan.ts @@ -79,6 +79,7 @@ function printPlanSummary(result: Plan, logger: Logger) { `${count("update") + count("drift-update")} to update`, `${count("drift-recreate")} to recreate`, `${count("delete-orphan")} to delete`, + `${count("indeterminate")} could not be checked`, `${count("noop")} unchanged`, ].join(", "); diff --git a/packages/reconciler/src/durable/operations.ts b/packages/reconciler/src/durable/operations.ts index c9bf94c..ec7027c 100644 --- a/packages/reconciler/src/durable/operations.ts +++ b/packages/reconciler/src/durable/operations.ts @@ -1,5 +1,5 @@ import { - findErrorMatcher, + ResourceNotReadyError, type BaseResource, type ResourceType, } from "@notation/resource"; @@ -9,7 +9,7 @@ import { createResourceRegistryFromResources, resolveResourceClass, } from "../resource-registry"; -import { decideAction, type ResourceAction } from "../plan"; +import { decideAction, type DriftRead, type ResourceAction } from "../plan"; import { emitEvent, emitLifecycle } from "./emit"; import type { DurableStateBackend } from "./state-backend"; import { @@ -129,11 +129,10 @@ export async function* reconcileResource( // Checkpoint successful provider calls. Provider mutations must be // idempotent because a crash before the checkpoint can repeat them. if (operation === "create") { - const primaryKey = yield* runProviderCall( + const primaryKey = yield* runResourceMutation( step, `${prefix}:create`, () => resource.create(params), - resource, opts.retryOptions, ); resource.setOutput(params); @@ -151,7 +150,7 @@ export async function* reconcileResource( ); return; } - yield* runProviderCall( + yield* runResourceMutation( step, `${prefix}:update`, () => @@ -161,7 +160,6 @@ export async function* reconcileResource( params, resource.toState(resource.output), ), - resource, opts.retryOptions, ); resource.setOutput({ ...resource.key, ...params }); @@ -174,9 +172,9 @@ export async function* reconcileResource( resource, opts, `${prefix}:read-after-write`, - { retryNotFound: true }, + { retryAbsent: true }, ); - if (read.status === "found") + if (read.kind === "present") resource.setOutput({ ...resource.output, ...read.output }); const nextState: StoredResourceState = { @@ -269,28 +267,12 @@ export async function* deleteResource( } try { - // An already-deleted remote is success, not failure: the goal state is - // absence, so a declared not-found error downgrades to a skip. - try { - yield* runProviderCall( - step, - `${prefix}:delete`, - () => resource.delete(resource.key, resource.toState(resource.output)), - resource, - opts.retryOptions, - ); - } catch (error) { - if (!findErrorMatcher(error, resource.notFoundOnError)) throw error; - yield* emitLifecycle( - step, - `${prefix}:delete:not-found`, - opts.emit, - "delete", - "skip", - resource, - { reason: "resource-not-found" }, - ); - } + yield* runResourceMutation( + step, + `${prefix}:delete`, + () => resource.delete(resource.key, resource.toState(resource.output)), + opts.retryOptions, + ); // State is removed only after the provider delete completes, and only if // the store still matches the snapshot read before deleting. @@ -327,13 +309,8 @@ export async function* readRemote( resource: BaseResource, opts: DurableOperationOptions, key: string, - behaviour: { retryNotFound?: boolean } = {}, -): AsyncGenerator< - any, - | { status: "found"; output: Record } - | { status: "not-found" }, - any -> { + behaviour: { retryAbsent?: boolean } = {}, +): AsyncGenerator { if (!resource.read) { yield* emitLifecycle( step, @@ -347,7 +324,7 @@ export async function* readRemote( }, ); return { - status: "found", + kind: "present", output: { ...(await resource.getParams()), ...resource.output }, }; } @@ -361,36 +338,40 @@ export async function* readRemote( resource, ); try { - const output = yield* step.run(key, async () => { - let value: Record; + const result = yield* step.run(key, async () => { try { - value = await resource.read!(resource.key); + const output = await resource.read!(resource.key); + if (output === undefined && behaviour.retryAbsent) { + throw new RetryableError("Waiting for resource to become visible", { + ...(opts.readPollOptions ?? DEFAULT_READ_POLL_OPTIONS), + }); + } + // Absence is null rather than undefined so that it survives the + // step's JSON round-trip when the run is replayed. + return output ?? null; } catch (error) { - const notFound = findErrorMatcher(error, resource.notFoundOnError); - if (behaviour.retryNotFound && notFound) { - throw new RetryableError(notFound.reason, { + // A tagged not-ready condition is the provider telling us to wait. + // Everything else is a genuine failure and must surface. + if (ResourceNotReadyError.is(error)) { + throw new RetryableError(error.message, { ...(opts.readPollOptions ?? DEFAULT_READ_POLL_OPTIONS), }); } throw error; } - // An unsettled read (a declared condition not yet met) re-polls - // durably instead of returning a half-provisioned remote. - const unsettled = (resource.retryReadOnCondition ?? []) - .filter(Boolean) - .find((condition) => { - const actual = value[condition!.key]; - return condition!.value === undefined - ? !actual - : actual !== condition!.value; - }); - if (unsettled) { - throw new RetryableError(unsettled.reason, { - ...(opts.readPollOptions ?? DEFAULT_READ_POLL_OPTIONS), - }); - } - return value; }); + if (result === null) { + yield* emitLifecycle( + step, + `${key}:absent`, + opts.emit, + "read", + "skip", + resource, + { reason: "resource-absent" }, + ); + return { kind: "absent" }; + } yield* emitLifecycle( step, `${key}:success`, @@ -399,20 +380,8 @@ export async function* readRemote( "success", resource, ); - return { status: "found", output }; + return { kind: "present", output: result }; } catch (error) { - if (findErrorMatcher(error, resource.notFoundOnError)) { - yield* emitLifecycle( - step, - `${key}:not-found`, - opts.emit, - "read", - "skip", - resource, - { reason: "resource-not-found" }, - ); - return { status: "not-found" }; - } yield* emitLifecycle( step, `${key}:error`, @@ -429,23 +398,21 @@ export async function* readRemote( } /** - * Runs a provider call in a durable step, converting errors the resource has - * declared retryable into durable retries. + * Runs a resource mutation in a durable step and adapts resource retry + * signals to the workflow runtime. */ -export function runProviderCall( +export function runResourceMutation( step: DurableStep, key: string, call: () => T | Promise, - resource: BaseResource, retryOptions?: PollOptions, ) { return step.run(key, async () => { try { return await call(); } catch (error) { - const matcher = findErrorMatcher(error, resource.retryLaterOnError); - if (matcher) { - throw new RetryableError(matcher.reason, { + if (ResourceNotReadyError.is(error)) { + throw new RetryableError(error.message, { ...(retryOptions ?? DEFAULT_RETRY_OPTIONS), }); } diff --git a/packages/reconciler/src/planner.ts b/packages/reconciler/src/planner.ts index 1826703..4b5a11e 100644 --- a/packages/reconciler/src/planner.ts +++ b/packages/reconciler/src/planner.ts @@ -1,4 +1,4 @@ -import { findErrorMatcher, type BaseResource } from "@notation/resource"; +import { ResourceNotReadyError, type BaseResource } from "@notation/resource"; import type { StateBackend } from "@notation/state"; import { buildResourceDepthLevels } from "./dependency-graph"; import { @@ -33,24 +33,31 @@ export async function createPlan({ let action = decideAction({ resource, stateNode, params }); if (action.decision === "noop" && driftDetection && resource.read) { + let output: Record | undefined; try { - const output = await resource.read(resource.key); - action = decideAction({ - resource, - stateNode, - params, - driftRead: { status: "found", output }, - }); + output = (await resource.read(resource.key)) as + Record | undefined; } catch (error) { - const notFound = findErrorMatcher(error, resource.notFoundOnError); - if (!notFound) throw error; - action = decideAction({ - resource, - stateNode, + // Planning cannot diff against a resource that has not settled, so + // it reports the condition rather than guessing at a decision. + if (!ResourceNotReadyError.is(error)) throw error; + nodes.push({ + id: resource.id, + type: resource.type, + decision: "indeterminate", + reason: error.message, params, - driftRead: { status: "not-found" }, + dependsOn: getDependencyIds(resource), }); + continue; } + + action = decideAction({ + resource, + stateNode, + params, + driftRead: output ? { kind: "present", output } : { kind: "absent" }, + }); } nodes.push({ diff --git a/packages/reconciler/test/durable-reconciliation.test.ts b/packages/reconciler/test/durable-reconciliation.test.ts index 78ea78c..f34e41b 100644 --- a/packages/reconciler/test/durable-reconciliation.test.ts +++ b/packages/reconciler/test/durable-reconciliation.test.ts @@ -8,7 +8,11 @@ import { SqliteStoreClient, createSqliteDb, } from "@yieldstar/sqlite-runtime/node"; -import { resource, type BaseResource } from "@notation/resource"; +import { + resource, + ResourceNotReadyError, + type BaseResource, +} from "@notation/resource"; import pino from "pino"; import { createWorkflowRouter, workflow } from "yieldstar"; import { describe, expect, it, vi } from "vitest"; @@ -30,15 +34,10 @@ describe("durable execution and replay", () => { create: async () => { attempts += 1; if (attempts === 1) { - const error = new Error("provider is pending"); - error.name = "ProviderPending"; - throw error; + throw new ResourceNotReadyError("provider is not ready"); } }, delete: async () => undefined, - retryLaterOnError: [ - { name: "ProviderPending", reason: "provider is pending" }, - ], }); const runtime = createRuntime( [new PendingResource({ id: "pending" })], @@ -116,14 +115,9 @@ describe("durable execution and replay", () => { delete: async () => { attempts += 1; if (attempts === 1) { - const error = new Error("delete is pending"); - error.name = "DeletePending"; - throw error; + throw new ResourceNotReadyError("delete is not ready"); } }, - retryLaterOnError: [ - { name: "DeletePending", reason: "delete is pending" }, - ], }); const runtime = createRuntime( [new PendingDelete({ id: "pending-delete" })], @@ -153,16 +147,11 @@ describe("durable execution and replay", () => { read: async () => { reads += 1; if (reads === 1) { - const error = new Error("not visible yet"); - error.name = "NotFound"; - throw error; + return undefined; } - return {}; + return {} as const; }, delete: async () => undefined, - notFoundOnError: [ - { name: "NotFound", reason: "resource is not visible yet" }, - ], }); const runtime = createRuntime( [new EventuallyReadable({ id: "eventually-readable" })], diff --git a/packages/reconciler/test/planner.test.ts b/packages/reconciler/test/planner.test.ts index f0011e5..35a854e 100644 --- a/packages/reconciler/test/planner.test.ts +++ b/packages/reconciler/test/planner.test.ts @@ -1,4 +1,4 @@ -import { resource } from "@notation/resource"; +import { ResourceNotReadyError, resource } from "@notation/resource"; import { MemoryStateBackend } from "@notation/state"; import { describe, expect, it } from "vitest"; import { createPlan } from "../src/planner"; @@ -34,24 +34,15 @@ describe("createPlan", () => { ]); }); - it("honours the message constraint in not-found matchers", async () => { - const TestResource = resource({ type: "test/planner/not-found" }) + it("propagates unexpected read failures", async () => { + const TestResource = resource({ type: "test/planner/read-failure" }) .defineSchema({}) .defineOperations({ create: async () => undefined, read: async () => { - const error = new Error("access denied"); - error.name = "ProviderError"; - throw error; + throw new Error("access denied"); }, delete: async () => undefined, - notFoundOnError: [ - { - name: "ProviderError", - message: "not found", - reason: "resource does not exist", - }, - ], }); const state = new MemoryStateBackend(); await state.update("existing", 0, { @@ -71,4 +62,67 @@ describe("createPlan", () => { }), ).rejects.toThrow("access denied"); }); + + it("plans recreation when the resource reports absence", async () => { + const TestResource = resource({ type: "test/planner/absent" }) + .defineSchema({}) + .defineOperations({ + create: async () => undefined, + read: async () => undefined, + delete: async () => undefined, + }); + const state = new MemoryStateBackend(); + await state.update("existing", 0, { + id: "existing", + type: TestResource.type, + config: {}, + params: {}, + output: {}, + lastOperation: "create", + lastOperationAt: "2026-07-22T00:00:00.000Z", + }); + + const plan = await createPlan({ + resources: [new TestResource({ id: "existing" })], + state, + }); + + expect(plan.nodes[0]).toMatchObject({ + id: "existing", + decision: "drift-recreate", + }); + }); + + it("reports an indeterminate decision while the resource is not ready", async () => { + const TestResource = resource({ type: "test/planner/pending" }) + .defineSchema({}) + .defineOperations({ + create: async () => undefined, + read: async () => { + throw new ResourceNotReadyError("Waiting for the provider"); + }, + delete: async () => undefined, + }); + const state = new MemoryStateBackend(); + await state.update("existing", 0, { + id: "existing", + type: TestResource.type, + config: {}, + params: {}, + output: {}, + lastOperation: "create", + lastOperationAt: "2026-07-22T00:00:00.000Z", + }); + + const plan = await createPlan({ + resources: [new TestResource({ id: "existing" })], + state, + }); + + expect(plan.nodes[0]).toMatchObject({ + id: "existing", + decision: "indeterminate", + reason: "Waiting for the provider", + }); + }); }); From 40d140c43106d0fc0d1ed9602415f856e3881df9 Mon Sep 17 00:00:00 2001 From: djgrant <1670902+djgrant@users.noreply.github.com> Date: Mon, 27 Jul 2026 17:40:10 +0100 Subject: [PATCH 06/34] Use resource operation signals in durable workflows --- packages/cli/src/plan.ts | 1 - packages/reconciler/src/durable/index.ts | 3 - packages/reconciler/src/durable/operations.ts | 136 ++++++------------ packages/reconciler/src/durable/types.ts | 18 +-- packages/reconciler/src/durable/yieldstar.ts | 2 +- packages/reconciler/src/planner.ts | 31 ++-- .../test/durable-reconciliation.test.ts | 70 ++++++--- packages/reconciler/test/planner.test.ts | 26 +++- 8 files changed, 135 insertions(+), 152 deletions(-) diff --git a/packages/cli/src/plan.ts b/packages/cli/src/plan.ts index 787eaf0..ef52ec8 100644 --- a/packages/cli/src/plan.ts +++ b/packages/cli/src/plan.ts @@ -79,7 +79,6 @@ function printPlanSummary(result: Plan, logger: Logger) { `${count("update") + count("drift-update")} to update`, `${count("drift-recreate")} to recreate`, `${count("delete-orphan")} to delete`, - `${count("indeterminate")} could not be checked`, `${count("noop")} unchanged`, ].join(", "); diff --git a/packages/reconciler/src/durable/index.ts b/packages/reconciler/src/durable/index.ts index e42707f..c896eb8 100644 --- a/packages/reconciler/src/durable/index.ts +++ b/packages/reconciler/src/durable/index.ts @@ -8,11 +8,8 @@ export { type StoredResourceState, } from "./stores"; export { - DEFAULT_READ_POLL_OPTIONS, - DEFAULT_RETRY_OPTIONS, type DurableDeployOptions, type DurableDestroyOptions, type DurableOperationOptions, - type PollOptions, } from "./types"; export type { DurableStep } from "./yieldstar"; diff --git a/packages/reconciler/src/durable/operations.ts b/packages/reconciler/src/durable/operations.ts index ec7027c..d945403 100644 --- a/packages/reconciler/src/durable/operations.ts +++ b/packages/reconciler/src/durable/operations.ts @@ -1,5 +1,5 @@ import { - ResourceNotReadyError, + ResourceNotFoundError, type BaseResource, type ResourceType, } from "@notation/resource"; @@ -9,6 +9,7 @@ import { createResourceRegistryFromResources, resolveResourceClass, } from "../resource-registry"; +import { runPendingOperation } from "../operations/operation.pending"; import { decideAction, type DriftRead, type ResourceAction } from "../plan"; import { emitEvent, emitLifecycle } from "./emit"; import type { DurableStateBackend } from "./state-backend"; @@ -17,18 +18,8 @@ import { toStateNode, type StoredResourceState, } from "./stores"; -import { - DEFAULT_READ_POLL_OPTIONS, - DEFAULT_RETRY_OPTIONS, - type DurableDeployOptions, - type DurableOperationOptions, - type PollOptions, -} from "./types"; -import { - RetryableError, - type DurableStep, - type WorkflowStore, -} from "./yieldstar"; +import type { DurableDeployOptions, DurableOperationOptions } from "./types"; +import type { DurableStep, WorkflowStore } from "./yieldstar"; export async function* reconcileResource( step: DurableStep, @@ -129,11 +120,11 @@ export async function* reconcileResource( // Checkpoint successful provider calls. Provider mutations must be // idempotent because a crash before the checkpoint can repeat them. if (operation === "create") { - const primaryKey = yield* runResourceMutation( + const primaryKey = yield* runPendingOperation( step, `${prefix}:create`, - () => resource.create(params), - opts.retryOptions, + (context) => resource.create(params, context), + opts.maxOperationAttempts, ); resource.setOutput(params); if (primaryKey) resource.setOutput({ ...primaryKey, ...resource.output }); @@ -150,32 +141,31 @@ export async function* reconcileResource( ); return; } - yield* runResourceMutation( + yield* runPendingOperation( step, `${prefix}:update`, - () => + (context) => resource.update!( resource.key, patch, params, resource.toState(resource.output), + context, ), - opts.retryOptions, + opts.maxOperationAttempts, ); resource.setOutput({ ...resource.key, ...params }); } // Read back the remote so persisted output reflects provider-assigned // values, then persist conditionally against the snapshot read above. - const read = yield* readRemote( + const read = yield* readResource( step, resource, opts, `${prefix}:read-after-write`, - { retryAbsent: true }, ); - if (read.kind === "present") - resource.setOutput({ ...resource.output, ...read.output }); + resource.setOutput({ ...resource.output, ...read }); const nextState: StoredResourceState = { id: resource.id, @@ -267,11 +257,16 @@ export async function* deleteResource( } try { - yield* runResourceMutation( + yield* runPendingOperation( step, `${prefix}:delete`, - () => resource.delete(resource.key, resource.toState(resource.output)), - opts.retryOptions, + (context) => + resource.delete( + resource.key, + resource.toState(resource.output), + context, + ), + opts.maxOperationAttempts, ); // State is removed only after the provider delete completes, and only if @@ -309,8 +304,24 @@ export async function* readRemote( resource: BaseResource, opts: DurableOperationOptions, key: string, - behaviour: { retryAbsent?: boolean } = {}, ): AsyncGenerator { + try { + return { + kind: "present", + output: yield* readResource(step, resource, opts, key), + }; + } catch (error) { + if (ResourceNotFoundError.is(error)) return { kind: "absent" }; + throw error; + } +} + +async function* readResource( + step: DurableStep, + resource: BaseResource, + opts: DurableOperationOptions, + key: string, +): AsyncGenerator, any> { if (!resource.read) { yield* emitLifecycle( step, @@ -323,10 +334,7 @@ export async function* readRemote( reason: "read-not-implemented", }, ); - return { - kind: "present", - output: { ...(await resource.getParams()), ...resource.output }, - }; + return { ...(await resource.getParams()), ...resource.output }; } yield* emitLifecycle( @@ -338,40 +346,12 @@ export async function* readRemote( resource, ); try { - const result = yield* step.run(key, async () => { - try { - const output = await resource.read!(resource.key); - if (output === undefined && behaviour.retryAbsent) { - throw new RetryableError("Waiting for resource to become visible", { - ...(opts.readPollOptions ?? DEFAULT_READ_POLL_OPTIONS), - }); - } - // Absence is null rather than undefined so that it survives the - // step's JSON round-trip when the run is replayed. - return output ?? null; - } catch (error) { - // A tagged not-ready condition is the provider telling us to wait. - // Everything else is a genuine failure and must surface. - if (ResourceNotReadyError.is(error)) { - throw new RetryableError(error.message, { - ...(opts.readPollOptions ?? DEFAULT_READ_POLL_OPTIONS), - }); - } - throw error; - } - }); - if (result === null) { - yield* emitLifecycle( - step, - `${key}:absent`, - opts.emit, - "read", - "skip", - resource, - { reason: "resource-absent" }, - ); - return { kind: "absent" }; - } + const result = yield* runPendingOperation( + step, + key, + (context) => resource.read!(resource.key, context), + opts.maxOperationAttempts, + ); yield* emitLifecycle( step, `${key}:success`, @@ -380,7 +360,7 @@ export async function* readRemote( "success", resource, ); - return { kind: "present", output: result }; + return result; } catch (error) { yield* emitLifecycle( step, @@ -397,30 +377,6 @@ export async function* readRemote( } } -/** - * Runs a resource mutation in a durable step and adapts resource retry - * signals to the workflow runtime. - */ -export function runResourceMutation( - step: DurableStep, - key: string, - call: () => T | Promise, - retryOptions?: PollOptions, -) { - return step.run(key, async () => { - try { - return await call(); - } catch (error) { - if (ResourceNotReadyError.is(error)) { - throw new RetryableError(error.message, { - ...(retryOptions ?? DEFAULT_RETRY_OPTIONS), - }); - } - throw error; - } - }); -} - /** * Deletes persisted resources that are no longer in the desired set. A state * node whose type has no registry entry is left in place and surfaced as a diff --git a/packages/reconciler/src/durable/types.ts b/packages/reconciler/src/durable/types.ts index c139348..76f7000 100644 --- a/packages/reconciler/src/durable/types.ts +++ b/packages/reconciler/src/durable/types.ts @@ -3,21 +3,6 @@ import type { ReconcilerEventEmitter } from "../events"; import type { ResourceRegistry } from "../resource-registry"; import type { DurableStateBackend } from "./state-backend"; -export type PollOptions = { - maxAttempts: number; - retryInterval: number; -}; - -export const DEFAULT_RETRY_OPTIONS: PollOptions = { - maxAttempts: 10, - retryInterval: 1_000, -}; - -export const DEFAULT_READ_POLL_OPTIONS: PollOptions = { - maxAttempts: 30, - retryInterval: 1_000, -}; - export type DurableOperationOptions = { deploymentId: string; executionId: string; @@ -26,8 +11,7 @@ export type DurableOperationOptions = { registry?: ResourceRegistry; dryRun?: boolean; emit?: ReconcilerEventEmitter; - retryOptions?: PollOptions; - readPollOptions?: PollOptions; + maxOperationAttempts?: number; }; export type DurableDeployOptions = DurableOperationOptions & { diff --git a/packages/reconciler/src/durable/yieldstar.ts b/packages/reconciler/src/durable/yieldstar.ts index 4fb26db..6bbba63 100644 --- a/packages/reconciler/src/durable/yieldstar.ts +++ b/packages/reconciler/src/durable/yieldstar.ts @@ -1,6 +1,6 @@ import type { WorkflowFn } from "yieldstar"; -export { RetryableError, defineStore } from "yieldstar"; +export { defineStore } from "yieldstar"; export type { WorkflowStore } from "yieldstar"; export type { StoreClient } from "@yieldstar/core"; diff --git a/packages/reconciler/src/planner.ts b/packages/reconciler/src/planner.ts index 4b5a11e..ebfa064 100644 --- a/packages/reconciler/src/planner.ts +++ b/packages/reconciler/src/planner.ts @@ -1,6 +1,7 @@ -import { ResourceNotReadyError, type BaseResource } from "@notation/resource"; +import { ResourceNotFoundError, type BaseResource } from "@notation/resource"; import type { StateBackend } from "@notation/state"; import { buildResourceDepthLevels } from "./dependency-graph"; +import { readResourceOperation } from "./operations"; import { decideAction, getDependencyIds, @@ -8,6 +9,7 @@ import { type Plan, type PlanNode, } from "./plan"; +import { createStepRunner, runOperation } from "./reconciler"; export type CreatePlanOptions = { resources: BaseResource[]; @@ -33,30 +35,25 @@ export async function createPlan({ let action = decideAction({ resource, stateNode, params }); if (action.decision === "noop" && driftDetection && resource.read) { - let output: Record | undefined; + let driftRead; try { - output = (await resource.read(resource.key)) as - Record | undefined; + const output = await runOperation( + readResourceOperation(createStepRunner(), { + resource, + state, + }), + ); + driftRead = { kind: "present" as const, output }; } catch (error) { - // Planning cannot diff against a resource that has not settled, so - // it reports the condition rather than guessing at a decision. - if (!ResourceNotReadyError.is(error)) throw error; - nodes.push({ - id: resource.id, - type: resource.type, - decision: "indeterminate", - reason: error.message, - params, - dependsOn: getDependencyIds(resource), - }); - continue; + if (!ResourceNotFoundError.is(error)) throw error; + driftRead = { kind: "absent" as const }; } action = decideAction({ resource, stateNode, params, - driftRead: output ? { kind: "present", output } : { kind: "absent" }, + driftRead, }); } diff --git a/packages/reconciler/test/durable-reconciliation.test.ts b/packages/reconciler/test/durable-reconciliation.test.ts index f34e41b..1a51cd7 100644 --- a/packages/reconciler/test/durable-reconciliation.test.ts +++ b/packages/reconciler/test/durable-reconciliation.test.ts @@ -10,9 +10,11 @@ import { } from "@yieldstar/sqlite-runtime/node"; import { resource, - ResourceNotReadyError, + ResourceNotFoundError, + ResourceOperationPendingError, type BaseResource, } from "@notation/resource"; +import { setTimeout as sleep } from "node:timers/promises"; import pino from "pino"; import { createWorkflowRouter, workflow } from "yieldstar"; import { describe, expect, it, vi } from "vitest"; @@ -31,24 +33,30 @@ describe("durable execution and replay", () => { const PendingResource = resource({ type: "test/durable/pending" }) .defineSchema({}) .defineOperations({ - create: async () => { + create: async (_params, context) => { attempts += 1; if (attempts === 1) { - throw new ResourceNotReadyError("provider is not ready"); + expect(context).toBeUndefined(); + throw new ResourceOperationPendingError("provider is not ready", { + retryAfterMs: 1, + callbackContext: { requestId: "request-123" }, + }); } + expect(context).toEqual({ requestId: "request-123" }); }, delete: async () => undefined, }); const runtime = createRuntime( [new PendingResource({ id: "pending" })], "durable-wait", - { retryOptions: { maxAttempts: 3, retryInterval: 1 } }, + { maxOperationAttempts: 3 }, ); await runtime.run("wait-execution"); expect(attempts).toBe(1); expect(runtime.scheduler.events).toHaveLength(1); + await sleep(5); await runtime.run("wait-execution"); expect(attempts).toBe(2); expect(await runtime.state.get("pending")).toMatchObject({ @@ -67,7 +75,7 @@ describe("durable execution and replay", () => { const runtime = createRuntime( [new TestResource({ id: "resume" })], "crash-resume", - { crashAfterStep: "notation:resource:resume:create" }, + { crashAfterStep: "notation:resource:resume:create:attempt:0" }, ); await expect(runtime.run("resume-execution")).rejects.toThrow( @@ -90,7 +98,7 @@ describe("durable execution and replay", () => { const runtime = createRuntime( [new TestResource({ id: "destroyed" })], "destroy-crash-resume", - { crashAfterStep: "notation:destroy:destroyed:delete" }, + { crashAfterStep: "notation:destroy:destroyed:delete:attempt:0" }, ); await runtime.run("deploy-before-destroy"); @@ -115,14 +123,16 @@ describe("durable execution and replay", () => { delete: async () => { attempts += 1; if (attempts === 1) { - throw new ResourceNotReadyError("delete is not ready"); + throw new ResourceOperationPendingError("delete is not ready", { + retryAfterMs: 1, + }); } }, }); const runtime = createRuntime( [new PendingDelete({ id: "pending-delete" })], "durable-destroy-wait", - { retryOptions: { maxAttempts: 3, retryInterval: 1 } }, + { maxOperationAttempts: 3 }, ); await runtime.run("deploy-before-wait"); @@ -130,13 +140,14 @@ describe("durable execution and replay", () => { expect(attempts).toBe(1); expect(await runtime.state.get("pending-delete")).toBeDefined(); + await sleep(5); await runtime.destroy("destroy-wait"); expect(attempts).toBe(2); expect(await runtime.state.get("pending-delete")).toBeUndefined(); runtime.close(); }); - it("retries a post-write not-found before persisting state", async () => { + it("waits when a resource reports that its post-write read is pending", async () => { let reads = 0; const EventuallyReadable = resource({ type: "test/durable/eventually-readable", @@ -147,7 +158,10 @@ describe("durable execution and replay", () => { read: async () => { reads += 1; if (reads === 1) { - return undefined; + throw new ResourceOperationPendingError( + "resource is not visible yet", + { retryAfterMs: 1 }, + ); } return {} as const; }, @@ -156,13 +170,14 @@ describe("durable execution and replay", () => { const runtime = createRuntime( [new EventuallyReadable({ id: "eventually-readable" })], "post-write-read", - { readPollOptions: { maxAttempts: 3, retryInterval: 1 } }, + { maxOperationAttempts: 3 }, ); await runtime.run("post-write-read-execution"); expect(reads).toBe(1); expect(await runtime.state.get("eventually-readable")).toBeUndefined(); + await sleep(5); await runtime.run("post-write-read-execution"); expect(reads).toBe(2); expect(await runtime.state.get("eventually-readable")).toMatchObject({ @@ -170,6 +185,30 @@ describe("durable execution and replay", () => { }); runtime.close(); }); + + it("does not infer that not-found after a write is pending", async () => { + const MissingAfterCreate = resource({ + type: "test/durable/missing-after-create", + }) + .defineSchema({}) + .defineOperations({ + create: async () => undefined, + read: async () => { + throw new ResourceNotFoundError("resource is absent"); + }, + delete: async () => undefined, + }); + const runtime = createRuntime( + [new MissingAfterCreate({ id: "missing-after-create" })], + "missing-after-create", + ); + + await expect(runtime.run("missing-after-create-execution")).rejects.toThrow( + "resource is absent", + ); + expect(await runtime.state.get("missing-after-create")).toBeUndefined(); + runtime.close(); + }); }); describe("dependency ordering", () => { @@ -452,8 +491,7 @@ function createRuntime( resources: BaseResource[], deploymentId: string, options: { - retryOptions?: { maxAttempts: number; retryInterval: number }; - readPollOptions?: { maxAttempts: number; retryInterval: number }; + maxOperationAttempts?: number; crashAfterStep?: string; registry?: ResourceRegistry; driftDetection?: boolean; @@ -480,8 +518,7 @@ function createRuntime( registry: options.registry, driftDetection: options.driftDetection ?? false, emit: options.emit, - retryOptions: options.retryOptions, - readPollOptions: options.readPollOptions, + maxOperationAttempts: options.maxOperationAttempts, }); }); const destroy = workflow(async function* (step, event) { @@ -492,8 +529,7 @@ function createRuntime( state, registry: options.registry, emit: options.emit, - retryOptions: options.retryOptions, - readPollOptions: options.readPollOptions, + maxOperationAttempts: options.maxOperationAttempts, }); }); const router = createWorkflowRouter({ deploy, destroy }); diff --git a/packages/reconciler/test/planner.test.ts b/packages/reconciler/test/planner.test.ts index 35a854e..8066578 100644 --- a/packages/reconciler/test/planner.test.ts +++ b/packages/reconciler/test/planner.test.ts @@ -1,4 +1,8 @@ -import { ResourceNotReadyError, resource } from "@notation/resource"; +import { + ResourceNotFoundError, + ResourceOperationPendingError, + resource, +} from "@notation/resource"; import { MemoryStateBackend } from "@notation/state"; import { describe, expect, it } from "vitest"; import { createPlan } from "../src/planner"; @@ -68,7 +72,9 @@ describe("createPlan", () => { .defineSchema({}) .defineOperations({ create: async () => undefined, - read: async () => undefined, + read: async () => { + throw new ResourceNotFoundError("resource is absent"); + }, delete: async () => undefined, }); const state = new MemoryStateBackend(); @@ -93,13 +99,21 @@ describe("createPlan", () => { }); }); - it("reports an indeterminate decision while the resource is not ready", async () => { + it("waits for a pending read before planning", async () => { + let attempts = 0; const TestResource = resource({ type: "test/planner/pending" }) .defineSchema({}) .defineOperations({ create: async () => undefined, read: async () => { - throw new ResourceNotReadyError("Waiting for the provider"); + attempts += 1; + if (attempts === 1) { + throw new ResourceOperationPendingError( + "Waiting for the provider", + { retryAfterMs: 0 }, + ); + } + return {}; }, delete: async () => undefined, }); @@ -121,8 +135,8 @@ describe("createPlan", () => { expect(plan.nodes[0]).toMatchObject({ id: "existing", - decision: "indeterminate", - reason: "Waiting for the provider", + decision: "noop", }); + expect(attempts).toBe(2); }); }); From 9167eb8ed15ee37866d70faaf77e8a16742876fc Mon Sep 17 00:00:00 2001 From: djgrant <1670902+djgrant@users.noreply.github.com> Date: Wed, 22 Jul 2026 14:14:27 +0100 Subject: [PATCH 07/34] Cut core and CLI over to the durable runtime Run deploy, plan, and destroy through NodeDurableRuntime, unify CLI error handling, and remove the legacy reconciler, operation workflows, and state leasing that the old runtime required. --- .changeset/reconciler.md | 4 +- docs/cli/dashboard.md | 16 +- docs/cli/deploy.md | 38 +- docs/cli/destroy.md | 10 +- docs/internals/reconciler.md | 111 +-- docs/internals/state.md | 122 +-- docs/manual/introduction.md | 3 +- docs/manual/reconciler.md | 78 +- docs/rfcs/reconciler.md | 102 +-- examples/reconciler/README.md | 14 +- examples/reconciler/package.json | 6 +- examples/reconciler/src/index.ts | 55 +- packages/cli/src/deploy.ts | 24 +- packages/cli/src/destroy.ts | 7 +- packages/cli/src/index.ts | 27 +- packages/cli/src/plan.ts | 60 +- packages/cli/src/run-with-error-handling.ts | 22 + .../cli/test/run-with-error-handling.test.ts | 41 + packages/core/package.json | 8 +- .../core/src/provisioner/durable-runtime.ts | 142 +++ packages/core/src/provisioner/index.ts | 2 +- .../core/src/provisioner/state-backend.ts | 16 - .../core/src/provisioner/workflows/index.ts | 1 - .../provisioner/workflows/workflow.deploy.ts | 49 +- .../provisioner/workflows/workflow.destroy.ts | 46 +- .../provisioner/workflows/workflow.plan.ts | 44 +- .../provisioner/workflows/workflow.refresh.ts | 39 - .../test/provisioner/durable-runtime.test.ts | 60 ++ .../test/provisioner/operation.create.test.ts | 54 -- .../test/provisioner/state-backend.test.ts | 59 -- packages/reconciler/src/durable/operations.ts | 2 +- packages/reconciler/src/index.ts | 5 +- packages/reconciler/src/logger-subscriber.ts | 4 +- packages/reconciler/src/operations/index.ts | 6 - .../src/operations/operation.create.ts | 82 -- .../src/operations/operation.delete.ts | 52 -- .../src/operations/operation.read.ts | 68 -- .../src/operations/operation.types.ts | 92 -- .../src/operations/operation.update.ts | 94 -- ...ration.pending.ts => pending-operation.ts} | 11 +- packages/reconciler/src/planner.ts | 43 +- packages/reconciler/src/protocol.ts | 2 +- packages/reconciler/src/reconciler.ts | 602 ------------- packages/reconciler/src/resource-registry.ts | 10 +- .../reconciler/test/logger-subscriber.test.ts | 14 +- .../test/operation.workflows.test.ts | 346 -------- .../reconciler/test/reconciler.deploy.test.ts | 807 ------------------ .../reconciler/test/reconciler.plan.test.ts | 463 ---------- packages/state-sqlite/src/index.ts | 96 +-- .../state-sqlite/test/state-sqlite.test.ts | 27 - packages/state/src/conflicts.ts | 11 - packages/state/src/state.ts | 130 +-- packages/state/test/state-backend.test.ts | 21 - pnpm-lock.yaml | 76 +- 54 files changed, 682 insertions(+), 3642 deletions(-) create mode 100644 packages/cli/src/run-with-error-handling.ts create mode 100644 packages/cli/test/run-with-error-handling.test.ts create mode 100644 packages/core/src/provisioner/durable-runtime.ts delete mode 100644 packages/core/src/provisioner/state-backend.ts delete mode 100644 packages/core/src/provisioner/workflows/workflow.refresh.ts create mode 100644 packages/core/test/provisioner/durable-runtime.test.ts delete mode 100644 packages/core/test/provisioner/operation.create.test.ts delete mode 100644 packages/core/test/provisioner/state-backend.test.ts delete mode 100644 packages/reconciler/src/operations/index.ts delete mode 100644 packages/reconciler/src/operations/operation.create.ts delete mode 100644 packages/reconciler/src/operations/operation.delete.ts delete mode 100644 packages/reconciler/src/operations/operation.read.ts delete mode 100644 packages/reconciler/src/operations/operation.types.ts delete mode 100644 packages/reconciler/src/operations/operation.update.ts rename packages/reconciler/src/{operations/operation.pending.ts => pending-operation.ts} (84%) delete mode 100644 packages/reconciler/src/reconciler.ts delete mode 100644 packages/reconciler/test/operation.workflows.test.ts delete mode 100644 packages/reconciler/test/reconciler.deploy.test.ts delete mode 100644 packages/reconciler/test/reconciler.plan.test.ts diff --git a/.changeset/reconciler.md b/.changeset/reconciler.md index 409f877..f28bafd 100644 --- a/.changeset/reconciler.md +++ b/.changeset/reconciler.md @@ -9,6 +9,4 @@ "@notation/state-sqlite": minor --- -Add the reconciler API, versioned event streams, renewable mutation leases, -SQLite state, backend-neutral dashboard state, and compiled infrastructure -graphs. +Add durable Yieldstar 0.5.0 deploy and destroy workflows, a resident Node SQLite runtime for CLI execution, versioned event streams, backend-neutral dashboard state, and compiled infrastructure graphs. diff --git a/docs/cli/dashboard.md b/docs/cli/dashboard.md index f630ad7..2baffaf 100644 --- a/docs/cli/dashboard.md +++ b/docs/cli/dashboard.md @@ -1,21 +1,13 @@ # notation dashboard ```sh -notation dashboard +notation dashboard ``` -Starts a local web dashboard for observing deployment state. +Starts a local web dashboard for observing the deployment's Yieldstar resource stores. ```sh -notation dashboard +notation dashboard infra/api.ts ``` -The dashboard uses the same state backend as deploy and destroy. Set -`NOTATION_STATE_PATH` to select SQLite: - -```sh -NOTATION_STATE_PATH=.notation/state.db notation dashboard -``` - -The server reads through `StateBackend`, so file and SQLite state produce the same -dashboard payload. +The dashboard reads `.notation/workflows.db`, the same database used by deploy, destroy, and plan. Set `NOTATION_STATE_PATH` to choose another SQLite database path. diff --git a/docs/cli/deploy.md b/docs/cli/deploy.md index 940cf7e..3e81379 100644 --- a/docs/cli/deploy.md +++ b/docs/cli/deploy.md @@ -4,7 +4,7 @@ notation deploy ``` -Compiles and deploys the stack to AWS. +Compiles and durably deploys the stack through the resident Yieldstar 0.5.0 Node runtime. ```sh notation deploy infra/api.ts @@ -12,32 +12,36 @@ notation deploy infra/api.ts ## Event stream -`--json` writes versioned reconciler events to stdout as newline-delimited JSON. Build -output and diagnostics move to stderr. +`--json` writes versioned reconciler events to stdout as newline-delimited JSON. Build output, the execution ID, and diagnostics move to stderr. ```sh notation deploy infra/api.ts --json > deploy.ndjson ``` +## Durable execution + +The command prints its Yieldstar execution ID before starting provider work. If the process crashes, resume the same durable heap with that ID: + +```sh +notation deploy infra/api.ts --execution-id +``` + +Do not reuse a completed execution ID for a new deploy or for destroy. + +Retryable provider conditions and consistency reads suspend on durable SQLite timers. The CLI stays resident until the scheduler wakes the execution and the workflow completes; completed provider calls are replayed from the heap rather than repeated. + ## What happens -1. **Compile** – esbuild compiles infra and runtime modules to `dist/`. +1. **Compile** – esbuild compiles infrastructure and runtime modules to `dist/`. -2. **Build resource graph** – imports the compiled output and collects the declared resources. +2. **Build resource graph** – the worker imports the compiled output and collects declared resources. -3. **Reconcile** – the reconciler compares desired state (graph) against current state (`.notation/state.json`): - - New resources → **create** - - Changed params → **update** - - No changes → **noop** - - Orphaned resources (in state but not in graph) → **delete** +3. **Reconcile** – Notation compares desired resources with Yieldstar stores, then creates, updates, recreates, or leaves each resource unchanged. -4. **Topological deployment** – resources deploy in dependency order (levels). Resources at the same level deploy concurrently. +4. **Order dependencies** – dependency levels run in topological order. -5. **Drift detection** – enabled by default. Reads actual AWS state and compares against stored state. If drifted, Notation updates to match your definition. +5. **Detect drift** – unchanged resources are read from the provider and repaired when their remote state differs. -State is persisted to `.notation/state.json` after each operation. Set -`NOTATION_STATE_PATH` to a path ending in `.db` or `.sqlite` to use SQLite: +6. **Delete orphans** – persisted resources absent from the graph are deleted when their resource type is registered. -```sh -NOTATION_STATE_PATH=.notation/state.db notation deploy infra/api.ts -``` +State, step results, timers, task coordination, and resource stores are persisted to `.notation/workflows.db`. Set `NOTATION_STATE_PATH` to choose another SQLite database path. diff --git a/docs/cli/destroy.md b/docs/cli/destroy.md index a74ff30..c25a386 100644 --- a/docs/cli/destroy.md +++ b/docs/cli/destroy.md @@ -4,7 +4,7 @@ notation destroy ``` -Removes all resources in the stack. Tears down runs in reverse dependency order, so routes are removed before APIs and Lambdas before IAM roles etc. +Compiles the application and runs durable destroy through the resident Yieldstar 0.5.0 Node runtime. Resources are removed in reverse dependency order, then registered persisted orphans are removed. ```sh notation destroy infra/api.ts @@ -15,3 +15,11 @@ notation destroy infra/api.ts ```sh notation destroy infra/api.ts --json > destroy.ndjson ``` + +The command prints its execution ID. Resume a crashed destroy with the same ID so a provider delete that already completed is replayed instead of repeated: + +```sh +notation destroy infra/api.ts --execution-id +``` + +Retryable deletes suspend on durable SQLite timers. Resource state is removed only after the provider delete succeeds or reports that the resource is already absent. diff --git a/docs/internals/reconciler.md b/docs/internals/reconciler.md index 6b0d413..05034a8 100644 --- a/docs/internals/reconciler.md +++ b/docs/internals/reconciler.md @@ -1,115 +1,50 @@ # Reconciler -The reconciler runs deployment operations to transition infrastructure from its current state to the state defined in the project. - -Source: `@notation/reconciler` +The reconciler expresses deployment and destruction as Yieldstar async generators. Notation owns desired-state decisions and provider lifecycle; the caller's Yieldstar runtime owns durable execution, waiting, shared state, and coordination. ## Deploy flow -```ts [packages/reconciler/src/index.ts] -const reconciler = new Reconciler({ state, registry, emit }); -await reconciler.deploy(resources, { dryRun, driftDetection }); -``` - -The reconciler walks the resource graph and, for each resource, determines an action: - -| Condition | Decision | -| --------------------------------------------- | ------------------ | -| Not in state | **create** | -| In state, params changed | **update** | -| In state, params unchanged, no drift | **noop** | -| In state, but deleted from AWS | **drift-recreate** | -| In state, AWS state differs from stored state | **drift-update** | -| In state, not in graph (orphan) | **delete** | - -The `dryRun` flag runs the full diffing pipeline without executing any operations, so you can preview what a deploy would do. - -## Topological ordering +`deploy` acquires the deployment coordination store, walks dependency levels in order, decides an action for every resource, executes provider calls as durable steps, persists the result in a resource store, and deletes registered orphans. -Resources are deployed in dependency order using `buildResourceDepthLevels()`. This function partitions the resource graph into levels – each level contains resources whose dependencies have all been satisfied by previous levels. +| Condition | Decision | +| --- | --- | +| Not in state | **create** | +| In state, params changed | **update** | +| In state, params unchanged, no drift | **noop** | +| In state, but deleted from the provider | **drift-recreate** | +| In state, provider state differs from stored state | **drift-update** | +| In state, not in graph | **delete** | -``` -Level 0: IAM Role, CloudWatch LogGroup -Level 1: Lambda Function (depends on Role, LogGroup) -Level 2: API Gateway Integration (depends on Lambda) -Level 3: API Gateway Route (depends on Integration) -``` +Dry-run deploy performs decisions and emits lifecycle events without calling providers or mutating state. -Resources within a level deploy concurrently, so independent resources like the IAM Role and LogGroup above are provisioned in parallel. Dependent resources wait for their dependencies. +## Destroy flow -Destroy operates in reverse order with dependents getting removed before their dependencies. +`destroy` is a first-class durable operation. It acquires the same deployment coordination store as deploy, deletes desired resources in reverse dependency order, deletes hydratable persisted orphans, and conditionally removes each resource store only after the provider delete succeeds or reports that the resource is already absent. -### Cycle detection +Provider delete is a stable durable step. If the process crashes after the provider acknowledges deletion but before state removal, replay uses the cached delete result and continues at the conditional store delete. -Cycle detection is built in. If resources form a circular dependency, the build fails with: +## Waiting and replay -``` -Resource dependency cycle detected -``` +A resource operation throws `ResourceOperationPendingError` when it has not finished. The error gives the reconciler a delay and optional callback context. The runtime stores the context, waits without keeping the process busy, and calls the same operation again. See [Operation errors](./resource.md#operation-errors) for the complete API. -This catches configuration errors before any cloud operations are attempted. +Each attempt, delay, event, state read, state write, and coordination change has a stable step key. A resumed execution must use the same execution ID. A new deploy or destroy must use a new execution ID. -## Drift detection +## State and coordination -Drift detection is enabled by default. After confirming no local changes to a resource, the reconciler reads the resource's current state from AWS (via the resource's `read()` operation) and diffs it against stored state. +Each resource is stored under `notation/resource-state` with a deployment-scoped ID. Conditional updates and deletes compare the snapshot's UUIDv7 `instanceId` and version, so a stale execution cannot modify a deleted and recreated store. -If AWS has drifted (e.g. someone changed a Lambda timeout in the console, or an IAM policy was modified by another tool), Notation updates the resource to match the canoncial definition in the source code. - -Properties marked as `volatile` in the schema (like `LastModified` timestamps) are excluded from drift comparison. +Deploy and destroy share one `notation/deployment-coordination` store per deployment. `store.take` suspends a competing execution as a durable waiter and wakes it after the holder releases. Before suspending, the waiter emits `reconciler.coordination.waiting` naming the holding execution ID, so a wait behind a crashed execution is visible instead of silent. ## Events -The reconciler emits events at each step of an operation's lifecycle. The default `createConsoleReconcilerSubscriber()` logs these to the console with formatted output. +The durable workflows emit these events: | Event | When | | ------------------------------------ | --------------------------------------------------- | | `reconciler.deploy.decision` | After deciding what action to take for a resource | | `reconciler.drift.detected` | When drift is found between stored and actual state | | `reconciler.operation.lifecycle` | When an operation starts, finishes, skips, or fails | +| `reconciler.coordination.waiting` | When another deployment holds the coordination store | | `reconciler.orphan-deletion.skipped` | When no registered class can delete an orphan | -Lifecycle events contain the operation (`create`, `read`, `update`, or `delete`) and its -status (`start`, `success`, `error`, `skip`, or `dry-run`). Events carry the resource ID, -type, and relevant diff or error details. - -## Operations - -Each CRUD operation is implemented as an async generator with retry support: - -- **`createResourceOperation`** – creates the resource, reads back its state, persists to state backend -- **`updateResourceOperation`** – applies the update, reads back new state, persists to state backend -- **`deleteResourceOperation`** – deletes the resource, removes the entry from state backend -- **`readResourceOperation`** – reads current state from the cloud provider (used for drift detection) - -### Pending operations - -A resource operation throws `ResourceOperationPendingError` when it has not finished. The reconciler reads two fields from the error: - -| Field | Action | -| ----- | ------ | -| `retryAfterMs` | Wait this many milliseconds. | -| `callbackContext` | Pass this value to the next call of the same operation. | - -The reconciler then calls the same operation again. Any other error fails the operation. See [Operation errors](./resource.md#operation-errors) for the complete API. - -The default limit is 30 calls to one operation: - -```ts [packages/reconciler/src/index.ts] -{ - maxOperationAttempts: 30, -} -``` - -The last pending error becomes a failure when the limit is reached. - -### Operation lifecycle - -Each operation follows the following pattern: - -1. Emit `started` event -2. Execute the cloud operation (with retries) -3. Read back the resource state -4. Persist to state backend -5. Emit `completed` event (or `failed` on error) - -State is updated after the provider operation and read-back complete. +Lifecycle events cover create, read, update, and delete with `start`, `success`, `error`, `skip`, or `dry-run` status. diff --git a/docs/internals/state.md b/docs/internals/state.md index 44d5df0..77647e4 100644 --- a/docs/internals/state.md +++ b/docs/internals/state.md @@ -1,129 +1,25 @@ # State -Notation tracks deployed resources in a state backend. State is the bridge between what is defined and what actually exists in the cloud. +Notation CLI deploy, destroy, plan, and dashboard use Yieldstar 0.5.0 stores in `.notation/workflows.db`. Override the database path with `NOTATION_STATE_PATH`. -Source: `@notation/state` - -## State file - -Default location: `.notation/state.json`. Override with the `NOTATION_STATE_PATH` environment variable. - -Each resource entry records everything needed to diff, update, or delete the resource: - -```json -{ - "my-api-lambda-getTodos": { - "rev": 3, - "id": "my-api-lambda-getTodos", - "type": "aws/lambda/LambdaFunction", - "config": { - "service": "aws/lambda", - "timeout": 5, - "memory": 64 - }, - "params": { - "FunctionName": "my-api-getTodos", - "Runtime": "nodejs18.x", - "Handler": "index.getTodos", - "MemorySize": 64, - "Timeout": 5 - }, - "output": { - "FunctionArn": "arn:aws:lambda:us-east-1:123456789:function:my-api-getTodos", - "FunctionUrl": "https://xyz.lambda-url.us-east-1.on.aws/" - }, - "lastOperation": "create", - "lastOperationAt": "2027-01-15T10:30:00.000Z" - } -} -``` - -Key fields: - -- **`id`** – unique identifier derived from the resource's position in the graph -- **`rev`** – monotonically increasing revision used for compare-and-swap writes -- **`type`** – the resource type string (e.g., `aws/lambda/LambdaFunction`) -- **`config`** – user-facing configuration values -- **`params`** – the full set of parameters sent to the cloud provider -- **`output`** – computed values returned by the provider after creation -- **`lastOperation`** – what the reconciler last did (`create`, `update`, `delete`) -- **`lastOperationAt`** – ISO timestamp of the last operation - -## Backends - -Three built-in backends: - -### `FileStateBackend` (default) - -Reads and writes JSON to disk. Uses atomic writes – writes to a temporary file first, then renames – to prevent corruption if the process is interrupted mid-write. - -```ts [packages/state/src/file.ts] -const state = new FileStateBackend(".notation/state.json"); -``` - -### `MemoryStateBackend` - -In-memory backend used for testing. Deep-clones on read and write to simulate persistence semantics (mutations to returned objects don't affect stored data). - -```ts [packages/state/src/memory.ts] -const state = new MemoryStateBackend(); -``` - -### `SqliteStateBackend` - -Stores state and leases in SQLite. Select it in the CLI by setting -`NOTATION_STATE_PATH` to a path ending in `.db` or `.sqlite`. +Each live resource is a `notation/resource-state` store scoped by deployment and resource ID. A missing store means the resource is absent. No application tombstone is written. ```ts -const state = new SqliteStateBackend(".notation/state.db"); +const state = new DurableStateBackend(storeClient, "infra/api.ts"); ``` -### `StateBackend` interface +The runtime assigns a UUIDv7 `instanceId` when a store is created and increments its version on update. Conditional workflow updates and deletes compare both values, preventing a stale snapshot from modifying a deleted and recreated resource. The one-based value exposed as `StateNode.rev` is derived from the authoritative Yieldstar store version. -All backends implement the same interface: - -```ts [@notation/state/src/backend.ts] +```ts interface StateBackend { get(id: string): Promise; has(id: string): Promise; - update( - id: string, - patch: Partial, - expectedRev?: number, - ): Promise<{ rev: number }>; - delete(id: string, expectedRev?: number): Promise; + update(id: string, expectedRev: number, patch: Partial): Promise<{ rev: number }>; + delete(id: string, expectedRev: number): Promise; values(): Promise; - lease(scope: string, ttl: number): Promise; } ``` -Every backend provides compare-and-swap writes and renewable exclusive leases. The -reconciler holds a per-resource lease across the provider operation and state write, so -concurrent deploys cannot both perform the same create or update. It renews long-running -leases until the mutation finishes. Orphan deletion additionally holds a snapshot lease -while it decides which state records no longer appear in the desired graph. - -## How state is used - -### Deploy - -The reconciler reads state to diff against the desired resource graph: - -1. For each resource in the graph, check if it exists in state -2. If it exists, compare `params` to detect changes -3. Execute the appropriate operation (create, update, noop) -4. After each operation, update the state entry with new params and output - -### Destroy - -The reconciler reads state to find resources to delete: - -1. Load all state entries -2. Delete resources in reverse dependency order -3. Remove each entry from state after successful deletion - -### Orphan detection - -The reconciler checks for orphaned resources – resources that exist in state but are no longer present in the resource graph. This happens when you remove a function export or delete a `.fn.ts` file. +Coordination is not part of the state backend contract. The outer Yieldstar workflow serializes deploy and destroy through a deployment coordination store and records applied store steps for crash-safe replay. -Orphaned resources are deleted from AWS and removed from state. +`MemoryStateBackend`, `FileStateBackend`, and `SqliteStateBackend` remain data adapters for tests and embedded read/write consumers. They are not CLI execution runtimes and do not provide mutation coordination. diff --git a/docs/manual/introduction.md b/docs/manual/introduction.md index bb629e0..41cafca 100644 --- a/docs/manual/introduction.md +++ b/docs/manual/introduction.md @@ -13,8 +13,7 @@ todoRouter.get("/todos", getTodos); Notation is a compiler, reconciler, and deployment engine. -The reconciler is also available as an embedded library. A Node.js host can construct -resources, choose a state backend, and run plan, deploy, or destroy without the CLI. +The reconciler is also available as an embedded library. A Node.js host can construct resources and compose durable reconciliation inside its own Yieldstar workflow without the CLI. The compiler runs two passes over your codebase: diff --git a/docs/manual/reconciler.md b/docs/manual/reconciler.md index 87b02f7..eee8d06 100644 --- a/docs/manual/reconciler.md +++ b/docs/manual/reconciler.md @@ -1,55 +1,45 @@ # Reconciler -Use the reconciler directly when a Node.js application needs to deploy resources without -starting the Notation CLI. - -This complete program deploys two static sites and keeps their deployment state in -SQLite: +Use `deploy` and `destroy` when a Node.js application needs durable resource lifecycle operations without starting the Notation CLI. Notation owns reconciliation intent, graph ordering, provider calls, and resource state; the application owns the outer Yieldstar workflow and runtime. ```ts -import { Reconciler, createResourceRegistry } from "@notation/reconciler"; -import { SqliteStateBackend } from "@notation/state-sqlite"; -import { StaticSite } from "./static-site"; - -const state = new SqliteStateBackend("sites.db"); - -const resources = [ - new StaticSite({ - id: "documentation", - config: { - siteDirectory: "sites/docs", - html: "

Documentation

\n", - }, - }), - new StaticSite({ - id: "status", - config: { - siteDirectory: "sites/status", - html: "

All systems operational

\n", - }, - }), -]; - -const reconciler = new Reconciler({ - state, - registry: createResourceRegistry([StaticSite]), +import { SqliteSchedulerClient, SqliteStoreClient, SqliteTaskQueueClient, SqliteTimersClient, createSqliteDb } from "@yieldstar/sqlite-runtime/node"; +import { DurableStateBackend, deploy, destroy } from "@notation/reconciler"; +import { workflow } from "yieldstar"; + +const database = createSqliteDb({ path: ".notation/workflows.db" }); +const schedulerClient = new SqliteSchedulerClient({ + taskQueueClient: new SqliteTaskQueueClient(database), + timersClient: new SqliteTimersClient(database), +}); +const storeClient = new SqliteStoreClient({ db: database, schedulerClient }); +const state = new DurableStateBackend(storeClient, "my-application"); + +export const deploy = workflow(async function* (step, event) { + yield* deploy(step, { + deploymentId: "my-application", + executionId: event.executionId, + resources, + state, + }); }); -try { - await reconciler.deploy(resources); -} finally { - state.close(); -} +export const destroy = workflow(async function* (step, event) { + yield* destroy(step, { + deploymentId: "my-application", + executionId: event.executionId, + resources, + state, + }); +}); ``` -`StaticSite` contains the provider operations which create, read, update, and delete a -site. A real provider would call its infrastructure API instead of writing local files. +The outer workflow supplies durable step execution, timers, shared stores, waiting, scheduling, and coordination. Completed provider calls are replayed from the heap after a crash, retryable provider conditions suspend on a durable timer, and conditional state writes use Yieldstar store identity and version. + +Each live resource is one Yieldstar store. Absence is represented by no store, not a tombstone. Yieldstar's UUIDv7 store `instanceId` and version are authoritative for conditional update and delete; Notation exposes the version as the resource state's `rev`. -Pass the complete desired set to `deploy`. A resource which remains in deployment state -but is absent from that set is deleted. The explicit registry lets the reconciler find -its delete operation. +Operations against the same `deploymentId` are serialized through a coordination store keyed by `executionId`. Resume a crashed operation with the same execution ID; use a new globally unique execution ID for every new deploy or destroy. An execution that must wait emits a `reconciler.coordination.waiting` event naming the holder before it suspends, which also identifies a crashed holder that should be resumed instead. -Notation's state records what was deployed. It does not replace application data which -owns the desired configuration. +Pass the complete desired set on every deployment. Persisted resources absent from that set are deleted through the supplied resource registry. Destroy removes current resources in reverse dependency order and then removes any persisted orphans that the registry can hydrate. -The runnable version is in `examples/reconciler`. +The runnable Node SQLite composition is in `examples/reconciler`. diff --git a/docs/rfcs/reconciler.md b/docs/rfcs/reconciler.md index dd33e92..7ec505a 100644 --- a/docs/rfcs/reconciler.md +++ b/docs/rfcs/reconciler.md @@ -1,102 +1,28 @@ -# RFC: Reconciler +# RFC: Durable Yieldstar reconciliation **Status:** implemented -**Scope:** `@notation/state`, `@notation/reconciler` +**Scope:** `@notation/reconciler`, `@notation/core`, Yieldstar 0.5.0 -Notation evaluates an infrastructure program into resources, then reconciles those -resources against recorded state. The same engine now runs behind the CLI, the dashboard, -and direct library integrations. +Notation describes reconciliation intent and resource lifecycle operations. An outer Yieldstar workflow supplies durable execution, waiting, state, and coordination by composing `deploy` or `destroy`. -```ts -import { Reconciler } from "@notation/reconciler"; -import { SqliteStateBackend } from "@notation/state-sqlite"; +## Boundary -const state = new SqliteStateBackend(".notation/state.db"); -const reconciler = new Reconciler({ state }); +Live resource objects remain in the workflow process. They are not serialized into workflow parameters. This keeps provider clients and operation closures under Notation's lifecycle control while Yieldstar persists step results and shared state. -await reconciler.deploy(resources); -state.close(); -``` +Provider create, update, read, and delete calls are durable steps with stable resource-scoped keys. A process crash after a completed provider call replays the cached result and continues at state persistence instead of repeating the call. Retryable provider conditions become Yieldstar delays, allowing the process to wait without polling the provider continuously. -The reconciler boundary consists of live resource objects, a state backend, and an event -subscriber. Resource operations run in the host process. +## State lifecycle -## State +`DurableStateBackend` stores one live resource per `notation/resource-state` store. The store ID is scoped by deployment and resource ID. Store absence is resource absence. -Each state record carries a revision. Updates and deletes can require the revision which -the caller previously read: +The runtime-assigned UUIDv7 `instanceId` distinguishes a deleted store from a later store created under the same logical ID. Yieldstar's version is the concurrency token and is exposed as Notation's one-based `rev`. Workflow updates use `store.updateFrom` and deletes use `store.deleteFrom`, so both the instance and version must match the snapshot that informed the operation. -```ts -await state.update(resource.id, patch, resource.rev); -``` +## Coordination -A stale writer receives `RevConflict`. A missing record has revision zero, so -`expectedRev: 0` means that the record must not exist. +Each deployment has a `notation/deployment-coordination` store shared by deploy and destroy. The workflow atomically claims it with `store.take`. A concurrent execution suspends as a durable waiter and is woken when the holder releases the store. The same execution can recover an acquisition across the store-commit and heap-write crash gap through Yieldstar's applied-step ledger. -The reconciler also takes a renewable per-resource lease before it reads a resource for -mutation. The lease remains held across the provider operation and state write. Two -hosts therefore cannot create or update the same resource concurrently through the same -backend. +## Node CLI runtime -Orphan deletion takes an additional snapshot lease. The snapshot remains stable while -the reconciler decides which state records no longer appear in the desired graph. +`NodeDurableRuntime` wires `WorkflowRunner`, `SqliteHeapClient`, `SqliteStoreClient`, `SqliteSchedulerClient`, and `SqliteEventLoop` against one Node SQLite database. CLI deploy and destroy run through this resident runtime and wait for a workflow result across timer and store wake-ups. -## Backends - -`@notation/state` provides file and memory backends. `@notation/state-sqlite` provides -the reference database backend. - -Every backend implements the same contract: - -```ts -interface StateBackend { - get(id: string): Promise; - has(id: string): Promise; - update( - id: string, - patch: Partial, - expectedRev?: number, - ): Promise<{ rev: number }>; - delete(id: string, expectedRev?: number): Promise; - values(): Promise; - lease(scope: string, ttl: number): Promise; -} -``` - -The dashboard reads this interface. It does not inspect a state file directly. - -## Events - -The reconciler accepts one subscriber: - -```ts -const reconciler = new Reconciler({ - state, - emit: async (event) => auditLog.write(event), -}); -``` - -`createNdjsonEventEmitter` adapts the subscriber to a versioned newline-delimited JSON -stream. The CLI uses the same adapter for `deploy --json` and `destroy --json`. - -## Package boundary - -The CLI creates resources from compiled Notation programs, then hands those live objects -to `Reconciler`. An application can construct the same resource classes directly. - -The reconciler does not serialise resource classes or execute operations in another -process. Detached execution needs manifests, resource-reference encoding, actuator -binding, and a runtime consumer. That work has its own RFC and release. - -## Acceptance - -The reconciler example is the compatibility test for this boundary. It must: - -1. Construct a resource without the CLI. -2. Plan and deploy it through `Reconciler`. -3. Close and reopen SQLite state. -4. Plan and apply an update. -5. Receive versioned events. -6. Destroy the resource and remove its state. - -The example lives in `examples/reconciler` and runs without cloud credentials. +The CLI prints a new execution ID for each operation. Re-running with `--execution-id ` resumes that operation from its durable heap after a process crash. diff --git a/examples/reconciler/README.md b/examples/reconciler/README.md index 428eea3..19a6ae0 100644 --- a/examples/reconciler/README.md +++ b/examples/reconciler/README.md @@ -1,12 +1,8 @@ -# Reconciler +# Durable reconciler -This example deploys two static sites from an ordinary Node.js program. It does not -compile a Notation project or start the Notation CLI. +This example deploys two static sites from an ordinary Node.js program using Yieldstar 0.5.0 for durable execution, state, retries, waiting, and deployment coordination. -[`src/index.ts`](./src/index.ts) is the complete program. It defines the desired -resources inline, opens a SQLite state backend, and passes the resources directly to the -reconciler. [`src/static-site.ts`](./src/static-site.ts) defines the local provider -operations used to create, read, update, and delete each site. +[`src/index.ts`](./src/index.ts) owns the outer workflow and Node SQLite runtime. It passes Yieldstar's `step` context to `deploy`, while [`src/static-site.ts`](./src/static-site.ts) contains only the desired resources and provider lifecycle operations. Run it from the repository root: @@ -14,9 +10,7 @@ Run it from the repository root: pnpm --filter reconciler-example demo ``` -The generated sites are written to `sites/`, and deployment state is stored in -`sites.db`. Change the resource configuration and run the command again to update the -sites. Remove a resource from the array and run it again to delete that site. +The generated sites are written to `sites/`, and the workflow heap, resource stores, timers, and coordination state are stored in `sites.db`. Change the resource configuration and run the command again to update the sites. Remove a resource from the array and run it again to delete that site. Run the integration test with: diff --git a/examples/reconciler/package.json b/examples/reconciler/package.json index 55cbaa1..c5d8f2e 100644 --- a/examples/reconciler/package.json +++ b/examples/reconciler/package.json @@ -11,8 +11,10 @@ "dependencies": { "@notation/reconciler": "workspace:*", "@notation/resource": "workspace:*", - "@notation/state-sqlite": "workspace:*", - "@notation/utils": "workspace:*" + "@yieldstar/core": "0.5.0", + "@yieldstar/sqlite-runtime": "0.5.0", + "pino": "^9.9.0", + "yieldstar": "0.5.0" }, "devDependencies": { "@types/node": "^22.13.4", diff --git a/examples/reconciler/src/index.ts b/examples/reconciler/src/index.ts index 4a85756..0253115 100644 --- a/examples/reconciler/src/index.ts +++ b/examples/reconciler/src/index.ts @@ -1,8 +1,26 @@ -import { Reconciler, createResourceRegistry } from "@notation/reconciler"; -import { SqliteStateBackend } from "@notation/state-sqlite"; +import { WorkflowRunner } from "@yieldstar/core"; +import { + SqliteHeapClient, + SqliteSchedulerClient, + SqliteStoreClient, + SqliteTaskQueueClient, + SqliteTimersClient, + createSqliteDb, +} from "@yieldstar/sqlite-runtime/node"; +import * as reconciler from "@notation/reconciler"; +import pino from "pino"; +import { createWorkflowRouter, workflow } from "yieldstar"; import { StaticSite } from "./static-site"; -const state = new SqliteStateBackend("sites.db"); +const logger = pino(); +const database = createSqliteDb({ path: "sites.db" }); +const taskQueueClient = new SqliteTaskQueueClient(database); +const schedulerClient = new SqliteSchedulerClient({ + taskQueueClient, + timersClient: new SqliteTimersClient(database), +}); +const storeClient = new SqliteStoreClient({ db: database, schedulerClient }); +const state = new reconciler.DurableStateBackend(storeClient, "static-sites"); const resources = [ new StaticSite({ @@ -21,13 +39,34 @@ const resources = [ }), ]; -const reconciler = new Reconciler({ - state, - registry: createResourceRegistry([StaticSite]), +const deploy = workflow(async function* (step, event) { + yield* reconciler.deploy(step, { + deploymentId: "static-sites", + executionId: event.executionId, + resources, + state, + registry: reconciler.createResourceRegistry([StaticSite]), + }); +}); + +const runner = new WorkflowRunner({ + router: createWorkflowRouter({ deploy }), + heapClient: new SqliteHeapClient(database), + storeClient, + schedulerClient, + logger, }); try { - await reconciler.deploy(resources); + await runner.run( + { + workflowId: "deploy", + executionId: crypto.randomUUID(), + params: {}, + context: new Map(), + }, + logger, + ); } finally { - state.close(); + database.close(); } diff --git a/packages/cli/src/deploy.ts b/packages/cli/src/deploy.ts index 07fdad1..3f141c1 100644 --- a/packages/cli/src/deploy.ts +++ b/packages/cli/src/deploy.ts @@ -3,12 +3,14 @@ import { createNdjsonEventEmitter, deployApp, } from "@notation/core"; +import { randomUUID } from "node:crypto"; import { compile } from "./compile"; import { defaultLogger, type Logger } from "./logger"; import { redirectStdoutToStderr } from "./stdio"; export type DeployCommandOptions = { json?: boolean; + executionId?: string; logger?: Logger; }; @@ -17,30 +19,14 @@ export async function deploy( opts: DeployCommandOptions = {}, ) { const logger = opts.logger ?? defaultLogger; - // In --json mode console output moves to stderr so stdout carries only the - // NDJSON event stream; capture the real stdout for the emitter first. const emit = opts.json ? createNdjsonEventEmitter(redirectStdoutToStderr().write) : createLoggerReconcilerSubscriber({ logger }); await compile(entryPoint, { logger }); logger.info(`Deploying ${entryPoint}`); + const executionId = opts.executionId ?? randomUUID(); + logger.info(`Yieldstar execution ${executionId}`); - try { - await deployApp({ - entryPoint, - emit, - }); - } catch (err: any) { - if (err.name === "CredentialsProviderError") { - logger.error( - "\nAWS credentials not found.", - "\n\nEnsure you have a default profile set up in ~/.aws/credentials.", - "\n\nIf using another profile run AWS_PROFILE=otherProfile notation deploy.\n", - ); - process.exit(1); - } - logger.error(err); - process.exit(1); - } + await deployApp({ entryPoint, emit, executionId }); } diff --git a/packages/cli/src/destroy.ts b/packages/cli/src/destroy.ts index acdc5b0..9352f3c 100644 --- a/packages/cli/src/destroy.ts +++ b/packages/cli/src/destroy.ts @@ -3,12 +3,14 @@ import { createNdjsonEventEmitter, destroyApp, } from "@notation/core"; +import { randomUUID } from "node:crypto"; import { compile } from "./compile"; import { defaultLogger, type Logger } from "./logger"; import { redirectStdoutToStderr } from "./stdio"; export type DestroyCommandOptions = { json?: boolean; + executionId?: string; logger?: Logger; }; @@ -23,5 +25,8 @@ export async function destroy( await compile(entryPoint, { logger }); logger.info(`Destroying ${entryPoint}\n`); - await destroyApp({ entryPoint, emit }); + const executionId = opts.executionId ?? randomUUID(); + logger.info(`Yieldstar execution ${executionId}`); + + await destroyApp({ entryPoint, emit, executionId }); } diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index 3ef90b7..eee95f2 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -4,10 +4,12 @@ import { compile } from "./compile"; import { deploy } from "./deploy"; import { destroy } from "./destroy"; import { plan } from "./plan"; +import { defaultLogger } from "./logger"; +import { runWithCliErrorHandling } from "./run-with-error-handling"; import { visualise } from "./visualise"; import { watch } from "./watch"; import { startDashboardServer } from "@notation/dashboard"; -import { createDefaultStateBackend } from "@notation/core"; +import { NodeDurableRuntime } from "@notation/core"; program .command("compile") @@ -19,9 +21,11 @@ program program .command("dashboard") + .argument("", "entryPoint") .description("Start Notation Dashboard") - .action(async () => { - await startDashboardServer({ state: createDefaultStateBackend() }); + .action(async (entryPoint) => { + const runtime = new NodeDurableRuntime({ deploymentId: entryPoint }); + await startDashboardServer({ state: runtime.state }); }); program @@ -29,8 +33,12 @@ program .argument("", "entryPoint") .description("Deploy Notation App") .option("--json", "stream reconciler events as NDJSON") + .option("--execution-id ", "resume a durable execution") .action(async (entryPoint, options) => { - await deploy(entryPoint, { json: options.json }); + await deploy(entryPoint, { + json: options.json, + executionId: options.executionId, + }); }); program @@ -38,8 +46,12 @@ program .argument("", "entryPoint") .description("Destroy Notation App") .option("--json", "stream reconciler events as NDJSON") + .option("--execution-id ", "resume a durable execution") .action(async (entryPoint, options) => { - await destroy(entryPoint, { json: options.json }); + await destroy(entryPoint, { + json: options.json, + executionId: options.executionId, + }); }); program @@ -67,4 +79,7 @@ program await watch(entryPoint); }); -program.parse(process.argv); +process.exitCode = await runWithCliErrorHandling( + () => program.parseAsync(process.argv), + { logger: defaultLogger, command: process.argv[2] ?? program.name() }, +); diff --git a/packages/cli/src/plan.ts b/packages/cli/src/plan.ts index ef52ec8..35841cc 100644 --- a/packages/cli/src/plan.ts +++ b/packages/cli/src/plan.ts @@ -1,9 +1,4 @@ -import { - createLoggerReconcilerSubscriber, - planApp, - type Plan, - type PlanNode, -} from "@notation/core"; +import { planApp, type Plan, type PlanNode } from "@notation/core"; import { compile } from "./compile"; import { defaultLogger, type Logger } from "./logger"; import { redirectStdoutToStderr } from "./stdio"; @@ -24,42 +19,27 @@ const decisionSymbols: Record = { export async function plan(entryPoint: string, opts: PlanCommandOptions = {}) { const logger = opts.logger ?? defaultLogger; - const emit = createLoggerReconcilerSubscriber({ logger }); - try { - if (opts.json) { - let result: Plan; - const { restore } = redirectStdoutToStderr(); - try { - await compile(entryPoint, { logger }); - result = await planApp({ - entryPoint, - emit, - }); - } finally { - restore(); - } - process.stdout.write(`${JSON.stringify(result, null, 2)}\n`); - return; + if (opts.json) { + let result: Plan; + const { restore } = redirectStdoutToStderr(); + try { + await compile(entryPoint, { logger }); + result = await planApp({ + entryPoint, + }); + } finally { + restore(); } - - await compile(entryPoint, { logger }); - logger.info(`Planning ${entryPoint}\n`); - const result = await planApp({ - entryPoint, - emit, - }); - printPlanSummary(result, logger); - } catch (err: any) { - if (err.name === "CredentialsProviderError") { - logger.error( - "\nAWS credentials not found.", - "\n\nEnsure you have a default profile set up in ~/.aws/credentials.", - "\n\nIf using another profile run AWS_PROFILE=otherProfile notation plan.\n", - ); - process.exit(1); - } - throw err; + process.stdout.write(`${JSON.stringify(result, null, 2)}\n`); + return; } + + await compile(entryPoint, { logger }); + logger.info(`Planning ${entryPoint}\n`); + const result = await planApp({ + entryPoint, + }); + printPlanSummary(result, logger); } function printPlanSummary(result: Plan, logger: Logger) { diff --git a/packages/cli/src/run-with-error-handling.ts b/packages/cli/src/run-with-error-handling.ts new file mode 100644 index 0000000..85f8f78 --- /dev/null +++ b/packages/cli/src/run-with-error-handling.ts @@ -0,0 +1,22 @@ +import type { Logger } from "./logger"; + +export async function runWithCliErrorHandling( + fn: () => Promise, + opts: { logger: Logger; command: string }, +): Promise<0 | 1> { + try { + await fn(); + return 0; + } catch (error: unknown) { + if (error instanceof Error && error.name === "CredentialsProviderError") { + opts.logger.error( + "\nAWS credentials not found.", + "\n\nEnsure you have a default profile set up in ~/.aws/credentials.", + `\n\nIf using another profile run AWS_PROFILE=otherProfile notation ${opts.command}.\n`, + ); + return 1; + } + opts.logger.error(error); + return 1; + } +} diff --git a/packages/cli/test/run-with-error-handling.test.ts b/packages/cli/test/run-with-error-handling.test.ts new file mode 100644 index 0000000..2ca64e9 --- /dev/null +++ b/packages/cli/test/run-with-error-handling.test.ts @@ -0,0 +1,41 @@ +import { describe, expect, it, vi } from "vitest"; +import { runWithCliErrorHandling } from "../src/run-with-error-handling"; + +describe("CLI error handling", () => { + it("reports credential failures with command-specific guidance", async () => { + const error = new Error("Could not load credentials"); + error.name = "CredentialsProviderError"; + const logger = { info: vi.fn(), warn: vi.fn(), error: vi.fn() }; + + const exitCode = await runWithCliErrorHandling( + async () => { + throw error; + }, + { logger, command: "deploy" }, + ); + + expect(exitCode).toBe(1); + expect(logger.error).toHaveBeenCalledOnce(); + expect(logger.error).toHaveBeenCalledWith( + "\nAWS credentials not found.", + "\n\nEnsure you have a default profile set up in ~/.aws/credentials.", + "\n\nIf using another profile run AWS_PROFILE=otherProfile notation deploy.\n", + ); + }); + + it("reports non-credential failures unchanged", async () => { + const error = new Error("deploy failed"); + const logger = { info: vi.fn(), warn: vi.fn(), error: vi.fn() }; + + const exitCode = await runWithCliErrorHandling( + async () => { + throw error; + }, + { logger, command: "deploy" }, + ); + + expect(exitCode).toBe(1); + expect(logger.error).toHaveBeenCalledOnce(); + expect(logger.error).toHaveBeenCalledWith(error); + }); +}); diff --git a/packages/core/package.json b/packages/core/package.json index 542912a..cc55f70 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -15,12 +15,14 @@ "dependencies": { "@notation/reconciler": "workspace:*", "@notation/resource": "workspace:*", - "@notation/state": "workspace:*", - "@notation/state-sqlite": "workspace:*", + "@yieldstar/core": "0.5.0", + "@yieldstar/sqlite-runtime": "0.5.0", "deep-object-diff": "^1.1.9", "js-base64": "^3.7.7", "lodash-es": "^4.17.21", - "pako": "^2.1.0" + "pako": "^2.1.0", + "pino": "^9.14.0", + "yieldstar": "0.5.0" }, "devDependencies": { "@types/common-tags": "^1.8.4", diff --git a/packages/core/src/provisioner/durable-runtime.ts b/packages/core/src/provisioner/durable-runtime.ts new file mode 100644 index 0000000..73face8 --- /dev/null +++ b/packages/core/src/provisioner/durable-runtime.ts @@ -0,0 +1,142 @@ +import { randomUUID } from "node:crypto"; +import { setImmediate } from "node:timers/promises"; +import { + WorkflowRunner, + type WorkflowEvent, + type WorkflowRouter, +} from "@yieldstar/core"; +import { + SqliteEventLoop, + SqliteHeapClient, + SqliteSchedulerClient, + SqliteStoreClient, + SqliteTaskQueueClient, + SqliteTimersClient, + createSqliteDb, +} from "@yieldstar/sqlite-runtime/node"; +import { DurableStateBackend } from "@notation/reconciler"; +import pino, { type Logger } from "pino"; + +export const DEFAULT_WORKFLOW_STATE_PATH = ".notation/workflows.db"; + +export function resolveWorkflowStatePath(): string { + return process.env.NOTATION_STATE_PATH ?? DEFAULT_WORKFLOW_STATE_PATH; +} + +export type NodeDurableRuntimeOptions = { + deploymentId: string; + databasePath?: string; + logger?: Logger; +}; + +export type RunWorkflowOptions = { + workflowId: string; + executionId?: string; + params?: Record; +}; + +/** Resident Yieldstar 0.5.0 Node runtime used by Notation application commands. */ +export class NodeDurableRuntime { + readonly deploymentId: string; + readonly state: DurableStateBackend; + readonly #database: ReturnType; + readonly #eventLoop: SqliteEventLoop; + readonly #heapClient: SqliteHeapClient; + readonly #schedulerClient: SqliteSchedulerClient; + readonly #storeClient: SqliteStoreClient; + readonly #logger: Logger; + #running = false; + + constructor(opts: NodeDurableRuntimeOptions) { + this.deploymentId = opts.deploymentId; + this.#logger = opts.logger ?? pino({ level: "silent" }); + this.#database = createSqliteDb({ + path: opts.databasePath ?? resolveWorkflowStatePath(), + }); + const taskQueueClient = new SqliteTaskQueueClient(this.#database); + this.#schedulerClient = new SqliteSchedulerClient({ + taskQueueClient, + timersClient: new SqliteTimersClient(this.#database), + }); + this.#storeClient = new SqliteStoreClient({ + db: this.#database, + schedulerClient: this.#schedulerClient, + }); + this.#heapClient = new SqliteHeapClient(this.#database); + this.#eventLoop = new SqliteEventLoop(this.#database); + this.state = new DurableStateBackend(this.#storeClient, this.deploymentId); + } + + async run( + router: WorkflowRouter, + opts: RunWorkflowOptions, + ): Promise { + if (this.#running) { + throw new Error( + "The Node Yieldstar runtime already has an active workflow", + ); + } + this.#running = true; + const event: WorkflowEvent = { + workflowId: opts.workflowId, + executionId: opts.executionId ?? randomUUID(), + params: opts.params ?? {}, + context: new Map(), + }; + const runner = new WorkflowRunner({ + router, + heapClient: this.#heapClient, + storeClient: this.#storeClient, + schedulerClient: this.#schedulerClient, + logger: this.#logger, + }); + + let resolveCompletion!: (value: unknown) => void; + let rejectCompletion!: (error: unknown) => void; + const completion = new Promise((resolve, reject) => { + resolveCompletion = resolve; + rejectCompletion = reject; + }); + const processEvent = async (nextEvent: WorkflowEvent, logger: Logger) => { + try { + const result = await runner.run(nextEvent, logger); + if (result && nextEvent.executionId === event.executionId) { + this.#eventLoop.stop(); + resolveCompletion(result.result); + } + } catch (error) { + if (nextEvent.executionId === event.executionId) { + this.#eventLoop.stop(); + rejectCompletion(error); + return; + } + this.#logger.error({ err: error }, "Yieldstar replay failed"); + } + }; + + try { + await processEvent(event, this.#logger); + this.#eventLoop.start({ onNewEvent: processEvent, logger: this.#logger }); + try { + return await completion; + } finally { + // Let SqliteEventLoop remove the completed queue item before callers + // close the shared database. + await setImmediate(); + } + } finally { + this.#eventLoop.stop(); + this.#running = false; + } + } + + close(): void { + if (this.#running) { + throw new Error( + "Cannot close the Node Yieldstar runtime while a workflow is active", + ); + } + this.#eventLoop.stop(); + this.#database.close(); + } +} diff --git a/packages/core/src/provisioner/index.ts b/packages/core/src/provisioner/index.ts index 89bf6e7..0b551ad 100644 --- a/packages/core/src/provisioner/index.ts +++ b/packages/core/src/provisioner/index.ts @@ -1,3 +1,3 @@ export * from "./workflows"; export * from "./resource-registry"; -export * from "./state-backend"; +export * from "./durable-runtime"; diff --git a/packages/core/src/provisioner/state-backend.ts b/packages/core/src/provisioner/state-backend.ts deleted file mode 100644 index 437e56c..0000000 --- a/packages/core/src/provisioner/state-backend.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { FileStateBackend, type StateBackend } from "@notation/state"; -import { SqliteStateBackend } from "@notation/state-sqlite"; - -export const DEFAULT_STATE_PATH = "./.notation/state.json"; - -export function resolveStatePath(): string { - return process.env.NOTATION_STATE_PATH ?? DEFAULT_STATE_PATH; -} - -export function createDefaultStateBackend(): StateBackend { - const statePath = resolveStatePath(); - if (statePath.endsWith(".db") || statePath.endsWith(".sqlite")) { - return new SqliteStateBackend(statePath); - } - return new FileStateBackend(statePath); -} diff --git a/packages/core/src/provisioner/workflows/index.ts b/packages/core/src/provisioner/workflows/index.ts index 9dd1579..835d222 100644 --- a/packages/core/src/provisioner/workflows/index.ts +++ b/packages/core/src/provisioner/workflows/index.ts @@ -7,4 +7,3 @@ export { export * from "./workflow.deploy"; export * from "./workflow.destroy"; export * from "./workflow.plan"; -export * from "./workflow.refresh"; diff --git a/packages/core/src/provisioner/workflows/workflow.deploy.ts b/packages/core/src/provisioner/workflows/workflow.deploy.ts index 3eaa88e..dc3b765 100644 --- a/packages/core/src/provisioner/workflows/workflow.deploy.ts +++ b/packages/core/src/provisioner/workflows/workflow.deploy.ts @@ -1,19 +1,22 @@ +import * as reconciler from "@notation/reconciler"; import { - Reconciler, createLoggerReconcilerSubscriber, type ReconcilerEventEmitter, type ResourceRegistry, } from "@notation/reconciler"; -import type { StateBackend } from "@notation/state"; +import { createWorkflowRouter, workflow } from "yieldstar"; import { getResourceGraph } from "src/orchestrator/graph"; -import { createDefaultStateBackend } from "../state-backend"; +import { NodeDurableRuntime } from "../durable-runtime"; export type DeployAppOptions = { entryPoint: string; driftDetection?: boolean; dryRun?: boolean; + maxOperationAttempts?: number; registry?: ResourceRegistry; - state?: StateBackend; + runtime?: NodeDurableRuntime; + executionId?: string; + databasePath?: string; emit?: ReconcilerEventEmitter; }; @@ -21,20 +24,36 @@ export async function deployApp({ entryPoint, driftDetection = true, dryRun = false, + maxOperationAttempts, registry, - state: stateBackend, + runtime: suppliedRuntime, + executionId, + databasePath, emit = createLoggerReconcilerSubscriber(), }: DeployAppOptions): Promise { const graph = await getResourceGraph(entryPoint); - const state = stateBackend ?? createDefaultStateBackend(); - const reconciler = new Reconciler({ - state, - registry, - emit, - }); - - await reconciler.deploy(graph.resources, { - dryRun, - driftDetection, + const runtime = + suppliedRuntime ?? + new NodeDurableRuntime({ deploymentId: entryPoint, databasePath }); + const deploy = workflow(async function* (step, event) { + yield* reconciler.deploy(step, { + deploymentId: runtime.deploymentId, + executionId: event.executionId, + resources: graph.resources, + state: runtime.state, + registry, + emit, + dryRun, + driftDetection, + maxOperationAttempts, + }); }); + try { + await runtime.run(createWorkflowRouter({ deploy }), { + workflowId: "deploy", + executionId, + }); + } finally { + if (!suppliedRuntime) runtime.close(); + } } diff --git a/packages/core/src/provisioner/workflows/workflow.destroy.ts b/packages/core/src/provisioner/workflows/workflow.destroy.ts index 813239a..9a534c4 100644 --- a/packages/core/src/provisioner/workflows/workflow.destroy.ts +++ b/packages/core/src/provisioner/workflows/workflow.destroy.ts @@ -1,35 +1,53 @@ +import * as reconciler from "@notation/reconciler"; import { - Reconciler, createLoggerReconcilerSubscriber, type ReconcilerEventEmitter, type ResourceRegistry, } from "@notation/reconciler"; -import type { StateBackend } from "@notation/state"; +import { createWorkflowRouter, workflow } from "yieldstar"; import { getResourceGraph } from "src/orchestrator/graph"; -import { createDefaultStateBackend } from "../state-backend"; -import { refreshState } from "./workflow.refresh"; +import { NodeDurableRuntime } from "../durable-runtime"; export type DestroyAppOptions = { entryPoint: string; + maxOperationAttempts?: number; registry?: ResourceRegistry; - state?: StateBackend; + runtime?: NodeDurableRuntime; + executionId?: string; + databasePath?: string; emit?: ReconcilerEventEmitter; }; export async function destroyApp({ entryPoint, + maxOperationAttempts, registry, - state: stateBackend, + runtime: suppliedRuntime, + executionId, + databasePath, emit = createLoggerReconcilerSubscriber(), }: DestroyAppOptions) { - const state = stateBackend ?? createDefaultStateBackend(); - await refreshState({ entryPoint, registry, state, emit }); - const graph = await getResourceGraph(entryPoint); - const reconciler = new Reconciler({ - state, - emit, + const runtime = + suppliedRuntime ?? + new NodeDurableRuntime({ deploymentId: entryPoint, databasePath }); + const destroy = workflow(async function* (step, event) { + yield* reconciler.destroy(step, { + deploymentId: runtime.deploymentId, + executionId: event.executionId, + resources: graph.resources, + state: runtime.state, + registry, + emit, + maxOperationAttempts, + }); }); - - await reconciler.destroy(graph.resources); + try { + await runtime.run(createWorkflowRouter({ destroy }), { + workflowId: "destroy", + executionId, + }); + } finally { + if (!suppliedRuntime) runtime.close(); + } } diff --git a/packages/core/src/provisioner/workflows/workflow.plan.ts b/packages/core/src/provisioner/workflows/workflow.plan.ts index 2e51cb8..b64db9d 100644 --- a/packages/core/src/provisioner/workflows/workflow.plan.ts +++ b/packages/core/src/provisioner/workflows/workflow.plan.ts @@ -1,38 +1,36 @@ -import { - Reconciler, - createLoggerReconcilerSubscriber, - type Plan, - type ReconcilerEventEmitter, - type ResourceRegistry, -} from "@notation/reconciler"; -import type { StateBackend } from "@notation/state"; +import { createPlan, type Plan } from "@notation/reconciler"; import { getResourceGraph } from "src/orchestrator/graph"; -import { createDefaultStateBackend } from "../state-backend"; +import { NodeDurableRuntime } from "../durable-runtime"; export type { Plan, PlanNode, PlanDecision } from "@notation/reconciler"; export type PlanAppOptions = { entryPoint: string; driftDetection?: boolean; - registry?: ResourceRegistry; - state?: StateBackend; - emit?: ReconcilerEventEmitter; + maxOperationAttempts?: number; + runtime?: NodeDurableRuntime; + databasePath?: string; }; export async function planApp({ entryPoint, driftDetection = true, - registry, - state: stateBackend, - emit = createLoggerReconcilerSubscriber(), + maxOperationAttempts, + runtime: suppliedRuntime, + databasePath, }: PlanAppOptions): Promise { const graph = await getResourceGraph(entryPoint); - const state = stateBackend ?? createDefaultStateBackend(); - const reconciler = new Reconciler({ - state, - registry, - emit, - }); - - return reconciler.plan(graph.resources, { driftDetection }); + const runtime = + suppliedRuntime ?? + new NodeDurableRuntime({ deploymentId: entryPoint, databasePath }); + try { + return await createPlan({ + resources: graph.resources, + state: runtime.state, + driftDetection, + maxOperationAttempts, + }); + } finally { + if (!suppliedRuntime) runtime.close(); + } } diff --git a/packages/core/src/provisioner/workflows/workflow.refresh.ts b/packages/core/src/provisioner/workflows/workflow.refresh.ts deleted file mode 100644 index b463a87..0000000 --- a/packages/core/src/provisioner/workflows/workflow.refresh.ts +++ /dev/null @@ -1,39 +0,0 @@ -import { - Reconciler, - createLoggerReconcilerSubscriber, - type ReconcilerEventEmitter, - type ResourceRegistry, -} from "@notation/reconciler"; -import type { StateBackend } from "@notation/state"; -import { getResourceGraph } from "src/orchestrator/graph"; -import { createDefaultStateBackend } from "../state-backend"; - -/** - * @description Destroy resources that are in state but not in the orchestration graph - */ -export type RefreshStateOptions = { - entryPoint: string; - dryRun?: boolean; - registry?: ResourceRegistry; - state?: StateBackend; - emit?: ReconcilerEventEmitter; -}; - -export async function refreshState({ - entryPoint, - dryRun = false, - registry, - state: stateBackend, - emit = createLoggerReconcilerSubscriber(), -}: RefreshStateOptions): Promise { - const graph = await getResourceGraph(entryPoint); - const state = stateBackend ?? createDefaultStateBackend(); - - const reconciler = new Reconciler({ - state, - registry, - emit, - }); - - await reconciler.refresh(graph.resources, { dryRun }); -} diff --git a/packages/core/test/provisioner/durable-runtime.test.ts b/packages/core/test/provisioner/durable-runtime.test.ts new file mode 100644 index 0000000..77a699b --- /dev/null +++ b/packages/core/test/provisioner/durable-runtime.test.ts @@ -0,0 +1,60 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import * as reconciler from "@notation/reconciler"; +import { + ResourceOperationPendingError, + resource, +} from "@notation/resource"; +import { createWorkflowRouter, workflow } from "yieldstar"; +import { describe, expect, it } from "vitest"; +import { NodeDurableRuntime } from "src/provisioner/durable-runtime"; + +describe("NodeDurableRuntime", () => { + it("stays resident across a provider delay and resumes from the SQLite event loop", async () => { + const directory = await mkdtemp(path.join(tmpdir(), "notation-runtime-")); + const runtime = new NodeDurableRuntime({ + deploymentId: "resident-wait", + databasePath: path.join(directory, "workflows.db"), + }); + let attempts = 0; + const PendingResource = resource({ type: "test/runtime/pending" }) + .defineSchema({}) + .defineOperations({ + create: async () => { + attempts += 1; + if (attempts === 1) { + throw new ResourceOperationPendingError("provider is not ready", { + retryAfterMs: 10, + }); + } + }, + delete: async () => undefined, + }); + const resources = [new PendingResource({ id: "pending" })]; + const deploy = workflow(async function* (step, event) { + yield* reconciler.deploy(step, { + deploymentId: runtime.deploymentId, + executionId: event.executionId, + resources, + state: runtime.state, + driftDetection: false, + maxOperationAttempts: 3, + }); + }); + + try { + await runtime.run(createWorkflowRouter({ deploy }), { + workflowId: "deploy", + executionId: "resident-execution", + }); + expect(attempts).toBe(2); + await expect(runtime.state.get("pending")).resolves.toMatchObject({ + lastOperation: "create", + }); + } finally { + runtime.close(); + await rm(directory, { recursive: true, force: true }); + } + }, 5_000); +}); diff --git a/packages/core/test/provisioner/operation.create.test.ts b/packages/core/test/provisioner/operation.create.test.ts deleted file mode 100644 index 2e4f506..0000000 --- a/packages/core/test/provisioner/operation.create.test.ts +++ /dev/null @@ -1,54 +0,0 @@ -import { describe, expect, it, vi } from "vitest"; -import { - createResourceOperation, - createStepRunner, - runOperation, -} from "@notation/reconciler"; -import { MemoryStateBackend } from "@notation/state"; -import { - TestResourceSchema, - testResourceConfig, - testOperations, - testResourceOutput, -} from "test/orchestrator/resource.doubles"; - -describe("resource creation", () => { - it("passes computed input to resource.create", async () => { - const stateBackend = new MemoryStateBackend(); - const readResult = { ...testResourceOutput, volatileComputed: "123" }; - const createMock = vi.fn(async () => ({ primaryKey: "" })); - const readMock = vi.fn(async () => readResult); - - const TestResource = TestResourceSchema.defineOperations({ - ...testOperations, - create: createMock, - read: readMock, - }); - - const testResource = new TestResource({ - id: "test-resource", - config: testResourceConfig, - }); - const step = createStepRunner(); - - await runOperation( - createResourceOperation(step, { - resource: testResource, - state: stateBackend, - expectedRev: 0, - }), - ); - - const params = await testResource.getParams(); - const persistedOutput = testResource.toState(readResult); - - expect(createMock.mock.calls[0]).toEqual([params, undefined]); - await expect(stateBackend.get(testResource.id)).resolves.toMatchObject({ - id: testResource.id, - output: persistedOutput, - lastOperation: "create", - }); - expect(testResource.output).not.toEqual(testResourceOutput); - expect(testResource.output).toEqual(readResult); - }); -}); diff --git a/packages/core/test/provisioner/state-backend.test.ts b/packages/core/test/provisioner/state-backend.test.ts deleted file mode 100644 index 5ac04a4..0000000 --- a/packages/core/test/provisioner/state-backend.test.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { mkdtempSync, rmSync } from "node:fs"; -import { tmpdir } from "node:os"; -import path from "node:path"; -import { afterEach, beforeEach, describe, expect, it } from "vitest"; -import { FileStateBackend } from "@notation/state"; -import { SqliteStateBackend } from "@notation/state-sqlite"; -import { createDefaultStateBackend } from "src/provisioner/state-backend"; - -describe("createDefaultStateBackend", () => { - let directory: string; - const originalStatePath = process.env.NOTATION_STATE_PATH; - - beforeEach(() => { - directory = mkdtempSync(path.join(tmpdir(), "notation-state-")); - }); - - afterEach(() => { - if (originalStatePath === undefined) { - delete process.env.NOTATION_STATE_PATH; - } else { - process.env.NOTATION_STATE_PATH = originalStatePath; - } - rmSync(directory, { recursive: true, force: true }); - }); - - it("uses the file backend for the default JSON path", () => { - delete process.env.NOTATION_STATE_PATH; - - expect(createDefaultStateBackend()).toBeInstanceOf(FileStateBackend); - }); - - it("uses the sqlite backend for .db paths", () => { - process.env.NOTATION_STATE_PATH = path.join(directory, "state.db"); - - const backend = createDefaultStateBackend(); - expect(backend).toBeInstanceOf(SqliteStateBackend); - (backend as SqliteStateBackend).close(); - }); - - it("uses the sqlite backend for .sqlite paths", () => { - process.env.NOTATION_STATE_PATH = path.join(directory, "state.sqlite"); - - const backend = createDefaultStateBackend(); - expect(backend).toBeInstanceOf(SqliteStateBackend); - (backend as SqliteStateBackend).close(); - }); - - it("creates missing parent directories for sqlite paths", () => { - process.env.NOTATION_STATE_PATH = path.join( - directory, - ".notation", - "state.db", - ); - - const backend = createDefaultStateBackend(); - expect(backend).toBeInstanceOf(SqliteStateBackend); - (backend as SqliteStateBackend).close(); - }); -}); diff --git a/packages/reconciler/src/durable/operations.ts b/packages/reconciler/src/durable/operations.ts index d945403..eeaa73a 100644 --- a/packages/reconciler/src/durable/operations.ts +++ b/packages/reconciler/src/durable/operations.ts @@ -9,7 +9,7 @@ import { createResourceRegistryFromResources, resolveResourceClass, } from "../resource-registry"; -import { runPendingOperation } from "../operations/operation.pending"; +import { runPendingOperation } from "../pending-operation"; import { decideAction, type DriftRead, type ResourceAction } from "../plan"; import { emitEvent, emitLifecycle } from "./emit"; import type { DurableStateBackend } from "./state-backend"; diff --git a/packages/reconciler/src/index.ts b/packages/reconciler/src/index.ts index 44fc402..1f6c3dc 100644 --- a/packages/reconciler/src/index.ts +++ b/packages/reconciler/src/index.ts @@ -4,9 +4,10 @@ export type DeepObjectDiffApi = typeof import("deep-object-diff"); export type YieldstarApi = typeof import("yieldstar"); export * from "./resource-registry"; -export * from "./operations"; export * from "./dependency-graph"; export * from "./plan"; -export * from "./reconciler"; +export * from "./planner"; +export * from "./events"; +export * from "./durable"; export * from "./logger-subscriber"; export * from "./protocol"; diff --git a/packages/reconciler/src/logger-subscriber.ts b/packages/reconciler/src/logger-subscriber.ts index 318a4cb..e756ff8 100644 --- a/packages/reconciler/src/logger-subscriber.ts +++ b/packages/reconciler/src/logger-subscriber.ts @@ -1,4 +1,4 @@ -import type { ReconcilerEvent, ReconcilerEventEmitter } from "./reconciler"; +import type { ReconcilerEvent, ReconcilerEventEmitter } from "./events"; export type Logger = Pick; @@ -17,7 +17,7 @@ export function createLoggerReconcilerSubscriber( return; } - if (event.event === "reconciler.orphan-deletion.skipped") { + if (event.level === "warn") { logger.warn(event.event, event); return; } diff --git a/packages/reconciler/src/operations/index.ts b/packages/reconciler/src/operations/index.ts deleted file mode 100644 index b0107e7..0000000 --- a/packages/reconciler/src/operations/index.ts +++ /dev/null @@ -1,6 +0,0 @@ -export * from "./operation.types"; -export * from "./operation.pending"; -export * from "./operation.create"; -export * from "./operation.read"; -export * from "./operation.update"; -export * from "./operation.delete"; diff --git a/packages/reconciler/src/operations/operation.create.ts b/packages/reconciler/src/operations/operation.create.ts deleted file mode 100644 index e0aee4f..0000000 --- a/packages/reconciler/src/operations/operation.create.ts +++ /dev/null @@ -1,82 +0,0 @@ -import { createWorkflow } from "yieldstar"; -import { - type CreateResourceParams, - type StepRunner, - emitLifecycleEvent, - getErrorDetails, -} from "./operation.types"; -import { runPendingOperation } from "./operation.pending"; -import { readResourceOperation } from "./operation.read"; - -export async function* createResourceOperation( - step: StepRunner, - params: CreateResourceParams, -): AsyncGenerator { - await emitLifecycleEvent(params, "create", "start"); - - if (params.dryRun) { - await emitLifecycleEvent(params, "create", "dry-run"); - return; - } - - try { - const resourceParams = yield* step.run("create:get-params", () => - params.resource.getParams(), - ); - - const computedPrimaryKey = yield* runPendingOperation( - step, - "create:remote", - (context) => params.resource.create(resourceParams, context), - params.maxOperationAttempts, - ); - - params.resource.setOutput(resourceParams); - if (computedPrimaryKey) { - params.resource.setOutput({ - ...computedPrimaryKey, - ...params.resource.output, - }); - } - - const readResult = yield* readResourceOperation(step, { - resource: params.resource, - state: params.state, - emit: params.emit, - maxOperationAttempts: params.maxOperationAttempts, - }); - - params.resource.setOutput({ - ...params.resource.output, - ...readResult, - }); - - yield* step.run("create:persist-state", async () => { - await params.state.update(params.resource.id, params.expectedRev, { - id: params.resource.id, - groupId: params.resource.groupId, - groupType: params.resource.groupType, - type: params.resource.type, - lastOperation: "create", - lastOperationAt: new Date().toISOString(), - config: params.resource.config, - params: params.resource.toState(resourceParams), - output: params.resource.toState(params.resource.output), - }); - }); - - await emitLifecycleEvent(params, "create", "success"); - } catch (err) { - await emitLifecycleEvent(params, "create", "error", getErrorDetails(err)); - throw err; - } -} - -export const createResourceWorkflow: unknown = createWorkflow( - async function* (step, event) { - return yield* createResourceOperation( - step as StepRunner, - event.params as CreateResourceParams, - ); - }, -); diff --git a/packages/reconciler/src/operations/operation.delete.ts b/packages/reconciler/src/operations/operation.delete.ts deleted file mode 100644 index 51975b6..0000000 --- a/packages/reconciler/src/operations/operation.delete.ts +++ /dev/null @@ -1,52 +0,0 @@ -import { createWorkflow } from "yieldstar"; -import { - type DeleteResourceParams, - type StepRunner, - emitLifecycleEvent, - getErrorDetails, -} from "./operation.types"; -import { runPendingOperation } from "./operation.pending"; - -export async function* deleteResourceOperation( - step: StepRunner, - params: DeleteResourceParams, -): AsyncGenerator { - await emitLifecycleEvent(params, "delete", "start"); - - if (params.dryRun) { - await emitLifecycleEvent(params, "delete", "dry-run"); - return; - } - - try { - yield* runPendingOperation( - step, - "delete:remote", - (context) => - params.resource.delete( - params.resource.key, - params.resource.toState(params.resource.output), - context, - ), - params.maxOperationAttempts, - ); - - yield* step.run("delete:persist-state", () => - params.state.delete(params.resource.id, params.expectedRev), - ); - - await emitLifecycleEvent(params, "delete", "success"); - } catch (err) { - await emitLifecycleEvent(params, "delete", "error", getErrorDetails(err)); - throw err; - } -} - -export const deleteResourceWorkflow: unknown = createWorkflow( - async function* (step, event) { - return yield* deleteResourceOperation( - step as StepRunner, - event.params as DeleteResourceParams, - ); - }, -); diff --git a/packages/reconciler/src/operations/operation.read.ts b/packages/reconciler/src/operations/operation.read.ts deleted file mode 100644 index e74196a..0000000 --- a/packages/reconciler/src/operations/operation.read.ts +++ /dev/null @@ -1,68 +0,0 @@ -import { createWorkflow } from "yieldstar"; -import { - type ReadResourceParams, - type StepRunner, - emitLifecycleEvent, - getErrorDetails, -} from "./operation.types"; -import { runPendingOperation } from "./operation.pending"; - -export async function* readResourceOperation( - step: StepRunner, - params: ReadResourceParams, -): AsyncGenerator, unknown> { - await emitLifecycleEvent(params, "read", "start"); - - if (params.dryRun) { - await emitLifecycleEvent(params, "read", "dry-run"); - return {}; - } - - try { - const resourceParams = yield* step.run("read:get-params", () => - params.resource.getParams(), - ); - - if (!params.resource.read) { - const stateNode = yield* step.run("read:get-state-node", () => - params.state.get(params.resource.id), - ); - const merged = stateNode - ? { ...stateNode.output, ...resourceParams } - : resourceParams; - - await emitLifecycleEvent(params, "read", "skip", { - reason: "read-not-implemented", - }); - await emitLifecycleEvent(params, "read", "success"); - return merged as Record; - } - - const remote = yield* runPendingOperation( - step, - "read:remote", - (context) => params.resource.read!(params.resource.key, context), - params.maxOperationAttempts, - ); - - const mergedOutput = { - ...resourceParams, - ...remote, - }; - - await emitLifecycleEvent(params, "read", "success"); - return mergedOutput; - } catch (err) { - await emitLifecycleEvent(params, "read", "error", getErrorDetails(err)); - throw err; - } -} - -export const readResourceWorkflow: unknown = createWorkflow( - async function* (step, event) { - return yield* readResourceOperation( - step as StepRunner, - event.params as ReadResourceParams, - ); - }, -); diff --git a/packages/reconciler/src/operations/operation.types.ts b/packages/reconciler/src/operations/operation.types.ts deleted file mode 100644 index 6cbcd1c..0000000 --- a/packages/reconciler/src/operations/operation.types.ts +++ /dev/null @@ -1,92 +0,0 @@ -import type { BaseResource, ResourceType } from "@notation/resource"; -import type { State } from "@notation/state"; - -export type OperationName = "create" | "read" | "update" | "delete"; - -export type OperationLifecycleStatus = - "start" | "success" | "error" | "skip" | "dry-run"; - -export type OperationLifecycleEvent = { - level: "info" | "error"; - event: "reconciler.operation.lifecycle"; - operation: OperationName; - status: OperationLifecycleStatus; - resourceId: string; - resourceType: ResourceType; - reason?: string; - errorName?: string; - errorMessage?: string; -}; - -export type OperationEventEmitter = ( - event: OperationLifecycleEvent, -) => void | Promise; - -export type StepRunner = { - run(fn: () => T | Promise): AsyncGenerator; - run( - key: string, - fn: () => T | Promise, - ): AsyncGenerator; - delay(ms: number): AsyncGenerator; - delay(key: string, ms: number): AsyncGenerator; -}; - -export type ResourceOperationBaseParams = { - resource: BaseResource; - state: Pick; - dryRun?: boolean; - emit?: OperationEventEmitter; - maxOperationAttempts?: number; -}; - -export type CreateResourceParams = ResourceOperationBaseParams & { - expectedRev: number; -}; - -export type ReadResourceParams = ResourceOperationBaseParams; - -export type UpdateResourceParams = ResourceOperationBaseParams & { - patch: Record; - expectedRev: number; -}; - -export type DeleteResourceParams = ResourceOperationBaseParams & { - expectedRev: number; -}; - -export function getErrorDetails(err: unknown): { - errorName: string; - errorMessage: string; -} { - if (err instanceof Error) { - return { - errorName: err.name, - errorMessage: err.message, - }; - } - - return { - errorName: "UnknownError", - errorMessage: String(err), - }; -} - -export async function emitLifecycleEvent( - params: ResourceOperationBaseParams, - operation: OperationName, - status: OperationLifecycleStatus, - extra: Partial = {}, -) { - if (!params.emit) return; - - await params.emit({ - level: status === "error" ? "error" : "info", - event: "reconciler.operation.lifecycle", - operation, - status, - resourceId: params.resource.id, - resourceType: params.resource.type, - ...extra, - }); -} diff --git a/packages/reconciler/src/operations/operation.update.ts b/packages/reconciler/src/operations/operation.update.ts deleted file mode 100644 index fc62a8c..0000000 --- a/packages/reconciler/src/operations/operation.update.ts +++ /dev/null @@ -1,94 +0,0 @@ -import { createWorkflow } from "yieldstar"; -import { - type StepRunner, - type UpdateResourceParams, - emitLifecycleEvent, - getErrorDetails, -} from "./operation.types"; -import { runPendingOperation } from "./operation.pending"; -import { readResourceOperation } from "./operation.read"; - -export async function* updateResourceOperation( - step: StepRunner, - params: UpdateResourceParams, -): AsyncGenerator { - await emitLifecycleEvent(params, "update", "start"); - - if (params.dryRun) { - await emitLifecycleEvent(params, "update", "dry-run"); - return; - } - - if (!params.resource.update) { - await emitLifecycleEvent(params, "update", "skip", { - reason: "update-not-implemented", - }); - await emitLifecycleEvent(params, "update", "success"); - return; - } - - try { - const resourceParams = yield* step.run("update:get-params", () => - params.resource.getParams(), - ); - - yield* runPendingOperation( - step, - "update:remote", - (context) => - params.resource.update!( - params.resource.key, - params.patch, - resourceParams, - params.resource.toState(params.resource.output), - context, - ), - params.maxOperationAttempts, - ); - - params.resource.setOutput({ - ...params.resource.key, - ...resourceParams, - }); - - const readResult = yield* readResourceOperation(step, { - resource: params.resource, - state: params.state, - emit: params.emit, - maxOperationAttempts: params.maxOperationAttempts, - }); - - params.resource.setOutput({ - ...params.resource.output, - ...readResult, - }); - - yield* step.run("update:persist-state", async () => { - await params.state.update(params.resource.id, params.expectedRev, { - id: params.resource.id, - groupId: params.resource.groupId, - groupType: params.resource.groupType, - type: params.resource.type, - lastOperation: "update", - lastOperationAt: new Date().toISOString(), - config: params.resource.config, - params: params.resource.toState(resourceParams), - output: params.resource.toState(params.resource.output), - }); - }); - - await emitLifecycleEvent(params, "update", "success"); - } catch (err) { - await emitLifecycleEvent(params, "update", "error", getErrorDetails(err)); - throw err; - } -} - -export const updateResourceWorkflow: unknown = createWorkflow( - async function* (step, event) { - return yield* updateResourceOperation( - step as StepRunner, - event.params as UpdateResourceParams, - ); - }, -); diff --git a/packages/reconciler/src/operations/operation.pending.ts b/packages/reconciler/src/pending-operation.ts similarity index 84% rename from packages/reconciler/src/operations/operation.pending.ts rename to packages/reconciler/src/pending-operation.ts index 04a6b7e..d9f2648 100644 --- a/packages/reconciler/src/operations/operation.pending.ts +++ b/packages/reconciler/src/pending-operation.ts @@ -2,12 +2,19 @@ import { ResourceOperationPendingError, type ResourceOperationContext, } from "@notation/resource"; -import type { StepRunner } from "./operation.types"; + +type OperationStep = { + run( + key: string, + operation: () => T | Promise, + ): AsyncGenerator; + delay(key: string, delayMs: number): AsyncGenerator; +}; export const DEFAULT_MAX_OPERATION_ATTEMPTS = 30; export async function* runPendingOperation( - step: StepRunner, + step: OperationStep, key: string, operation: (context?: ResourceOperationContext) => T | Promise, maxAttempts = DEFAULT_MAX_OPERATION_ATTEMPTS, diff --git a/packages/reconciler/src/planner.ts b/packages/reconciler/src/planner.ts index ebfa064..5dfed90 100644 --- a/packages/reconciler/src/planner.ts +++ b/packages/reconciler/src/planner.ts @@ -1,7 +1,8 @@ import { ResourceNotFoundError, type BaseResource } from "@notation/resource"; import type { StateBackend } from "@notation/state"; +import { setTimeout as sleep } from "node:timers/promises"; import { buildResourceDepthLevels } from "./dependency-graph"; -import { readResourceOperation } from "./operations"; +import { runPendingOperation } from "./pending-operation"; import { decideAction, getDependencyIds, @@ -9,18 +10,19 @@ import { type Plan, type PlanNode, } from "./plan"; -import { createStepRunner, runOperation } from "./reconciler"; export type CreatePlanOptions = { resources: BaseResource[]; state: StateBackend; driftDetection?: boolean; + maxOperationAttempts?: number; }; export async function createPlan({ resources, state, driftDetection = true, + maxOperationAttempts, }: CreatePlanOptions): Promise { const resourceById = new Map( resources.map((resource) => [resource.id, resource]), @@ -38,10 +40,12 @@ export async function createPlan({ let driftRead; try { const output = await runOperation( - readResourceOperation(createStepRunner(), { - resource, - state, - }), + runPendingOperation( + createStepRunner(), + `plan:${resource.id}:read`, + (context) => resource.read!(resource.key, context), + maxOperationAttempts, + ), ); driftRead = { kind: "present" as const, output }; } catch (error) { @@ -81,3 +85,30 @@ export async function createPlan({ return { createdAt: new Date().toISOString(), nodes }; } + +async function runOperation( + operation: AsyncGenerator, +) { + let next = await operation.next(); + while (!next.done) { + next = await operation.next(); + } + return next.value; +} + +function createStepRunner() { + return { + async *run( + _key: string, + operation: () => T | Promise, + ): AsyncGenerator { + return await operation(); + }, + async *delay( + _key: string, + delayMs: number, + ): AsyncGenerator { + await sleep(delayMs); + }, + }; +} diff --git a/packages/reconciler/src/protocol.ts b/packages/reconciler/src/protocol.ts index bf3706e..a8ab407 100644 --- a/packages/reconciler/src/protocol.ts +++ b/packages/reconciler/src/protocol.ts @@ -1,4 +1,4 @@ -import type { ReconcilerEvent, ReconcilerEventEmitter } from "./reconciler"; +import type { ReconcilerEvent, ReconcilerEventEmitter } from "./events"; export const EVENT_STREAM_VERSION = 1 as const; diff --git a/packages/reconciler/src/reconciler.ts b/packages/reconciler/src/reconciler.ts deleted file mode 100644 index fab6c0b..0000000 --- a/packages/reconciler/src/reconciler.ts +++ /dev/null @@ -1,602 +0,0 @@ -import { ResourceNotFoundError } from "@notation/resource"; -import type { BaseResource, ResourceType } from "@notation/resource"; -import { RevConflict, type State, type StateNode } from "@notation/state"; -import { setTimeout as sleep } from "node:timers/promises"; -import { buildResourceDepthLevels } from "./dependency-graph"; -import { - decideAction, - getDependencyIds, - resolvePlanParams, - type DriftRead, - type Plan, - type PlanNode, - type ResourceAction, -} from "./plan"; -import { - createResourceOperation, - deleteResourceOperation, - readResourceOperation, - type OperationLifecycleEvent, - type StepRunner, - updateResourceOperation, -} from "./operations"; -import { - createMissingResourceRegistryMatchWarningEvent, - createResourceRegistryFromResources, - resolveResourceClass, - type MissingResourceRegistryMatchWarningEvent, - type ResourceRegistry, -} from "./resource-registry"; - -export type ReconcilerDeployEvent = { - level: "info"; - event: "reconciler.deploy.decision"; - resourceId: string; - resourceType: string; - decision: "create" | "update" | "drift-update" | "drift-recreate" | "noop"; -}; - -export type ReconcilerDriftDetectedEvent = { - level: "info"; - event: "reconciler.drift.detected"; - resourceId: string; - resourceType: string; - diff: Record; -}; - -export type ReconcilerEvent = - | OperationLifecycleEvent - | ReconcilerDeployEvent - | ReconcilerDriftDetectedEvent - | MissingResourceRegistryMatchWarningEvent; - -export type ReconcilerEventEmitter = ( - event: ReconcilerEvent, -) => void | Promise; - -export type ReconcilerState = Pick< - State, - "get" | "update" | "delete" | "values" | "lease" ->; - -export type ReconcilerOptions = { - state: ReconcilerState; - registry?: ResourceRegistry; - dryRun?: boolean; - driftDetection?: boolean; - emit?: ReconcilerEventEmitter; - maxOperationAttempts?: number; - mutationLeaseTtl?: number; -}; - -export type DeployOptions = { - dryRun?: boolean; - driftDetection?: boolean; -}; - -export type DestroyOptions = { - dryRun?: boolean; -}; - -export type RefreshOptions = { - dryRun?: boolean; -}; - -export type PlanOptions = { - driftDetection?: boolean; -}; - -export class Reconciler { - readonly #state: ReconcilerState; - readonly #registry?: ResourceRegistry; - readonly #defaultDryRun: boolean; - readonly #defaultDriftDetection: boolean; - readonly #emit?: ReconcilerEventEmitter; - readonly #maxOperationAttempts?: number; - readonly #mutationLeaseTtl: number; - readonly #stepRunner: StepRunner; - - constructor(opts: ReconcilerOptions) { - this.#state = opts.state; - this.#registry = opts.registry; - this.#defaultDryRun = opts.dryRun ?? false; - this.#defaultDriftDetection = opts.driftDetection ?? true; - this.#emit = opts.emit; - this.#maxOperationAttempts = opts.maxOperationAttempts; - this.#mutationLeaseTtl = opts.mutationLeaseTtl ?? 30_000; - this.#stepRunner = createStepRunner(); - } - - async deploy( - resources: BaseResource[], - opts: DeployOptions = {}, - ): Promise { - const dryRun = opts.dryRun ?? this.#defaultDryRun; - const driftDetection = opts.driftDetection ?? this.#defaultDriftDetection; - const resourceById = new Map( - resources.map((resource) => [resource.id, resource]), - ); - - const dependencyLevels = buildResourceDepthLevels(resources); - for (const level of dependencyLevels) { - await Promise.all( - level.map((resource) => - this.#deployResource(resource, dryRun, driftDetection), - ), - ); - } - - await this.#deleteOrphans(resources, resourceById, dryRun, "deploy"); - } - - async plan(resources: BaseResource[], opts: PlanOptions = {}): Promise { - const driftDetection = opts.driftDetection ?? this.#defaultDriftDetection; - const resourceById = new Map( - resources.map((resource) => [resource.id, resource]), - ); - const nodes: PlanNode[] = []; - - const dependencyLevels = buildResourceDepthLevels(resources); - for (const level of dependencyLevels) { - for (const resource of level) { - nodes.push(await this.#planResource(resource, driftDetection)); - } - } - - const stateNodes = await this.#state.values(); - for (const stateNode of stateNodes) { - if (resourceById.has(stateNode.id)) continue; - - nodes.push({ - id: stateNode.id, - type: stateNode.type, - decision: "delete-orphan", - params: stateNode.params, - dependsOn: [], - }); - } - - return { - createdAt: new Date().toISOString(), - nodes, - }; - } - - async destroy( - resources: BaseResource[], - opts: DestroyOptions = {}, - ): Promise { - const dryRun = opts.dryRun ?? this.#defaultDryRun; - const dependencyLevels = buildResourceDepthLevels(resources); - - for ( - let levelIndex = dependencyLevels.length - 1; - levelIndex >= 0; - levelIndex -= 1 - ) { - const level = dependencyLevels[levelIndex]!; - await Promise.all( - level.map((resource) => this.#destroyResource(resource, dryRun)), - ); - } - } - - async refresh( - resources: BaseResource[], - opts: RefreshOptions = {}, - ): Promise { - const dryRun = opts.dryRun ?? this.#defaultDryRun; - const resourceById = new Map( - resources.map((resource) => [resource.id, resource]), - ); - - await this.#deleteOrphans(resources, resourceById, dryRun, "refresh"); - } - - async #deployResource( - resource: BaseResource, - dryRun: boolean, - driftDetection: boolean, - ) { - await this.#withMutationLease(resource.id, () => - this.#retryOnRevConflict((conflict) => - this.#deployResourceOnce(resource, dryRun, driftDetection, conflict), - ), - ); - } - - async #withMutationLease(resourceId: string, fn: () => Promise) { - return this.#withLease(`reconciler:resource:${resourceId}`, fn); - } - - async #withLease(scope: string, fn: () => Promise): Promise { - const lease = await this.#state.lease(scope, this.#mutationLeaseTtl); - const controller = new AbortController(); - let renewalError: unknown; - const heartbeat = (async () => { - try { - while (!controller.signal.aborted) { - await sleep( - Math.max(1, Math.floor(this.#mutationLeaseTtl / 3)), - undefined, - { - signal: controller.signal, - }, - ); - await lease.renew(this.#mutationLeaseTtl); - } - } catch (error) { - if (!controller.signal.aborted) renewalError = error; - } - })(); - - try { - const result = await fn(); - if (renewalError) throw renewalError; - return result; - } finally { - controller.abort(); - await heartbeat; - await lease.release(); - } - } - - async #retryOnRevConflict(fn: (conflict?: RevConflict) => Promise) { - let conflict: RevConflict | undefined; - for (let attempt = 0; attempt < 3; attempt += 1) { - try { - await fn(conflict); - return; - } catch (error) { - if (!(error instanceof RevConflict) || attempt === 2) throw error; - // Re-throwing the conflict supplied for recovery means the resource - // cannot be recovered safely (for example, it has no read operation). - if (error === conflict) throw error; - conflict = error; - } - } - } - - async #deployResourceOnce( - resource: BaseResource, - dryRun: boolean, - driftDetection: boolean, - conflict?: RevConflict, - ) { - if (conflict) { - await this.#recoverDeployResource(resource, dryRun, conflict); - return; - } - - const stateNode = await this.#state.get(resource.id); - - let action: ResourceAction; - if (!stateNode) { - action = decideAction({ resource }); - } else { - resource.setOutput(stateNode.output); - const params = await resource.getParams(); - action = decideAction({ resource, stateNode, params }); - - if (action.decision === "noop" && driftDetection) { - const driftRead = await this.#readForDrift(resource); - action = decideAction({ resource, stateNode, params, driftRead }); - } - } - - if (action.decision === "drift-update") { - await this.#emit?.({ - level: "info", - event: "reconciler.drift.detected", - resourceId: resource.id, - resourceType: resource.type, - diff: action.patch, - }); - } - - await this.#emit?.({ - level: "info", - event: "reconciler.deploy.decision", - resourceId: resource.id, - resourceType: resource.type, - decision: action.decision, - }); - - switch (action.decision) { - case "create": - case "drift-recreate": - await runOperation( - createResourceOperation(this.#stepRunner, { - resource, - state: this.#state, - dryRun, - emit: this.#emit, - maxOperationAttempts: this.#maxOperationAttempts, - expectedRev: stateNode?.rev ?? 0, - }), - ); - return; - case "update": - case "drift-update": - // decideAction only returns update decisions for an existing stateNode - await runOperation( - updateResourceOperation(this.#stepRunner, { - resource, - state: this.#state, - patch: action.patch, - dryRun, - emit: this.#emit, - maxOperationAttempts: this.#maxOperationAttempts, - expectedRev: stateNode!.rev, - }), - ); - return; - case "noop": - return; - } - } - - async #recoverDeployResource( - resource: BaseResource, - dryRun: boolean, - conflict: RevConflict, - ) { - if (!resource.read) throw conflict; - - const stateNode = await this.#state.get(resource.id); - if (stateNode) resource.setOutput(stateNode.output); - - const params = await resource.getParams(); - const remote = await this.#readForDrift(resource); - const action = decideAction({ - resource, - stateNode, - params, - driftRead: remote, - }); - if (remote.kind === "present") resource.setOutput(remote.output); - - await this.#emit?.({ - level: "info", - event: "reconciler.deploy.decision", - resourceId: resource.id, - resourceType: resource.type, - decision: action.decision, - }); - - switch (action.decision) { - case "create": - case "drift-recreate": - await runOperation( - createResourceOperation(this.#stepRunner, { - resource, - state: this.#state, - dryRun, - emit: this.#emit, - maxOperationAttempts: this.#maxOperationAttempts, - expectedRev: stateNode?.rev ?? 0, - }), - ); - return; - case "update": - case "drift-update": - await runOperation( - updateResourceOperation(this.#stepRunner, { - resource, - state: this.#state, - patch: action.patch, - dryRun, - emit: this.#emit, - maxOperationAttempts: this.#maxOperationAttempts, - expectedRev: stateNode?.rev ?? 0, - }), - ); - return; - case "noop": - if (dryRun) return; - await this.#state.update(resource.id, stateNode?.rev ?? 0, { - id: resource.id, - groupId: resource.groupId, - groupType: resource.groupType, - type: resource.type, - lastOperation: "drift", - lastOperationAt: new Date().toISOString(), - config: resource.config, - params: resource.toState(params), - output: resource.toState(resource.output), - }); - return; - } - } - - async #planResource( - resource: BaseResource, - driftDetection: boolean, - ): Promise { - const stateNode = await this.#state.get(resource.id); - if (stateNode) { - resource.setOutput(stateNode.output); - } - - const params = await resolvePlanParams(resource); - let action = decideAction({ resource, stateNode, params }); - - if (action.decision === "noop" && driftDetection) { - const driftRead = await this.#readForDrift(resource); - action = decideAction({ resource, stateNode, params, driftRead }); - } - - return { - id: resource.id, - type: resource.type, - decision: action.decision, - ...("diff" in action ? { diff: action.diff } : {}), - params, - dependsOn: getDependencyIds(resource), - }; - } - - async #readForDrift(resource: BaseResource): Promise { - try { - const output = await runOperation( - readResourceOperation(this.#stepRunner, { - resource, - state: this.#state, - emit: this.#emit, - maxOperationAttempts: this.#maxOperationAttempts, - }), - ); - return { kind: "present", output }; - } catch (error) { - if (ResourceNotFoundError.is(error)) return { kind: "absent" }; - throw error; - } - } - - async #deleteOrphans( - resources: BaseResource[], - resourceById: Map, - dryRun: boolean, - workflow: "deploy" | "refresh", - ) { - await this.#withLease("reconciler:orphan-deletion", async () => { - const stateNodes = await this.#state.values(); - const registry = - this.#registry ?? createResourceRegistryFromResources(resources); - - for (const stateNode of stateNodes) { - if (resourceById.has(stateNode.id)) continue; - - const stateNodeResourceType = stateNode.type as ResourceType; - - const Resource = resolveResourceClass(registry, stateNodeResourceType); - if (!Resource) { - await this.#emit?.( - createMissingResourceRegistryMatchWarningEvent({ - workflow, - resourceId: stateNode.id, - resourceType: stateNodeResourceType, - }), - ); - continue; - } - - await this.#withMutationLease(stateNode.id, () => - this.#retryOnRevConflict(async (conflict) => { - const currentNode = await this.#state.get(stateNode.id); - if (!currentNode) return; - - const orphanResource = hydrateResourceFromState( - Resource, - currentNode, - ); - - await this.#deleteResourceOnce( - orphanResource, - currentNode, - dryRun, - conflict, - ); - }), - ); - } - }); - } - - async #destroyResource(resource: BaseResource, dryRun: boolean) { - await this.#withMutationLease(resource.id, () => - this.#retryOnRevConflict(async (conflict) => { - const stateNode = await this.#state.get(resource.id); - if (!stateNode) { - return; - } - - resource.setOutput(stateNode.output); - await this.#deleteResourceOnce(resource, stateNode, dryRun, conflict); - }), - ); - } - - async #deleteResourceOnce( - resource: BaseResource, - stateNode: StateNode, - dryRun: boolean, - conflict?: RevConflict, - ) { - if (conflict) { - if (!resource.read) throw conflict; - - const remote = await this.#readForDrift(resource); - if (remote.kind !== "present") { - if (!dryRun) await this.#state.delete(resource.id, stateNode.rev); - return; - } - resource.setOutput(remote.output); - } - - await runOperation( - deleteResourceOperation(this.#stepRunner, { - resource, - state: this.#state, - dryRun, - emit: this.#emit, - maxOperationAttempts: this.#maxOperationAttempts, - expectedRev: stateNode.rev, - }), - ); - } -} - -export async function runOperation( - operation: AsyncGenerator, -) { - let next = await operation.next(); - while (!next.done) { - next = await operation.next(); - } - return next.value; -} - -function hydrateResourceFromState( - Resource: new (opts: { - id: string; - config: Record; - }) => BaseResource, - stateNode: StateNode, -): BaseResource { - const resource = new Resource({ - id: stateNode.id, - config: stateNode.config, - }); - resource.setOutput(stateNode.output); - return resource; -} - -export function createStepRunner(): StepRunner { - return { - async *run( - arg1: string | (() => T | Promise), - arg2?: () => T | Promise, - ): AsyncGenerator { - const fn = (typeof arg1 === "string" ? arg2 : arg1) as - (() => T | Promise) | undefined; - - if (!fn) { - throw new Error("Missing run function"); - } - - return await fn(); - }, - async *delay( - arg1: string | number, - arg2?: number, - ): AsyncGenerator { - const ms = typeof arg1 === "number" ? arg1 : arg2; - if (ms === undefined) { - throw new Error("Missing delay duration"); - } - - await new Promise((resolve) => setTimeout(resolve, ms)); - }, - }; -} diff --git a/packages/reconciler/src/resource-registry.ts b/packages/reconciler/src/resource-registry.ts index 6916caa..34dabfc 100644 --- a/packages/reconciler/src/resource-registry.ts +++ b/packages/reconciler/src/resource-registry.ts @@ -1,4 +1,8 @@ -import type { BaseResource, ResourceClass, ResourceType } from "@notation/resource"; +import type { + BaseResource, + ResourceClass, + ResourceType, +} from "@notation/resource"; export type ResourceRegistry = Map>; @@ -6,7 +10,7 @@ export type MissingResourceRegistryMatchWarningEvent = { level: "warn"; event: "reconciler.orphan-deletion.skipped"; reason: "resource-type-not-registered"; - workflow: "deploy" | "refresh" | "destroy"; + workflow: "deploy" | "destroy"; resourceId: string; resourceType: ResourceType; }; @@ -46,7 +50,7 @@ export function resolveResourceClass( } export function createMissingResourceRegistryMatchWarningEvent(opts: { - workflow: "deploy" | "refresh" | "destroy"; + workflow: "deploy" | "destroy"; resourceId: string; resourceType: ResourceType; }): MissingResourceRegistryMatchWarningEvent { diff --git a/packages/reconciler/test/logger-subscriber.test.ts b/packages/reconciler/test/logger-subscriber.test.ts index 49569dd..721e884 100644 --- a/packages/reconciler/test/logger-subscriber.test.ts +++ b/packages/reconciler/test/logger-subscriber.test.ts @@ -27,6 +27,13 @@ describe("logger reconciler subscriber", () => { resourceId: "resource-2", resourceType: "test/service/subscriber", }); + await emit({ + level: "warn", + event: "reconciler.coordination.waiting", + deploymentId: "deployment-1", + executionId: "execution-2", + holderExecutionId: "execution-1", + }); await emit({ level: "error", event: "reconciler.operation.lifecycle", @@ -39,7 +46,12 @@ describe("logger reconciler subscriber", () => { }); expect(info).toHaveBeenCalledOnce(); - expect(warn).toHaveBeenCalledOnce(); + expect(warn).toHaveBeenCalledTimes(2); + expect(warn).toHaveBeenNthCalledWith( + 2, + "reconciler.coordination.waiting", + expect.objectContaining({ level: "warn" }), + ); expect(error).toHaveBeenCalledOnce(); }); }); diff --git a/packages/reconciler/test/operation.workflows.test.ts b/packages/reconciler/test/operation.workflows.test.ts deleted file mode 100644 index 2afc6df..0000000 --- a/packages/reconciler/test/operation.workflows.test.ts +++ /dev/null @@ -1,346 +0,0 @@ -import { describe, expect, it, vi } from "vitest"; -import { - resource, - ResourceNotFoundError, - ResourceOperationPendingError, -} from "@notation/resource"; -import { - createResourceOperation, - deleteResourceOperation, - readResourceOperation, - type OperationLifecycleEvent, - type StepRunner, -} from "../src/operations"; - -function createStepRunnerDouble(): StepRunner { - const run = vi.fn(async function* ( - arg1: string | (() => T | Promise), - arg2?: () => T | Promise, - ): AsyncGenerator { - const fn = (typeof arg1 === "string" ? arg2 : arg1) as () => T | Promise; - if (!fn) { - throw new Error("Missing run function"); - } - - return await fn(); - }); - - const delay = vi.fn(async function* (): AsyncGenerator< - unknown, - void, - unknown - > { - return; - }); - - return { - run, - delay, - }; -} - -async function runOperation(operation: AsyncGenerator) { - let next = await operation.next(); - while (!next.done) { - next = await operation.next(); - } - return next.value; -} - -describe("operation workflows", () => { - it("create performs create + read-after-create + state persistence", async () => { - const step = createStepRunnerDouble(); - const events: OperationLifecycleEvent[] = []; - const state = { - get: vi.fn(async () => undefined), - update: vi.fn(async () => undefined), - delete: vi.fn(async () => undefined), - }; - - let createAttempts = 0; - const createMock = vi.fn(async (_params, context) => { - createAttempts += 1; - if (createAttempts === 1) { - expect(context).toBeUndefined(); - throw Object.assign(new Error("retry create"), { - _tag: "ResourceOperationPendingError", - retryAfterMs: 25, - callbackContext: { operationId: "create-123" }, - }); - } - expect(context).toEqual({ operationId: "create-123" }); - return { remoteId: "abc" }; - }); - - const TestResource = resource({ type: "test/service/create" }) - .defineSchema({}) - .defineOperations({ - create: createMock, - read: async () => ({ remoteId: "abc", status: "ready" }), - delete: async () => undefined, - }); - - const testResource = new TestResource({ id: "test-create" }); - - await runOperation( - createResourceOperation(step, { - resource: testResource, - state, - expectedRev: 0, - emit: async (event) => { - events.push(event); - }, - }), - ); - - expect(createAttempts).toBe(2); - expect(state.update).toHaveBeenCalledOnce(); - expect(createMock).toHaveBeenNthCalledWith( - 1, - await testResource.getParams(), - undefined, - ); - expect(createMock).toHaveBeenNthCalledWith( - 2, - await testResource.getParams(), - { operationId: "create-123" }, - ); - expect(testResource.output).toEqual({ remoteId: "abc", status: "ready" }); - expect(events.map((event) => `${event.operation}:${event.status}`)).toEqual( - ["create:start", "read:start", "read:success", "create:success"], - ); - expect(events[0]).toMatchObject({ - resourceId: "test-create", - resourceType: TestResource.type, - event: "reconciler.operation.lifecycle", - }); - }); - - it("read follows pending retry instructions", async () => { - const step = createStepRunnerDouble(); - const state = { - get: vi.fn(async () => undefined), - update: vi.fn(async () => undefined), - delete: vi.fn(async () => undefined), - }; - - let readAttempts = 0; - const TestResource = resource({ type: "test/service/read" }) - .defineSchema({}) - .defineOperations({ - create: async () => ({}), - read: async (_key, context) => { - readAttempts += 1; - if (readAttempts < 3) { - throw new ResourceOperationPendingError("resource is not ready", { - retryAfterMs: readAttempts * 10, - callbackContext: { readAttempts }, - }); - } - expect(context).toEqual({ readAttempts: 2 }); - return { status: "ready" } as const; - }, - delete: async () => undefined, - }); - - const testResource = new TestResource({ id: "test-read" }); - - const result = await runOperation( - readResourceOperation(step, { - resource: testResource, - state, - }), - ); - - expect(readAttempts).toBe(3); - expect(result).toEqual({ status: "ready" }); - expect(step.delay).toHaveBeenNthCalledWith( - 1, - "read:remote:retry-delay:0", - 10, - ); - expect(step.delay).toHaveBeenNthCalledWith( - 2, - "read:remote:retry-delay:1", - 20, - ); - }); - - it("fails when an operation remains pending past the safety limit", async () => { - const step = createStepRunnerDouble(); - const state = { - get: vi.fn(async () => undefined), - update: vi.fn(async () => undefined), - delete: vi.fn(async () => undefined), - }; - const read = vi.fn(async () => { - throw new ResourceOperationPendingError("still pending", { - retryAfterMs: 10, - }); - }); - const TestResource = resource({ type: "test/service/pending-limit" }) - .defineSchema({}) - .defineOperations({ - create: async () => ({}), - read, - delete: async () => undefined, - }); - - await expect( - runOperation( - readResourceOperation(step, { - resource: new TestResource({ id: "pending-limit" }), - state, - maxOperationAttempts: 2, - }), - ), - ).rejects.toThrowError("still pending after 2 attempts"); - expect(read).toHaveBeenCalledTimes(2); - expect(step.delay).toHaveBeenCalledOnce(); - }); - - it("does not infer that not-found after creation is retryable", async () => { - const step = createStepRunnerDouble(); - const state = { - get: vi.fn(async () => undefined), - update: vi.fn(async () => undefined), - delete: vi.fn(async () => undefined), - }; - const TestResource = resource({ type: "test/service/eventually-visible" }) - .defineSchema({}) - .defineOperations({ - create: async () => ({}), - read: async () => { - throw new ResourceNotFoundError("resource is absent"); - }, - delete: async () => undefined, - }); - - await expect( - runOperation( - createResourceOperation(step, { - resource: new TestResource({ id: "eventually-visible" }), - state, - expectedRev: 0, - }), - ), - ).rejects.toThrowError("resource is absent"); - expect(state.update).not.toHaveBeenCalled(); - }); - - it("delete treats an already-absent remote as success through its idempotent resource contract", async () => { - const step = createStepRunnerDouble(); - const events: OperationLifecycleEvent[] = []; - const state = { - get: vi.fn(async () => undefined), - update: vi.fn(async () => undefined), - delete: vi.fn(async () => undefined), - }; - - const TestResource = resource({ type: "test/service/delete" }) - .defineSchema({}) - .defineOperations({ - create: async () => ({}), - delete: async () => undefined, - }); - - const testResource = new TestResource({ id: "test-delete" }); - - await runOperation( - deleteResourceOperation(step, { - resource: testResource, - state, - expectedRev: 1, - emit: async (event) => { - events.push(event); - }, - }), - ); - - expect(state.delete).toHaveBeenCalledWith("test-delete", 1); - expect(events.map((event) => event.status)).toEqual(["start", "success"]); - }); - - it("delete rethrows an unclassified resource error", async () => { - const step = createStepRunnerDouble(); - const state = { - get: vi.fn(async () => undefined), - update: vi.fn(async () => undefined), - delete: vi.fn(async () => undefined), - }; - - const TestResource = resource({ type: "test/service/delete-miss" }) - .defineSchema({}) - .defineOperations({ - create: async () => ({}), - delete: async () => { - const err = new Error("still exists"); - err.name = "DifferentError"; - throw err; - }, - }); - - const testResource = new TestResource({ id: "test-delete-miss" }); - - await expect( - runOperation( - deleteResourceOperation(step, { - resource: testResource, - state, - expectedRev: 1, - }), - ), - ).rejects.toMatchObject({ - name: "DifferentError", - message: "still exists", - }); - - expect(state.delete).not.toHaveBeenCalled(); - }); - - it("emits structured error details on operation failure", async () => { - const step = createStepRunnerDouble(); - const events: OperationLifecycleEvent[] = []; - const state = { - get: vi.fn(async () => undefined), - update: vi.fn(async () => undefined), - delete: vi.fn(async () => undefined), - }; - - const TestResource = resource({ type: "test/service/create-error" }) - .defineSchema({}) - .defineOperations({ - create: async () => { - const err = new Error("boom"); - err.name = "CreateFailed"; - throw err; - }, - delete: async () => undefined, - }); - - const testResource = new TestResource({ id: "test-create-error" }); - - await expect( - runOperation( - createResourceOperation(step, { - resource: testResource, - state, - expectedRev: 0, - emit: async (event) => { - events.push(event); - }, - }), - ), - ).rejects.toMatchObject({ name: "CreateFailed", message: "boom" }); - - expect(events.map((event) => event.status)).toEqual(["start", "error"]); - expect(events[1]).toMatchObject({ - operation: "create", - status: "error", - resourceId: "test-create-error", - resourceType: TestResource.type, - errorName: "CreateFailed", - errorMessage: "boom", - }); - }); -}); diff --git a/packages/reconciler/test/reconciler.deploy.test.ts b/packages/reconciler/test/reconciler.deploy.test.ts deleted file mode 100644 index eaa5ce5..0000000 --- a/packages/reconciler/test/reconciler.deploy.test.ts +++ /dev/null @@ -1,807 +0,0 @@ -import { describe, expect, it, vi } from "vitest"; -import { resource, ResourceNotFoundError } from "@notation/resource"; -import { - LeaseConflict, - MemoryStateBackend, - RevConflict, - type StateNode, -} from "@notation/state"; -import { Reconciler, createResourceRegistry } from "../src"; - -function createMemoryState(initial: Record = {}) { - const store: Record = { ...initial }; - - return { - store, - get: vi.fn(async (id: string) => store[id]), - update: vi.fn( - async (id: string, expectedRev: number, patch: Partial) => { - const actualRev = store[id]?.rev ?? 0; - if (actualRev !== expectedRev) { - throw new RevConflict(id, expectedRev, store[id]?.rev); - } - const rev = actualRev + 1; - store[id] = { - ...(store[id] ?? {}), - ...patch, - rev, - } as StateNode; - return { rev }; - }, - ), - delete: vi.fn(async (id: string, expectedRev: number) => { - const actualRev = store[id]?.rev ?? 0; - if (actualRev !== expectedRev) { - throw new RevConflict(id, expectedRev, store[id]?.rev); - } - delete store[id]; - }), - values: vi.fn(async () => Object.values(store)), - lease: vi.fn(async (scope: string, ttl: number) => { - let expiresAt = new Date(Date.now() + ttl).toISOString(); - return { - scope, - get expiresAt() { - return expiresAt; - }, - renew: vi.fn(async (nextTtl: number) => { - expiresAt = new Date(Date.now() + nextTtl).toISOString(); - return expiresAt; - }), - release: vi.fn(async () => undefined), - }; - }), - }; -} - -function createTestResourceClass(opts: { - type: `${string}/${string}/${string}`; - create?: ( - params: Record, - ) => Promise | void>; - read?: (key: Record) => Promise>; - update?: ( - key: Record, - patch: Record, - params: Record, - state: Record, - ) => Promise; - delete?: ( - key: Record, - state: Record, - ) => Promise; -}) { - return resource({ type: opts.type }) - .defineSchema({ - name: { - presence: "required", - propertyType: "param", - valueType: "string" as any, - }, - }) - .defineOperations({ - create: opts.create ?? (async () => ({})), - read: opts.read, - update: opts.update, - delete: opts.delete ?? (async () => undefined), - }); -} - -const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); -const found = (output: Record) => output; - -describe("reconciler deploy", () => { - it("chooses create vs update from desired params vs state", async () => { - const createSpy = vi.fn(async () => ({ name: "new" })); - const updateSpy = vi.fn(async () => undefined); - - const CreateResource = createTestResourceClass({ - type: "test/service/create-choice", - create: createSpy, - read: async () => found({ name: "new" }), - }); - const UpdateResource = createTestResourceClass({ - type: "test/service/update-choice", - update: updateSpy, - read: async () => found({ name: "new" }), - }); - - const state = createMemoryState({ - existing: { - rev: 1, - id: "existing", - groupId: -1, - groupType: "", - type: UpdateResource.type, - config: { name: "old" }, - params: { name: "old" }, - output: { name: "old" }, - lastOperation: "create", - lastOperationAt: new Date().toISOString(), - }, - }); - - const events: string[] = []; - const reconciler = new Reconciler({ - state, - driftDetection: false, - emit: async (event) => { - if ("operation" in event) { - events.push(`${event.operation}:${event.status}:${event.resourceId}`); - } - }, - }); - - await reconciler.deploy([ - new CreateResource({ id: "new", config: { name: "new" } }), - new UpdateResource({ id: "existing", config: { name: "new" } }), - ]); - - expect(createSpy).toHaveBeenCalledOnce(); - expect(updateSpy).toHaveBeenCalledOnce(); - expect(updateSpy.mock.calls[0]?.[1]).toEqual({ name: "new" }); - expect(events).toContain("create:success:new"); - expect(events).toContain("update:success:existing"); - }); - - it("persists first-time creates with an expect-absent revision", async () => { - const CreateResource = createTestResourceClass({ - type: "test/service/first-create", - create: async () => ({ name: "new" }), - read: async () => found({ name: "new" }), - }); - const state = createMemoryState(); - const reconciler = new Reconciler({ state, driftDetection: false }); - - await reconciler.deploy([ - new CreateResource({ id: "new", config: { name: "new" } }), - ]); - - expect(state.update).toHaveBeenCalledWith("new", 0, expect.any(Object)); - }); - - it("leases a resource before remote create so concurrent deploys cannot duplicate it", async () => { - let signalCreateStarted!: () => void; - const createStarted = new Promise((resolve) => { - signalCreateStarted = resolve; - }); - let allowCreateToFinish!: () => void; - const createCanFinish = new Promise((resolve) => { - allowCreateToFinish = resolve; - }); - const createSpy = vi.fn(async () => { - signalCreateStarted(); - await createCanFinish; - return { name: "new" }; - }); - const CreateResource = createTestResourceClass({ - type: "test/service/concurrent-create", - create: createSpy, - read: async () => found({ name: "new" }), - }); - const state = new MemoryStateBackend(); - const first = new Reconciler({ state, driftDetection: false }); - const second = new Reconciler({ state, driftDetection: false }); - - const firstDeploy = first.deploy([ - new CreateResource({ id: "new", config: { name: "new" } }), - ]); - await createStarted; - - await expect( - second.deploy([ - new CreateResource({ id: "new", config: { name: "new" } }), - ]), - ).rejects.toBeInstanceOf(LeaseConflict); - - allowCreateToFinish(); - await firstDeploy; - expect(createSpy).toHaveBeenCalledOnce(); - }); - - it("reads remote state after an update conflict instead of repeating the update", async () => { - let remoteName = "old"; - const readSpy = vi.fn(async () => found({ name: remoteName })); - const updateSpy = vi.fn(async (_key, _patch, params) => { - remoteName = params.name as string; - }); - const UpdateResource = createTestResourceClass({ - type: "test/service/update-conflict", - read: readSpy, - update: updateSpy, - }); - const state = createMemoryState({ - existing: { - rev: 1, - id: "existing", - groupId: -1, - groupType: "", - type: UpdateResource.type, - config: { name: "old" }, - params: { name: "old" }, - output: { name: "old" }, - lastOperation: "create", - lastOperationAt: new Date().toISOString(), - }, - }); - const updateState = state.update.getMockImplementation()!; - state.update - .mockImplementationOnce(async () => { - state.store.existing = { - ...state.store.existing!, - rev: 2, - config: { name: "concurrent" }, - params: { name: "concurrent" }, - output: { name: "concurrent" }, - }; - throw new RevConflict("existing", 1, 2); - }) - .mockImplementation(updateState); - - const reconciler = new Reconciler({ state, driftDetection: false }); - await reconciler.deploy([ - new UpdateResource({ id: "existing", config: { name: "new" } }), - ]); - - expect(updateSpy).toHaveBeenCalledOnce(); - expect(readSpy).toHaveBeenCalledTimes(2); - expect(state.update).toHaveBeenLastCalledWith( - "existing", - 2, - expect.objectContaining({ - params: { name: "new" }, - output: { name: "new" }, - lastOperation: "drift", - }), - ); - expect(state.store.existing).toMatchObject({ - rev: 3, - params: { name: "new" }, - output: { name: "new" }, - }); - }); - - it("reads remote state after a create conflict instead of creating twice", async () => { - let remoteName: string | undefined; - const createSpy = vi.fn(async (params) => { - remoteName = params.name as string; - return { name: remoteName }; - }); - const readSpy = vi.fn(async () => found({ name: remoteName! })); - const CreateResource = createTestResourceClass({ - type: "test/service/create-conflict", - create: createSpy, - read: readSpy, - }); - const state = createMemoryState(); - const updateState = state.update.getMockImplementation()!; - state.update - .mockImplementationOnce(async () => { - state.store.new = { - rev: 1, - id: "new", - groupId: -1, - groupType: "", - type: CreateResource.type, - config: { name: "concurrent" }, - params: { name: "concurrent" }, - output: { name: "concurrent" }, - lastOperation: "create", - lastOperationAt: new Date().toISOString(), - }; - throw new RevConflict("new", 0, 1); - }) - .mockImplementation(updateState); - - const reconciler = new Reconciler({ state, driftDetection: false }); - await reconciler.deploy([ - new CreateResource({ id: "new", config: { name: "new" } }), - ]); - - expect(createSpy).toHaveBeenCalledOnce(); - expect(readSpy).toHaveBeenCalledTimes(2); - expect(state.store.new).toMatchObject({ - rev: 2, - params: { name: "new" }, - output: { name: "new" }, - lastOperation: "drift", - }); - }); - - it("does not blindly retry a conflicted mutation without a read operation", async () => { - const updateSpy = vi.fn(async () => undefined); - const UpdateResource = createTestResourceClass({ - type: "test/service/unreadable-conflict", - update: updateSpy, - }); - const state = createMemoryState({ - existing: { - rev: 1, - id: "existing", - groupId: -1, - groupType: "", - type: UpdateResource.type, - config: { name: "old" }, - params: { name: "old" }, - output: { name: "old" }, - lastOperation: "create", - lastOperationAt: new Date().toISOString(), - }, - }); - state.update.mockImplementationOnce(async () => { - state.store.existing = { ...state.store.existing!, rev: 2 }; - throw new RevConflict("existing", 1, 2); - }); - - const reconciler = new Reconciler({ state, driftDetection: false }); - await expect( - reconciler.deploy([ - new UpdateResource({ id: "existing", config: { name: "new" } }), - ]), - ).rejects.toMatchObject({ - id: "existing", - expectedRev: 1, - actualRev: 2, - }); - - expect(updateSpy).toHaveBeenCalledOnce(); - expect(state.update).toHaveBeenCalledOnce(); - }); - - it("runs independent resources concurrently per dependency depth", async () => { - const marks: Record = {}; - - const AResource = createTestResourceClass({ - type: "test/service/a", - create: async () => { - marks.aStart = Date.now(); - await sleep(60); - marks.aEnd = Date.now(); - return { name: "a" }; - }, - read: async () => found({ name: "a" }), - }); - const CResource = createTestResourceClass({ - type: "test/service/c", - create: async () => { - marks.cStart = Date.now(); - await sleep(60); - marks.cEnd = Date.now(); - return { name: "c" }; - }, - read: async () => found({ name: "c" }), - }); - const BResource = createTestResourceClass({ - type: "test/service/b", - create: async () => { - marks.bStart = Date.now(); - return { name: "b" }; - }, - read: async () => found({ name: "b" }), - }); - - const state = createMemoryState(); - const resourceA = new AResource({ id: "a", config: { name: "a" } }); - const resourceB = new BResource({ - id: "b", - config: { name: "b" }, - dependencies: { a: resourceA }, - }); - const resourceC = new CResource({ id: "c", config: { name: "c" } }); - - const reconciler = new Reconciler({ state, driftDetection: false }); - await reconciler.deploy([resourceA, resourceB, resourceC]); - - expect(Math.abs(marks.aStart - marks.cStart)).toBeLessThan(40); - expect(marks.bStart).toBeGreaterThanOrEqual(marks.aEnd); - }); - - it("detects drift using live read output and converges with update", async () => { - const updateSpy = vi.fn(async () => undefined); - const events: Array> = []; - const TestResource = createTestResourceClass({ - type: "test/service/drift", - read: async () => found({ name: "drifted" }), - update: updateSpy, - }); - - const state = createMemoryState({ - resource: { - rev: 1, - id: "resource", - groupId: -1, - groupType: "", - type: TestResource.type, - config: { name: "desired" }, - params: { name: "desired" }, - output: { name: "desired" }, - lastOperation: "create", - lastOperationAt: new Date().toISOString(), - }, - }); - - const reconciler = new Reconciler({ - state, - driftDetection: true, - emit: async (event) => { - events.push(event as unknown as Record); - }, - }); - await reconciler.deploy([ - new TestResource({ id: "resource", config: { name: "desired" } }), - ]); - - expect(updateSpy).toHaveBeenCalledOnce(); - expect(updateSpy.mock.calls[0]?.[1]).toEqual({ name: "desired" }); - expect(events).toContainEqual({ - level: "info", - event: "reconciler.drift.detected", - resourceId: "resource", - resourceType: TestResource.type, - diff: { name: "desired" }, - }); - }); - - it("deletes orphaned state entries by reconstructing from registry", async () => { - const deleteSpy = vi.fn(async () => undefined); - const OrphanResource = createTestResourceClass({ - type: "test/service/orphan", - delete: deleteSpy, - }); - - const state = createMemoryState({ - orphan: { - rev: 1, - id: "orphan", - groupId: -1, - groupType: "", - type: OrphanResource.type, - config: { name: "from-state" }, - params: { name: "from-state" }, - output: { name: "from-state" }, - lastOperation: "create", - lastOperationAt: new Date().toISOString(), - }, - }); - - const reconciler = new Reconciler({ - state, - registry: createResourceRegistry([OrphanResource]), - driftDetection: false, - }); - - await reconciler.deploy([]); - - expect(deleteSpy).toHaveBeenCalledOnce(); - expect(state.delete).toHaveBeenCalledWith("orphan", 1); - }); - - it("dryRun emits operation intent without applying side effects", async () => { - const createSpy = vi.fn(async () => ({ name: "new" })); - const deleteSpy = vi.fn(async () => undefined); - - const CreateResource = createTestResourceClass({ - type: "test/service/dry-run-create", - create: createSpy, - read: async () => found({ name: "new" }), - }); - const OrphanResource = createTestResourceClass({ - type: "test/service/dry-run-orphan", - delete: deleteSpy, - }); - - const state = createMemoryState({ - orphan: { - rev: 1, - id: "orphan", - groupId: -1, - groupType: "", - type: OrphanResource.type, - config: { name: "orphan" }, - params: { name: "orphan" }, - output: { name: "orphan" }, - lastOperation: "create", - lastOperationAt: new Date().toISOString(), - }, - }); - - const operationEvents: string[] = []; - const reconciler = new Reconciler({ - state, - registry: createResourceRegistry([OrphanResource]), - dryRun: true, - driftDetection: false, - emit: async (event) => { - if ("operation" in event) { - operationEvents.push( - `${event.operation}:${event.status}:${event.resourceId}`, - ); - } - }, - }); - - await reconciler.deploy([ - new CreateResource({ id: "new", config: { name: "new" } }), - ]); - - expect(createSpy).not.toHaveBeenCalled(); - expect(deleteSpy).not.toHaveBeenCalled(); - expect(state.update).not.toHaveBeenCalled(); - expect(state.delete).not.toHaveBeenCalled(); - expect(operationEvents).toContain("create:dry-run:new"); - expect(operationEvents).toContain("delete:dry-run:orphan"); - }); -}); - -describe("reconciler destroy + refresh", () => { - it("reads remote state after a delete conflict instead of deleting twice", async () => { - let remoteExists = true; - const deleteSpy = vi.fn(async () => { - remoteExists = false; - }); - const readSpy = vi.fn(async () => { - if (!remoteExists) { - throw new ResourceNotFoundError("resource is absent"); - } - return found({ name: "doomed" }); - }); - const DestroyResource = createTestResourceClass({ - type: "test/service/destroy-retry", - read: readSpy, - delete: deleteSpy, - }); - const state = createMemoryState({ - doomed: { - rev: 1, - id: "doomed", - groupId: -1, - groupType: "", - type: DestroyResource.type, - config: { name: "doomed" }, - params: { name: "doomed" }, - output: { name: "doomed" }, - lastOperation: "create", - lastOperationAt: new Date().toISOString(), - }, - }); - const deleteState = state.delete.getMockImplementation()!; - state.delete - .mockImplementationOnce(async () => { - state.store.doomed = { ...state.store.doomed!, rev: 2 }; - throw new RevConflict("doomed", 1, 2); - }) - .mockImplementation(deleteState); - - const reconciler = new Reconciler({ state }); - await reconciler.destroy([ - new DestroyResource({ id: "doomed", config: { name: "doomed" } }), - ]); - - expect(deleteSpy).toHaveBeenCalledOnce(); - expect(readSpy).toHaveBeenCalledOnce(); - expect(state.delete).toHaveBeenCalledTimes(2); - expect(state.store.doomed).toBeUndefined(); - }); - - it("holds a backend lease for the orphan snapshot", async () => { - const state = createMemoryState(); - const release = vi.fn(async () => undefined); - const lease = vi.fn(async () => ({ - scope: "reconciler:orphan-deletion", - expiresAt: new Date(Date.now() + 10_000).toISOString(), - renew: vi.fn(async () => new Date(Date.now() + 10_000).toISOString()), - release, - })); - const reconciler = new Reconciler({ - state: { ...state, lease }, - mutationLeaseTtl: 10_000, - }); - - await reconciler.refresh([]); - - expect(lease).toHaveBeenCalledWith("reconciler:orphan-deletion", 10_000); - expect(state.values).toHaveBeenCalledOnce(); - expect(release).toHaveBeenCalledOnce(); - }); - - it("destroys resources in reverse dependency order", async () => { - const destroyOrder: string[] = []; - const deleteA = vi.fn(async () => { - destroyOrder.push("a"); - }); - const deleteB = vi.fn(async () => { - destroyOrder.push("b"); - }); - const deleteC = vi.fn(async () => { - destroyOrder.push("c"); - }); - - const AResource = createTestResourceClass({ - type: "test/service/destroy-a", - delete: deleteA, - }); - const BResource = createTestResourceClass({ - type: "test/service/destroy-b", - delete: deleteB, - }); - const CResource = createTestResourceClass({ - type: "test/service/destroy-c", - delete: deleteC, - }); - - const resourceA = new AResource({ id: "a", config: { name: "a" } }); - const resourceB = new BResource({ - id: "b", - config: { name: "b" }, - dependencies: { a: resourceA }, - }); - const resourceC = new CResource({ - id: "c", - config: { name: "c" }, - dependencies: { b: resourceB }, - }); - - const state = createMemoryState({ - a: { - rev: 1, - id: "a", - groupId: -1, - groupType: "", - type: AResource.type, - config: { name: "a" }, - params: { name: "a" }, - output: { name: "a" }, - lastOperation: "create", - lastOperationAt: new Date().toISOString(), - }, - b: { - rev: 1, - id: "b", - groupId: -1, - groupType: "", - type: BResource.type, - config: { name: "b" }, - params: { name: "b" }, - output: { name: "b" }, - lastOperation: "create", - lastOperationAt: new Date().toISOString(), - }, - c: { - rev: 1, - id: "c", - groupId: -1, - groupType: "", - type: CResource.type, - config: { name: "c" }, - params: { name: "c" }, - output: { name: "c" }, - lastOperation: "create", - lastOperationAt: new Date().toISOString(), - }, - }); - - const reconciler = new Reconciler({ state }); - await reconciler.destroy([resourceA, resourceB, resourceC]); - - expect(destroyOrder).toEqual(["c", "b", "a"]); - expect(state.delete).toHaveBeenCalledWith("a", 1); - expect(state.delete).toHaveBeenCalledWith("b", 1); - expect(state.delete).toHaveBeenCalledWith("c", 1); - }); - - it("refresh removes orphan state entries", async () => { - const deleteSpy = vi.fn(async () => undefined); - const OrphanResource = createTestResourceClass({ - type: "test/service/refresh-orphan", - delete: deleteSpy, - }); - const KeepResource = createTestResourceClass({ - type: "test/service/refresh-keep", - }); - - const keep = new KeepResource({ id: "keep", config: { name: "keep" } }); - const state = createMemoryState({ - keep: { - rev: 1, - id: "keep", - groupId: -1, - groupType: "", - type: KeepResource.type, - config: { name: "keep" }, - params: { name: "keep" }, - output: { name: "keep" }, - lastOperation: "create", - lastOperationAt: new Date().toISOString(), - }, - orphan: { - rev: 1, - id: "orphan", - groupId: -1, - groupType: "", - type: OrphanResource.type, - config: { name: "orphan" }, - params: { name: "orphan" }, - output: { name: "orphan" }, - lastOperation: "create", - lastOperationAt: new Date().toISOString(), - }, - }); - - const reconciler = new Reconciler({ - state, - registry: createResourceRegistry([OrphanResource]), - }); - - await reconciler.refresh([keep]); - - expect(deleteSpy).toHaveBeenCalledOnce(); - expect(state.delete).toHaveBeenCalledWith("orphan", 1); - expect(state.delete).not.toHaveBeenCalledWith("keep", expect.anything()); - }); - - it("destroy and refresh dryRun emit operation events without side effects", async () => { - const deleteSpy = vi.fn(async () => undefined); - const DestroyResource = createTestResourceClass({ - type: "test/service/dry-run-destroy", - delete: deleteSpy, - }); - const OrphanResource = createTestResourceClass({ - type: "test/service/dry-run-refresh", - delete: deleteSpy, - }); - - const destroyResource = new DestroyResource({ - id: "destroy-me", - config: { name: "destroy-me" }, - }); - - const state = createMemoryState({ - "destroy-me": { - rev: 1, - id: "destroy-me", - groupId: -1, - groupType: "", - type: DestroyResource.type, - config: { name: "destroy-me" }, - params: { name: "destroy-me" }, - output: { name: "destroy-me" }, - lastOperation: "create", - lastOperationAt: new Date().toISOString(), - }, - orphan: { - rev: 1, - id: "orphan", - groupId: -1, - groupType: "", - type: OrphanResource.type, - config: { name: "orphan" }, - params: { name: "orphan" }, - output: { name: "orphan" }, - lastOperation: "create", - lastOperationAt: new Date().toISOString(), - }, - }); - - const operationEvents: string[] = []; - const reconciler = new Reconciler({ - state, - dryRun: true, - registry: createResourceRegistry([OrphanResource]), - emit: async (event) => { - if ("operation" in event) { - operationEvents.push( - `${event.operation}:${event.status}:${event.resourceId}`, - ); - } - }, - }); - - await reconciler.destroy([destroyResource]); - await reconciler.refresh([destroyResource]); - - expect(deleteSpy).not.toHaveBeenCalled(); - expect(state.delete).not.toHaveBeenCalled(); - expect(operationEvents).toContain("delete:dry-run:destroy-me"); - expect(operationEvents).toContain("delete:dry-run:orphan"); - }); -}); diff --git a/packages/reconciler/test/reconciler.plan.test.ts b/packages/reconciler/test/reconciler.plan.test.ts deleted file mode 100644 index d98e55a..0000000 --- a/packages/reconciler/test/reconciler.plan.test.ts +++ /dev/null @@ -1,463 +0,0 @@ -import { describe, expect, it, vi } from "vitest"; -import { - ResourceNotFoundError, - ResourceOperationPendingError, - resource, - type BaseResource, -} from "@notation/resource"; -import type { StateNode } from "@notation/state"; -import { Reconciler, UNKNOWN_AFTER_APPLY } from "../src"; - -function createMemoryState(initial: Record = {}) { - const store: Record = { ...initial }; - - return { - store, - get: vi.fn(async (id: string) => store[id]), - update: vi.fn( - async (id: string, expectedRev: number, patch: Partial) => { - store[id] = { - ...(store[id] ?? {}), - ...patch, - } as StateNode; - }, - ), - delete: vi.fn(async (id: string) => { - delete store[id]; - }), - values: vi.fn(async () => Object.values(store)), - lease: vi.fn(async (scope: string, ttl: number) => ({ - scope, - expiresAt: new Date(Date.now() + ttl).toISOString(), - renew: vi.fn(async (nextTtl: number) => - new Date(Date.now() + nextTtl).toISOString(), - ), - release: vi.fn(async () => undefined), - })), - }; -} - -function createTestResourceClass(opts: { - type: `${string}/${string}/${string}`; - create?: ( - params: Record, - ) => Promise | void>; - read?: (key: Record) => Promise>; - update?: ( - key: Record, - patch: Record, - params: Record, - state: Record, - ) => Promise; - delete?: ( - key: Record, - state: Record, - ) => Promise; -}) { - return resource({ type: opts.type }) - .defineSchema({ - name: { - presence: "required", - propertyType: "param", - valueType: "string" as any, - }, - tag: { - presence: "optional", - propertyType: "param", - valueType: "string" as any, - }, - }) - .defineOperations({ - create: opts.create ?? (async () => ({})), - read: opts.read, - update: opts.update, - delete: opts.delete ?? (async () => undefined), - }); -} - -function createStateNode( - id: string, - type: string, - params: Record, - output: Record = params, -): StateNode { - return { - id, - groupId: -1, - groupType: "", - type, - config: params, - params, - output, - lastOperation: "create", - lastOperationAt: new Date().toISOString(), - }; -} - -describe("reconciler plan", () => { - it("plans create for resources without state", async () => { - const TestResource = createTestResourceClass({ - type: "test/service/plan-create", - }); - - const state = createMemoryState(); - const reconciler = new Reconciler({ state, driftDetection: false }); - - const plan = await reconciler.plan([ - new TestResource({ id: "new", config: { name: "new" } }), - ]); - - expect(plan.nodes).toEqual([ - { - id: "new", - type: TestResource.type, - decision: "create", - params: { name: "new" }, - dependsOn: [], - }, - ]); - }); - - it("plans update with the detailed diff that justified it", async () => { - const TestResource = createTestResourceClass({ - type: "test/service/plan-update", - }); - - const state = createMemoryState({ - existing: createStateNode("existing", "test/service/plan-update", { - name: "old", - tag: "keep", - }), - }); - const reconciler = new Reconciler({ state, driftDetection: false }); - - const plan = await reconciler.plan([ - new TestResource({ id: "existing", config: { name: "new" } }), - ]); - - expect(plan.nodes).toEqual([ - { - id: "existing", - type: TestResource.type, - decision: "update", - diff: { - added: {}, - deleted: { tag: null }, - updated: { name: "new" }, - }, - params: { name: "new" }, - dependsOn: [], - }, - ]); - }); - - it("plans noop when params match state", async () => { - const TestResource = createTestResourceClass({ - type: "test/service/plan-noop", - }); - - const state = createMemoryState({ - unchanged: createStateNode("unchanged", "test/service/plan-noop", { - name: "same", - }), - }); - const reconciler = new Reconciler({ state, driftDetection: false }); - - const plan = await reconciler.plan([ - new TestResource({ id: "unchanged", config: { name: "same" } }), - ]); - - expect(plan.nodes[0]).toMatchObject({ id: "unchanged", decision: "noop" }); - }); - - it("plans drift-update from live read output when drift detection is on", async () => { - const readSpy = vi.fn(async () => ({ name: "drifted" })); - const TestResource = createTestResourceClass({ - type: "test/service/plan-drift-update", - read: readSpy, - }); - - const state = createMemoryState({ - resource: createStateNode("resource", "test/service/plan-drift-update", { - name: "desired", - }), - }); - const reconciler = new Reconciler({ state, driftDetection: true }); - - const plan = await reconciler.plan([ - new TestResource({ id: "resource", config: { name: "desired" } }), - ]); - - expect(readSpy).toHaveBeenCalledOnce(); - expect(plan.nodes[0]).toEqual({ - id: "resource", - type: TestResource.type, - decision: "drift-update", - diff: { - added: {}, - deleted: {}, - updated: { name: "desired" }, - }, - params: { name: "desired" }, - dependsOn: [], - }); - }); - - it("plans drift-recreate when the remote resource is gone", async () => { - const TestResource = createTestResourceClass({ - type: "test/service/plan-drift-recreate", - read: async () => { - throw new ResourceNotFoundError("resource is absent"); - }, - }); - - const state = createMemoryState({ - resource: createStateNode( - "resource", - "test/service/plan-drift-recreate", - { name: "desired" }, - ), - }); - const reconciler = new Reconciler({ state, driftDetection: true }); - - const plan = await reconciler.plan([ - new TestResource({ id: "resource", config: { name: "desired" } }), - ]); - - expect(plan.nodes[0]).toMatchObject({ - id: "resource", - decision: "drift-recreate", - }); - }); - - it("waits for a pending read before planning", async () => { - let attempts = 0; - const TestResource = createTestResourceClass({ - type: "test/service/plan-not-ready", - read: async () => { - attempts += 1; - if (attempts === 1) { - throw new ResourceOperationPendingError( - "Waiting for Lambda to become active", - { retryAfterMs: 0 }, - ); - } - return { name: "desired" }; - }, - }); - - const state = createMemoryState({ - resource: createStateNode("resource", "test/service/plan-not-ready", { - name: "desired", - }), - }); - const reconciler = new Reconciler({ state, driftDetection: true }); - - const plan = await reconciler.plan([ - new TestResource({ id: "resource", config: { name: "desired" } }), - ]); - - expect(plan.nodes[0]).toMatchObject({ - id: "resource", - decision: "noop", - }); - expect(attempts).toBe(2); - }); - - it("skips remote reads when drift detection is off", async () => { - const readSpy = vi.fn(async () => ({ name: "drifted" })); - const TestResource = createTestResourceClass({ - type: "test/service/plan-no-read", - read: readSpy, - }); - - const state = createMemoryState({ - resource: createStateNode("resource", "test/service/plan-no-read", { - name: "desired", - }), - }); - const reconciler = new Reconciler({ state, driftDetection: false }); - - await reconciler.plan([ - new TestResource({ id: "resource", config: { name: "desired" } }), - ]); - - expect(readSpy).not.toHaveBeenCalled(); - }); - - it("plans delete-orphan for state nodes without a matching resource", async () => { - const state = createMemoryState({ - orphan: createStateNode("orphan", "test/service/plan-orphan", { - name: "orphan", - }), - }); - const reconciler = new Reconciler({ state, driftDetection: false }); - - const plan = await reconciler.plan([]); - - expect(plan.nodes).toEqual([ - { - id: "orphan", - type: "test/service/plan-orphan", - decision: "delete-orphan", - params: { name: "orphan" }, - dependsOn: [], - }, - ]); - }); - - it("populates dependsOn from resource dependencies", async () => { - const AResource = createTestResourceClass({ - type: "test/service/plan-dep-a", - }); - const BResource = createTestResourceClass({ - type: "test/service/plan-dep-b", - }); - - const resourceA = new AResource({ id: "a", config: { name: "a" } }); - const resourceB = new BResource({ - id: "b", - config: { name: "b" }, - dependencies: { a: resourceA }, - }); - - const state = createMemoryState(); - const reconciler = new Reconciler({ state, driftDetection: false }); - - const plan = await reconciler.plan([resourceA, resourceB]); - - const nodeB = plan.nodes.find((node) => node.id === "b"); - expect(nodeB?.dependsOn).toEqual(["a"]); - }); - - it("marks params derived from uncreated dependencies as unknown after apply", async () => { - const AResource = createTestResourceClass({ - type: "test/service/plan-unknown-a", - }); - const BResource = createTestResourceClass({ - type: "test/service/plan-unknown-b", - }) - .requireDependencies<{ a: BaseResource }>() - .deriveParams(({ deps }) => ({ - name: (deps.a.output as { name: string }).name, - })); - - const resourceA = new AResource({ id: "a", config: { name: "a" } }); - const resourceB = new BResource({ - id: "b", - config: { tag: "known" }, - dependencies: { a: resourceA }, - }); - - const state = createMemoryState(); - const reconciler = new Reconciler({ state, driftDetection: false }); - - const plan = await reconciler.plan([resourceA, resourceB]); - - const nodeB = plan.nodes.find((node) => node.id === "b"); - expect(nodeB).toMatchObject({ - decision: "create", - params: { - name: UNKNOWN_AFTER_APPLY, - tag: "known", - }, - }); - }); - - it("does not disguise parameter derivation failures as unknown values", async () => { - const TestResource = resource({ - type: "test/service/plan-derive-failure", - }) - .defineSchema({ - name: { - presence: "required", - propertyType: "param", - valueType: "string" as any, - }, - }) - .defineOperations({ - create: async () => ({}), - delete: async () => undefined, - deriveParams: () => { - throw new Error("invalid derived configuration"); - }, - }); - - const reconciler = new Reconciler({ - state: createMemoryState(), - driftDetection: false, - }); - - await expect( - reconciler.plan([new TestResource({ id: "broken" })]), - ).rejects.toThrow("invalid derived configuration"); - }); - - it("produces a JSON-round-trippable plan", async () => { - const CreateResource = createTestResourceClass({ - type: "test/service/plan-json-create", - }); - const UpdateResource = createTestResourceClass({ - type: "test/service/plan-json-update", - }); - - const state = createMemoryState({ - existing: createStateNode("existing", "test/service/plan-json-update", { - name: "old", - tag: "gone", - }), - orphan: createStateNode("orphan", "test/service/plan-json-orphan", { - name: "orphan", - }), - }); - const reconciler = new Reconciler({ state, driftDetection: false }); - - const plan = await reconciler.plan([ - new CreateResource({ id: "new", config: { name: "new" } }), - new UpdateResource({ id: "existing", config: { name: "new" } }), - ]); - - expect(JSON.parse(JSON.stringify(plan))).toStrictEqual(plan); - }); - - it("performs no state writes or resource operations", async () => { - const createSpy = vi.fn(async () => ({ name: "new" })); - const updateSpy = vi.fn(async () => undefined); - const deleteSpy = vi.fn(async () => undefined); - - const CreateResource = createTestResourceClass({ - type: "test/service/plan-pure-create", - create: createSpy, - update: updateSpy, - delete: deleteSpy, - }); - const UpdateResource = createTestResourceClass({ - type: "test/service/plan-pure-update", - create: createSpy, - update: updateSpy, - delete: deleteSpy, - read: async () => ({ name: "drifted" }), - }); - - const state = createMemoryState({ - existing: createStateNode("existing", "test/service/plan-pure-update", { - name: "same", - }), - orphan: createStateNode("orphan", "test/service/plan-pure-orphan", { - name: "orphan", - }), - }); - const reconciler = new Reconciler({ state, driftDetection: true }); - - await reconciler.plan([ - new CreateResource({ id: "new", config: { name: "new" } }), - new UpdateResource({ id: "existing", config: { name: "same" } }), - ]); - - expect(createSpy).not.toHaveBeenCalled(); - expect(updateSpy).not.toHaveBeenCalled(); - expect(deleteSpy).not.toHaveBeenCalled(); - expect(state.update).not.toHaveBeenCalled(); - expect(state.delete).not.toHaveBeenCalled(); - }); -}); diff --git a/packages/state-sqlite/src/index.ts b/packages/state-sqlite/src/index.ts index 057628f..c01775b 100644 --- a/packages/state-sqlite/src/index.ts +++ b/packages/state-sqlite/src/index.ts @@ -1,11 +1,8 @@ -import { randomUUID } from "node:crypto"; import { mkdirSync } from "node:fs"; import { dirname } from "node:path"; import { DatabaseSync } from "node:sqlite"; import { - LeaseConflict, RevConflict, - type Lease, type StateBackend, type StateNode, } from "@notation/state"; @@ -24,13 +21,6 @@ export class SqliteStateBackend implements StateBackend { value TEXT NOT NULL ) `); - this.#database.exec(` - CREATE TABLE IF NOT EXISTS resource_leases ( - scope TEXT PRIMARY KEY, - owner TEXT NOT NULL, - expires_at INTEGER NOT NULL - ) - `); } close(): void { @@ -46,9 +36,7 @@ export class SqliteStateBackend implements StateBackend { async has(id: string): Promise { return Boolean( - this.#database - .prepare("SELECT 1 FROM resources WHERE id = ?") - .get(id), + this.#database.prepare("SELECT 1 FROM resources WHERE id = ?").get(id), ); } @@ -79,9 +67,7 @@ export class SqliteStateBackend implements StateBackend { } } else { this.#database - .prepare( - "INSERT INTO resources (id, rev, value) VALUES (?, ?, ?)", - ) + .prepare("INSERT INTO resources (id, rev, value) VALUES (?, ?, ?)") .run(id, rev, JSON.stringify(node)); } this.#database.exec("COMMIT"); @@ -114,82 +100,4 @@ export class SqliteStateBackend implements StateBackend { .all() as { value: string }[]; return rows.map(({ value }) => JSON.parse(value) as StateNode); } - - async lease(scope: string, ttl: number): Promise { - if (!Number.isFinite(ttl) || ttl <= 0) { - throw new RangeError( - "Lease TTL must be a positive number of milliseconds", - ); - } - - const owner = randomUUID(); - const expiresAtMs = Date.now() + ttl; - this.#database.exec("BEGIN IMMEDIATE"); - try { - this.#database - .prepare( - "DELETE FROM resource_leases WHERE scope = ? AND expires_at <= ?", - ) - .run(scope, Date.now()); - const current = this.#database - .prepare("SELECT expires_at FROM resource_leases WHERE scope = ?") - .get(scope) as { expires_at: number } | undefined; - if (current) { - throw new LeaseConflict( - scope, - new Date(current.expires_at).toISOString(), - ); - } - this.#database - .prepare( - "INSERT INTO resource_leases (scope, owner, expires_at) VALUES (?, ?, ?)", - ) - .run(scope, owner, expiresAtMs); - this.#database.exec("COMMIT"); - } catch (error) { - this.#database.exec("ROLLBACK"); - throw error; - } - - let released = false; - let currentExpiresAtMs = expiresAtMs; - return { - scope, - get expiresAt() { - return new Date(currentExpiresAtMs).toISOString(); - }, - renew: async (nextTtl) => { - if (!Number.isFinite(nextTtl) || nextTtl <= 0) { - throw new RangeError( - "Lease TTL must be a positive number of milliseconds", - ); - } - const now = Date.now(); - const nextExpiresAtMs = now + nextTtl; - const result = this.#database - .prepare( - "UPDATE resource_leases SET expires_at = ? WHERE scope = ? AND owner = ? AND expires_at > ?", - ) - .run(nextExpiresAtMs, scope, owner, now); - if (result.changes !== 1) { - const current = this.#database - .prepare("SELECT expires_at FROM resource_leases WHERE scope = ?") - .get(scope) as { expires_at: number } | undefined; - throw new LeaseConflict( - scope, - new Date(current?.expires_at ?? 0).toISOString(), - ); - } - currentExpiresAtMs = nextExpiresAtMs; - return new Date(nextExpiresAtMs).toISOString(); - }, - release: async () => { - if (released) return; - this.#database - .prepare("DELETE FROM resource_leases WHERE scope = ? AND owner = ?") - .run(scope, owner); - released = true; - }, - }; - } } diff --git a/packages/state-sqlite/test/state-sqlite.test.ts b/packages/state-sqlite/test/state-sqlite.test.ts index 31eb35d..f50456e 100644 --- a/packages/state-sqlite/test/state-sqlite.test.ts +++ b/packages/state-sqlite/test/state-sqlite.test.ts @@ -46,33 +46,6 @@ describe("SqliteStateBackend", () => { }); }); - it("coordinates leases across backend instances and releases by owner", async () => { - const directory = await mkdtemp( - path.join(tmpdir(), "notation-sqlite-lease-"), - ); - const databasePath = path.join(directory, "state.db"); - const first = new SqliteStateBackend(databasePath); - const second = new SqliteStateBackend(databasePath); - cleanups.push(async () => { - first.close(); - second.close(); - await rm(directory, { recursive: true, force: true }); - }); - - const lease = await first.lease("orphans", 10_000); - await expect(second.lease("orphans", 10_000)).rejects.toMatchObject({ - name: "LeaseConflict", - scope: "orphans", - }); - const firstExpiry = lease.expiresAt; - await lease.renew(20_000); - expect(lease.expiresAt).not.toBe(firstExpiry); - await lease.release(); - const nextLease = await second.lease("orphans", 10_000); - expect(nextLease).toMatchObject({ scope: "orphans" }); - await nextLease.release(); - }); - it("waits for a concurrent writer instead of raising database locked", async () => { const directory = await mkdtemp( path.join(tmpdir(), "notation-sqlite-busy-"), diff --git a/packages/state/src/conflicts.ts b/packages/state/src/conflicts.ts index 81559dc..a1e666c 100644 --- a/packages/state/src/conflicts.ts +++ b/packages/state/src/conflicts.ts @@ -11,14 +11,3 @@ export class RevConflict extends Error { ); } } - -export class LeaseConflict extends Error { - readonly name = "LeaseConflict"; - - constructor( - readonly scope: string, - readonly expiresAt: string, - ) { - super(`State lease conflict for ${scope}: held until ${expiresAt}`); - } -} diff --git a/packages/state/src/state.ts b/packages/state/src/state.ts index d0c10c1..971f802 100644 --- a/packages/state/src/state.ts +++ b/packages/state/src/state.ts @@ -10,7 +10,7 @@ import { import path from "node:path"; import { setTimeout as sleep } from "node:timers/promises"; import { isErrorWithCode } from "@notation/utils"; -import { LeaseConflict, RevConflict } from "./conflicts"; +import { RevConflict } from "./conflicts"; export type StateNode = { rev: number; @@ -38,21 +38,12 @@ export interface StateBackend { ): Promise<{ rev: number }>; delete(id: string, expectedRev: number): Promise; values(): Promise; - lease(scope: string, ttl: number): Promise; -} - -export interface Lease { - readonly scope: string; - readonly expiresAt: string; - renew(ttl: number): Promise; - release(): Promise; } export type State = StateBackend; export class MemoryStateBackend implements StateBackend { #state: Record; - #leases = new Map(); constructor(initialState: Record = {}) { this.#state = cloneAsPersistedState(initialState); @@ -109,46 +100,6 @@ export class MemoryStateBackend implements StateBackend { .map(([, value]) => value); } - async lease(scope: string, ttl: number): Promise { - assertLeaseTtl(ttl); - const now = Date.now(); - const current = this.#leases.get(scope); - if (current && current.expiresAtMs > now) { - throw new LeaseConflict( - scope, - new Date(current.expiresAtMs).toISOString(), - ); - } - - const owner = randomUUID(); - let expiresAtMs = now + ttl; - this.#leases.set(scope, { owner, expiresAtMs }); - - return { - scope, - get expiresAt() { - return new Date(expiresAtMs).toISOString(); - }, - renew: async (nextTtl) => { - assertLeaseTtl(nextTtl); - const held = this.#leases.get(scope); - if (!held || held.owner !== owner || held.expiresAtMs <= Date.now()) { - throw new LeaseConflict( - scope, - new Date(held?.expiresAtMs ?? 0).toISOString(), - ); - } - expiresAtMs = Date.now() + nextTtl; - held.expiresAtMs = expiresAtMs; - return new Date(expiresAtMs).toISOString(); - }, - release: async () => { - if (this.#leases.get(scope)?.owner === owner) - this.#leases.delete(scope); - }, - }; - } - private async readState(): Promise> { return cloneAsPersistedState(this.#state); } @@ -208,65 +159,6 @@ export class FileStateBackend implements StateBackend { return Object.values(state); } - async lease(scope: string, ttl: number): Promise { - assertLeaseTtl(ttl); - const leaseFilePath = `${this.stateFilePath}.${encodeURIComponent(scope)}.lease`; - const owner = randomUUID(); - let expiresAtMs: number; - await mkdir(path.dirname(this.stateFilePath), { recursive: true }); - - for (;;) { - expiresAtMs = Date.now() + ttl; - try { - await writeFile(leaseFilePath, JSON.stringify({ owner, expiresAtMs }), { - flag: "wx", - }); - break; - } catch (error) { - if (!isErrorWithCode(error, "EEXIST")) throw error; - const current = await readFileLease(leaseFilePath); - if (!current || current.expiresAtMs <= Date.now()) { - await unlink(leaseFilePath).catch(() => undefined); - continue; - } - throw new LeaseConflict( - scope, - new Date(current.expiresAtMs).toISOString(), - ); - } - } - - return { - scope, - get expiresAt() { - return new Date(expiresAtMs).toISOString(); - }, - renew: async (nextTtl) => { - assertLeaseTtl(nextTtl); - const current = await readFileLease(leaseFilePath); - if ( - !current || - current.owner !== owner || - current.expiresAtMs <= Date.now() - ) { - throw new LeaseConflict( - scope, - new Date(current?.expiresAtMs ?? 0).toISOString(), - ); - } - expiresAtMs = Date.now() + nextTtl; - await writeFile(leaseFilePath, JSON.stringify({ owner, expiresAtMs })); - return new Date(expiresAtMs).toISOString(); - }, - release: async () => { - const current = await readFileLease(leaseFilePath); - if (current?.owner === owner) { - await unlink(leaseFilePath).catch(() => undefined); - } - }, - }; - } - private async readState(): Promise> { try { const file = await readFile(this.stateFilePath, "utf8"); @@ -354,26 +246,6 @@ function assertExpectedRev( } } -function assertLeaseTtl(ttl: number): void { - if (!Number.isFinite(ttl) || ttl <= 0) { - throw new RangeError("Lease TTL must be a positive number of milliseconds"); - } -} - -type FileLeaseRecord = { owner: string; expiresAtMs: number }; - -async function readFileLease( - filePath: string, -): Promise { - try { - return JSON.parse(await readFile(filePath, "utf8")) as FileLeaseRecord; - } catch (error) { - if (isErrorWithCode(error, "ENOENT") || error instanceof SyntaxError) - return undefined; - throw error; - } -} - function cloneAsPersistedState( state: Record, ): Record { diff --git a/packages/state/test/state-backend.test.ts b/packages/state/test/state-backend.test.ts index fde5e5b..888055a 100644 --- a/packages/state/test/state-backend.test.ts +++ b/packages/state/test/state-backend.test.ts @@ -157,27 +157,6 @@ function runStateBackendContractTests( await fixture.cleanup(); } }); - - it("holds and renews an exclusive lease", async () => { - const fixture = await createBackend(); - - try { - const lease = await fixture.backend.lease("resource:a", 1_000); - const firstExpiry = lease.expiresAt; - await expect( - fixture.backend.lease("resource:a", 1_000), - ).rejects.toMatchObject({ name: "LeaseConflict" }); - - await lease.renew(2_000); - expect(lease.expiresAt).not.toBe(firstExpiry); - await lease.release(); - - const next = await fixture.backend.lease("resource:a", 1_000); - await next.release(); - } finally { - await fixture.cleanup(); - } - }); }); } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index fd80fe6..02ef3b9 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -123,12 +123,18 @@ importers: '@notation/resource': specifier: workspace:* version: link:../../packages/resource - '@notation/state-sqlite': - specifier: workspace:* - version: link:../../packages/state-sqlite - '@notation/utils': - specifier: workspace:* - version: link:../../packages/utils + '@yieldstar/core': + specifier: 0.5.0 + version: 0.5.0 + '@yieldstar/sqlite-runtime': + specifier: 0.5.0 + version: 0.5.0 + pino: + specifier: ^9.9.0 + version: 9.14.0 + yieldstar: + specifier: 0.5.0 + version: 0.5.0 devDependencies: '@types/node': specifier: ^22.13.4 @@ -224,12 +230,12 @@ importers: '@notation/resource': specifier: workspace:* version: link:../resource - '@notation/state': - specifier: workspace:* - version: link:../state - '@notation/state-sqlite': - specifier: workspace:* - version: link:../state-sqlite + '@yieldstar/core': + specifier: 0.5.0 + version: 0.5.0 + '@yieldstar/sqlite-runtime': + specifier: 0.5.0 + version: 0.5.0 deep-object-diff: specifier: ^1.1.9 version: 1.1.9 @@ -242,6 +248,12 @@ importers: pako: specifier: ^2.1.0 version: 2.1.0 + pino: + specifier: ^9.14.0 + version: 9.14.0 + yieldstar: + specifier: 0.5.0 + version: 0.5.0 devDependencies: '@types/common-tags': specifier: ^1.8.4 @@ -2242,16 +2254,9 @@ packages: pino-abstract-transport@2.0.0: resolution: {integrity: sha512-F63x5tizV6WCh4R6RHyi2Ml+M70DNRXt/+HANowMflpgGFMAym/VKm6G7ZOQRjqN7XbGxK1Lg9t6ZrtzOaivMw==} - pino-abstract-transport@3.0.0: - resolution: {integrity: sha512-wlfUczU+n7Hy/Ha5j9a/gZNy7We5+cXp8YL+X+PG8S0KXxw7n/JXA3c46Y0zQznIJ83URJiwy7Lh56WLokNuxg==} - pino-std-serializers@7.1.0: resolution: {integrity: sha512-BndPH67/JxGExRgiX1dX0w1FvZck5Wa4aal9198SrRhZjH3GxKQUKIBnYJTdj2HDN3UQAS06HlfcSbQj2OHmaw==} - pino@10.3.1: - resolution: {integrity: sha512-r34yH/GlQpKZbU1BvFFqOjhISRo1MNx1tWYsYvmj6KIRHSPMT2+yHOEb1SG6NMvRoHRF0a07kCOox/9yakl1vg==} - hasBin: true - pino@9.14.0: resolution: {integrity: sha512-8OEwKp5juEvb/MjpIc4hjqfgCNysrS94RIOMXYvpYCdm/jglrKEiAYmiumbmGhCvs+IcInsphYDFwqrjr7398w==} hasBin: true @@ -2340,9 +2345,6 @@ packages: resolution: {integrity: sha512-57frrGM/OCTLqLOAh0mhVA9VBMHd+9U7Zb2THMGdBUoZVOtGbJzjxsYGDJ3A9AYYCP4hn6y1TVbaOfzWtm5GFg==} engines: {node: '>= 12.13.0'} - real-require@1.0.0: - resolution: {integrity: sha512-P4nbQYQfePJxRSmY+v/KINxVucm4NF3p3s7pJveMTtom52FR4YGltUQLB8idDXwDDWW+eYrWDFbuzUnjoWHF7g==} - require-from-string@2.0.2: resolution: {integrity: sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==} engines: {node: '>=0.10.0'} @@ -2504,10 +2506,6 @@ packages: thread-stream@3.1.0: resolution: {integrity: sha512-OqyPZ9u96VohAyMfJykzmivOrY2wfMSf3C5TtFJVgN+Hm6aj+voFhlK+kZEIv2FBh1X6Xp3DlnCOfEQ3B2J86A==} - thread-stream@4.2.0: - resolution: {integrity: sha512-e2zZ96wSChazBsbENf/Pcm/4swHt2cEKQ92rhUjkL9GCKiTDJIaTBenjE/m9DXi0QBmTMDkFDdOomUy20A1tDQ==} - engines: {node: '>=20'} - tinybench@2.9.0: resolution: {integrity: sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==} @@ -4139,7 +4137,7 @@ snapshots: fast-json-stringify: 7.0.0 find-my-way: 9.6.0 light-my-request: 6.6.0 - pino: 10.3.1 + pino: 9.14.0 process-warning: 5.0.0 rfdc: 1.4.1 secure-json-parse: 4.1.0 @@ -4488,26 +4486,8 @@ snapshots: dependencies: split2: 4.2.0 - pino-abstract-transport@3.0.0: - dependencies: - split2: 4.2.0 - pino-std-serializers@7.1.0: {} - pino@10.3.1: - dependencies: - '@pinojs/redact': 0.4.0 - atomic-sleep: 1.0.0 - on-exit-leak-free: 2.1.2 - pino-abstract-transport: 3.0.0 - pino-std-serializers: 7.1.0 - process-warning: 5.0.0 - quick-format-unescaped: 4.0.4 - real-require: 0.2.0 - safe-stable-stringify: 2.5.0 - sonic-boom: 4.2.1 - thread-stream: 4.2.0 - pino@9.14.0: dependencies: '@pinojs/redact': 0.4.0 @@ -4584,8 +4564,6 @@ snapshots: real-require@0.2.0: {} - real-require@1.0.0: {} - require-from-string@2.0.2: {} resolve-from@5.0.0: {} @@ -4749,10 +4727,6 @@ snapshots: dependencies: real-require: 0.2.0 - thread-stream@4.2.0: - dependencies: - real-require: 1.0.0 - tinybench@2.9.0: {} tinyexec@0.3.2: {} From a53ce8df5c3e9e4e7b0aece2033da203d948fa3d Mon Sep 17 00:00:00 2001 From: djgrant <1670902+djgrant@users.noreply.github.com> Date: Wed, 22 Jul 2026 16:05:02 +0100 Subject: [PATCH 08/34] Harden durable runtime cutover --- docs/cli/deploy.md | 4 +- docs/cli/destroy.md | 4 +- docs/internals/reconciler.md | 4 +- docs/internals/resource.md | 2 +- docs/manual/reconciler.md | 8 +- docs/rfcs/reconciler.md | 2 +- examples/reconciler/package.json | 2 + examples/reconciler/src/index.ts | 46 +---- packages/cli/src/index.ts | 7 +- packages/core/package.json | 1 + .../core/src/provisioner/durable-runtime.ts | 172 +++++++++++++++++- .../provisioner/workflows/workflow.deploy.ts | 7 +- .../provisioner/workflows/workflow.destroy.ts | 7 +- .../provisioner/workflows/workflow.plan.ts | 8 +- .../test/provisioner/durable-runtime.test.ts | 155 +++++++++++++++- .../test/durable-reconciliation.test.ts | 12 +- pnpm-lock.yaml | 9 + 17 files changed, 374 insertions(+), 76 deletions(-) diff --git a/docs/cli/deploy.md b/docs/cli/deploy.md index 3e81379..4df0e43 100644 --- a/docs/cli/deploy.md +++ b/docs/cli/deploy.md @@ -28,7 +28,7 @@ notation deploy infra/api.ts --execution-id Do not reuse a completed execution ID for a new deploy or for destroy. -Retryable provider conditions and consistency reads suspend on durable SQLite timers. The CLI stays resident until the scheduler wakes the execution and the workflow completes; completed provider calls are replayed from the heap rather than repeated. +Retryable provider conditions and consistency reads suspend on durable SQLite timers. The CLI stays resident until the scheduler wakes the execution and the workflow completes. Provider results are replayed after their heap checkpoint, but a crash after the provider accepts a create or update and before that checkpoint repeats the call, so provider mutations must be idempotent. Reconciler event consumers must tolerate the equivalent duplicate-delivery window. ## What happens @@ -45,3 +45,5 @@ Retryable provider conditions and consistency reads suspend on durable SQLite ti 6. **Delete orphans** – persisted resources absent from the graph are deleted when their resource type is registered. State, step results, timers, task coordination, and resource stores are persisted to `.notation/workflows.db`. Set `NOTATION_STATE_PATH` to choose another SQLite database path. + +On first use, Notation imports resource state from the legacy `.notation/state.json` file and archives it as `.notation/state.json.migrated`. If the durable database already contains conflicting resource state, Notation stops with recovery instructions instead of attempting to create resources from an empty namespace. diff --git a/docs/cli/destroy.md b/docs/cli/destroy.md index c25a386..aa1be47 100644 --- a/docs/cli/destroy.md +++ b/docs/cli/destroy.md @@ -16,10 +16,12 @@ notation destroy infra/api.ts notation destroy infra/api.ts --json > destroy.ndjson ``` -The command prints its execution ID. Resume a crashed destroy with the same ID so a provider delete that already completed is replayed instead of repeated: +The command prints its execution ID. Resume a crashed destroy with the same ID so checkpointed work can be replayed: ```sh notation destroy infra/api.ts --execution-id ``` Retryable deletes suspend on durable SQLite timers. Resource state is removed only after the provider delete succeeds or reports that the resource is already absent. + +The provider acknowledgement and heap checkpoint are not atomic. A crash between them repeats the delete, so provider delete operations must be idempotent and event consumers must tolerate duplicate delivery. diff --git a/docs/internals/reconciler.md b/docs/internals/reconciler.md index 05034a8..41de5e3 100644 --- a/docs/internals/reconciler.md +++ b/docs/internals/reconciler.md @@ -15,13 +15,13 @@ The reconciler expresses deployment and destruction as Yieldstar async generator | In state, provider state differs from stored state | **drift-update** | | In state, not in graph | **delete** | -Dry-run deploy performs decisions and emits lifecycle events without calling providers or mutating state. +Dry-run deploy performs decisions and emits lifecycle events without provider mutations or state mutations. When drift detection is enabled, it can still call provider read operations to decide whether a nominal noop has drifted. ## Destroy flow `destroy` is a first-class durable operation. It acquires the same deployment coordination store as deploy, deletes desired resources in reverse dependency order, deletes hydratable persisted orphans, and conditionally removes each resource store only after the provider delete succeeds or reports that the resource is already absent. -Provider delete is a stable durable step. If the process crashes after the provider acknowledges deletion but before state removal, replay uses the cached delete result and continues at the conditional store delete. +Provider delete is a stable durable step, but the provider acknowledgement and Yieldstar heap checkpoint are not atomic. If the process crashes between them, replay repeats the delete, so provider create, update, and delete operations must be idempotent. Event subscribers must likewise tolerate duplicate delivery when a crash occurs before the event checkpoint. ## Waiting and replay diff --git a/docs/internals/resource.md b/docs/internals/resource.md index baae261..851be55 100644 --- a/docs/internals/resource.md +++ b/docs/internals/resource.md @@ -187,7 +187,7 @@ new ResourceOperationPendingError(message: string, { | `callbackContext` | `Readonly>` | no | Plain serializable data for the next attempt. | | `cause` | `unknown` | no | The provider error that caused this result. | -The default limit is 30 attempts. Set `maxOperationAttempts` on the reconciler to change it. Reaching the limit fails the operation. +The default limit is 30 attempts. Set `maxOperationAttempts` in the deploy, plan, or destroy options to change it. Reaching the limit fails the operation. ```ts read: async (key, context) => { diff --git a/docs/manual/reconciler.md b/docs/manual/reconciler.md index eee8d06..9d4435f 100644 --- a/docs/manual/reconciler.md +++ b/docs/manual/reconciler.md @@ -4,7 +4,7 @@ Use `deploy` and `destroy` when a Node.js application needs durable resource lif ```ts import { SqliteSchedulerClient, SqliteStoreClient, SqliteTaskQueueClient, SqliteTimersClient, createSqliteDb } from "@yieldstar/sqlite-runtime/node"; -import { DurableStateBackend, deploy, destroy } from "@notation/reconciler"; +import { DurableStateBackend, deploy as deployResources, destroy as destroyResources } from "@notation/reconciler"; import { workflow } from "yieldstar"; const database = createSqliteDb({ path: ".notation/workflows.db" }); @@ -16,7 +16,7 @@ const storeClient = new SqliteStoreClient({ db: database, schedulerClient }); const state = new DurableStateBackend(storeClient, "my-application"); export const deploy = workflow(async function* (step, event) { - yield* deploy(step, { + yield* deployResources(step, { deploymentId: "my-application", executionId: event.executionId, resources, @@ -25,7 +25,7 @@ export const deploy = workflow(async function* (step, event) { }); export const destroy = workflow(async function* (step, event) { - yield* destroy(step, { + yield* destroyResources(step, { deploymentId: "my-application", executionId: event.executionId, resources, @@ -34,7 +34,7 @@ export const destroy = workflow(async function* (step, event) { }); ``` -The outer workflow supplies durable step execution, timers, shared stores, waiting, scheduling, and coordination. Completed provider calls are replayed from the heap after a crash, retryable provider conditions suspend on a durable timer, and conditional state writes use Yieldstar store identity and version. +The outer workflow supplies durable step execution, timers, shared stores, waiting, scheduling, and coordination. Checkpointed provider results are replayed from the heap after a crash, retryable provider conditions suspend on a durable timer, and conditional state writes use Yieldstar store identity and version. Provider mutations must be idempotent because a crash after provider acknowledgement but before the heap checkpoint repeats the call; event consumers must tolerate the same duplicate-delivery window. Each live resource is one Yieldstar store. Absence is represented by no store, not a tombstone. Yieldstar's UUIDv7 store `instanceId` and version are authoritative for conditional update and delete; Notation exposes the version as the resource state's `rev`. diff --git a/docs/rfcs/reconciler.md b/docs/rfcs/reconciler.md index 7ec505a..6cc9d1c 100644 --- a/docs/rfcs/reconciler.md +++ b/docs/rfcs/reconciler.md @@ -9,7 +9,7 @@ Notation describes reconciliation intent and resource lifecycle operations. An o Live resource objects remain in the workflow process. They are not serialized into workflow parameters. This keeps provider clients and operation closures under Notation's lifecycle control while Yieldstar persists step results and shared state. -Provider create, update, read, and delete calls are durable steps with stable resource-scoped keys. A process crash after a completed provider call replays the cached result and continues at state persistence instead of repeating the call. Retryable provider conditions become Yieldstar delays, allowing the process to wait without polling the provider continuously. +Provider create, update, read, and delete calls are durable steps with stable resource-scoped keys. Once a result reaches the heap checkpoint, replay uses the cached result and continues at state persistence. Provider mutations must be idempotent because a crash after provider acknowledgement but before that checkpoint repeats the call. Retryable provider conditions become Yieldstar delays, allowing the process to wait without polling the provider continuously. ## State lifecycle diff --git a/examples/reconciler/package.json b/examples/reconciler/package.json index c5d8f2e..8a26c85 100644 --- a/examples/reconciler/package.json +++ b/examples/reconciler/package.json @@ -9,8 +9,10 @@ "typecheck": "tsc --noEmit" }, "dependencies": { + "@notation/core": "workspace:*", "@notation/reconciler": "workspace:*", "@notation/resource": "workspace:*", + "@notation/utils": "workspace:*", "@yieldstar/core": "0.5.0", "@yieldstar/sqlite-runtime": "0.5.0", "pino": "^9.9.0", diff --git a/examples/reconciler/src/index.ts b/examples/reconciler/src/index.ts index 0253115..ee4360a 100644 --- a/examples/reconciler/src/index.ts +++ b/examples/reconciler/src/index.ts @@ -1,26 +1,12 @@ -import { WorkflowRunner } from "@yieldstar/core"; -import { - SqliteHeapClient, - SqliteSchedulerClient, - SqliteStoreClient, - SqliteTaskQueueClient, - SqliteTimersClient, - createSqliteDb, -} from "@yieldstar/sqlite-runtime/node"; +import { NodeDurableRuntime } from "@notation/core"; import * as reconciler from "@notation/reconciler"; -import pino from "pino"; import { createWorkflowRouter, workflow } from "yieldstar"; import { StaticSite } from "./static-site"; -const logger = pino(); -const database = createSqliteDb({ path: "sites.db" }); -const taskQueueClient = new SqliteTaskQueueClient(database); -const schedulerClient = new SqliteSchedulerClient({ - taskQueueClient, - timersClient: new SqliteTimersClient(database), +const runtime = new NodeDurableRuntime({ + deploymentId: "static-sites", + databasePath: "sites.db", }); -const storeClient = new SqliteStoreClient({ db: database, schedulerClient }); -const state = new reconciler.DurableStateBackend(storeClient, "static-sites"); const resources = [ new StaticSite({ @@ -44,29 +30,15 @@ const deploy = workflow(async function* (step, event) { deploymentId: "static-sites", executionId: event.executionId, resources, - state, + state: runtime.state, registry: reconciler.createResourceRegistry([StaticSite]), }); }); -const runner = new WorkflowRunner({ - router: createWorkflowRouter({ deploy }), - heapClient: new SqliteHeapClient(database), - storeClient, - schedulerClient, - logger, -}); - try { - await runner.run( - { - workflowId: "deploy", - executionId: crypto.randomUUID(), - params: {}, - context: new Map(), - }, - logger, - ); + await runtime.run(createWorkflowRouter({ deploy }), { + workflowId: "deploy", + }); } finally { - database.close(); + runtime.close(); } diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index eee95f2..1bfade1 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -9,7 +9,7 @@ import { runWithCliErrorHandling } from "./run-with-error-handling"; import { visualise } from "./visualise"; import { watch } from "./watch"; import { startDashboardServer } from "@notation/dashboard"; -import { NodeDurableRuntime } from "@notation/core"; +import { NodeDurableRuntime, resolveDeploymentId } from "@notation/core"; program .command("compile") @@ -24,7 +24,10 @@ program .argument("", "entryPoint") .description("Start Notation Dashboard") .action(async (entryPoint) => { - const runtime = new NodeDurableRuntime({ deploymentId: entryPoint }); + const runtime = new NodeDurableRuntime({ + deploymentId: resolveDeploymentId(entryPoint), + }); + await runtime.initialize(); await startDashboardServer({ state: runtime.state }); }); diff --git a/packages/core/package.json b/packages/core/package.json index cc55f70..c3e0fae 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -15,6 +15,7 @@ "dependencies": { "@notation/reconciler": "workspace:*", "@notation/resource": "workspace:*", + "@notation/state": "workspace:*", "@yieldstar/core": "0.5.0", "@yieldstar/sqlite-runtime": "0.5.0", "deep-object-diff": "^1.1.9", diff --git a/packages/core/src/provisioner/durable-runtime.ts b/packages/core/src/provisioner/durable-runtime.ts index 73face8..10d902c 100644 --- a/packages/core/src/provisioner/durable-runtime.ts +++ b/packages/core/src/provisioner/durable-runtime.ts @@ -1,5 +1,8 @@ import { randomUUID } from "node:crypto"; +import { access, rename } from "node:fs/promises"; +import path from "node:path"; import { setImmediate } from "node:timers/promises"; +import { isDeepStrictEqual } from "node:util"; import { WorkflowRunner, type WorkflowEvent, @@ -15,17 +18,25 @@ import { createSqliteDb, } from "@yieldstar/sqlite-runtime/node"; import { DurableStateBackend } from "@notation/reconciler"; +import { FileStateBackend, type StateNode } from "@notation/state"; import pino, { type Logger } from "pino"; +import { defineStore } from "yieldstar"; export const DEFAULT_WORKFLOW_STATE_PATH = ".notation/workflows.db"; +export const DEFAULT_LEGACY_STATE_PATH = ".notation/state.json"; export function resolveWorkflowStatePath(): string { return process.env.NOTATION_STATE_PATH ?? DEFAULT_WORKFLOW_STATE_PATH; } +export function resolveDeploymentId(entryPoint: string): string { + return path.resolve(entryPoint); +} + export type NodeDurableRuntimeOptions = { deploymentId: string; databasePath?: string; + legacyStatePath?: string | false; logger?: Logger; }; @@ -35,6 +46,31 @@ export type RunWorkflowOptions = { params?: Record; }; +type ExecutionBinding = { + deploymentId: string; + workflowId: string; +}; + +const executionBindingStore = defineStore("notation/execution-binding", { + "~standard": { + version: 1 as const, + vendor: "notation", + validate(value: unknown) { + if ( + typeof value === "object" && + value !== null && + "deploymentId" in value && + typeof value.deploymentId === "string" && + "workflowId" in value && + typeof value.workflowId === "string" + ) { + return { value: value as ExecutionBinding }; + } + return { issues: [{ message: "Execution binding is invalid" }] }; + }, + }, +}); + /** Resident Yieldstar 0.5.0 Node runtime used by Notation application commands. */ export class NodeDurableRuntime { readonly deploymentId: string; @@ -45,13 +81,22 @@ export class NodeDurableRuntime { readonly #schedulerClient: SqliteSchedulerClient; readonly #storeClient: SqliteStoreClient; readonly #logger: Logger; + readonly #legacyStatePath: string | undefined; #running = false; constructor(opts: NodeDurableRuntimeOptions) { this.deploymentId = opts.deploymentId; this.#logger = opts.logger ?? pino({ level: "silent" }); + const databasePath = opts.databasePath ?? resolveWorkflowStatePath(); + this.#legacyStatePath = + opts.legacyStatePath === false + ? undefined + : (opts.legacyStatePath ?? + (databasePath === DEFAULT_WORKFLOW_STATE_PATH + ? DEFAULT_LEGACY_STATE_PATH + : undefined)); this.#database = createSqliteDb({ - path: opts.databasePath ?? resolveWorkflowStatePath(), + path: databasePath, }); const taskQueueClient = new SqliteTaskQueueClient(this.#database); this.#schedulerClient = new SqliteSchedulerClient({ @@ -77,9 +122,10 @@ export class NodeDurableRuntime { ); } this.#running = true; + const executionId = opts.executionId ?? randomUUID(); const event: WorkflowEvent = { workflowId: opts.workflowId, - executionId: opts.executionId ?? randomUUID(), + executionId, params: opts.params ?? {}, context: new Map(), }; @@ -93,6 +139,7 @@ export class NodeDurableRuntime { let resolveCompletion!: (value: unknown) => void; let rejectCompletion!: (error: unknown) => void; + let completed = false; const completion = new Promise((resolve, reject) => { resolveCompletion = resolve; rejectCompletion = reject; @@ -101,12 +148,12 @@ export class NodeDurableRuntime { try { const result = await runner.run(nextEvent, logger); if (result && nextEvent.executionId === event.executionId) { - this.#eventLoop.stop(); + completed = true; resolveCompletion(result.result); } } catch (error) { if (nextEvent.executionId === event.executionId) { - this.#eventLoop.stop(); + completed = true; rejectCompletion(error); return; } @@ -115,28 +162,135 @@ export class NodeDurableRuntime { }; try { + await this.initialize(); + await this.#bindExecution(executionId, opts.workflowId); await processEvent(event, this.#logger); - this.#eventLoop.start({ onNewEvent: processEvent, logger: this.#logger }); + const eventPump = this.#processQueuedEvents( + executionId, + processEvent, + rejectCompletion, + () => completed, + ); try { return await completion; } finally { - // Let SqliteEventLoop remove the completed queue item before callers - // close the shared database. + await eventPump; + // Let the queue transaction finish before callers close the shared database. await setImmediate(); } } finally { - this.#eventLoop.stop(); this.#running = false; } } + async initialize(): Promise { + await this.#migrateLegacyState(); + } + + async #bindExecution(executionId: string, workflowId: string): Promise { + const expected: ExecutionBinding = { + deploymentId: this.deploymentId, + workflowId, + }; + const binding = await this.#storeClient.getOrCreateStore({ + definition: executionBindingStore, + id: executionId, + initial: expected, + }); + const existing = binding.state as ExecutionBinding; + if (!isDeepStrictEqual(existing, expected)) { + throw new Error( + `Yieldstar execution ${executionId} is bound to deployment ${existing.deploymentId} workflow ${existing.workflowId}, not deployment ${this.deploymentId} workflow ${workflowId}`, + ); + } + } + + async #processQueuedEvents( + executionId: string, + processEvent: (event: WorkflowEvent, logger: Logger) => Promise, + rejectCompletion: (error: unknown) => void, + isCompleted: () => boolean, + ): Promise { + const deferredTaskIds: number[] = []; + try { + while (this.#running && !isCompleted()) { + let task = this.#eventLoop.taskQueue.process(); + while (task) { + if (task.event.executionId !== executionId) { + deferredTaskIds.push(task.taskId); + } else { + try { + await this.#bindExecution( + task.event.executionId, + task.event.workflowId, + ); + await processEvent(task.event, this.#logger); + } finally { + this.#eventLoop.taskQueue.remove(task.taskId); + } + } + if (!this.#running || isCompleted()) return; + task = this.#eventLoop.taskQueue.process(); + } + this.#eventLoop.timers.processTimers(); + await new Promise((resolve) => setTimeout(resolve, 10)); + } + } catch (error) { + rejectCompletion(error); + } finally { + for (const taskId of deferredTaskIds) { + this.#eventLoop.taskQueue.makeVisible(taskId); + } + } + } + + async #migrateLegacyState(): Promise { + const legacyStatePath = this.#legacyStatePath; + if (!legacyStatePath) return; + try { + await access(legacyStatePath); + } catch { + return; + } + + const legacyState = await new FileStateBackend(legacyStatePath).values(); + const durableState = await this.state.values(); + const legacyById = new Map(legacyState.map((node) => [node.id, node])); + for (const current of durableState) { + const legacy = legacyById.get(current.id); + if (!legacy || !statesMatchIgnoringRevision(current, legacy)) { + throw legacyMigrationConflict(legacyStatePath); + } + } + for (const node of legacyState) { + const current = await this.state.get(node.id); + if (!current) { + await this.state.update(node.id, 0, node); + } else if (!statesMatchIgnoringRevision(current, node)) { + throw legacyMigrationConflict(legacyStatePath); + } + } + await rename(legacyStatePath, `${legacyStatePath}.migrated`); + } + close(): void { if (this.#running) { throw new Error( "Cannot close the Node Yieldstar runtime while a workflow is active", ); } - this.#eventLoop.stop(); this.#database.close(); } } + +function statesMatchIgnoringRevision(left: StateNode, right: StateNode) { + const { rev: _leftRev, ...leftState } = left; + const { rev: _rightRev, ...rightState } = right; + return isDeepStrictEqual(leftState, rightState); +} + +function legacyMigrationConflict(legacyStatePath: string) { + return new Error( + `Cannot migrate legacy state from ${legacyStatePath} because the durable database already contains different resource state. Back up both files, then remove the new durable database and retry the command to import the legacy state.`, + ); +} diff --git a/packages/core/src/provisioner/workflows/workflow.deploy.ts b/packages/core/src/provisioner/workflows/workflow.deploy.ts index dc3b765..ab11ecf 100644 --- a/packages/core/src/provisioner/workflows/workflow.deploy.ts +++ b/packages/core/src/provisioner/workflows/workflow.deploy.ts @@ -6,7 +6,7 @@ import { } from "@notation/reconciler"; import { createWorkflowRouter, workflow } from "yieldstar"; import { getResourceGraph } from "src/orchestrator/graph"; -import { NodeDurableRuntime } from "../durable-runtime"; +import { NodeDurableRuntime, resolveDeploymentId } from "../durable-runtime"; export type DeployAppOptions = { entryPoint: string; @@ -32,9 +32,10 @@ export async function deployApp({ emit = createLoggerReconcilerSubscriber(), }: DeployAppOptions): Promise { const graph = await getResourceGraph(entryPoint); + const deploymentId = + suppliedRuntime?.deploymentId ?? resolveDeploymentId(entryPoint); const runtime = - suppliedRuntime ?? - new NodeDurableRuntime({ deploymentId: entryPoint, databasePath }); + suppliedRuntime ?? new NodeDurableRuntime({ deploymentId, databasePath }); const deploy = workflow(async function* (step, event) { yield* reconciler.deploy(step, { deploymentId: runtime.deploymentId, diff --git a/packages/core/src/provisioner/workflows/workflow.destroy.ts b/packages/core/src/provisioner/workflows/workflow.destroy.ts index 9a534c4..c5d9303 100644 --- a/packages/core/src/provisioner/workflows/workflow.destroy.ts +++ b/packages/core/src/provisioner/workflows/workflow.destroy.ts @@ -6,7 +6,7 @@ import { } from "@notation/reconciler"; import { createWorkflowRouter, workflow } from "yieldstar"; import { getResourceGraph } from "src/orchestrator/graph"; -import { NodeDurableRuntime } from "../durable-runtime"; +import { NodeDurableRuntime, resolveDeploymentId } from "../durable-runtime"; export type DestroyAppOptions = { entryPoint: string; @@ -28,9 +28,10 @@ export async function destroyApp({ emit = createLoggerReconcilerSubscriber(), }: DestroyAppOptions) { const graph = await getResourceGraph(entryPoint); + const deploymentId = + suppliedRuntime?.deploymentId ?? resolveDeploymentId(entryPoint); const runtime = - suppliedRuntime ?? - new NodeDurableRuntime({ deploymentId: entryPoint, databasePath }); + suppliedRuntime ?? new NodeDurableRuntime({ deploymentId, databasePath }); const destroy = workflow(async function* (step, event) { yield* reconciler.destroy(step, { deploymentId: runtime.deploymentId, diff --git a/packages/core/src/provisioner/workflows/workflow.plan.ts b/packages/core/src/provisioner/workflows/workflow.plan.ts index b64db9d..5b06aba 100644 --- a/packages/core/src/provisioner/workflows/workflow.plan.ts +++ b/packages/core/src/provisioner/workflows/workflow.plan.ts @@ -1,6 +1,6 @@ import { createPlan, type Plan } from "@notation/reconciler"; import { getResourceGraph } from "src/orchestrator/graph"; -import { NodeDurableRuntime } from "../durable-runtime"; +import { NodeDurableRuntime, resolveDeploymentId } from "../durable-runtime"; export type { Plan, PlanNode, PlanDecision } from "@notation/reconciler"; @@ -20,10 +20,12 @@ export async function planApp({ databasePath, }: PlanAppOptions): Promise { const graph = await getResourceGraph(entryPoint); + const deploymentId = + suppliedRuntime?.deploymentId ?? resolveDeploymentId(entryPoint); const runtime = - suppliedRuntime ?? - new NodeDurableRuntime({ deploymentId: entryPoint, databasePath }); + suppliedRuntime ?? new NodeDurableRuntime({ deploymentId, databasePath }); try { + await runtime.initialize(); return await createPlan({ resources: graph.resources, state: runtime.state, diff --git a/packages/core/test/provisioner/durable-runtime.test.ts b/packages/core/test/provisioner/durable-runtime.test.ts index 77a699b..09aa6d8 100644 --- a/packages/core/test/provisioner/durable-runtime.test.ts +++ b/packages/core/test/provisioner/durable-runtime.test.ts @@ -1,4 +1,4 @@ -import { mkdtemp, rm } from "node:fs/promises"; +import { access, mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; import * as reconciler from "@notation/reconciler"; @@ -6,9 +6,17 @@ import { ResourceOperationPendingError, resource, } from "@notation/resource"; -import { createWorkflowRouter, workflow } from "yieldstar"; +import { + SqliteEventLoop, + SqliteTaskQueueClient, + createSqliteDb, +} from "@yieldstar/sqlite-runtime/node"; +import { RetryableError, createWorkflowRouter, workflow } from "yieldstar"; import { describe, expect, it } from "vitest"; -import { NodeDurableRuntime } from "src/provisioner/durable-runtime"; +import { + NodeDurableRuntime, + resolveDeploymentId, +} from "src/provisioner/durable-runtime"; describe("NodeDurableRuntime", () => { it("stays resident across a provider delay and resumes from the SQLite event loop", async () => { @@ -57,4 +65,145 @@ describe("NodeDurableRuntime", () => { await rm(directory, { recursive: true, force: true }); } }, 5_000); + + it("binds an execution ID to its deployment and workflow", async () => { + const directory = await mkdtemp(path.join(tmpdir(), "notation-binding-")); + const databasePath = path.join(directory, "workflows.db"); + const completed = workflow(async function* () {}); + const router = createWorkflowRouter({ + deploy: completed, + destroy: completed, + }); + const first = new NodeDurableRuntime({ + deploymentId: "first-deployment", + databasePath, + }); + + try { + await first.run(router, { + workflowId: "deploy", + executionId: "bound-execution", + }); + await expect( + first.run(router, { + workflowId: "destroy", + executionId: "bound-execution", + }), + ).rejects.toThrow("bound to deployment first-deployment workflow deploy"); + } finally { + first.close(); + } + + const second = new NodeDurableRuntime({ + deploymentId: "second-deployment", + databasePath, + }); + try { + await expect( + second.run(router, { + workflowId: "deploy", + executionId: "bound-execution", + }), + ).rejects.toThrow("bound to deployment first-deployment workflow deploy"); + } finally { + second.close(); + await rm(directory, { recursive: true, force: true }); + } + }); + + it("does not acknowledge queued events from another execution", async () => { + const directory = await mkdtemp(path.join(tmpdir(), "notation-queue-")); + const databasePath = path.join(directory, "workflows.db"); + const database = createSqliteDb({ path: databasePath }); + new SqliteEventLoop(database); + new SqliteTaskQueueClient(database).add({ + workflowId: "deploy", + executionId: "unrelated-execution", + params: {}, + context: new Map(), + }); + database.close(); + + let attempts = 0; + const delayed = workflow(async function* (step) { + yield* step.run("delay", async () => { + attempts += 1; + if (attempts === 1) { + throw new RetryableError("not ready", { + maxAttempts: 2, + retryInterval: 10, + }); + } + }); + }); + const runtime = new NodeDurableRuntime({ + deploymentId: "queue-test", + databasePath, + }); + try { + await runtime.run(createWorkflowRouter({ deploy: delayed }), { + workflowId: "deploy", + executionId: "current-execution", + }); + } finally { + runtime.close(); + } + + const reopened = createSqliteDb({ path: databasePath }); + const queued = new SqliteEventLoop(reopened).taskQueue.process(); + expect(queued?.event.executionId).toBe("unrelated-execution"); + reopened.close(); + await rm(directory, { recursive: true, force: true }); + }, 5_000); + + it("imports and archives legacy JSON state before running", async () => { + const directory = await mkdtemp(path.join(tmpdir(), "notation-migrate-")); + const databasePath = path.join(directory, "workflows.db"); + const legacyStatePath = path.join(directory, "state.json"); + await writeFile( + legacyStatePath, + JSON.stringify({ + existing: { + rev: 7, + id: "existing", + type: "test/legacy", + config: {}, + params: {}, + output: { remoteId: "provider-123" }, + lastOperation: "create", + lastOperationAt: "2026-07-22T00:00:00.000Z", + }, + }), + ); + const runtime = new NodeDurableRuntime({ + deploymentId: "legacy-deployment", + databasePath, + legacyStatePath, + }); + const completed = workflow(async function* () {}); + + try { + await runtime.run(createWorkflowRouter({ deploy: completed }), { + workflowId: "deploy", + executionId: "migration-execution", + }); + await expect(runtime.state.get("existing")).resolves.toMatchObject({ + output: { remoteId: "provider-123" }, + }); + await expect(access(legacyStatePath)).rejects.toThrow(); + await expect( + access(`${legacyStatePath}.migrated`), + ).resolves.toBeUndefined(); + } finally { + runtime.close(); + await rm(directory, { recursive: true, force: true }); + } + }); + + it("canonicalises equivalent entry-point spellings", () => { + const absolute = path.resolve("infra/api.ts"); + expect(resolveDeploymentId("infra/api.ts")).toBe(absolute); + expect(resolveDeploymentId("./infra/api.ts")).toBe(absolute); + expect(resolveDeploymentId(absolute)).toBe(absolute); + }); }); diff --git a/packages/reconciler/test/durable-reconciliation.test.ts b/packages/reconciler/test/durable-reconciliation.test.ts index 1a51cd7..47f471e 100644 --- a/packages/reconciler/test/durable-reconciliation.test.ts +++ b/packages/reconciler/test/durable-reconciliation.test.ts @@ -38,7 +38,7 @@ describe("durable execution and replay", () => { if (attempts === 1) { expect(context).toBeUndefined(); throw new ResourceOperationPendingError("provider is not ready", { - retryAfterMs: 1, + retryAfterMs: 250, callbackContext: { requestId: "request-123" }, }); } @@ -56,7 +56,7 @@ describe("durable execution and replay", () => { expect(attempts).toBe(1); expect(runtime.scheduler.events).toHaveLength(1); - await sleep(5); + await sleep(275); await runtime.run("wait-execution"); expect(attempts).toBe(2); expect(await runtime.state.get("pending")).toMatchObject({ @@ -124,7 +124,7 @@ describe("durable execution and replay", () => { attempts += 1; if (attempts === 1) { throw new ResourceOperationPendingError("delete is not ready", { - retryAfterMs: 1, + retryAfterMs: 250, }); } }, @@ -140,7 +140,7 @@ describe("durable execution and replay", () => { expect(attempts).toBe(1); expect(await runtime.state.get("pending-delete")).toBeDefined(); - await sleep(5); + await sleep(275); await runtime.destroy("destroy-wait"); expect(attempts).toBe(2); expect(await runtime.state.get("pending-delete")).toBeUndefined(); @@ -160,7 +160,7 @@ describe("durable execution and replay", () => { if (reads === 1) { throw new ResourceOperationPendingError( "resource is not visible yet", - { retryAfterMs: 1 }, + { retryAfterMs: 250 }, ); } return {} as const; @@ -177,7 +177,7 @@ describe("durable execution and replay", () => { expect(reads).toBe(1); expect(await runtime.state.get("eventually-readable")).toBeUndefined(); - await sleep(5); + await sleep(275); await runtime.run("post-write-read-execution"); expect(reads).toBe(2); expect(await runtime.state.get("eventually-readable")).toMatchObject({ diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 02ef3b9..5f1ee6d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -117,12 +117,18 @@ importers: examples/reconciler: dependencies: + '@notation/core': + specifier: workspace:* + version: link:../../packages/core '@notation/reconciler': specifier: workspace:* version: link:../../packages/reconciler '@notation/resource': specifier: workspace:* version: link:../../packages/resource + '@notation/utils': + specifier: workspace:* + version: link:../../packages/utils '@yieldstar/core': specifier: 0.5.0 version: 0.5.0 @@ -230,6 +236,9 @@ importers: '@notation/resource': specifier: workspace:* version: link:../resource + '@notation/state': + specifier: workspace:* + version: link:../state '@yieldstar/core': specifier: 0.5.0 version: 0.5.0 From c871428a6417d4257ec5776b9169d6126227c2c5 Mon Sep 17 00:00:00 2001 From: djgrant <1670902+djgrant@users.noreply.github.com> Date: Mon, 27 Jul 2026 19:08:51 +0100 Subject: [PATCH 09/34] Share one operation library across reconciler drivers The durable path reimplemented create, update, read and delete inline rather than using the StepRunner seam those operations were built for, leaving two copies of the same provider protocol to keep in step. Persistence becomes a driver-supplied step. In process that is a compare-and-set against the revision read before the operation; in a workflow it is a store write stamped with the step that made it, so the applied-step ledger and the state change commit together and a replay returns the recorded result instead of retrying a stale compare-and-set. Event emission becomes a step for the same reason. Scoped step keys let one operation body run at several call sites, which replaces the hand-threaded key prefixes and the parameter bags that carried them. With writes owned by the workflow, DurableStateBackend is read-only for the planner and reporting, so the revision arithmetic and the creation token that guarded create-if-absent both go. Also collapses the three copies of the reconciler event types, delegates Reconciler.plan to createPlan, and exports the durable subsystem, which no consumer could previously import. --- packages/reconciler/package.json | 10 + .../reconciler/src/durable/coordination.ts | 24 +- packages/reconciler/src/durable/deploy.ts | 29 +- packages/reconciler/src/durable/destroy.ts | 45 +- packages/reconciler/src/durable/emit.ts | 64 --- packages/reconciler/src/durable/operations.ts | 493 ++++++------------ .../reconciler/src/durable/state-backend.ts | 148 ++---- packages/reconciler/src/durable/step.ts | 84 +++ packages/reconciler/src/durable/stores.ts | 23 +- packages/reconciler/src/durable/yieldstar.ts | 2 +- packages/reconciler/src/events.ts | 22 +- packages/reconciler/src/index.ts | 2 + .../src/operations/operation.create.ts | 30 +- .../src/operations/operation.delete.ts | 12 +- .../src/operations/operation.read.ts | 32 +- .../src/operations/operation.types.ts | 77 ++- .../src/operations/operation.update.ts | 34 +- packages/reconciler/src/planner.ts | 40 +- packages/reconciler/src/protocol.ts | 2 +- packages/reconciler/src/reconciler.ts | 203 ++------ packages/reconciler/src/step-runner.ts | 40 ++ .../test/durable-reconciliation.test.ts | 153 +++--- .../test/operation.workflows.test.ts | 49 +- packages/reconciler/tsup.config.ts | 2 +- 24 files changed, 696 insertions(+), 924 deletions(-) delete mode 100644 packages/reconciler/src/durable/emit.ts create mode 100644 packages/reconciler/src/durable/step.ts create mode 100644 packages/reconciler/src/step-runner.ts diff --git a/packages/reconciler/package.json b/packages/reconciler/package.json index d1ec1db..58b0ab0 100644 --- a/packages/reconciler/package.json +++ b/packages/reconciler/package.json @@ -2,6 +2,16 @@ "type": "module", "name": "@notation/reconciler", "version": "0.12.0", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "default": "./dist/index.js" + }, + "./durable": { + "types": "./dist/durable/index.d.ts", + "default": "./dist/durable/index.js" + } + }, "main": "./dist/index.js", "types": "./dist/index.d.ts", "files": [ diff --git a/packages/reconciler/src/durable/coordination.ts b/packages/reconciler/src/durable/coordination.ts index 2fb2579..5c7af65 100644 --- a/packages/reconciler/src/durable/coordination.ts +++ b/packages/reconciler/src/durable/coordination.ts @@ -1,5 +1,5 @@ import type { ReconcilerEventEmitter } from "../events"; -import { emitEvent } from "./emit"; +import { durableEmitter, scopeStep } from "./step"; import { deploymentCoordinationStore, type CoordinationState } from "./stores"; import type { DurableStep, WorkflowStore } from "./yieldstar"; @@ -13,7 +13,7 @@ type CoordinationOptions = { * Prevents concurrent executions from mutating the same deployment. Names * the holder so an operator can resume it after a crash. */ -export async function* acquireDeploymentCoordination( +async function* acquireDeploymentCoordination( step: DurableStep, opts: CoordinationOptions, ): AsyncGenerator, any> { @@ -25,13 +25,13 @@ export async function* acquireDeploymentCoordination( const snapshot = yield* coordination.get("notation:coordination:inspect"); const holder = snapshot.state.holder; if (holder !== null && holder !== opts.executionId) { - yield* emitEvent(step, "notation:coordination:waiting", opts.emit, () => ({ + yield* durableEmitter(scopeStep(step, "notation:coordination"), opts.emit)({ level: "warn", event: "reconciler.coordination.waiting", deploymentId: opts.deploymentId, executionId: opts.executionId, holderExecutionId: holder, - })); + }); } yield* coordination.take( @@ -45,7 +45,7 @@ export async function* acquireDeploymentCoordination( return coordination; } -export function releaseDeploymentCoordination( +function releaseDeploymentCoordination( coordination: WorkflowStore, executionId: string, ) { @@ -53,3 +53,17 @@ export function releaseDeploymentCoordination( if (draft.holder === executionId) draft.holder = null; }); } + +/** Runs `body` while holding the deployment, releasing it even on error. */ +export async function* withDeploymentHold( + step: DurableStep, + opts: CoordinationOptions, + body: () => AsyncGenerator, +): AsyncGenerator { + const coordination = yield* acquireDeploymentCoordination(step, opts); + try { + yield* body(); + } finally { + yield* releaseDeploymentCoordination(coordination, opts.executionId); + } +} diff --git a/packages/reconciler/src/durable/deploy.ts b/packages/reconciler/src/durable/deploy.ts index 3b93520..b28c273 100644 --- a/packages/reconciler/src/durable/deploy.ts +++ b/packages/reconciler/src/durable/deploy.ts @@ -1,9 +1,7 @@ import { buildResourceDepthLevels } from "../dependency-graph"; -import { - acquireDeploymentCoordination, - releaseDeploymentCoordination, -} from "./coordination"; +import { withDeploymentHold } from "./coordination"; import { reconcileResource, sweepOrphans } from "./operations"; +import { scopeStep } from "./step"; import type { DurableDeployOptions } from "./types"; import type { DurableStep } from "./yieldstar"; @@ -11,27 +9,16 @@ export async function* deploy( step: DurableStep, opts: DurableDeployOptions, ): AsyncGenerator { - // Phase 1: take exclusive hold of the deployment. - const coordination = yield* acquireDeploymentCoordination(step, opts); - - try { - // Phase 2: reconcile in dependency order, so a resource only runs once - // its dependencies have converged. + yield* withDeploymentHold(step, opts, async function* () { + // Reconcile in dependency order, so a resource only runs once its + // dependencies have converged. for (const level of buildResourceDepthLevels(opts.resources)) { for (const resource of level) { yield* reconcileResource(step, resource, opts); } } - // Phase 3: delete resources that are in state but no longer declared. - yield* sweepOrphans(step, opts, { - workflow: "deploy", - listKey: "notation:orphans:list", - warningKey: (nodeId) => `notation:orphan:${nodeId}:warning`, - deleteSuffix: "orphan", - }); - } finally { - // Phase 4: release the hold, even on error. - yield* releaseDeploymentCoordination(coordination, opts.executionId); - } + // Then delete resources that are in state but no longer declared. + yield* sweepOrphans(scopeStep(step, "notation:orphans"), opts, "deploy"); + }); } diff --git a/packages/reconciler/src/durable/destroy.ts b/packages/reconciler/src/durable/destroy.ts index 8f01316..fa6158a 100644 --- a/packages/reconciler/src/durable/destroy.ts +++ b/packages/reconciler/src/durable/destroy.ts @@ -1,9 +1,7 @@ import { buildResourceDepthLevels } from "../dependency-graph"; -import { - acquireDeploymentCoordination, - releaseDeploymentCoordination, -} from "./coordination"; +import { withDeploymentHold } from "./coordination"; import { deleteResource, sweepOrphans } from "./operations"; +import { scopeStep } from "./step"; import type { DurableDestroyOptions } from "./types"; import type { DurableStep } from "./yieldstar"; @@ -12,35 +10,26 @@ export async function* destroy( step: DurableStep, opts: DurableDestroyOptions, ): AsyncGenerator { - // Phase 1: take exclusive hold of the deployment. - const coordination = yield* acquireDeploymentCoordination(step, opts); - - try { - // Phase 2: delete in reverse dependency order, so dependents are gone - // before the resources they depend on. Resources with no persisted state - // were never created (or are already deleted) and are skipped. + yield* withDeploymentHold(step, opts, async function* () { + // Delete in reverse dependency order, so dependents are gone before the + // resources they depend on. Resources with no persisted state were never + // created (or are already deleted) and are skipped by deleteResource. const levels = buildResourceDepthLevels(opts.resources); for (let index = levels.length - 1; index >= 0; index -= 1) { for (const resource of levels[index]!) { - const stateNode = yield* step.run( - `notation:destroy:${resource.id}:state:lookup`, - () => opts.state.get(resource.id), + yield* deleteResource( + scopeStep(step, `notation:destroy:${resource.id}`), + resource, + opts, ); - if (!stateNode) continue; - resource.setOutput(stateNode.output); - yield* deleteResource(step, resource, opts, "destroy"); } } - // Phase 3: delete resources that are in state but no longer declared. - yield* sweepOrphans(step, opts, { - workflow: "destroy", - listKey: "notation:destroy:orphans:list", - warningKey: (nodeId) => `notation:destroy:orphan:${nodeId}:warning`, - deleteSuffix: "destroy-orphan", - }); - } finally { - // Phase 4: release the hold, even on error. - yield* releaseDeploymentCoordination(coordination, opts.executionId); - } + // Then delete resources that are in state but no longer declared. + yield* sweepOrphans( + scopeStep(step, "notation:destroy:orphans"), + opts, + "destroy", + ); + }); } diff --git a/packages/reconciler/src/durable/emit.ts b/packages/reconciler/src/durable/emit.ts deleted file mode 100644 index 6cc14a9..0000000 --- a/packages/reconciler/src/durable/emit.ts +++ /dev/null @@ -1,64 +0,0 @@ -import type { - OperationLifecycleEvent, - OperationLifecycleStatus, - OperationName, - ReconcilerEventEmitter, -} from "../events"; -import type { DurableStep } from "./yieldstar"; - -/** Checkpoints delivery after the emitter returns. Emitters must tolerate a duplicate if the process crashes before that checkpoint. */ -export function emitEvent( - step: DurableStep, - key: string, - emit: ReconcilerEventEmitter | undefined, - event: () => Parameters[0], -) { - return step.run(key, async () => { - await emit?.(event()); - }); -} - -export function emitLifecycle( - step: DurableStep, - key: string, - emit: ReconcilerEventEmitter | undefined, - operation: OperationName, - status: OperationLifecycleStatus, - resource: LifecycleResource, - extra: { reason?: string; error?: unknown } = {}, -) { - return emitEvent(step, key, emit, () => - createLifecycleEvent(operation, status, resource, extra), - ); -} - -type LifecycleResource = { - id: string; - type: OperationLifecycleEvent["resourceType"]; -}; - -export function createLifecycleEvent( - operation: OperationName, - status: OperationLifecycleStatus, - resource: LifecycleResource, - extra: { reason?: string; error?: unknown } = {}, -): OperationLifecycleEvent { - const error = extra.error; - const details = - error === undefined - ? {} - : error instanceof Error - ? { errorName: error.name, errorMessage: error.message } - : { errorName: "UnknownError", errorMessage: String(error) }; - - return { - level: status === "error" ? "error" : "info", - event: "reconciler.operation.lifecycle", - operation, - status, - resourceId: resource.id, - resourceType: resource.type, - ...(extra.reason ? { reason: extra.reason } : {}), - ...details, - }; -} diff --git a/packages/reconciler/src/durable/operations.ts b/packages/reconciler/src/durable/operations.ts index d945403..e439df1 100644 --- a/packages/reconciler/src/durable/operations.ts +++ b/packages/reconciler/src/durable/operations.ts @@ -1,380 +1,107 @@ -import { - ResourceNotFoundError, - type BaseResource, - type ResourceType, -} from "@notation/resource"; -import { RevConflict } from "@notation/state"; +import type { BaseResource, ResourceType } from "@notation/resource"; +import { RevConflict, type StateNode } from "@notation/state"; import { createMissingResourceRegistryMatchWarningEvent, createResourceRegistryFromResources, resolveResourceClass, } from "../resource-registry"; -import { runPendingOperation } from "../operations/operation.pending"; -import { decideAction, type DriftRead, type ResourceAction } from "../plan"; -import { emitEvent, emitLifecycle } from "./emit"; +import { + createResourceOperation, + deleteResourceOperation, + readDriftOperation, + updateResourceOperation, + type PersistState, + type RemoveState, + type ResourceOperationBaseParams, +} from "../operations"; +import { decideAction, type ResourceAction } from "../plan"; import type { DurableStateBackend } from "./state-backend"; +import { durableEmitter, scopeStep } from "./step"; import { resourceStateStore, toStateNode, - type StoredResourceState, + type ResourceSnapshot, } from "./stores"; import type { DurableDeployOptions, DurableOperationOptions } from "./types"; -import type { DurableStep, WorkflowStore } from "./yieldstar"; +import type { DurableStep } from "./yieldstar"; export async function* reconcileResource( step: DurableStep, resource: BaseResource, opts: DurableDeployOptions, ): AsyncGenerator { - const prefix = `notation:resource:${resource.id}`; + const scope = scopeStep(step, `notation:resource:${resource.id}`); + const emit = durableEmitter(scope, opts.emit); - // Hydrate the resource from persisted state. The snapshot is kept so later - // writes can be conditional on the exact instance identity and version that - // was read here. - let stateNode = yield* step.run(`${prefix}:state:lookup`, () => - opts.state.get(resource.id), - ); - let stateStore: WorkflowStore | undefined; - let snapshot: - Awaited> | undefined; - if (stateNode) { - stateStore = yield* openResourceState(step, opts.state, resource.id); - snapshot = yield* stateStore.get(`${prefix}:state:get`); - stateNode = toStateNode(snapshot); - } - if (stateNode) resource.setOutput(stateNode.output); + const { stateNode, snapshot } = yield* hydrateResource(scope, resource, opts); + const shared = operationParams(scope, resource, opts, stateNode); // Decide the operation from desired params vs persisted state. - const params = yield* step.run(`${prefix}:params`, () => - resource.getParams(), - ); - let action: ResourceAction = decideAction({ - resource, - stateNode: stateNode ?? undefined, - params, - }); + const params = yield* scope.run("params", () => resource.getParams()); + let action: ResourceAction = decideAction({ resource, stateNode, params }); // A noop is only trusted after the remote is read back: the provider may // have drifted from persisted state, which upgrades the decision. if (action.decision === "noop" && (opts.driftDetection ?? true)) { - const remote = yield* readRemote( - step, - resource, - opts, - `${prefix}:drift-read`, + // Its own scope: the operation that follows reads the remote again, and + // the two reads must not share step keys. + const driftScope = scopeStep(scope, "drift-read"); + const driftRead = yield* readDriftOperation( + driftScope, + operationParams(driftScope, resource, opts, stateNode), ); - action = decideAction({ - resource, - stateNode: stateNode ?? undefined, - params, - driftRead: remote, - }); + action = decideAction({ resource, stateNode, params, driftRead }); } if (action.decision === "drift-update") { - const diff = action.patch; - yield* emitEvent(step, `${prefix}:drift-detected`, opts.emit, () => ({ + yield* emit({ level: "info", event: "reconciler.drift.detected", resourceId: resource.id, resourceType: resource.type, - diff, - })); + diff: action.patch, + }); } - yield* emitEvent(step, `${prefix}:decision`, opts.emit, () => ({ + yield* emit({ level: "info", event: "reconciler.deploy.decision", resourceId: resource.id, resourceType: resource.type, decision: action.decision, - })); + }); if (action.decision === "noop") return; - const operation = - action.decision === "create" || action.decision === "drift-recreate" - ? "create" - : "update"; - const patch = "patch" in action ? action.patch : {}; - yield* emitLifecycle( - step, - `${prefix}:${operation}:start`, - opts.emit, - operation, - "start", - resource, - ); - if (opts.dryRun) { - yield* emitLifecycle( - step, - `${prefix}:${operation}:dry-run`, - opts.emit, - operation, - "dry-run", - resource, - ); + + const persist = persistResourceState(scope, opts, resource, snapshot); + if (action.decision === "create" || action.decision === "drift-recreate") { + yield* createResourceOperation(scope, { ...shared, persist }); return; } - try { - // Checkpoint successful provider calls. Provider mutations must be - // idempotent because a crash before the checkpoint can repeat them. - if (operation === "create") { - const primaryKey = yield* runPendingOperation( - step, - `${prefix}:create`, - (context) => resource.create(params, context), - opts.maxOperationAttempts, - ); - resource.setOutput(params); - if (primaryKey) resource.setOutput({ ...primaryKey, ...resource.output }); - } else { - if (!resource.update) { - yield* emitLifecycle( - step, - `${prefix}:update:skip`, - opts.emit, - "update", - "skip", - resource, - { reason: "update-not-implemented" }, - ); - return; - } - yield* runPendingOperation( - step, - `${prefix}:update`, - (context) => - resource.update!( - resource.key, - patch, - params, - resource.toState(resource.output), - context, - ), - opts.maxOperationAttempts, - ); - resource.setOutput({ ...resource.key, ...params }); - } - - // Read back the remote so persisted output reflects provider-assigned - // values, then persist conditionally against the snapshot read above. - const read = yield* readResource( - step, - resource, - opts, - `${prefix}:read-after-write`, - ); - resource.setOutput({ ...resource.output, ...read }); - - const nextState: StoredResourceState = { - id: resource.id, - groupId: resource.groupId, - groupType: resource.groupType, - type: resource.type, - lastOperation: operation, - lastOperationAt: new Date().toISOString(), - config: resource.config, - params: resource.toState(params), - output: resource.toState(resource.output), - }; - - if (!stateStore || !snapshot) { - yield* step.store(resourceStateStore, { - id: opts.state.storeId(resource.id), - initial: nextState, - }); - } else { - const result = yield* stateStore.updateFrom( - `${prefix}:state:persist`, - snapshot, - () => nextState, - ); - if (!result.updated) - throw new RevConflict( - resource.id, - stateNode?.rev ?? 0, - result.actualVersion + 1, - ); - } - - yield* emitLifecycle( - step, - `${prefix}:${operation}:success`, - opts.emit, - operation, - "success", - resource, - ); - } catch (error) { - yield* emitLifecycle( - step, - `${prefix}:${operation}:error`, - opts.emit, - operation, - "error", - resource, - { error }, - ); - throw error; - } + yield* updateResourceOperation(scope, { + ...shared, + patch: action.patch, + persist, + }); } export async function* deleteResource( step: DurableStep, resource: BaseResource, opts: DurableOperationOptions, - suffix: string, ): AsyncGenerator { - const prefix = `notation:${suffix}:${resource.id}`; - // Hydrate output from persisted state; the delete call needs the primary // key and the state removal must be conditional on this exact snapshot. - const stateStore = yield* openResourceState(step, opts.state, resource.id); - const snapshot = yield* stateStore.get(`${prefix}:state:get`); + const snapshot = yield* readSnapshot(step, opts.state, resource.id); + if (!snapshot) return; const stateNode = toStateNode(snapshot); resource.setOutput(stateNode.output); - yield* emitLifecycle( - step, - `${prefix}:delete:start`, - opts.emit, - "delete", - "start", - resource, - ); - - if (opts.dryRun) { - yield* emitLifecycle( - step, - `${prefix}:delete:dry-run`, - opts.emit, - "delete", - "dry-run", - resource, - ); - return; - } - - try { - yield* runPendingOperation( - step, - `${prefix}:delete`, - (context) => - resource.delete( - resource.key, - resource.toState(resource.output), - context, - ), - opts.maxOperationAttempts, - ); - - // State is removed only after the provider delete completes, and only if - // the store still matches the snapshot read before deleting. - const deleted = yield* stateStore.deleteFrom( - `${prefix}:state:delete`, - snapshot, - ); - if (!deleted.deleted) - throw new RevConflict(resource.id, stateNode.rev, undefined); - yield* emitLifecycle( - step, - `${prefix}:delete:success`, - opts.emit, - "delete", - "success", - resource, - ); - } catch (error) { - yield* emitLifecycle( - step, - `${prefix}:delete:error`, - opts.emit, - "delete", - "error", - resource, - { error }, - ); - throw error; - } -} - -export async function* readRemote( - step: DurableStep, - resource: BaseResource, - opts: DurableOperationOptions, - key: string, -): AsyncGenerator { - try { - return { - kind: "present", - output: yield* readResource(step, resource, opts, key), - }; - } catch (error) { - if (ResourceNotFoundError.is(error)) return { kind: "absent" }; - throw error; - } -} - -async function* readResource( - step: DurableStep, - resource: BaseResource, - opts: DurableOperationOptions, - key: string, -): AsyncGenerator, any> { - if (!resource.read) { - yield* emitLifecycle( - step, - `${key}:skip`, - opts.emit, - "read", - "skip", - resource, - { - reason: "read-not-implemented", - }, - ); - return { ...(await resource.getParams()), ...resource.output }; - } - - yield* emitLifecycle( - step, - `${key}:start`, - opts.emit, - "read", - "start", - resource, - ); - try { - const result = yield* runPendingOperation( - step, - key, - (context) => resource.read!(resource.key, context), - opts.maxOperationAttempts, - ); - yield* emitLifecycle( - step, - `${key}:success`, - opts.emit, - "read", - "success", - resource, - ); - return result; - } catch (error) { - yield* emitLifecycle( - step, - `${key}:error`, - opts.emit, - "read", - "error", - resource, - { - error, - }, - ); - throw error; - } + yield* deleteResourceOperation(step, { + ...operationParams(step, resource, opts, stateNode), + remove: removeResourceState(step, opts, resource, snapshot), + }); } /** @@ -385,28 +112,25 @@ async function* readResource( export async function* sweepOrphans( step: DurableStep, opts: DurableOperationOptions, - params: { - workflow: "deploy" | "destroy"; - listKey: string; - warningKey: (nodeId: string) => string; - deleteSuffix: string; - }, + workflow: "deploy" | "destroy", ): AsyncGenerator { const resourceById = new Map( opts.resources.map((resource) => [resource.id, resource]), ); - const persisted = yield* step.run(params.listKey, () => opts.state.values()); + const persisted = yield* step.run("list", () => opts.state.values()); const registry = opts.registry ?? createResourceRegistryFromResources(opts.resources); for (const node of persisted) { if (resourceById.has(node.id)) continue; + const nodeScope = scopeStep(step, node.id); const Resource = resolveResourceClass(registry, node.type as ResourceType); if (!Resource) { - yield* emitEvent(step, params.warningKey(node.id), opts.emit, () => + const emit = durableEmitter(nodeScope, opts.emit); + yield* emit( createMissingResourceRegistryMatchWarningEvent({ - workflow: params.workflow, + workflow, resourceId: node.id, resourceType: node.type as ResourceType, }), @@ -416,16 +140,113 @@ export async function* sweepOrphans( const resource = new Resource({ id: node.id, config: node.config }); resource.setOutput(node.output); - yield* deleteResource(step, resource, opts, params.deleteSuffix); + yield* deleteResource(nodeScope, resource, opts); } } -export function openResourceState( +/** + * Reads the persisted record once. The snapshot is kept so that later writes + * can be made conditional on the exact instance identity and version read + * here, and is re-served to the operations so they need no second read. + */ +async function* hydrateResource( + step: DurableStep, + resource: BaseResource, + opts: DurableOperationOptions, +): AsyncGenerator< + any, + { stateNode?: StateNode; snapshot?: ResourceSnapshot }, + any +> { + const snapshot = yield* readSnapshot(step, opts.state, resource.id); + if (!snapshot) return {}; + + const stateNode = toStateNode(snapshot); + resource.setOutput(stateNode.output); + return { stateNode, snapshot }; +} + +function readSnapshot( step: DurableStep, state: DurableStateBackend, resourceId: string, -) { - return step.store(resourceStateStore, { - id: state.storeId(resourceId), - }); +): AsyncGenerator { + return step.run("state:snapshot", () => state.snapshot(resourceId)); +} + +/** The half of the operation params every durable driver call site shares. */ +function operationParams( + step: DurableStep, + resource: BaseResource, + opts: DurableOperationOptions, + stateNode: StateNode | undefined, +): ResourceOperationBaseParams { + return { + resource, + // Serve the record already read during hydration rather than reading it + // again; a workflow must see the same value on every replay anyway. + state: { get: async () => stateNode }, + dryRun: opts.dryRun, + emit: durableEmitter(step, opts.emit), + maxOperationAttempts: opts.maxOperationAttempts, + }; +} + +/** + * State writes go through the workflow store, never through the state backend: + * the store stamps the write with the step that made it, so the applied-step + * ledger and the state change commit together. Replaying then returns the + * recorded result instead of retrying a compare-and-set that would now fail. + */ +function persistResourceState( + step: DurableStep, + opts: DurableOperationOptions, + resource: BaseResource, + snapshot: ResourceSnapshot | undefined, +): PersistState { + return async function* (next) { + if (!snapshot) { + // Create-if-absent. A racing writer would win here and this record would + // be silently adopted rather than written, which is safe only because a + // deployment is held exclusively for the length of the workflow. + yield* step.store(resourceStateStore, { + id: opts.state.storeId(resource.id), + initial: next, + }); + return; + } + + const store = yield* step.store(resourceStateStore, { + id: opts.state.storeId(resource.id), + }); + const result = yield* store.updateFrom( + "state:persist", + snapshot, + () => next, + ); + if (!result.updated) { + throw new RevConflict( + resource.id, + snapshot.version + 1, + result.actualVersion + 1, + ); + } + }; +} + +function removeResourceState( + step: DurableStep, + opts: DurableOperationOptions, + resource: BaseResource, + snapshot: ResourceSnapshot, +): RemoveState { + return async function* () { + const store = yield* step.store(resourceStateStore, { + id: opts.state.storeId(resource.id), + }); + const result = yield* store.deleteFrom("state:delete", snapshot); + if (!result.deleted) { + throw new RevConflict(resource.id, snapshot.version + 1, undefined); + } + }; } diff --git a/packages/reconciler/src/durable/state-backend.ts b/packages/reconciler/src/durable/state-backend.ts index 88fa35c..6e1130d 100644 --- a/packages/reconciler/src/durable/state-backend.ts +++ b/packages/reconciler/src/durable/state-backend.ts @@ -1,14 +1,20 @@ -import { RevConflict, type StateNode } from "@notation/state"; -import { randomUUID } from "node:crypto"; +import type { StateNode } from "@notation/state"; import { - RESOURCE_CREATION_TOKEN, resourceStateStore, toStateNode, - withoutRev, - type StoredResourceState, + type ResourceSnapshot, } from "./stores"; import type { StoreClient } from "./yieldstar"; +/** + * Reads deployment state from outside a workflow, for the planner and for + * anything reporting on a deployment. + * + * Deliberately read-only. Writes belong to the workflow, which makes them + * through the store handle so they are stamped with the step that made them. + * There is nowhere on this interface to carry that idempotency key, so a + * write made here would be repeated on replay rather than recognised. + */ export class DurableStateBackend { readonly #client: StoreClient; readonly #prefix: string; @@ -25,95 +31,16 @@ export class DurableStateBackend { } async get(id: string): Promise { - const snapshot = await this.#tryGetSnapshot(this.storeId(id)); + const snapshot = await this.snapshot(id); return snapshot ? toStateNode(snapshot) : undefined; } - async #tryGetSnapshot( - storeId: string, - ): Promise< - | { state: StoredResourceState; instanceId: string; version: number } - | undefined - > { - try { - return await this.#client.getStore({ - definition: resourceStateStore, - id: storeId, - }); - } catch (error) { - const ids = await this.#client.listStores(resourceStateStore); - if (!ids.includes(storeId)) return undefined; - throw error; - } - } - async has(id: string): Promise { return (await this.get(id)) !== undefined; } - async update( - id: string, - expectedRev: number, - patch: Partial, - ): Promise<{ rev: number }> { - const storeId = this.storeId(id); - const snapshot = await this.#tryGetSnapshot(storeId); - if (!snapshot) { - if (expectedRev !== 0) throw new RevConflict(id, expectedRev, undefined); - const creationToken = randomUUID(); - const initial = { - ...withoutRev(patch), - id, - [RESOURCE_CREATION_TOKEN]: creationToken, - } as StoredResourceState; - const created = await this.#client.getOrCreateStore({ - definition: resourceStateStore, - id: storeId, - initial, - }); - if (created.state[RESOURCE_CREATION_TOKEN] !== creationToken) { - throw new RevConflict(id, expectedRev, created.version + 1); - } - return { rev: created.version + 1 }; - } - - const actualRev = snapshot.version + 1; - if (actualRev !== expectedRev) - throw new RevConflict(id, expectedRev, actualRev); - const result = await this.#client.updateStoreFrom({ - definition: resourceStateStore, - id: storeId, - snapshot, - updater: (draft) => { - Object.assign(draft, withoutRev(patch)); - }, - }); - if (!result.updated) - throw new RevConflict(id, expectedRev, result.actualVersion + 1); - return { rev: result.version + 1 }; - } - - async delete(id: string, expectedRev: number): Promise { - const storeId = this.storeId(id); - const snapshot = await this.#tryGetSnapshot(storeId); - if (!snapshot) { - if (expectedRev !== 0) throw new RevConflict(id, expectedRev, undefined); - return; - } - const actualRev = snapshot.version + 1; - if (actualRev !== expectedRev) - throw new RevConflict(id, expectedRev, actualRev); - const result = await this.#client.deleteStoreFrom({ - definition: resourceStateStore, - id: storeId, - snapshot, - }); - if (!result.deleted) - throw new RevConflict( - id, - expectedRev, - result.reason === "conflict" ? result.actualVersion + 1 : undefined, - ); + snapshot(id: string): Promise { + return this.#read(this.storeId(id)); } async values(): Promise { @@ -121,43 +48,26 @@ export class DurableStateBackend { const snapshots = await Promise.all( ids .filter((id) => id.startsWith(this.#prefix)) - .map((id) => this.#tryGetSnapshot(id)), + .map((id) => this.#read(id)), ); return snapshots .filter((snapshot) => snapshot !== undefined) .map(toStateNode); } - snapshot(id: string) { - return this.#client.getStore({ - definition: resourceStateStore, - id: this.storeId(id), - }); - } - - async clear(): Promise { - const ids = await this.#client.listStores(resourceStateStore); - const scopedIds = ids.filter((id) => id.startsWith(this.#prefix)); - const snapshots = await Promise.all( - scopedIds.map((id) => this.#tryGetSnapshot(id)), - ); - await Promise.all( - scopedIds.map(async (id, index) => { - const snapshot = snapshots[index]; - if (!snapshot) return; - const result = await this.#client.deleteStoreFrom({ - definition: resourceStateStore, - id, - snapshot, - }); - if (!result.deleted && result.reason === "conflict") { - throw new RevConflict( - id.slice(this.#prefix.length), - snapshot.version + 1, - result.actualVersion + 1, - ); - } - }), - ); + // getStore throws for a store that does not exist rather than returning + // undefined, and the error is not distinguishable from a real failure, so + // absence is confirmed by listing. Kept here so no caller has to know that. + async #read(storeId: string): Promise { + try { + return await this.#client.getStore({ + definition: resourceStateStore, + id: storeId, + }); + } catch (error) { + const ids = await this.#client.listStores(resourceStateStore); + if (!ids.includes(storeId)) return undefined; + throw error; + } } } diff --git a/packages/reconciler/src/durable/step.ts b/packages/reconciler/src/durable/step.ts new file mode 100644 index 0000000..ad52ade --- /dev/null +++ b/packages/reconciler/src/durable/step.ts @@ -0,0 +1,84 @@ +import type { + EmitStep, + ReconcilerEvent, + ReconcilerEventEmitter, +} from "../events"; +import type { DurableStep, WorkflowStore } from "./yieldstar"; + +/** + * Namespaces the step keys of `step` so an operation can be written once and + * replayed at several call sites without its keys colliding. + * + * Opening a store is not prefixed: yieldstar derives that key from the store + * name and store id, which is already unique. The keys the store *handle* + * takes are caller-supplied, so those are scoped like any other step. + */ +export function scopeStep(step: DurableStep, prefix: string): DurableStep { + const scoped = (key: string) => `${prefix}:${key}`; + + return { + ...step, + // The keyless overloads fall through untouched; yieldstar hashes the call + // site for those, and a prefix would not make them any more unique. + run: ((arg1: unknown, arg2?: unknown) => + typeof arg1 === "string" + ? (step.run as any)(scoped(arg1), arg2) + : (step.run as any)(arg1)) as DurableStep["run"], + delay: ((arg1: unknown, arg2?: unknown) => + typeof arg1 === "string" + ? (step.delay as any)(scoped(arg1), arg2) + : (step.delay as any)(arg1)) as DurableStep["delay"], + store: ((definition: any, params: any) => + (async function* () { + const store = yield* step.store(definition, params); + return scopeStore(store, prefix); + })()) as DurableStep["store"], + }; +} + +/** + * Checkpoints delivery so that replaying a workflow does not re-emit. The key + * is derived from the event itself, which keeps it deterministic across a + * replay; the enclosing scope is what keeps it unique, since an operation + * emits each (operation, status) pair at most once. + * + * Emitters must still tolerate a duplicate: the process can crash after the + * event is delivered but before the checkpoint is written. + */ +export function durableEmitter( + step: DurableStep, + emit: ReconcilerEventEmitter | undefined, +): EmitStep { + return async function* (event) { + if (!emit) return; + yield* step.run(emitKey(event), () => emit(event)); + }; +} + +function emitKey(event: ReconcilerEvent): string { + return event.event === "reconciler.operation.lifecycle" + ? `emit:${event.event}:${event.operation}:${event.status}` + : `emit:${event.event}`; +} + +function scopeStore( + store: WorkflowStore, + prefix: string, +): WorkflowStore { + const scoped = (key: string) => `${prefix}:${key}`; + + return { + ...store, + get: (key?: string) => store.get(key === undefined ? key : scoped(key)), + select: (key, selector) => store.select(scoped(key), selector), + update: (key, updater) => store.update(scoped(key), updater), + updateFrom: (key, snapshot, updater) => + store.updateFrom(scoped(key), snapshot, updater), + deleteFrom: (key, snapshot) => store.deleteFrom(scoped(key), snapshot), + when: ((arg1: any, arg2?: any) => + typeof arg1 === "string" + ? store.when(scoped(arg1), arg2) + : store.when(arg1)) as WorkflowStore["when"], + take: (key, selector, claim) => store.take(scoped(key), selector, claim), + }; +} diff --git a/packages/reconciler/src/durable/stores.ts b/packages/reconciler/src/durable/stores.ts index d25f3ab..aca56fd 100644 --- a/packages/reconciler/src/durable/stores.ts +++ b/packages/reconciler/src/durable/stores.ts @@ -1,8 +1,6 @@ import type { StateNode } from "@notation/state"; import * as v from "valibot"; -import { defineStore } from "./yieldstar"; - -export const RESOURCE_CREATION_TOKEN = "$notationCreateToken"; +import { defineStore, type StoreSnapshot } from "./yieldstar"; export const resourceStateStore = defineStore( "resource-state", @@ -14,7 +12,6 @@ export const resourceStateStore = defineStore( output: v.record(v.string(), v.unknown()), lastOperation: v.picklist(["drift", "create", "update", "delete"]), lastOperationAt: v.string(), - [RESOURCE_CREATION_TOKEN]: v.optional(v.string()), }), ); @@ -30,18 +27,10 @@ export type CoordinationState = v.InferOutput< typeof deploymentCoordinationStore.schema >; -export function toStateNode(snapshot: { - state: StoredResourceState; - version: number; -}): StateNode { - const { [RESOURCE_CREATION_TOKEN]: _creationToken, ...state } = - snapshot.state; - return { ...state, rev: snapshot.version + 1 }; -} +/** A read of a resource record, carrying the identity a write is made against. */ +export type ResourceSnapshot = StoreSnapshot; -export function withoutRev( - patch: Partial, -): Partial { - const { rev: _rev, ...stored } = patch; - return stored; +/** Store versions count from zero, state revisions from one. */ +export function toStateNode(snapshot: ResourceSnapshot): StateNode { + return { ...snapshot.state, rev: snapshot.version + 1 }; } diff --git a/packages/reconciler/src/durable/yieldstar.ts b/packages/reconciler/src/durable/yieldstar.ts index 6bbba63..afa9871 100644 --- a/packages/reconciler/src/durable/yieldstar.ts +++ b/packages/reconciler/src/durable/yieldstar.ts @@ -2,7 +2,7 @@ import type { WorkflowFn } from "yieldstar"; export { defineStore } from "yieldstar"; export type { WorkflowStore } from "yieldstar"; -export type { StoreClient } from "@yieldstar/core"; +export type { StoreClient, StoreSnapshot } from "@yieldstar/core"; /** The durable step primitive the runtime hands to workflow functions. */ export type DurableStep = Parameters>[0]; diff --git a/packages/reconciler/src/events.ts b/packages/reconciler/src/events.ts index 2cad300..46aacaf 100644 --- a/packages/reconciler/src/events.ts +++ b/packages/reconciler/src/events.ts @@ -1,4 +1,5 @@ import type { ResourceType } from "@notation/resource"; +import type { MissingResourceRegistryMatchWarningEvent } from "./resource-registry"; export type OperationName = "create" | "read" | "update" | "delete"; @@ -46,8 +47,27 @@ export type ReconcilerEvent = | CoordinationWaitingEvent | ReconcilerDeployEvent | ReconcilerDriftDetectedEvent - | import("./resource-registry").MissingResourceRegistryMatchWarningEvent; + | MissingResourceRegistryMatchWarningEvent; export type ReconcilerEventEmitter = ( event: ReconcilerEvent, ) => void | Promise; + +/** + * The seam a driver fills in to deliver an event. Emission is a step so that + * each driver decides how it is recorded: the in-process driver simply awaits + * the emitter, while the durable driver checkpoints it so that replaying a + * workflow does not re-emit events it has already delivered. + */ +export type EmitStep = ( + event: TEvent, +) => AsyncGenerator; + +/** Adapts a plain emitter to the driver seam, for drivers that just await. */ +export function toEmitStep( + emit: ((event: TEvent) => void | Promise) | undefined, +): EmitStep { + return async function* (event) { + await emit?.(event); + }; +} diff --git a/packages/reconciler/src/index.ts b/packages/reconciler/src/index.ts index 44fc402..b4e749a 100644 --- a/packages/reconciler/src/index.ts +++ b/packages/reconciler/src/index.ts @@ -6,7 +6,9 @@ export type YieldstarApi = typeof import("yieldstar"); export * from "./resource-registry"; export * from "./operations"; export * from "./dependency-graph"; +export * from "./events"; export * from "./plan"; +export * from "./planner"; export * from "./reconciler"; export * from "./logger-subscriber"; export * from "./protocol"; diff --git a/packages/reconciler/src/operations/operation.create.ts b/packages/reconciler/src/operations/operation.create.ts index e0aee4f..5cecb14 100644 --- a/packages/reconciler/src/operations/operation.create.ts +++ b/packages/reconciler/src/operations/operation.create.ts @@ -12,10 +12,10 @@ export async function* createResourceOperation( step: StepRunner, params: CreateResourceParams, ): AsyncGenerator { - await emitLifecycleEvent(params, "create", "start"); + yield* emitLifecycleEvent(params, "create", "start"); if (params.dryRun) { - await emitLifecycleEvent(params, "create", "dry-run"); + yield* emitLifecycleEvent(params, "create", "dry-run"); return; } @@ -51,23 +51,21 @@ export async function* createResourceOperation( ...readResult, }); - yield* step.run("create:persist-state", async () => { - await params.state.update(params.resource.id, params.expectedRev, { - id: params.resource.id, - groupId: params.resource.groupId, - groupType: params.resource.groupType, - type: params.resource.type, - lastOperation: "create", - lastOperationAt: new Date().toISOString(), - config: params.resource.config, - params: params.resource.toState(resourceParams), - output: params.resource.toState(params.resource.output), - }); + yield* params.persist({ + id: params.resource.id, + groupId: params.resource.groupId, + groupType: params.resource.groupType, + type: params.resource.type, + lastOperation: "create", + lastOperationAt: new Date().toISOString(), + config: params.resource.config, + params: params.resource.toState(resourceParams), + output: params.resource.toState(params.resource.output), }); - await emitLifecycleEvent(params, "create", "success"); + yield* emitLifecycleEvent(params, "create", "success"); } catch (err) { - await emitLifecycleEvent(params, "create", "error", getErrorDetails(err)); + yield* emitLifecycleEvent(params, "create", "error", getErrorDetails(err)); throw err; } } diff --git a/packages/reconciler/src/operations/operation.delete.ts b/packages/reconciler/src/operations/operation.delete.ts index 51975b6..170927f 100644 --- a/packages/reconciler/src/operations/operation.delete.ts +++ b/packages/reconciler/src/operations/operation.delete.ts @@ -11,10 +11,10 @@ export async function* deleteResourceOperation( step: StepRunner, params: DeleteResourceParams, ): AsyncGenerator { - await emitLifecycleEvent(params, "delete", "start"); + yield* emitLifecycleEvent(params, "delete", "start"); if (params.dryRun) { - await emitLifecycleEvent(params, "delete", "dry-run"); + yield* emitLifecycleEvent(params, "delete", "dry-run"); return; } @@ -31,13 +31,11 @@ export async function* deleteResourceOperation( params.maxOperationAttempts, ); - yield* step.run("delete:persist-state", () => - params.state.delete(params.resource.id, params.expectedRev), - ); + yield* params.remove(); - await emitLifecycleEvent(params, "delete", "success"); + yield* emitLifecycleEvent(params, "delete", "success"); } catch (err) { - await emitLifecycleEvent(params, "delete", "error", getErrorDetails(err)); + yield* emitLifecycleEvent(params, "delete", "error", getErrorDetails(err)); throw err; } } diff --git a/packages/reconciler/src/operations/operation.read.ts b/packages/reconciler/src/operations/operation.read.ts index e74196a..327e405 100644 --- a/packages/reconciler/src/operations/operation.read.ts +++ b/packages/reconciler/src/operations/operation.read.ts @@ -1,4 +1,6 @@ +import { ResourceNotFoundError } from "@notation/resource"; import { createWorkflow } from "yieldstar"; +import type { DriftRead } from "../plan"; import { type ReadResourceParams, type StepRunner, @@ -11,10 +13,10 @@ export async function* readResourceOperation( step: StepRunner, params: ReadResourceParams, ): AsyncGenerator, unknown> { - await emitLifecycleEvent(params, "read", "start"); + yield* emitLifecycleEvent(params, "read", "start"); if (params.dryRun) { - await emitLifecycleEvent(params, "read", "dry-run"); + yield* emitLifecycleEvent(params, "read", "dry-run"); return {}; } @@ -31,10 +33,10 @@ export async function* readResourceOperation( ? { ...stateNode.output, ...resourceParams } : resourceParams; - await emitLifecycleEvent(params, "read", "skip", { + yield* emitLifecycleEvent(params, "read", "skip", { reason: "read-not-implemented", }); - await emitLifecycleEvent(params, "read", "success"); + yield* emitLifecycleEvent(params, "read", "success"); return merged as Record; } @@ -50,14 +52,32 @@ export async function* readResourceOperation( ...remote, }; - await emitLifecycleEvent(params, "read", "success"); + yield* emitLifecycleEvent(params, "read", "success"); return mergedOutput; } catch (err) { - await emitLifecycleEvent(params, "read", "error", getErrorDetails(err)); + yield* emitLifecycleEvent(params, "read", "error", getErrorDetails(err)); throw err; } } +/** + * Reads the remote to compare it against persisted state. An absent resource + * is a fact about the world rather than a failure, so it is reported as such; + * every other error still propagates. + */ +export async function* readDriftOperation( + step: StepRunner, + params: ReadResourceParams, +): AsyncGenerator { + try { + const output = yield* readResourceOperation(step, params); + return { kind: "present", output }; + } catch (error) { + if (ResourceNotFoundError.is(error)) return { kind: "absent" }; + throw error; + } +} + export const readResourceWorkflow: unknown = createWorkflow( async function* (step, event) { return yield* readResourceOperation( diff --git a/packages/reconciler/src/operations/operation.types.ts b/packages/reconciler/src/operations/operation.types.ts index 6cbcd1c..5d9d640 100644 --- a/packages/reconciler/src/operations/operation.types.ts +++ b/packages/reconciler/src/operations/operation.types.ts @@ -1,26 +1,19 @@ -import type { BaseResource, ResourceType } from "@notation/resource"; -import type { State } from "@notation/state"; +import type { BaseResource } from "@notation/resource"; +import type { State, StateNode } from "@notation/state"; +import type { + EmitStep, + OperationLifecycleEvent, + OperationLifecycleStatus, + OperationName, +} from "../events"; -export type OperationName = "create" | "read" | "update" | "delete"; +export type { + OperationLifecycleEvent, + OperationLifecycleStatus, + OperationName, +} from "../events"; -export type OperationLifecycleStatus = - "start" | "success" | "error" | "skip" | "dry-run"; - -export type OperationLifecycleEvent = { - level: "info" | "error"; - event: "reconciler.operation.lifecycle"; - operation: OperationName; - status: OperationLifecycleStatus; - resourceId: string; - resourceType: ResourceType; - reason?: string; - errorName?: string; - errorMessage?: string; -}; - -export type OperationEventEmitter = ( - event: OperationLifecycleEvent, -) => void | Promise; +export type OperationEventEmitter = EmitStep; export type StepRunner = { run(fn: () => T | Promise): AsyncGenerator; @@ -32,27 +25,55 @@ export type StepRunner = { delay(key: string, ms: number): AsyncGenerator; }; +/** + * The record an operation wants persisted; the driver owns the revision. + * Spelled out rather than derived with Omit, which would collapse against + * StateNode's index signature and widen every field to unknown. + */ +export type PersistedResourceState = Pick< + StateNode, + | "id" + | "type" + | "config" + | "params" + | "output" + | "lastOperation" + | "lastOperationAt" +> & { [key: string]: unknown }; + +/** + * How a driver writes state. Both are steps so that each driver can carry its + * own concurrency control: in process that is a compare-and-set against the + * revision read before the operation, and in a workflow it is a store write + * stamped with the step that made it, so a replay does not repeat it. + */ +export type PersistState = ( + next: PersistedResourceState, +) => AsyncGenerator; + +export type RemoveState = () => AsyncGenerator; + export type ResourceOperationBaseParams = { resource: BaseResource; - state: Pick; + state: Pick; dryRun?: boolean; emit?: OperationEventEmitter; maxOperationAttempts?: number; }; export type CreateResourceParams = ResourceOperationBaseParams & { - expectedRev: number; + persist: PersistState; }; export type ReadResourceParams = ResourceOperationBaseParams; export type UpdateResourceParams = ResourceOperationBaseParams & { patch: Record; - expectedRev: number; + persist: PersistState; }; export type DeleteResourceParams = ResourceOperationBaseParams & { - expectedRev: number; + remove: RemoveState; }; export function getErrorDetails(err: unknown): { @@ -72,15 +93,15 @@ export function getErrorDetails(err: unknown): { }; } -export async function emitLifecycleEvent( +export async function* emitLifecycleEvent( params: ResourceOperationBaseParams, operation: OperationName, status: OperationLifecycleStatus, extra: Partial = {}, -) { +): AsyncGenerator { if (!params.emit) return; - await params.emit({ + yield* params.emit({ level: status === "error" ? "error" : "info", event: "reconciler.operation.lifecycle", operation, diff --git a/packages/reconciler/src/operations/operation.update.ts b/packages/reconciler/src/operations/operation.update.ts index fc62a8c..5f07609 100644 --- a/packages/reconciler/src/operations/operation.update.ts +++ b/packages/reconciler/src/operations/operation.update.ts @@ -12,18 +12,18 @@ export async function* updateResourceOperation( step: StepRunner, params: UpdateResourceParams, ): AsyncGenerator { - await emitLifecycleEvent(params, "update", "start"); + yield* emitLifecycleEvent(params, "update", "start"); if (params.dryRun) { - await emitLifecycleEvent(params, "update", "dry-run"); + yield* emitLifecycleEvent(params, "update", "dry-run"); return; } if (!params.resource.update) { - await emitLifecycleEvent(params, "update", "skip", { + yield* emitLifecycleEvent(params, "update", "skip", { reason: "update-not-implemented", }); - await emitLifecycleEvent(params, "update", "success"); + yield* emitLifecycleEvent(params, "update", "success"); return; } @@ -63,23 +63,21 @@ export async function* updateResourceOperation( ...readResult, }); - yield* step.run("update:persist-state", async () => { - await params.state.update(params.resource.id, params.expectedRev, { - id: params.resource.id, - groupId: params.resource.groupId, - groupType: params.resource.groupType, - type: params.resource.type, - lastOperation: "update", - lastOperationAt: new Date().toISOString(), - config: params.resource.config, - params: params.resource.toState(resourceParams), - output: params.resource.toState(params.resource.output), - }); + yield* params.persist({ + id: params.resource.id, + groupId: params.resource.groupId, + groupType: params.resource.groupType, + type: params.resource.type, + lastOperation: "update", + lastOperationAt: new Date().toISOString(), + config: params.resource.config, + params: params.resource.toState(resourceParams), + output: params.resource.toState(params.resource.output), }); - await emitLifecycleEvent(params, "update", "success"); + yield* emitLifecycleEvent(params, "update", "success"); } catch (err) { - await emitLifecycleEvent(params, "update", "error", getErrorDetails(err)); + yield* emitLifecycleEvent(params, "update", "error", getErrorDetails(err)); throw err; } } diff --git a/packages/reconciler/src/planner.ts b/packages/reconciler/src/planner.ts index ebfa064..ef7faca 100644 --- a/packages/reconciler/src/planner.ts +++ b/packages/reconciler/src/planner.ts @@ -1,7 +1,8 @@ -import { ResourceNotFoundError, type BaseResource } from "@notation/resource"; -import type { StateBackend } from "@notation/state"; +import type { BaseResource } from "@notation/resource"; +import type { State } from "@notation/state"; import { buildResourceDepthLevels } from "./dependency-graph"; -import { readResourceOperation } from "./operations"; +import { toEmitStep, type ReconcilerEventEmitter } from "./events"; +import { readDriftOperation } from "./operations"; import { decideAction, getDependencyIds, @@ -9,22 +10,30 @@ import { type Plan, type PlanNode, } from "./plan"; -import { createStepRunner, runOperation } from "./reconciler"; +import { createStepRunner, runOperation } from "./step-runner"; + +/** Planning reads state and never mutates it, so no lease is required. */ +export type PlannerState = Pick; export type CreatePlanOptions = { resources: BaseResource[]; - state: StateBackend; + state: PlannerState; driftDetection?: boolean; + emit?: ReconcilerEventEmitter; + maxOperationAttempts?: number; }; export async function createPlan({ resources, state, driftDetection = true, + emit, + maxOperationAttempts, }: CreatePlanOptions): Promise { const resourceById = new Map( resources.map((resource) => [resource.id, resource]), ); + const emitStep = emit ? toEmitStep(emit) : undefined; const nodes: PlanNode[] = []; for (const level of buildResourceDepthLevels(resources)) { @@ -35,19 +44,14 @@ export async function createPlan({ let action = decideAction({ resource, stateNode, params }); if (action.decision === "noop" && driftDetection && resource.read) { - let driftRead; - try { - const output = await runOperation( - readResourceOperation(createStepRunner(), { - resource, - state, - }), - ); - driftRead = { kind: "present" as const, output }; - } catch (error) { - if (!ResourceNotFoundError.is(error)) throw error; - driftRead = { kind: "absent" as const }; - } + const driftRead = await runOperation( + readDriftOperation(createStepRunner(), { + resource, + state, + emit: emitStep, + maxOperationAttempts, + }), + ); action = decideAction({ resource, diff --git a/packages/reconciler/src/protocol.ts b/packages/reconciler/src/protocol.ts index bf3706e..a8ab407 100644 --- a/packages/reconciler/src/protocol.ts +++ b/packages/reconciler/src/protocol.ts @@ -1,4 +1,4 @@ -import type { ReconcilerEvent, ReconcilerEventEmitter } from "./reconciler"; +import type { ReconcilerEvent, ReconcilerEventEmitter } from "./events"; export const EVENT_STREAM_VERSION = 1 as const; diff --git a/packages/reconciler/src/reconciler.ts b/packages/reconciler/src/reconciler.ts index fab6c0b..8db798b 100644 --- a/packages/reconciler/src/reconciler.ts +++ b/packages/reconciler/src/reconciler.ts @@ -1,4 +1,3 @@ -import { ResourceNotFoundError } from "@notation/resource"; import type { BaseResource, ResourceType } from "@notation/resource"; import { RevConflict, type State, type StateNode } from "@notation/state"; import { setTimeout as sleep } from "node:timers/promises"; @@ -15,8 +14,10 @@ import { import { createResourceOperation, deleteResourceOperation, - readResourceOperation, - type OperationLifecycleEvent, + readDriftOperation, + type OperationEventEmitter, + type PersistState, + type RemoveState, type StepRunner, updateResourceOperation, } from "./operations"; @@ -24,35 +25,19 @@ import { createMissingResourceRegistryMatchWarningEvent, createResourceRegistryFromResources, resolveResourceClass, - type MissingResourceRegistryMatchWarningEvent, type ResourceRegistry, } from "./resource-registry"; - -export type ReconcilerDeployEvent = { - level: "info"; - event: "reconciler.deploy.decision"; - resourceId: string; - resourceType: string; - decision: "create" | "update" | "drift-update" | "drift-recreate" | "noop"; -}; - -export type ReconcilerDriftDetectedEvent = { - level: "info"; - event: "reconciler.drift.detected"; - resourceId: string; - resourceType: string; - diff: Record; -}; - -export type ReconcilerEvent = - | OperationLifecycleEvent - | ReconcilerDeployEvent - | ReconcilerDriftDetectedEvent - | MissingResourceRegistryMatchWarningEvent; - -export type ReconcilerEventEmitter = ( - event: ReconcilerEvent, -) => void | Promise; +import { toEmitStep, type ReconcilerEventEmitter } from "./events"; +import { createPlan } from "./planner"; +import { createStepRunner, runOperation } from "./step-runner"; + +export type { + ReconcilerDeployEvent, + ReconcilerDriftDetectedEvent, + ReconcilerEvent, + ReconcilerEventEmitter, +} from "./events"; +export { createStepRunner, runOperation } from "./step-runner"; export type ReconcilerState = Pick< State, @@ -92,6 +77,7 @@ export class Reconciler { readonly #defaultDryRun: boolean; readonly #defaultDriftDetection: boolean; readonly #emit?: ReconcilerEventEmitter; + readonly #emitStep?: OperationEventEmitter; readonly #maxOperationAttempts?: number; readonly #mutationLeaseTtl: number; readonly #stepRunner: StepRunner; @@ -102,6 +88,8 @@ export class Reconciler { this.#defaultDryRun = opts.dryRun ?? false; this.#defaultDriftDetection = opts.driftDetection ?? true; this.#emit = opts.emit; + // Operations emit through a step seam; in process that is a plain await. + this.#emitStep = opts.emit ? toEmitStep(opts.emit) : undefined; this.#maxOperationAttempts = opts.maxOperationAttempts; this.#mutationLeaseTtl = opts.mutationLeaseTtl ?? 30_000; this.#stepRunner = createStepRunner(); @@ -130,36 +118,13 @@ export class Reconciler { } async plan(resources: BaseResource[], opts: PlanOptions = {}): Promise { - const driftDetection = opts.driftDetection ?? this.#defaultDriftDetection; - const resourceById = new Map( - resources.map((resource) => [resource.id, resource]), - ); - const nodes: PlanNode[] = []; - - const dependencyLevels = buildResourceDepthLevels(resources); - for (const level of dependencyLevels) { - for (const resource of level) { - nodes.push(await this.#planResource(resource, driftDetection)); - } - } - - const stateNodes = await this.#state.values(); - for (const stateNode of stateNodes) { - if (resourceById.has(stateNode.id)) continue; - - nodes.push({ - id: stateNode.id, - type: stateNode.type, - decision: "delete-orphan", - params: stateNode.params, - dependsOn: [], - }); - } - - return { - createdAt: new Date().toISOString(), - nodes, - }; + return createPlan({ + resources, + state: this.#state, + driftDetection: opts.driftDetection ?? this.#defaultDriftDetection, + emit: this.#emit, + maxOperationAttempts: this.#maxOperationAttempts, + }); } async destroy( @@ -310,9 +275,9 @@ export class Reconciler { resource, state: this.#state, dryRun, - emit: this.#emit, + emit: this.#emitStep, maxOperationAttempts: this.#maxOperationAttempts, - expectedRev: stateNode?.rev ?? 0, + persist: this.#persist(resource.id, stateNode?.rev ?? 0), }), ); return; @@ -325,9 +290,9 @@ export class Reconciler { state: this.#state, patch: action.patch, dryRun, - emit: this.#emit, + emit: this.#emitStep, maxOperationAttempts: this.#maxOperationAttempts, - expectedRev: stateNode!.rev, + persist: this.#persist(resource.id, stateNode!.rev), }), ); return; @@ -372,9 +337,9 @@ export class Reconciler { resource, state: this.#state, dryRun, - emit: this.#emit, + emit: this.#emitStep, maxOperationAttempts: this.#maxOperationAttempts, - expectedRev: stateNode?.rev ?? 0, + persist: this.#persist(resource.id, stateNode?.rev ?? 0), }), ); return; @@ -386,9 +351,9 @@ export class Reconciler { state: this.#state, patch: action.patch, dryRun, - emit: this.#emit, + emit: this.#emitStep, maxOperationAttempts: this.#maxOperationAttempts, - expectedRev: stateNode?.rev ?? 0, + persist: this.#persist(resource.id, stateNode?.rev ?? 0), }), ); return; @@ -409,48 +374,31 @@ export class Reconciler { } } - async #planResource( - resource: BaseResource, - driftDetection: boolean, - ): Promise { - const stateNode = await this.#state.get(resource.id); - if (stateNode) { - resource.setOutput(stateNode.output); - } - - const params = await resolvePlanParams(resource); - let action = decideAction({ resource, stateNode, params }); - - if (action.decision === "noop" && driftDetection) { - const driftRead = await this.#readForDrift(resource); - action = decideAction({ resource, stateNode, params, driftRead }); - } + // In process, concurrency control is a compare-and-set against the revision + // read before the operation started; the mutation lease keeps writers apart. + #persist(resourceId: string, expectedRev: number): PersistState { + const state = this.#state; + return async function* (next) { + await state.update(resourceId, expectedRev, next); + }; + } - return { - id: resource.id, - type: resource.type, - decision: action.decision, - ...("diff" in action ? { diff: action.diff } : {}), - params, - dependsOn: getDependencyIds(resource), + #remove(resourceId: string, expectedRev: number): RemoveState { + const state = this.#state; + return async function* () { + await state.delete(resourceId, expectedRev); }; } - async #readForDrift(resource: BaseResource): Promise { - try { - const output = await runOperation( - readResourceOperation(this.#stepRunner, { - resource, - state: this.#state, - emit: this.#emit, - maxOperationAttempts: this.#maxOperationAttempts, - }), - ); - return { kind: "present", output }; - } catch (error) { - if (ResourceNotFoundError.is(error)) return { kind: "absent" }; - throw error; - } + #readForDrift(resource: BaseResource): Promise { + return runOperation( + readDriftOperation(this.#stepRunner, { + resource, + state: this.#state, + emit: this.#emitStep, + maxOperationAttempts: this.#maxOperationAttempts, + }), + ); } async #deleteOrphans( @@ -539,24 +487,14 @@ export class Reconciler { resource, state: this.#state, dryRun, - emit: this.#emit, + emit: this.#emitStep, maxOperationAttempts: this.#maxOperationAttempts, - expectedRev: stateNode.rev, + remove: this.#remove(resource.id, stateNode.rev), }), ); } } -export async function runOperation( - operation: AsyncGenerator, -) { - let next = await operation.next(); - while (!next.done) { - next = await operation.next(); - } - return next.value; -} - function hydrateResourceFromState( Resource: new (opts: { id: string; @@ -571,32 +509,3 @@ function hydrateResourceFromState( resource.setOutput(stateNode.output); return resource; } - -export function createStepRunner(): StepRunner { - return { - async *run( - arg1: string | (() => T | Promise), - arg2?: () => T | Promise, - ): AsyncGenerator { - const fn = (typeof arg1 === "string" ? arg2 : arg1) as - (() => T | Promise) | undefined; - - if (!fn) { - throw new Error("Missing run function"); - } - - return await fn(); - }, - async *delay( - arg1: string | number, - arg2?: number, - ): AsyncGenerator { - const ms = typeof arg1 === "number" ? arg1 : arg2; - if (ms === undefined) { - throw new Error("Missing delay duration"); - } - - await new Promise((resolve) => setTimeout(resolve, ms)); - }, - }; -} diff --git a/packages/reconciler/src/step-runner.ts b/packages/reconciler/src/step-runner.ts new file mode 100644 index 0000000..61cc6d5 --- /dev/null +++ b/packages/reconciler/src/step-runner.ts @@ -0,0 +1,40 @@ +import type { StepRunner } from "./operations"; + +export async function runOperation( + operation: AsyncGenerator, +) { + let next = await operation.next(); + while (!next.done) { + next = await operation.next(); + } + return next.value; +} + +export function createStepRunner(): StepRunner { + return { + async *run( + arg1: string | (() => T | Promise), + arg2?: () => T | Promise, + ): AsyncGenerator { + const fn = (typeof arg1 === "string" ? arg2 : arg1) as + (() => T | Promise) | undefined; + + if (!fn) { + throw new Error("Missing run function"); + } + + return await fn(); + }, + async *delay( + arg1: string | number, + arg2?: number, + ): AsyncGenerator { + const ms = typeof arg1 === "number" ? arg1 : arg2; + if (ms === undefined) { + throw new Error("Missing delay duration"); + } + + await new Promise((resolve) => setTimeout(resolve, ms)); + }, + }; +} diff --git a/packages/reconciler/test/durable-reconciliation.test.ts b/packages/reconciler/test/durable-reconciliation.test.ts index 1a51cd7..96e2ee9 100644 --- a/packages/reconciler/test/durable-reconciliation.test.ts +++ b/packages/reconciler/test/durable-reconciliation.test.ts @@ -75,7 +75,7 @@ describe("durable execution and replay", () => { const runtime = createRuntime( [new TestResource({ id: "resume" })], "crash-resume", - { crashAfterStep: "notation:resource:resume:create:attempt:0" }, + { crashAfterStep: "notation:resource:resume:create:remote:attempt:0" }, ); await expect(runtime.run("resume-execution")).rejects.toThrow( @@ -98,7 +98,7 @@ describe("durable execution and replay", () => { const runtime = createRuntime( [new TestResource({ id: "destroyed" })], "destroy-crash-resume", - { crashAfterStep: "notation:destroy:destroyed:delete:attempt:0" }, + { crashAfterStep: "notation:destroy:destroyed:delete:remote:attempt:0" }, ); await runtime.run("deploy-before-destroy"); @@ -242,69 +242,75 @@ describe("dependency ordering", () => { }); describe("conditional state persistence", () => { - it("allows only one concurrent create-if-absent", async () => { - const runtime = createRuntime([], "conditional-create"); - const first = runtime.state.update("resource", 0, { - ...statePatch("resource"), - rev: 41, - output: { winner: "first" }, - }); - const second = runtime.state.update("resource", 0, { - ...statePatch("resource"), - rev: 42, - output: { winner: "second" }, - }); + it("rejects a state write whose snapshot another writer has moved past", async () => { + const RaceResource = resource({ type: "test/durable/write-race" }) + .defineSchema({ + name: { + presence: "required", + propertyType: "param", + valueType: "string" as any, + }, + }) + .defineOperations({ + create: async () => undefined, + // Moves the store on between the workflow reading its snapshot and + // persisting against it, which is what the conditional write guards. + update: async () => { + await runtime.storeClient.updateStore({ + definition: durable.resourceStateStore, + id: runtime.state.storeId("raced"), + updater: (draft: any) => { + draft.lastOperationAt = "1999-01-01T00:00:00.000Z"; + }, + }); + }, + delete: async () => undefined, + }); + const resources = [ + new RaceResource({ id: "raced", config: { name: "before" } }), + ]; + const runtime = createRuntime(resources, "write-race"); - const results = await Promise.allSettled([first, second]); + await runtime.run("deploy-1"); + resources[0] = new RaceResource({ id: "raced", config: { name: "after" } }); - expect( - results.filter((result) => result.status === "fulfilled"), - ).toHaveLength(1); - expect( - results.filter((result) => result.status === "rejected"), - ).toHaveLength(1); - const state = await runtime.state.get("resource"); - expect(state).toMatchObject({ rev: 1 }); - expect(state).not.toHaveProperty("$notationCreateToken"); + await expect(runtime.run("deploy-2")).rejects.toMatchObject({ + name: "RevConflict", + }); + // The losing write left the other writer's record intact. + expect(await runtime.state.get("raced")).toMatchObject({ + rev: 2, + lastOperationAt: "1999-01-01T00:00:00.000Z", + }); runtime.close(); }); - it("uses store identity and version for conditional update and delete", async () => { - const runtime = createRuntime([], "conditional-state"); - await runtime.state.update("resource", 0, statePatch("resource")); - const originalSnapshot = await runtime.state.snapshot("resource"); - - const first = runtime.state.update("resource", 1, { - output: { winner: "first" }, - }); - const second = runtime.state.update("resource", 1, { - output: { winner: "second" }, - }); - const results = await Promise.allSettled([first, second]); + it("rejects a state removal whose snapshot another writer has moved past", async () => { + const RaceResource = resource({ type: "test/durable/delete-race" }) + .defineSchema({}) + .defineOperations({ + create: async () => undefined, + delete: async () => { + await runtime.storeClient.updateStore({ + definition: durable.resourceStateStore, + id: runtime.state.storeId("delete-raced"), + updater: (draft: any) => { + draft.lastOperationAt = "1999-01-01T00:00:00.000Z"; + }, + }); + }, + }); + const runtime = createRuntime( + [new RaceResource({ id: "delete-raced" })], + "delete-race", + ); - expect( - results.filter((result) => result.status === "fulfilled"), - ).toHaveLength(1); - expect( - results.filter((result) => result.status === "rejected"), - ).toHaveLength(1); - await expect(runtime.state.delete("resource", 1)).rejects.toMatchObject({ + await runtime.run("deploy-1"); + await expect(runtime.destroy("destroy-1")).rejects.toMatchObject({ name: "RevConflict", }); - expect(await runtime.state.get("resource")).toMatchObject({ rev: 2 }); - - await runtime.state.clear(); - await runtime.state.update("resource", 0, statePatch("resource")); - const staleDelete = await runtime.storeClient.deleteStoreFrom({ - definition: durable.resourceStateStore, - id: runtime.state.storeId("resource"), - snapshot: originalSnapshot, - }); - expect(staleDelete).toMatchObject({ - deleted: false, - reason: "conflict", - }); - expect(await runtime.state.get("resource")).toMatchObject({ rev: 1 }); + // State survives a removal that could not be proven safe. + expect(await runtime.state.get("delete-raced")).toBeDefined(); runtime.close(); }); }); @@ -403,16 +409,17 @@ describe("deployment scoping", () => { const app = new durable.DurableStateBackend(storeClient, "app"); const appBlue = new durable.DurableStateBackend(storeClient, "app:blue"); - await app.update("site", 0, statePatch("site")); - await appBlue.update("site", 0, statePatch("site")); - - expect(await app.values()).toHaveLength(1); - expect(await appBlue.values()).toHaveLength(1); - - await app.clear(); - expect(await app.values()).toHaveLength(0); - expect(await appBlue.values()).toHaveLength(1); - expect(await appBlue.get("site")).toBeDefined(); + await seedResourceState(storeClient, app.storeId("site"), "site"); + await seedResourceState(storeClient, appBlue.storeId("site"), "blue-site"); + + // A deployment named "app:blue" falls inside a naive "app:" prefix scan; + // encoding the deployment id is what keeps the two listings disjoint. + expect((await app.values()).map((node) => node.id)).toEqual(["site"]); + expect((await appBlue.values()).map((node) => node.id)).toEqual([ + "blue-site", + ]); + expect(await app.get("site")).toMatchObject({ id: "site" }); + expect(await appBlue.get("site")).toMatchObject({ id: "blue-site" }); database.close(); }); }); @@ -613,6 +620,18 @@ class CrashAfterWriteHeap implements HeapClient { } } +function seedResourceState( + storeClient: SqliteStoreClient, + storeId: string, + resourceId: string, +) { + return storeClient.getOrCreateStore({ + definition: durable.resourceStateStore, + id: storeId, + initial: statePatch(resourceId), + }); +} + function statePatch(id: string) { return { id, diff --git a/packages/reconciler/test/operation.workflows.test.ts b/packages/reconciler/test/operation.workflows.test.ts index 2afc6df..a721a29 100644 --- a/packages/reconciler/test/operation.workflows.test.ts +++ b/packages/reconciler/test/operation.workflows.test.ts @@ -11,6 +11,7 @@ import { type OperationLifecycleEvent, type StepRunner, } from "../src/operations"; +import { toEmitStep } from "../src/events"; function createStepRunnerDouble(): StepRunner { const run = vi.fn(async function* ( @@ -51,11 +52,8 @@ describe("operation workflows", () => { it("create performs create + read-after-create + state persistence", async () => { const step = createStepRunnerDouble(); const events: OperationLifecycleEvent[] = []; - const state = { - get: vi.fn(async () => undefined), - update: vi.fn(async () => undefined), - delete: vi.fn(async () => undefined), - }; + const state = { get: vi.fn(async () => undefined) }; + const persist = vi.fn(async function* () {}); let createAttempts = 0; const createMock = vi.fn(async (_params, context) => { @@ -86,15 +84,25 @@ describe("operation workflows", () => { createResourceOperation(step, { resource: testResource, state, - expectedRev: 0, - emit: async (event) => { - events.push(event); - }, + persist, + emit: toEmitStep((event) => void events.push(event)), }), ); expect(createAttempts).toBe(2); - expect(state.update).toHaveBeenCalledOnce(); + expect(persist).toHaveBeenCalledOnce(); + expect(persist).toHaveBeenCalledWith({ + id: "test-create", + groupId: testResource.groupId, + groupType: testResource.groupType, + type: TestResource.type, + lastOperation: "create", + lastOperationAt: expect.any(String), + config: testResource.config, + params: {}, + // The resource declares no schema, so nothing survives toState. + output: {}, + }); expect(createMock).toHaveBeenNthCalledWith( 1, await testResource.getParams(), @@ -231,11 +239,8 @@ describe("operation workflows", () => { it("delete treats an already-absent remote as success through its idempotent resource contract", async () => { const step = createStepRunnerDouble(); const events: OperationLifecycleEvent[] = []; - const state = { - get: vi.fn(async () => undefined), - update: vi.fn(async () => undefined), - delete: vi.fn(async () => undefined), - }; + const state = { get: vi.fn(async () => undefined) }; + const remove = vi.fn(async function* () {}); const TestResource = resource({ type: "test/service/delete" }) .defineSchema({}) @@ -250,14 +255,14 @@ describe("operation workflows", () => { deleteResourceOperation(step, { resource: testResource, state, - expectedRev: 1, - emit: async (event) => { - events.push(event); - }, + remove, + emit: toEmitStep((event) => void events.push(event)), }), ); - expect(state.delete).toHaveBeenCalledWith("test-delete", 1); + // State is removed only after the provider delete resolves; which record + // and revision that targets is the driver's concern, not the operation's. + expect(remove).toHaveBeenCalledOnce(); expect(events.map((event) => event.status)).toEqual(["start", "success"]); }); @@ -326,9 +331,7 @@ describe("operation workflows", () => { resource: testResource, state, expectedRev: 0, - emit: async (event) => { - events.push(event); - }, + emit: toEmitStep((event) => void events.push(event)), }), ), ).rejects.toMatchObject({ name: "CreateFailed", message: "boom" }); diff --git a/packages/reconciler/tsup.config.ts b/packages/reconciler/tsup.config.ts index f0ac238..2834ee4 100644 --- a/packages/reconciler/tsup.config.ts +++ b/packages/reconciler/tsup.config.ts @@ -1,7 +1,7 @@ import { defineConfig } from "tsup"; export default defineConfig({ - entry: ["src/index.ts"], + entry: ["src/index.ts", "src/durable/index.ts"], dts: true, format: ["esm"], }); From 551466a52ccf73e4919efa60f742093a9e83814a Mon Sep 17 00:00:00 2001 From: djgrant <1670902+djgrant@users.noreply.github.com> Date: Mon, 27 Jul 2026 19:15:33 +0100 Subject: [PATCH 10/34] Stop the durable retry tests racing the workflow loop A delay whose deadline has already passed is continued inline rather than suspended, so a one millisecond retry could resume inside the execution that scheduled it once the heap write outlived it. The tests then saw two provider calls where they expected the workflow to have parked after one. Names the interval and puts it comfortably beyond a SQLite write. The assertions are unchanged: the provider is still called once before the workflow parks, and once more after it resumes. --- .../test/durable-reconciliation.test.ts | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/packages/reconciler/test/durable-reconciliation.test.ts b/packages/reconciler/test/durable-reconciliation.test.ts index 96e2ee9..4cf80cd 100644 --- a/packages/reconciler/test/durable-reconciliation.test.ts +++ b/packages/reconciler/test/durable-reconciliation.test.ts @@ -27,6 +27,15 @@ import { const logger = pino({ level: "silent" }); +/** + * A retry delay has to outlive the heap write that follows it. The workflow + * loop continues inline for a delay that has already elapsed by the time it + * is reached, so a delay shorter than a SQLite write runs the retry in the + * same execution — which is the opposite of what these tests assert. + */ +const RETRY_AFTER_MS = 50; +const PAST_RETRY_MS = RETRY_AFTER_MS + 25; + describe("durable execution and replay", () => { it("waits durably for a retryable provider and persists after success", async () => { let attempts = 0; @@ -38,7 +47,7 @@ describe("durable execution and replay", () => { if (attempts === 1) { expect(context).toBeUndefined(); throw new ResourceOperationPendingError("provider is not ready", { - retryAfterMs: 1, + retryAfterMs: RETRY_AFTER_MS, callbackContext: { requestId: "request-123" }, }); } @@ -56,7 +65,7 @@ describe("durable execution and replay", () => { expect(attempts).toBe(1); expect(runtime.scheduler.events).toHaveLength(1); - await sleep(5); + await sleep(PAST_RETRY_MS); await runtime.run("wait-execution"); expect(attempts).toBe(2); expect(await runtime.state.get("pending")).toMatchObject({ @@ -124,7 +133,7 @@ describe("durable execution and replay", () => { attempts += 1; if (attempts === 1) { throw new ResourceOperationPendingError("delete is not ready", { - retryAfterMs: 1, + retryAfterMs: RETRY_AFTER_MS, }); } }, @@ -140,7 +149,7 @@ describe("durable execution and replay", () => { expect(attempts).toBe(1); expect(await runtime.state.get("pending-delete")).toBeDefined(); - await sleep(5); + await sleep(PAST_RETRY_MS); await runtime.destroy("destroy-wait"); expect(attempts).toBe(2); expect(await runtime.state.get("pending-delete")).toBeUndefined(); @@ -160,7 +169,7 @@ describe("durable execution and replay", () => { if (reads === 1) { throw new ResourceOperationPendingError( "resource is not visible yet", - { retryAfterMs: 1 }, + { retryAfterMs: RETRY_AFTER_MS }, ); } return {} as const; @@ -177,7 +186,7 @@ describe("durable execution and replay", () => { expect(reads).toBe(1); expect(await runtime.state.get("eventually-readable")).toBeUndefined(); - await sleep(5); + await sleep(PAST_RETRY_MS); await runtime.run("post-write-read-execution"); expect(reads).toBe(2); expect(await runtime.state.get("eventually-readable")).toMatchObject({ From 1fb03c4d4826112dd92a8d548bc71c4d26e52ead Mon Sep 17 00:00:00 2001 From: djgrant <1670902+djgrant@users.noreply.github.com> Date: Mon, 27 Jul 2026 20:40:30 +0100 Subject: [PATCH 11/34] Release the deployment hold only on success MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The hold was released in a finally, so every failure mode released it — except a crash, where the runner abandons the workflow generator without calling .throw() and the finally never runs. That split matters because yieldstar caches steps per execution: a resumed execution replays `take` from the cache without re-acquiring, and replays `release` as a cached no-op, so a resumption that followed the release path proceeds holding nothing while it mutates the deployment. Unify every failure mode on the crash path's semantics. The hold is released in exactly one place, after the body completes. An execution that will never resume now holds its deployment forever, which is why takeOverDeploymentHold lands with it: force-clearing the coordination store through storeClient.updateStore is unconditional and undocumented, so without a named, compare-holder takeover this commit would turn any transient provider error into a wedge with no runbook. The test cannot be written against a failing step: a cached StepError is rethrown before the step's function runs again, so the resumption would never reach uncached work. It fails in plain generator code between two resources instead, and asserts the holder observed from inside the second resource's create — null before this change, the execution id after it. --- .../reconciler/src/durable/coordination.ts | 73 +++++++++++++++++-- packages/reconciler/src/durable/index.ts | 4 + .../test/durable-reconciliation.test.ts | 45 ++++++++++++ 3 files changed, 116 insertions(+), 6 deletions(-) diff --git a/packages/reconciler/src/durable/coordination.ts b/packages/reconciler/src/durable/coordination.ts index 5c7af65..094e6f3 100644 --- a/packages/reconciler/src/durable/coordination.ts +++ b/packages/reconciler/src/durable/coordination.ts @@ -1,7 +1,7 @@ import type { ReconcilerEventEmitter } from "../events"; import { durableEmitter, scopeStep } from "./step"; import { deploymentCoordinationStore, type CoordinationState } from "./stores"; -import type { DurableStep, WorkflowStore } from "./yieldstar"; +import type { DurableStep, StoreClient, WorkflowStore } from "./yieldstar"; type CoordinationOptions = { deploymentId: string; @@ -54,16 +54,77 @@ function releaseDeploymentCoordination( }); } -/** Runs `body` while holding the deployment, releasing it even on error. */ +/** + * Runs `body` while holding the deployment, releasing the hold only once + * `body` has completed. A failed or suspended execution keeps the hold, which + * is what makes it safe to resume: the resumed execution replays `take` from + * the step cache and so never re-acquires anything, so a hold released on the + * way out would leave the resumption mutating a deployment it does not hold. + * + * The cost is that an execution which will never be resumed holds its + * deployment indefinitely. That is deliberate — nothing here can tell "will + * retry" from "abandoned" — and it is resolved by an operator calling + * `takeOverDeploymentHold`. + */ export async function* withDeploymentHold( step: DurableStep, opts: CoordinationOptions, body: () => AsyncGenerator, ): AsyncGenerator { const coordination = yield* acquireDeploymentCoordination(step, opts); - try { - yield* body(); - } finally { - yield* releaseDeploymentCoordination(coordination, opts.executionId); + yield* body(); + yield* releaseDeploymentCoordination(coordination, opts.executionId); +} + +export type DeploymentHoldTakeover = + | { taken: true; previousHolder: string } + | { taken: false; holder: string | null }; + +/** + * Clears a deployment hold left by an execution that will not be resumed, so + * that later deployments are not blocked behind it. Named separately from the + * workflow path because it is the only supported way out of that state: the + * hold is otherwise released solely by an execution completing. + * + * The write is conditional on `fromExecutionId` still being the named holder, + * so it cannot clear a hold that has since been released and re-taken by + * another execution. Confirm the holder is genuinely dead first: it may still + * be mid-flight, and taking its hold away permits a concurrent mutation of + * the same deployment. + * + * Throws if the deployment has no coordination store, i.e. if it has never + * been deployed. + */ +export async function takeOverDeploymentHold(params: { + storeClient: StoreClient; + deploymentId: string; + fromExecutionId: string; + toExecutionId?: string | null; +}): Promise { + const { storeClient, deploymentId, fromExecutionId } = params; + const read = () => + storeClient.getStore({ + definition: deploymentCoordinationStore, + id: deploymentId, + }); + + const snapshot = await read(); + if (snapshot.state.holder !== fromExecutionId) { + return { taken: false, holder: snapshot.state.holder }; + } + + const result = await storeClient.updateStoreFrom({ + definition: deploymentCoordinationStore, + id: deploymentId, + snapshot, + updater: (draft) => { + draft.holder = params.toExecutionId ?? null; + }, + }); + + if (!result.updated) { + return { taken: false, holder: (await read()).state.holder }; } + + return { taken: true, previousHolder: fromExecutionId }; } diff --git a/packages/reconciler/src/durable/index.ts b/packages/reconciler/src/durable/index.ts index c896eb8..74c9a22 100644 --- a/packages/reconciler/src/durable/index.ts +++ b/packages/reconciler/src/durable/index.ts @@ -1,5 +1,9 @@ export { deploy } from "./deploy"; export { destroy } from "./destroy"; +export { + takeOverDeploymentHold, + type DeploymentHoldTakeover, +} from "./coordination"; export { DurableStateBackend } from "./state-backend"; export { deploymentCoordinationStore, diff --git a/packages/reconciler/test/durable-reconciliation.test.ts b/packages/reconciler/test/durable-reconciliation.test.ts index 4cf80cd..23e65b8 100644 --- a/packages/reconciler/test/durable-reconciliation.test.ts +++ b/packages/reconciler/test/durable-reconciliation.test.ts @@ -364,6 +364,51 @@ describe("deployment coordination", () => { runtime.close(); }); + it("still holds the deployment when a failed execution is resumed", async () => { + // The failure has to live in plain generator code. A step that fails + // caches a StepError, and a cached StepError is rethrown on replay before + // the step's function is reached, so no later work would ever run + // uncached. decideAction calls toComparable outside any step, after the + // resource's reads have been checkpointed. + let failBeforeSecond = true; + const holders: Array = []; + const Resource = resource({ type: "test/durable/hold-replay" }) + .defineSchema({}) + .defineOperations({ + create: async () => { + const snapshot = await runtime.storeClient.getStore({ + definition: durable.deploymentCoordinationStore, + id: "hold-replay", + }); + holders.push(snapshot.state.holder); + }, + delete: async () => undefined, + }); + + const first = new Resource({ id: "first" }); + const second = new Resource({ id: "second" }); + const toComparable = second.toComparable.bind(second); + second.toComparable = (output) => { + if (failBeforeSecond) throw new Error("simulated mid-deployment failure"); + return toComparable(output); + }; + const runtime = createRuntime([first, second], "hold-replay"); + + await expect(runtime.run("replayed-execution")).rejects.toThrow( + "simulated mid-deployment failure", + ); + expect(holders).toEqual(["replayed-execution"]); + + failBeforeSecond = false; + await runtime.run("replayed-execution"); + + // The second resource's create is the first uncached work after the + // failure, so it observes whichever hold the resumed execution is running + // under. Releasing on the way out of a failure would leave it null here. + expect(holders).toEqual(["replayed-execution", "replayed-execution"]); + runtime.close(); + }); + it("emits a coordination waiting event when another execution holds the deployment", async () => { let unblockCreate!: () => void; const blocked = new Promise((resolve) => { From 355c3634d8cd885b7e308ed904c25abf2f48479e Mon Sep 17 00:00:00 2001 From: djgrant <1670902+djgrant@users.noreply.github.com> Date: Mon, 27 Jul 2026 20:41:19 +0100 Subject: [PATCH 12/34] Delete the unused per-operation workflow exports createResourceWorkflow, readResourceWorkflow, updateResourceWorkflow and deleteResourceWorkflow had no consumers, and each was typed `unknown`, so nothing could have called one without casting anyway. They were also the only runtime yieldstar import reachable from the base entry point, which made the durable runtime look like a dependency of the in-process driver. YieldstarApi goes with them: `export type`, so never a runtime import, and equally unused. operation.workflows.test.ts exercises the operations directly despite its name, so it keeps its coverage. After this, the only "yieldstar" imports under src are in durable/. --- packages/reconciler/src/index.ts | 1 - packages/reconciler/src/operations/operation.create.ts | 10 ---------- packages/reconciler/src/operations/operation.delete.ts | 10 ---------- packages/reconciler/src/operations/operation.read.ts | 10 ---------- packages/reconciler/src/operations/operation.update.ts | 10 ---------- 5 files changed, 41 deletions(-) diff --git a/packages/reconciler/src/index.ts b/packages/reconciler/src/index.ts index b4e749a..cdc37c8 100644 --- a/packages/reconciler/src/index.ts +++ b/packages/reconciler/src/index.ts @@ -1,7 +1,6 @@ export type ResourceApi = typeof import("@notation/resource"); export type StateApi = typeof import("@notation/state"); export type DeepObjectDiffApi = typeof import("deep-object-diff"); -export type YieldstarApi = typeof import("yieldstar"); export * from "./resource-registry"; export * from "./operations"; diff --git a/packages/reconciler/src/operations/operation.create.ts b/packages/reconciler/src/operations/operation.create.ts index 5cecb14..2a1c8d1 100644 --- a/packages/reconciler/src/operations/operation.create.ts +++ b/packages/reconciler/src/operations/operation.create.ts @@ -1,4 +1,3 @@ -import { createWorkflow } from "yieldstar"; import { type CreateResourceParams, type StepRunner, @@ -69,12 +68,3 @@ export async function* createResourceOperation( throw err; } } - -export const createResourceWorkflow: unknown = createWorkflow( - async function* (step, event) { - return yield* createResourceOperation( - step as StepRunner, - event.params as CreateResourceParams, - ); - }, -); diff --git a/packages/reconciler/src/operations/operation.delete.ts b/packages/reconciler/src/operations/operation.delete.ts index 170927f..3bb4e8b 100644 --- a/packages/reconciler/src/operations/operation.delete.ts +++ b/packages/reconciler/src/operations/operation.delete.ts @@ -1,4 +1,3 @@ -import { createWorkflow } from "yieldstar"; import { type DeleteResourceParams, type StepRunner, @@ -39,12 +38,3 @@ export async function* deleteResourceOperation( throw err; } } - -export const deleteResourceWorkflow: unknown = createWorkflow( - async function* (step, event) { - return yield* deleteResourceOperation( - step as StepRunner, - event.params as DeleteResourceParams, - ); - }, -); diff --git a/packages/reconciler/src/operations/operation.read.ts b/packages/reconciler/src/operations/operation.read.ts index 327e405..306af17 100644 --- a/packages/reconciler/src/operations/operation.read.ts +++ b/packages/reconciler/src/operations/operation.read.ts @@ -1,5 +1,4 @@ import { ResourceNotFoundError } from "@notation/resource"; -import { createWorkflow } from "yieldstar"; import type { DriftRead } from "../plan"; import { type ReadResourceParams, @@ -77,12 +76,3 @@ export async function* readDriftOperation( throw error; } } - -export const readResourceWorkflow: unknown = createWorkflow( - async function* (step, event) { - return yield* readResourceOperation( - step as StepRunner, - event.params as ReadResourceParams, - ); - }, -); diff --git a/packages/reconciler/src/operations/operation.update.ts b/packages/reconciler/src/operations/operation.update.ts index 5f07609..84698ee 100644 --- a/packages/reconciler/src/operations/operation.update.ts +++ b/packages/reconciler/src/operations/operation.update.ts @@ -1,4 +1,3 @@ -import { createWorkflow } from "yieldstar"; import { type StepRunner, type UpdateResourceParams, @@ -81,12 +80,3 @@ export async function* updateResourceOperation( throw err; } } - -export const updateResourceWorkflow: unknown = createWorkflow( - async function* (step, event) { - return yield* updateResourceOperation( - step as StepRunner, - event.params as UpdateResourceParams, - ); - }, -); From 2e84f7029429efdd8211cb46532a0900666b12f4 Mon Sep 17 00:00:00 2001 From: djgrant <1670902+djgrant@users.noreply.github.com> Date: Mon, 27 Jul 2026 20:42:04 +0100 Subject: [PATCH 13/34] Declare groupId and groupType in the durable state schema Both are persisted by create and update, but the schema was loose, so neither appeared in StoredResourceState and nothing checked that a durable record carried the same fields as an in-process one. The store stays loose. PersistedResourceState deliberately carries an index signature, and v.object strips unknown entries on parse, so tightening the object would silently drop forward-compatible fields at the store boundary. The schema is a floor on the record, not a description of it. Validation runs on write rather than read, so records persisted before this commit are still readable; the next write to one supplies both fields. The test fixture that seeds a record directly now supplies them. --- packages/reconciler/src/durable/stores.ts | 11 +++++++++++ packages/reconciler/src/operations/operation.types.ts | 7 ++++++- .../reconciler/test/durable-reconciliation.test.ts | 2 ++ 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/packages/reconciler/src/durable/stores.ts b/packages/reconciler/src/durable/stores.ts index aca56fd..c5f03b5 100644 --- a/packages/reconciler/src/durable/stores.ts +++ b/packages/reconciler/src/durable/stores.ts @@ -2,11 +2,22 @@ import type { StateNode } from "@notation/state"; import * as v from "valibot"; import { defineStore, type StoreSnapshot } from "./yieldstar"; +/** + * Loose on purpose: PersistedResourceState carries an index signature so a + * driver can persist fields this schema does not name yet, and v.object would + * strip those at the store boundary rather than carry them forward. Every + * field an operation actually writes is still declared, so the schema is a + * floor on the record rather than a description of one. + */ export const resourceStateStore = defineStore( "resource-state", v.looseObject({ id: v.string(), type: v.string(), + // Written by create and update. -1 and "" are BaseResource's defaults for + // a resource that belongs to no group. + groupId: v.number(), + groupType: v.string(), config: v.record(v.string(), v.unknown()), params: v.record(v.string(), v.unknown()), output: v.record(v.string(), v.unknown()), diff --git a/packages/reconciler/src/operations/operation.types.ts b/packages/reconciler/src/operations/operation.types.ts index 5d9d640..ad11676 100644 --- a/packages/reconciler/src/operations/operation.types.ts +++ b/packages/reconciler/src/operations/operation.types.ts @@ -39,7 +39,12 @@ export type PersistedResourceState = Pick< | "output" | "lastOperation" | "lastOperationAt" -> & { [key: string]: unknown }; +> & { + // Not on StateNode itself, where they arrive through its index signature. + groupId: number; + groupType: string; + [key: string]: unknown; +}; /** * How a driver writes state. Both are steps so that each driver can carry its diff --git a/packages/reconciler/test/durable-reconciliation.test.ts b/packages/reconciler/test/durable-reconciliation.test.ts index 23e65b8..595ab82 100644 --- a/packages/reconciler/test/durable-reconciliation.test.ts +++ b/packages/reconciler/test/durable-reconciliation.test.ts @@ -690,6 +690,8 @@ function statePatch(id: string) { return { id, type: "test/durable/state", + groupId: -1, + groupType: "", config: {}, params: {}, output: {}, From 7ab8201453a4688f4f8b146e5088448af5dc7f9d Mon Sep 17 00:00:00 2001 From: djgrant <1670902+djgrant@users.noreply.github.com> Date: Mon, 27 Jul 2026 20:49:47 +0100 Subject: [PATCH 14/34] Freeze an operation's inputs before it starts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Operations resolved two things for themselves that the driver had already decided against: the desired params, through resource.getParams(), and the persisted record, through a `state: Pick` seam that existed for a single read in operation.read.ts. Both are now passed in. getParams is resolved once per scheduled reconciliation — conflict recovery included, so recovery compares the remote against the same params the failed attempt decided on — and persistedOutput replaces the state seam, which also deletes the fake `state: { get: async () => stateNode }` adapter the durable driver had to construct. This is not a live correctness fix. getParams spreads config over deriveParams(id, config, deps) and never reads this.output, so repeated calls do not diverge today. The win is that deriveParams is user code: a non-deterministic one would let the emitted decision diff and the persisted params record disagree with nothing able to reconcile them. Deletion keeps resolving params lazily, on the recovery path only, since that is the sole consumer there and an orphan hydrated from state should not be asked to derive params just to be deleted. Three construction sites changed: reconciler.ts, durable/operations.ts and planner.ts, which already had both the state node and the resolved plan params in hand and was resolving params a second time inside the drift read. Also adds the typecheck script the package never had, which is how the call sites still passing the long-deleted expectedRev field surfaced. Making the package typecheck exposed two more things: the durable store schema now requires groupId and groupType that PersistedResourceState did not declare (declared here, since StateNode only carries them through its index signature), and a batch of test fixtures that were never type checked. The fixture casts are annotated: a resource declared without API types cannot express a named schema key at all. --- .../test/provisioner/operation.create.test.ts | 6 +- packages/reconciler/package.json | 1 + packages/reconciler/src/durable/operations.ts | 26 ++++--- .../src/operations/operation.create.ts | 7 +- .../src/operations/operation.read.ts | 11 +-- .../src/operations/operation.types.ts | 20 +++-- .../src/operations/operation.update.ts | 7 +- packages/reconciler/src/planner.ts | 3 +- packages/reconciler/src/reconciler.ts | 73 ++++++++++++++----- .../test/durable-reconciliation.test.ts | 22 +++--- .../test/operation.workflows.test.ts | 67 ++++++----------- .../reconciler/test/reconciler.deploy.test.ts | 22 +++--- .../reconciler/test/reconciler.plan.test.ts | 40 +++++----- .../reconciler/test/resource-registry.test.ts | 4 +- 14 files changed, 162 insertions(+), 147 deletions(-) diff --git a/packages/core/test/provisioner/operation.create.test.ts b/packages/core/test/provisioner/operation.create.test.ts index 2e4f506..84ed0e3 100644 --- a/packages/core/test/provisioner/operation.create.test.ts +++ b/packages/core/test/provisioner/operation.create.test.ts @@ -34,8 +34,10 @@ describe("resource creation", () => { await runOperation( createResourceOperation(step, { resource: testResource, - state: stateBackend, - expectedRev: 0, + resourceParams: await testResource.getParams(), + persist: async function* (next) { + await stateBackend.update(testResource.id, 0, next); + }, }), ); diff --git a/packages/reconciler/package.json b/packages/reconciler/package.json index 58b0ab0..c36012c 100644 --- a/packages/reconciler/package.json +++ b/packages/reconciler/package.json @@ -19,6 +19,7 @@ ], "scripts": { "build": "tsup --clean", + "typecheck": "tsc --noEmit", "dev": "tsup --watch" }, "dependencies": { diff --git a/packages/reconciler/src/durable/operations.ts b/packages/reconciler/src/durable/operations.ts index e439df1..fca0036 100644 --- a/packages/reconciler/src/durable/operations.ts +++ b/packages/reconciler/src/durable/operations.ts @@ -34,10 +34,17 @@ export async function* reconcileResource( const emit = durableEmitter(scope, opts.emit); const { stateNode, snapshot } = yield* hydrateResource(scope, resource, opts); - const shared = operationParams(scope, resource, opts, stateNode); - // Decide the operation from desired params vs persisted state. + // Decide the operation from desired params vs persisted state. The params + // are resolved once and then carried: deriveParams is user code and need + // not be deterministic, so an operation resolving them again could persist + // params other than the ones the decision was taken against. const params = yield* scope.run("params", () => resource.getParams()); + const shared = { + ...operationParams(scope, resource, opts), + resourceParams: params, + persistedOutput: stateNode?.output, + }; let action: ResourceAction = decideAction({ resource, stateNode, params }); // A noop is only trusted after the remote is read back: the provider may @@ -46,10 +53,11 @@ export async function* reconcileResource( // Its own scope: the operation that follows reads the remote again, and // the two reads must not share step keys. const driftScope = scopeStep(scope, "drift-read"); - const driftRead = yield* readDriftOperation( - driftScope, - operationParams(driftScope, resource, opts, stateNode), - ); + const driftRead = yield* readDriftOperation(driftScope, { + ...operationParams(driftScope, resource, opts), + resourceParams: params, + persistedOutput: stateNode?.output, + }); action = decideAction({ resource, stateNode, params, driftRead }); } @@ -99,7 +107,7 @@ export async function* deleteResource( resource.setOutput(stateNode.output); yield* deleteResourceOperation(step, { - ...operationParams(step, resource, opts, stateNode), + ...operationParams(step, resource, opts), remove: removeResourceState(step, opts, resource, snapshot), }); } @@ -179,13 +187,9 @@ function operationParams( step: DurableStep, resource: BaseResource, opts: DurableOperationOptions, - stateNode: StateNode | undefined, ): ResourceOperationBaseParams { return { resource, - // Serve the record already read during hydration rather than reading it - // again; a workflow must see the same value on every replay anyway. - state: { get: async () => stateNode }, dryRun: opts.dryRun, emit: durableEmitter(step, opts.emit), maxOperationAttempts: opts.maxOperationAttempts, diff --git a/packages/reconciler/src/operations/operation.create.ts b/packages/reconciler/src/operations/operation.create.ts index 2a1c8d1..d76d06f 100644 --- a/packages/reconciler/src/operations/operation.create.ts +++ b/packages/reconciler/src/operations/operation.create.ts @@ -19,9 +19,7 @@ export async function* createResourceOperation( } try { - const resourceParams = yield* step.run("create:get-params", () => - params.resource.getParams(), - ); + const resourceParams = params.resourceParams; const computedPrimaryKey = yield* runPendingOperation( step, @@ -40,7 +38,8 @@ export async function* createResourceOperation( const readResult = yield* readResourceOperation(step, { resource: params.resource, - state: params.state, + resourceParams, + persistedOutput: params.persistedOutput, emit: params.emit, maxOperationAttempts: params.maxOperationAttempts, }); diff --git a/packages/reconciler/src/operations/operation.read.ts b/packages/reconciler/src/operations/operation.read.ts index 306af17..76e02da 100644 --- a/packages/reconciler/src/operations/operation.read.ts +++ b/packages/reconciler/src/operations/operation.read.ts @@ -20,16 +20,11 @@ export async function* readResourceOperation( } try { - const resourceParams = yield* step.run("read:get-params", () => - params.resource.getParams(), - ); + const resourceParams = params.resourceParams; if (!params.resource.read) { - const stateNode = yield* step.run("read:get-state-node", () => - params.state.get(params.resource.id), - ); - const merged = stateNode - ? { ...stateNode.output, ...resourceParams } + const merged = params.persistedOutput + ? { ...params.persistedOutput, ...resourceParams } : resourceParams; yield* emitLifecycleEvent(params, "read", "skip", { diff --git a/packages/reconciler/src/operations/operation.types.ts b/packages/reconciler/src/operations/operation.types.ts index ad11676..1c247dc 100644 --- a/packages/reconciler/src/operations/operation.types.ts +++ b/packages/reconciler/src/operations/operation.types.ts @@ -1,5 +1,5 @@ import type { BaseResource } from "@notation/resource"; -import type { State, StateNode } from "@notation/state"; +import type { StateNode } from "@notation/state"; import type { EmitStep, OperationLifecycleEvent, @@ -60,19 +60,27 @@ export type RemoveState = () => AsyncGenerator; export type ResourceOperationBaseParams = { resource: BaseResource; - state: Pick; dryRun?: boolean; emit?: OperationEventEmitter; maxOperationAttempts?: number; }; -export type CreateResourceParams = ResourceOperationBaseParams & { - persist: PersistState; +/** + * Everything a read needs is resolved before the operation starts: the + * desired params, and — for a resource with no read operation — the output + * the last write persisted. An operation that resolved either itself could + * see a different answer from the one the decision was taken against. + */ +export type ReadResourceParams = ResourceOperationBaseParams & { + resourceParams: Record; + persistedOutput?: Record; }; -export type ReadResourceParams = ResourceOperationBaseParams; +export type CreateResourceParams = ReadResourceParams & { + persist: PersistState; +}; -export type UpdateResourceParams = ResourceOperationBaseParams & { +export type UpdateResourceParams = ReadResourceParams & { patch: Record; persist: PersistState; }; diff --git a/packages/reconciler/src/operations/operation.update.ts b/packages/reconciler/src/operations/operation.update.ts index 84698ee..838161e 100644 --- a/packages/reconciler/src/operations/operation.update.ts +++ b/packages/reconciler/src/operations/operation.update.ts @@ -27,9 +27,7 @@ export async function* updateResourceOperation( } try { - const resourceParams = yield* step.run("update:get-params", () => - params.resource.getParams(), - ); + const resourceParams = params.resourceParams; yield* runPendingOperation( step, @@ -52,7 +50,8 @@ export async function* updateResourceOperation( const readResult = yield* readResourceOperation(step, { resource: params.resource, - state: params.state, + resourceParams, + persistedOutput: params.persistedOutput, emit: params.emit, maxOperationAttempts: params.maxOperationAttempts, }); diff --git a/packages/reconciler/src/planner.ts b/packages/reconciler/src/planner.ts index ef7faca..8ac8b27 100644 --- a/packages/reconciler/src/planner.ts +++ b/packages/reconciler/src/planner.ts @@ -47,7 +47,8 @@ export async function createPlan({ const driftRead = await runOperation( readDriftOperation(createStepRunner(), { resource, - state, + resourceParams: params, + persistedOutput: stateNode?.output, emit: emitStep, maxOperationAttempts, }), diff --git a/packages/reconciler/src/reconciler.ts b/packages/reconciler/src/reconciler.ts index 8db798b..634bf73 100644 --- a/packages/reconciler/src/reconciler.ts +++ b/packages/reconciler/src/reconciler.ts @@ -163,11 +163,23 @@ export class Reconciler { dryRun: boolean, driftDetection: boolean, ) { - await this.#withMutationLease(resource.id, () => - this.#retryOnRevConflict((conflict) => - this.#deployResourceOnce(resource, dryRun, driftDetection, conflict), - ), - ); + await this.#withMutationLease(resource.id, async () => { + // Resolved once for the whole scheduled reconciliation, recovery + // included: deriveParams is user code and need not be deterministic, so + // a second resolution could decide against one set of params and + // persist another, with nothing to reconcile the two afterwards. + const params = (await resource.getParams()) as Record; + + await this.#retryOnRevConflict((conflict) => + this.#deployResourceOnce( + resource, + params, + dryRun, + driftDetection, + conflict, + ), + ); + }); } async #withMutationLease(resourceId: string, fn: () => Promise) { @@ -224,12 +236,13 @@ export class Reconciler { async #deployResourceOnce( resource: BaseResource, + params: Record, dryRun: boolean, driftDetection: boolean, conflict?: RevConflict, ) { if (conflict) { - await this.#recoverDeployResource(resource, dryRun, conflict); + await this.#recoverDeployResource(resource, params, dryRun, conflict); return; } @@ -237,14 +250,17 @@ export class Reconciler { let action: ResourceAction; if (!stateNode) { - action = decideAction({ resource }); + action = decideAction({ resource, params }); } else { resource.setOutput(stateNode.output); - const params = await resource.getParams(); action = decideAction({ resource, stateNode, params }); if (action.decision === "noop" && driftDetection) { - const driftRead = await this.#readForDrift(resource); + const driftRead = await this.#readForDrift( + resource, + params, + stateNode.output, + ); action = decideAction({ resource, stateNode, params, driftRead }); } } @@ -273,7 +289,8 @@ export class Reconciler { await runOperation( createResourceOperation(this.#stepRunner, { resource, - state: this.#state, + resourceParams: params, + persistedOutput: stateNode?.output, dryRun, emit: this.#emitStep, maxOperationAttempts: this.#maxOperationAttempts, @@ -287,7 +304,8 @@ export class Reconciler { await runOperation( updateResourceOperation(this.#stepRunner, { resource, - state: this.#state, + resourceParams: params, + persistedOutput: stateNode?.output, patch: action.patch, dryRun, emit: this.#emitStep, @@ -303,6 +321,7 @@ export class Reconciler { async #recoverDeployResource( resource: BaseResource, + params: Record, dryRun: boolean, conflict: RevConflict, ) { @@ -311,8 +330,11 @@ export class Reconciler { const stateNode = await this.#state.get(resource.id); if (stateNode) resource.setOutput(stateNode.output); - const params = await resource.getParams(); - const remote = await this.#readForDrift(resource); + const remote = await this.#readForDrift( + resource, + params, + stateNode?.output, + ); const action = decideAction({ resource, stateNode, @@ -335,7 +357,8 @@ export class Reconciler { await runOperation( createResourceOperation(this.#stepRunner, { resource, - state: this.#state, + resourceParams: params, + persistedOutput: stateNode?.output, dryRun, emit: this.#emitStep, maxOperationAttempts: this.#maxOperationAttempts, @@ -348,7 +371,8 @@ export class Reconciler { await runOperation( updateResourceOperation(this.#stepRunner, { resource, - state: this.#state, + resourceParams: params, + persistedOutput: stateNode?.output, patch: action.patch, dryRun, emit: this.#emitStep, @@ -390,11 +414,16 @@ export class Reconciler { }; } - #readForDrift(resource: BaseResource): Promise { + #readForDrift( + resource: BaseResource, + params: Record, + persistedOutput: Record | undefined, + ): Promise { return runOperation( readDriftOperation(this.#stepRunner, { resource, - state: this.#state, + resourceParams: params, + persistedOutput, emit: this.#emitStep, maxOperationAttempts: this.#maxOperationAttempts, }), @@ -474,7 +503,14 @@ export class Reconciler { if (conflict) { if (!resource.read) throw conflict; - const remote = await this.#readForDrift(resource); + // Deletion never needs the desired params, so they are resolved here + // rather than for every delete: only the recovery read consumes them. + const params = (await resource.getParams()) as Record; + const remote = await this.#readForDrift( + resource, + params, + stateNode.output, + ); if (remote.kind !== "present") { if (!dryRun) await this.#state.delete(resource.id, stateNode.rev); return; @@ -485,7 +521,6 @@ export class Reconciler { await runOperation( deleteResourceOperation(this.#stepRunner, { resource, - state: this.#state, dryRun, emit: this.#emitStep, maxOperationAttempts: this.#maxOperationAttempts, diff --git a/packages/reconciler/test/durable-reconciliation.test.ts b/packages/reconciler/test/durable-reconciliation.test.ts index 595ab82..7b87cca 100644 --- a/packages/reconciler/test/durable-reconciliation.test.ts +++ b/packages/reconciler/test/durable-reconciliation.test.ts @@ -253,13 +253,11 @@ describe("dependency ordering", () => { describe("conditional state persistence", () => { it("rejects a state write whose snapshot another writer has moved past", async () => { const RaceResource = resource({ type: "test/durable/write-race" }) + // Cast: a resource declared without API types constrains every schema + // key to be a key of an `any` API schema, which no named key satisfies. .defineSchema({ - name: { - presence: "required", - propertyType: "param", - valueType: "string" as any, - }, - }) + name: { presence: "required", propertyType: "param" }, + } as any) .defineOperations({ create: async () => undefined, // Moves the store on between the workflow reading its snapshot and @@ -509,15 +507,13 @@ describe("drift detection and repair", () => { remote = { name: "expected" }; }); const DriftResource = resource({ type: "test/durable/drift" }) + // Cast: a resource declared without API types constrains every schema + // key to be a key of an `any` API schema, which no named key satisfies. .defineSchema({ - name: { - presence: "required", - propertyType: "param", - valueType: "string" as any, - }, - }) + name: { presence: "required", propertyType: "param" }, + } as any) .defineOperations({ - create: async () => remote, + create: (async () => remote) as any, read: async () => remote, update: updateSpy, delete: async () => undefined, diff --git a/packages/reconciler/test/operation.workflows.test.ts b/packages/reconciler/test/operation.workflows.test.ts index a721a29..245cb00 100644 --- a/packages/reconciler/test/operation.workflows.test.ts +++ b/packages/reconciler/test/operation.workflows.test.ts @@ -52,11 +52,10 @@ describe("operation workflows", () => { it("create performs create + read-after-create + state persistence", async () => { const step = createStepRunnerDouble(); const events: OperationLifecycleEvent[] = []; - const state = { get: vi.fn(async () => undefined) }; const persist = vi.fn(async function* () {}); let createAttempts = 0; - const createMock = vi.fn(async (_params, context) => { + const createMock = vi.fn(async (_params: unknown, context: unknown) => { createAttempts += 1; if (createAttempts === 1) { expect(context).toBeUndefined(); @@ -73,7 +72,8 @@ describe("operation workflows", () => { const TestResource = resource({ type: "test/service/create" }) .defineSchema({}) .defineOperations({ - create: createMock, + // Cast: an empty schema declares no primary key to return. + create: createMock as any, read: async () => ({ remoteId: "abc", status: "ready" }), delete: async () => undefined, }); @@ -83,7 +83,7 @@ describe("operation workflows", () => { await runOperation( createResourceOperation(step, { resource: testResource, - state, + resourceParams: await testResource.getParams(), persist, emit: toEmitStep((event) => void events.push(event)), }), @@ -126,17 +126,12 @@ describe("operation workflows", () => { it("read follows pending retry instructions", async () => { const step = createStepRunnerDouble(); - const state = { - get: vi.fn(async () => undefined), - update: vi.fn(async () => undefined), - delete: vi.fn(async () => undefined), - }; let readAttempts = 0; const TestResource = resource({ type: "test/service/read" }) .defineSchema({}) .defineOperations({ - create: async () => ({}), + create: (async () => ({})) as any, read: async (_key, context) => { readAttempts += 1; if (readAttempts < 3) { @@ -156,7 +151,7 @@ describe("operation workflows", () => { const result = await runOperation( readResourceOperation(step, { resource: testResource, - state, + resourceParams: await testResource.getParams(), }), ); @@ -176,11 +171,6 @@ describe("operation workflows", () => { it("fails when an operation remains pending past the safety limit", async () => { const step = createStepRunnerDouble(); - const state = { - get: vi.fn(async () => undefined), - update: vi.fn(async () => undefined), - delete: vi.fn(async () => undefined), - }; const read = vi.fn(async () => { throw new ResourceOperationPendingError("still pending", { retryAfterMs: 10, @@ -189,7 +179,7 @@ describe("operation workflows", () => { const TestResource = resource({ type: "test/service/pending-limit" }) .defineSchema({}) .defineOperations({ - create: async () => ({}), + create: (async () => ({})) as any, read, delete: async () => undefined, }); @@ -198,7 +188,7 @@ describe("operation workflows", () => { runOperation( readResourceOperation(step, { resource: new TestResource({ id: "pending-limit" }), - state, + resourceParams: {}, maxOperationAttempts: 2, }), ), @@ -209,15 +199,11 @@ describe("operation workflows", () => { it("does not infer that not-found after creation is retryable", async () => { const step = createStepRunnerDouble(); - const state = { - get: vi.fn(async () => undefined), - update: vi.fn(async () => undefined), - delete: vi.fn(async () => undefined), - }; + const persist = vi.fn(async function* () {}); const TestResource = resource({ type: "test/service/eventually-visible" }) .defineSchema({}) .defineOperations({ - create: async () => ({}), + create: (async () => ({})) as any, read: async () => { throw new ResourceNotFoundError("resource is absent"); }, @@ -228,24 +214,23 @@ describe("operation workflows", () => { runOperation( createResourceOperation(step, { resource: new TestResource({ id: "eventually-visible" }), - state, - expectedRev: 0, + resourceParams: {}, + persist, }), ), ).rejects.toThrowError("resource is absent"); - expect(state.update).not.toHaveBeenCalled(); + expect(persist).not.toHaveBeenCalled(); }); it("delete treats an already-absent remote as success through its idempotent resource contract", async () => { const step = createStepRunnerDouble(); const events: OperationLifecycleEvent[] = []; - const state = { get: vi.fn(async () => undefined) }; const remove = vi.fn(async function* () {}); const TestResource = resource({ type: "test/service/delete" }) .defineSchema({}) .defineOperations({ - create: async () => ({}), + create: (async () => ({})) as any, delete: async () => undefined, }); @@ -254,7 +239,6 @@ describe("operation workflows", () => { await runOperation( deleteResourceOperation(step, { resource: testResource, - state, remove, emit: toEmitStep((event) => void events.push(event)), }), @@ -268,16 +252,12 @@ describe("operation workflows", () => { it("delete rethrows an unclassified resource error", async () => { const step = createStepRunnerDouble(); - const state = { - get: vi.fn(async () => undefined), - update: vi.fn(async () => undefined), - delete: vi.fn(async () => undefined), - }; + const remove = vi.fn(async function* () {}); const TestResource = resource({ type: "test/service/delete-miss" }) .defineSchema({}) .defineOperations({ - create: async () => ({}), + create: (async () => ({})) as any, delete: async () => { const err = new Error("still exists"); err.name = "DifferentError"; @@ -291,8 +271,7 @@ describe("operation workflows", () => { runOperation( deleteResourceOperation(step, { resource: testResource, - state, - expectedRev: 1, + remove, }), ), ).rejects.toMatchObject({ @@ -300,17 +279,13 @@ describe("operation workflows", () => { message: "still exists", }); - expect(state.delete).not.toHaveBeenCalled(); + expect(remove).not.toHaveBeenCalled(); }); it("emits structured error details on operation failure", async () => { const step = createStepRunnerDouble(); const events: OperationLifecycleEvent[] = []; - const state = { - get: vi.fn(async () => undefined), - update: vi.fn(async () => undefined), - delete: vi.fn(async () => undefined), - }; + const persist = vi.fn(async function* () {}); const TestResource = resource({ type: "test/service/create-error" }) .defineSchema({}) @@ -329,8 +304,8 @@ describe("operation workflows", () => { runOperation( createResourceOperation(step, { resource: testResource, - state, - expectedRev: 0, + resourceParams: {}, + persist, emit: toEmitStep((event) => void events.push(event)), }), ), diff --git a/packages/reconciler/test/reconciler.deploy.test.ts b/packages/reconciler/test/reconciler.deploy.test.ts index eaa5ce5..61d0209 100644 --- a/packages/reconciler/test/reconciler.deploy.test.ts +++ b/packages/reconciler/test/reconciler.deploy.test.ts @@ -72,15 +72,15 @@ function createTestResourceClass(opts: { ) => Promise; }) { return resource({ type: opts.type }) + // Cast: a resource declared without API types constrains every schema key + // to be a key of an `any` API schema, which no named key can satisfy. .defineSchema({ - name: { - presence: "required", - propertyType: "param", - valueType: "string" as any, - }, - }) + name: { presence: "required", propertyType: "param" }, + } as any) .defineOperations({ - create: opts.create ?? (async () => ({})), + // Cast: with no API types the schema resolves to no primary key, so the + // inferred create signature returns void. + create: (opts.create ?? (async () => ({}))) as any, read: opts.read, update: opts.update, delete: opts.delete ?? (async () => undefined), @@ -93,7 +93,9 @@ const found = (output: Record) => output; describe("reconciler deploy", () => { it("chooses create vs update from desired params vs state", async () => { const createSpy = vi.fn(async () => ({ name: "new" })); - const updateSpy = vi.fn(async () => undefined); + const updateSpy = vi.fn( + async (_key: unknown, _patch: Record) => undefined, + ); const CreateResource = createTestResourceClass({ type: "test/service/create-choice", @@ -397,7 +399,9 @@ describe("reconciler deploy", () => { }); it("detects drift using live read output and converges with update", async () => { - const updateSpy = vi.fn(async () => undefined); + const updateSpy = vi.fn( + async (_key: unknown, _patch: Record) => undefined, + ); const events: Array> = []; const TestResource = createTestResourceClass({ type: "test/service/drift", diff --git a/packages/reconciler/test/reconciler.plan.test.ts b/packages/reconciler/test/reconciler.plan.test.ts index d98e55a..3ea8752 100644 --- a/packages/reconciler/test/reconciler.plan.test.ts +++ b/packages/reconciler/test/reconciler.plan.test.ts @@ -13,16 +13,19 @@ function createMemoryState(initial: Record = {}) { return { store, - get: vi.fn(async (id: string) => store[id]), + get: vi.fn(async (id: string): Promise => store[id]), update: vi.fn( - async (id: string, expectedRev: number, patch: Partial) => { + async (id: string, _expectedRev: number, patch: Partial) => { + const rev = (store[id]?.rev ?? 0) + 1; store[id] = { ...(store[id] ?? {}), ...patch, + rev, } as StateNode; + return { rev }; }, ), - delete: vi.fn(async (id: string) => { + delete: vi.fn(async (id: string, _expectedRev: number) => { delete store[id]; }), values: vi.fn(async () => Object.values(store)), @@ -55,20 +58,16 @@ function createTestResourceClass(opts: { ) => Promise; }) { return resource({ type: opts.type }) + // Cast: a resource declared without API types constrains every schema key + // to be a key of an `any` API schema, which no named key can satisfy. .defineSchema({ - name: { - presence: "required", - propertyType: "param", - valueType: "string" as any, - }, - tag: { - presence: "optional", - propertyType: "param", - valueType: "string" as any, - }, - }) + name: { presence: "required", propertyType: "param" }, + tag: { presence: "optional", propertyType: "param" }, + } as any) .defineOperations({ - create: opts.create ?? (async () => ({})), + // Cast: with no API types the schema resolves to no primary key, so the + // inferred create signature returns void. + create: (opts.create ?? (async () => ({}))) as any, read: opts.read, update: opts.update, delete: opts.delete ?? (async () => undefined), @@ -82,6 +81,7 @@ function createStateNode( output: Record = params, ): StateNode { return { + rev: 1, id, groupId: -1, groupType: "", @@ -369,14 +369,10 @@ describe("reconciler plan", () => { type: "test/service/plan-derive-failure", }) .defineSchema({ - name: { - presence: "required", - propertyType: "param", - valueType: "string" as any, - }, - }) + name: { presence: "required", propertyType: "param" }, + } as any) .defineOperations({ - create: async () => ({}), + create: (async () => ({})) as any, delete: async () => undefined, deriveParams: () => { throw new Error("invalid derived configuration"); diff --git a/packages/reconciler/test/resource-registry.test.ts b/packages/reconciler/test/resource-registry.test.ts index dd63f48..de3fdf8 100644 --- a/packages/reconciler/test/resource-registry.test.ts +++ b/packages/reconciler/test/resource-registry.test.ts @@ -9,14 +9,14 @@ import { const TestResourceA = resource({ type: "test/service/a" }) .defineSchema({}) .defineOperations({ - create: async () => ({}), + create: (async () => ({})) as any, delete: async () => undefined, }); const TestResourceB = resource({ type: "test/service/b" }) .defineSchema({}) .defineOperations({ - create: async () => ({}), + create: (async () => ({})) as any, delete: async () => undefined, }); From f2959144d6f1e61a8334133eb7d3646aebc890c9 Mon Sep 17 00:00:00 2001 From: djgrant <1670902+djgrant@users.noreply.github.com> Date: Mon, 27 Jul 2026 20:53:10 +0100 Subject: [PATCH 15/34] Move step scoping onto the step runner seam MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Scoping was a durable-only wrapper that rebuilt a step object. It is now part of the StepRunner contract: scope(prefix) is the identity in process, because createStepRunner ignores keys entirely, and the prefixing wrapper in a workflow. scopeStore goes with it. A store outlives the scope that opened it, so prefixing its caller-supplied keys with whichever scope happened to open the handle was arbitrary; the three call sites now qualify their own keys, as state:persist:, state:delete: and the notation:coordination:* keys, which were already fully qualified. The keyless run/delay overloads are gone. Yieldstar derives a key from the call site for those, and through a scoping wrapper that call site is the wrapper's, not the caller's — so any two keyless steps reached through one scope would have collided. Nothing called them; removing them makes that structural rather than a convention. Keys cost nothing in process, where they are ignored. BLAST RADIUS. Store keys change, so an execution that was in flight before this commit and resumes after it must be drained or abandoned first. It will not refuse cleanly: plain run steps have no idempotency ledger, so a resumed execution takes a cache miss and RE-EXECUTES the provider mutations behind those keys. It then fails safe at the first conditional state write, because updateFrom/deleteFrom carry a snapshot that is now stale and abort with RevConflict — but the remote calls will already have happened. Registered store waiters are keyed by stepKey too, so a `take` suspended on the deployment hold re-registers under its new key when it next wakes. The key shapes now in use are documented at the durable entry point, where they can be read as the contract they are. --- packages/reconciler/src/durable/index.ts | 19 +++++ packages/reconciler/src/durable/operations.ts | 27 ++++--- packages/reconciler/src/durable/step.ts | 77 ++++++++----------- .../src/operations/operation.types.ts | 15 +++- packages/reconciler/src/step-runner.ts | 29 ++----- .../test/operation.workflows.test.ts | 30 ++++---- 6 files changed, 103 insertions(+), 94 deletions(-) diff --git a/packages/reconciler/src/durable/index.ts b/packages/reconciler/src/durable/index.ts index 74c9a22..336d548 100644 --- a/packages/reconciler/src/durable/index.ts +++ b/packages/reconciler/src/durable/index.ts @@ -1,3 +1,22 @@ +/** + * Step keys are a public contract: they are what a resumed execution matches + * its cached work against, so changing one re-executes the work behind it. + * The shapes in use are: + * + * notation:resource::* per-resource reconciliation steps + * notation:destroy::* per-resource deletion steps + * notation:orphans::* orphan sweep, per persisted record + * *:remote:attempt: one provider call attempt + * *:remote:retry-delay: the wait between two attempts + * emit:[::] event delivery checkpoint + * notation:coordination:* deployment hold: inspect/acquire/release + * state:persist: conditional write of a resource record + * state:delete: conditional removal of one + * + * The state: keys are store-handle keys and so are not scope-prefixed: a + * store outlives the scope that opened it, which is why they carry the + * resource id themselves. + */ export { deploy } from "./deploy"; export { destroy } from "./destroy"; export { diff --git a/packages/reconciler/src/durable/operations.ts b/packages/reconciler/src/durable/operations.ts index fca0036..2912922 100644 --- a/packages/reconciler/src/durable/operations.ts +++ b/packages/reconciler/src/durable/operations.ts @@ -16,7 +16,7 @@ import { } from "../operations"; import { decideAction, type ResourceAction } from "../plan"; import type { DurableStateBackend } from "./state-backend"; -import { durableEmitter, scopeStep } from "./step"; +import { durableEmitter, scopeStep, type DurableStepRunner } from "./step"; import { resourceStateStore, toStateNode, @@ -52,7 +52,7 @@ export async function* reconcileResource( if (action.decision === "noop" && (opts.driftDetection ?? true)) { // Its own scope: the operation that follows reads the remote again, and // the two reads must not share step keys. - const driftScope = scopeStep(scope, "drift-read"); + const driftScope = scope.scope("drift-read"); const driftRead = yield* readDriftOperation(driftScope, { ...operationParams(driftScope, resource, opts), resourceParams: params, @@ -95,7 +95,7 @@ export async function* reconcileResource( } export async function* deleteResource( - step: DurableStep, + step: DurableStepRunner, resource: BaseResource, opts: DurableOperationOptions, ): AsyncGenerator { @@ -118,7 +118,7 @@ export async function* deleteResource( * warning, because deleting it would need a resource class we cannot resolve. */ export async function* sweepOrphans( - step: DurableStep, + step: DurableStepRunner, opts: DurableOperationOptions, workflow: "deploy" | "destroy", ): AsyncGenerator { @@ -131,7 +131,7 @@ export async function* sweepOrphans( for (const node of persisted) { if (resourceById.has(node.id)) continue; - const nodeScope = scopeStep(step, node.id); + const nodeScope = step.scope(node.id); const Resource = resolveResourceClass(registry, node.type as ResourceType); if (!Resource) { @@ -158,7 +158,7 @@ export async function* sweepOrphans( * here, and is re-served to the operations so they need no second read. */ async function* hydrateResource( - step: DurableStep, + step: DurableStepRunner, resource: BaseResource, opts: DurableOperationOptions, ): AsyncGenerator< @@ -175,7 +175,7 @@ async function* hydrateResource( } function readSnapshot( - step: DurableStep, + step: DurableStepRunner, state: DurableStateBackend, resourceId: string, ): AsyncGenerator { @@ -184,7 +184,7 @@ function readSnapshot( /** The half of the operation params every durable driver call site shares. */ function operationParams( - step: DurableStep, + step: DurableStepRunner, resource: BaseResource, opts: DurableOperationOptions, ): ResourceOperationBaseParams { @@ -203,7 +203,7 @@ function operationParams( * recorded result instead of retrying a compare-and-set that would now fail. */ function persistResourceState( - step: DurableStep, + step: DurableStepRunner, opts: DurableOperationOptions, resource: BaseResource, snapshot: ResourceSnapshot | undefined, @@ -224,7 +224,7 @@ function persistResourceState( id: opts.state.storeId(resource.id), }); const result = yield* store.updateFrom( - "state:persist", + `state:persist:${resource.id}`, snapshot, () => next, ); @@ -239,7 +239,7 @@ function persistResourceState( } function removeResourceState( - step: DurableStep, + step: DurableStepRunner, opts: DurableOperationOptions, resource: BaseResource, snapshot: ResourceSnapshot, @@ -248,7 +248,10 @@ function removeResourceState( const store = yield* step.store(resourceStateStore, { id: opts.state.storeId(resource.id), }); - const result = yield* store.deleteFrom("state:delete", snapshot); + const result = yield* store.deleteFrom( + `state:delete:${resource.id}`, + snapshot, + ); if (!result.deleted) { throw new RevConflict(resource.id, snapshot.version + 1, undefined); } diff --git a/packages/reconciler/src/durable/step.ts b/packages/reconciler/src/durable/step.ts index ad52ade..a3c78ae 100644 --- a/packages/reconciler/src/durable/step.ts +++ b/packages/reconciler/src/durable/step.ts @@ -3,36 +3,49 @@ import type { ReconcilerEvent, ReconcilerEventEmitter, } from "../events"; -import type { DurableStep, WorkflowStore } from "./yieldstar"; +import type { StepRunner } from "../operations"; +import type { DurableStep } from "./yieldstar"; + +/** + * A durable step runner: the operation seam plus the store handle, which only + * the durable driver uses. + */ +export type DurableStepRunner = { + run( + key: string, + fn: () => T | Promise, + ): AsyncGenerator; + delay(key: string, ms: number): AsyncGenerator; + /** Narrower than StepRunner's, so a scope keeps its store handle. */ + scope(prefix: string): DurableStepRunner; + store: DurableStep["store"]; +}; + +// A durable runner is one of the step runners the operations accept. +type AssertStepRunner = DurableStepRunner extends StepRunner ? true : never; +export type DurableStepRunnerIsStepRunner = AssertStepRunner; /** * Namespaces the step keys of `step` so an operation can be written once and * replayed at several call sites without its keys colliding. * * Opening a store is not prefixed: yieldstar derives that key from the store - * name and store id, which is already unique. The keys the store *handle* - * takes are caller-supplied, so those are scoped like any other step. + * name and store id, which is already unique. The keys a store *handle* takes + * are caller-supplied and are left alone too — a store outlives the scope + * that opened it, so its call sites qualify their own keys. */ -export function scopeStep(step: DurableStep, prefix: string): DurableStep { +export function scopeStep( + step: DurableStep, + prefix: string, +): DurableStepRunner { const scoped = (key: string) => `${prefix}:${key}`; return { - ...step, - // The keyless overloads fall through untouched; yieldstar hashes the call - // site for those, and a prefix would not make them any more unique. - run: ((arg1: unknown, arg2?: unknown) => - typeof arg1 === "string" - ? (step.run as any)(scoped(arg1), arg2) - : (step.run as any)(arg1)) as DurableStep["run"], - delay: ((arg1: unknown, arg2?: unknown) => - typeof arg1 === "string" - ? (step.delay as any)(scoped(arg1), arg2) - : (step.delay as any)(arg1)) as DurableStep["delay"], - store: ((definition: any, params: any) => - (async function* () { - const store = yield* step.store(definition, params); - return scopeStore(store, prefix); - })()) as DurableStep["store"], + run: ((key: string, fn: any) => + step.run(scoped(key), fn)) as DurableStepRunner["run"], + delay: (key: string, ms: number) => step.delay(scoped(key), ms), + store: step.store, + scope: (nested: string) => scopeStep(step, scoped(nested)), }; } @@ -46,7 +59,7 @@ export function scopeStep(step: DurableStep, prefix: string): DurableStep { * event is delivered but before the checkpoint is written. */ export function durableEmitter( - step: DurableStep, + step: DurableStepRunner, emit: ReconcilerEventEmitter | undefined, ): EmitStep { return async function* (event) { @@ -60,25 +73,3 @@ function emitKey(event: ReconcilerEvent): string { ? `emit:${event.event}:${event.operation}:${event.status}` : `emit:${event.event}`; } - -function scopeStore( - store: WorkflowStore, - prefix: string, -): WorkflowStore { - const scoped = (key: string) => `${prefix}:${key}`; - - return { - ...store, - get: (key?: string) => store.get(key === undefined ? key : scoped(key)), - select: (key, selector) => store.select(scoped(key), selector), - update: (key, updater) => store.update(scoped(key), updater), - updateFrom: (key, snapshot, updater) => - store.updateFrom(scoped(key), snapshot, updater), - deleteFrom: (key, snapshot) => store.deleteFrom(scoped(key), snapshot), - when: ((arg1: any, arg2?: any) => - typeof arg1 === "string" - ? store.when(scoped(arg1), arg2) - : store.when(arg1)) as WorkflowStore["when"], - take: (key, selector, claim) => store.take(scoped(key), selector, claim), - }; -} diff --git a/packages/reconciler/src/operations/operation.types.ts b/packages/reconciler/src/operations/operation.types.ts index 1c247dc..492f4c9 100644 --- a/packages/reconciler/src/operations/operation.types.ts +++ b/packages/reconciler/src/operations/operation.types.ts @@ -15,14 +15,25 @@ export type { export type OperationEventEmitter = EmitStep; +/** + * How an operation runs a step, and how it namespaces the steps it runs. + * + * Keys are mandatory. A driver may ignore them — the in-process one does — + * but a keyless step would have to derive a key from its call site, which for + * a step reached through `scope` is the scoping wrapper's call site rather + * than the caller's, so any two keyless steps under one scope would collide. + * + * `scope` is the seam that lets one operation run at several call sites in a + * single execution: in process it is the identity, and in a workflow it + * prefixes the keys the runtime caches against. + */ export type StepRunner = { - run(fn: () => T | Promise): AsyncGenerator; run( key: string, fn: () => T | Promise, ): AsyncGenerator; - delay(ms: number): AsyncGenerator; delay(key: string, ms: number): AsyncGenerator; + scope(prefix: string): StepRunner; }; /** diff --git a/packages/reconciler/src/step-runner.ts b/packages/reconciler/src/step-runner.ts index 61cc6d5..ff39fdc 100644 --- a/packages/reconciler/src/step-runner.ts +++ b/packages/reconciler/src/step-runner.ts @@ -11,30 +11,17 @@ export async function runOperation( } export function createStepRunner(): StepRunner { - return { - async *run( - arg1: string | (() => T | Promise), - arg2?: () => T | Promise, - ): AsyncGenerator { - const fn = (typeof arg1 === "string" ? arg2 : arg1) as - (() => T | Promise) | undefined; - - if (!fn) { - throw new Error("Missing run function"); - } - + const runner: StepRunner = { + async *run(_key: string, fn: () => T | Promise) { return await fn(); }, - async *delay( - arg1: string | number, - arg2?: number, - ): AsyncGenerator { - const ms = typeof arg1 === "number" ? arg1 : arg2; - if (ms === undefined) { - throw new Error("Missing delay duration"); - } - + async *delay(_key: string, ms: number) { await new Promise((resolve) => setTimeout(resolve, ms)); }, + // Nothing is replayed in process, so no step key is ever read and a scope + // has nothing to namespace: one runner serves every scope. + scope: () => runner, }; + + return runner; } diff --git a/packages/reconciler/test/operation.workflows.test.ts b/packages/reconciler/test/operation.workflows.test.ts index 245cb00..1d4c410 100644 --- a/packages/reconciler/test/operation.workflows.test.ts +++ b/packages/reconciler/test/operation.workflows.test.ts @@ -13,31 +13,29 @@ import { } from "../src/operations"; import { toEmitStep } from "../src/events"; -function createStepRunnerDouble(): StepRunner { +function createStepRunnerDouble() { const run = vi.fn(async function* ( - arg1: string | (() => T | Promise), - arg2?: () => T | Promise, + _key: string, + fn: () => T | Promise, ): AsyncGenerator { - const fn = (typeof arg1 === "string" ? arg2 : arg1) as () => T | Promise; - if (!fn) { - throw new Error("Missing run function"); - } - return await fn(); }); - const delay = vi.fn(async function* (): AsyncGenerator< - unknown, - void, - unknown - > { + const delay = vi.fn(async function* ( + _key: string, + _ms: number, + ): AsyncGenerator { return; }); - return { - run, - delay, + // vi.fn erases the generic, so the seam's signature is restored here. + const runner: StepRunner = { + run: run as unknown as StepRunner["run"], + delay: delay as unknown as StepRunner["delay"], + scope: () => runner, }; + + return runner; } async function runOperation(operation: AsyncGenerator) { From 4b3bd59e86b06b3344eb8c03e7926915e9ae3b14 Mon Sep 17 00:00:00 2001 From: djgrant <1670902+djgrant@users.noreply.github.com> Date: Mon, 27 Jul 2026 20:54:58 +0100 Subject: [PATCH 16/34] Read the remote during a durable dry run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The durable driver threaded dryRun into the drift read's params, so readResourceOperation returned {} without calling the provider. decideAction then diffed toComparable({}) against the desired params, found every param "missing" from the remote, and upgraded a noop into a drift-update carrying a fabricated diff. Every dry-run test set driftDetection: false, so nothing caught it. The in-process driver never had this: #readForDrift deliberately omits dryRun, and createPlan does the same. The contract is that a dry run suppresses mutations, not reads — a dry run that cannot read the remote cannot report drift, which is most of what it is for. So this matches the in-process behaviour rather than guarding the read. The new test pins both halves: the provider read happens during the dry run, and the decision stays noop with no drift event. --- packages/reconciler/src/durable/operations.ts | 5 ++ .../test/durable-reconciliation.test.ts | 49 +++++++++++++++++++ 2 files changed, 54 insertions(+) diff --git a/packages/reconciler/src/durable/operations.ts b/packages/reconciler/src/durable/operations.ts index 2912922..b33c49c 100644 --- a/packages/reconciler/src/durable/operations.ts +++ b/packages/reconciler/src/durable/operations.ts @@ -55,6 +55,11 @@ export async function* reconcileResource( const driftScope = scope.scope("drift-read"); const driftRead = yield* readDriftOperation(driftScope, { ...operationParams(driftScope, resource, opts), + // A dry run suppresses mutations, not reads. Threading dryRun in here + // would make the read return {} without touching the provider, which + // decideAction would then diff against the desired params and report as + // drift that is not there. + dryRun: undefined, resourceParams: params, persistedOutput: stateNode?.output, }); diff --git a/packages/reconciler/test/durable-reconciliation.test.ts b/packages/reconciler/test/durable-reconciliation.test.ts index 7b87cca..f4e3ba3 100644 --- a/packages/reconciler/test/durable-reconciliation.test.ts +++ b/packages/reconciler/test/durable-reconciliation.test.ts @@ -542,6 +542,52 @@ describe("drift detection and repair", () => { ).toHaveLength(1); runtime.close(); }); + + it("reads the remote during a dry run rather than reporting false drift", async () => { + const remote = { name: "expected" }; + const read = vi.fn(async () => remote); + const DryRunDriftResource = resource({ type: "test/durable/dry-run-drift" }) + .defineSchema({ + name: { presence: "required", propertyType: "param" }, + } as any) + .defineOperations({ + create: (async () => remote) as any, + read, + update: async () => undefined, + delete: async () => undefined, + }); + const events: ReconcilerEvent[] = []; + const options = { + driftDetection: true, + dryRun: false, + emit: (event: ReconcilerEvent) => void events.push(event), + }; + const runtime = createRuntime( + [new DryRunDriftResource({ id: "steady", config: { name: "expected" } })], + "dry-run-drift", + options, + ); + + await runtime.run("deploy-1"); + const readsAfterDeploy = read.mock.calls.length; + + options.dryRun = true; + events.length = 0; + await runtime.run("dry-run"); + + // Suppressing the drift read under dryRun would leave decideAction + // diffing an empty read against the desired params, which reports every + // param as drift; skipping it entirely would make a dry run unable to + // report the drift it exists to report. + expect(read.mock.calls.length).toBeGreaterThan(readsAfterDeploy); + expect( + events.filter((event) => event.event === "reconciler.drift.detected"), + ).toEqual([]); + expect( + events.find((event) => event.event === "reconciler.deploy.decision"), + ).toMatchObject({ resourceId: "steady", decision: "noop" }); + runtime.close(); + }); }); function createRuntime( @@ -552,6 +598,7 @@ function createRuntime( crashAfterStep?: string; registry?: ResourceRegistry; driftDetection?: boolean; + dryRun?: boolean; emit?: (event: ReconcilerEvent) => void; } = {}, ) { @@ -574,6 +621,8 @@ function createRuntime( state, registry: options.registry, driftDetection: options.driftDetection ?? false, + // Read at execution time, so a test can switch it between runs. + dryRun: options.dryRun, emit: options.emit, maxOperationAttempts: options.maxOperationAttempts, }); From 1c4de0e3bf8f9ce4e5a7a0279fdaa54eb964a19f Mon Sep 17 00:00:00 2001 From: djgrant <1670902+djgrant@users.noreply.github.com> Date: Mon, 27 Jul 2026 21:20:24 +0100 Subject: [PATCH 17/34] Emit drift detection when recovery adopts drift MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Conflict recovery re-reads the remote, re-decides against it, and can land on drift-update — but it emitted only deploy.decision, so the drift.detected event that the same decision produces on a first pass was missing. Recovery after a conflict is drift adoption, so it is owed. This is a visible change for anything diffing event sequences: a recovering deployment now emits reconciler.drift.detected before its deploy.decision, where before it emitted only the decision. It lands on its own so that it is not buried in the driver unification that follows. --- packages/reconciler/src/reconciler.ts | 13 ++++ .../reconciler/test/reconciler.deploy.test.ts | 73 +++++++++++++++++++ 2 files changed, 86 insertions(+) diff --git a/packages/reconciler/src/reconciler.ts b/packages/reconciler/src/reconciler.ts index 634bf73..e2b4a79 100644 --- a/packages/reconciler/src/reconciler.ts +++ b/packages/reconciler/src/reconciler.ts @@ -343,6 +343,19 @@ export class Reconciler { }); if (remote.kind === "present") resource.setOutput(remote.output); + // Recovery is drift adoption: the remote moved while the attempt that + // conflicted was in flight, so the same event a first-pass drift-update + // emits is owed here too. + if (action.decision === "drift-update") { + await this.#emit?.({ + level: "info", + event: "reconciler.drift.detected", + resourceId: resource.id, + resourceType: resource.type, + diff: action.patch, + }); + } + await this.#emit?.({ level: "info", event: "reconciler.deploy.decision", diff --git a/packages/reconciler/test/reconciler.deploy.test.ts b/packages/reconciler/test/reconciler.deploy.test.ts index 61d0209..9987a90 100644 --- a/packages/reconciler/test/reconciler.deploy.test.ts +++ b/packages/reconciler/test/reconciler.deploy.test.ts @@ -263,6 +263,79 @@ describe("reconciler deploy", () => { }); }); + it("reports the drift it adopts when recovering from a conflict", async () => { + // The provider ignored the update, so the remote still holds the old + // value while another writer has moved persisted state to the desired + // one: recovery re-decides as drift rather than as a repeat update. + const readSpy = vi.fn(async () => found({ name: "stale" })); + const updateSpy = vi.fn( + async (_key: unknown, _patch: Record) => undefined, + ); + const UpdateResource = createTestResourceClass({ + type: "test/service/recovery-drift", + read: readSpy, + update: updateSpy, + }); + const state = createMemoryState({ + existing: { + rev: 1, + id: "existing", + groupId: -1, + groupType: "", + type: UpdateResource.type, + config: { name: "old" }, + params: { name: "old" }, + output: { name: "old" }, + lastOperation: "create", + lastOperationAt: new Date().toISOString(), + }, + }); + const updateState = state.update.getMockImplementation()!; + state.update + .mockImplementationOnce(async () => { + state.store.existing = { + ...state.store.existing!, + rev: 2, + config: { name: "new" }, + params: { name: "new" }, + output: { name: "new" }, + }; + throw new RevConflict("existing", 1, 2); + }) + .mockImplementation(updateState); + + const events: Array> = []; + const reconciler = new Reconciler({ + state, + driftDetection: false, + emit: async (event) => { + events.push(event as unknown as Record); + }, + }); + await reconciler.deploy([ + new UpdateResource({ id: "existing", config: { name: "new" } }), + ]); + + expect( + events.filter((event) => event.event === "reconciler.drift.detected"), + ).toEqual([ + { + level: "info", + event: "reconciler.drift.detected", + resourceId: "existing", + resourceType: UpdateResource.type, + diff: { name: "new" }, + }, + ]); + expect( + events.filter( + (event) => + event.event === "reconciler.deploy.decision" && + event.decision === "drift-update", + ), + ).toHaveLength(1); + }); + it("reads remote state after a create conflict instead of creating twice", async () => { let remoteName: string | undefined; const createSpy = vi.fn(async (params) => { From efd75c502c4c28eab94f8c6027cbdfa37cbdb487 Mon Sep 17 00:00:00 2001 From: djgrant <1670902+djgrant@users.noreply.github.com> Date: Mon, 27 Jul 2026 21:20:41 +0100 Subject: [PATCH 18/34] Test the deployment hold takeover, and finish the key map MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit takeOverDeploymentHold is the only exit from a hold left by an execution that will never resume, so it ships tested: it takes when the named holder still holds, and refuses — reporting the actual holder, leaving the record alone — when the hold has moved on. The key map gains the destroy-path orphan sweep, which it omitted, and a note that resource ids are spliced in unescaped. That ambiguity predates the map and is recorded rather than fixed, since changing key composition invalidates in-flight executions. --- packages/reconciler/src/durable/index.ts | 8 ++- .../test/durable-reconciliation.test.ts | 52 +++++++++++++++++++ 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/packages/reconciler/src/durable/index.ts b/packages/reconciler/src/durable/index.ts index 336d548..72646b2 100644 --- a/packages/reconciler/src/durable/index.ts +++ b/packages/reconciler/src/durable/index.ts @@ -5,7 +5,8 @@ * * notation:resource::* per-resource reconciliation steps * notation:destroy::* per-resource deletion steps - * notation:orphans::* orphan sweep, per persisted record + * notation:orphans::* orphan sweep on deploy, per record + * notation:destroy:orphans::* orphan sweep on destroy, per record * *:remote:attempt: one provider call attempt * *:remote:retry-delay: the wait between two attempts * emit:[::] event delivery checkpoint @@ -16,6 +17,11 @@ * The state: keys are store-handle keys and so are not scope-prefixed: a * store outlives the scope that opened it, which is why they carry the * resource id themselves. + * + * Resource ids are spliced in unescaped, so the delimiter is ambiguous: a + * resource named "orphans" sits in the same key space as the sweep's own + * segment. That predates the key map and is recorded here rather than fixed, + * since changing the composition invalidates in-flight executions. */ export { deploy } from "./deploy"; export { destroy } from "./destroy"; diff --git a/packages/reconciler/test/durable-reconciliation.test.ts b/packages/reconciler/test/durable-reconciliation.test.ts index f4e3ba3..516663b 100644 --- a/packages/reconciler/test/durable-reconciliation.test.ts +++ b/packages/reconciler/test/durable-reconciliation.test.ts @@ -451,6 +451,58 @@ describe("deployment coordination", () => { }); }); +describe("deployment hold takeover", () => { + it("clears a hold its named holder still has, and unblocks the deployment", async () => { + const create = vi.fn(async () => undefined); + const Resource = resource({ type: "test/durable/takeover" }) + .defineSchema({}) + .defineOperations({ create, delete: async () => undefined }); + const runtime = createRuntime([new Resource({ id: "held" })], "takeover"); + await runtime.storeClient.getOrCreateStore({ + definition: durable.deploymentCoordinationStore, + id: "takeover", + initial: { holder: "abandoned-execution" }, + }); + + const result = await durable.takeOverDeploymentHold({ + storeClient: runtime.storeClient, + deploymentId: "takeover", + fromExecutionId: "abandoned-execution", + }); + + expect(result).toEqual({ + taken: true, + previousHolder: "abandoned-execution", + }); + await runtime.run("later-execution"); + expect(create).toHaveBeenCalledOnce(); + runtime.close(); + }); + + it("refuses to clear a hold that has moved to another execution", async () => { + const runtime = createRuntime([], "takeover-race"); + await runtime.storeClient.getOrCreateStore({ + definition: durable.deploymentCoordinationStore, + id: "takeover-race", + initial: { holder: "current-execution" }, + }); + + const result = await durable.takeOverDeploymentHold({ + storeClient: runtime.storeClient, + deploymentId: "takeover-race", + fromExecutionId: "abandoned-execution", + }); + + expect(result).toEqual({ taken: false, holder: "current-execution" }); + const snapshot = await runtime.storeClient.getStore({ + definition: durable.deploymentCoordinationStore, + id: "takeover-race", + }); + expect(snapshot.state.holder).toBe("current-execution"); + runtime.close(); + }); +}); + describe("deployment scoping", () => { it("scopes store listing to the exact deployment despite prefix-like IDs", async () => { const database = createSqliteDb({ path: ":memory:" }); From 8b0f0e445f1001f30a797ba1bd37e44f1e85bef2 Mon Sep 17 00:00:00 2001 From: djgrant <1670902+djgrant@users.noreply.github.com> Date: Mon, 27 Jul 2026 21:20:58 +0100 Subject: [PATCH 19/34] Run one reconciliation algorithm in both drivers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The per-resource algorithm existed twice — #deployResourceOnce plus #recoverDeployResource in process, reconcileResource in the workflow — and the copies had already drifted apart. There is now one generator, src/reconcile.ts, that owns hydration, the decision, the drift read, event emission, dispatch, recovery adoption and deletion recovery. Both drivers run it. What stays with a driver is what genuinely differs: dependency-level scheduling and concurrency (Promise.all under a mutation lease against a sequential yield* under a deployment hold), the conflict retry policy, how a state session is opened, and how a step is run. Two seams carry that split. The driver passes openSession(resource), a factory rather than an open session, because recovery re-reads and persists against what the winning writer left; the first pass opens once and a recovering pass opens again. And ResourceStateSession is a discriminated union, so `remove` exists only alongside a `node` — the precondition deleteResource expressed as an early return and destroy expressed as skip-when-absent, now expressed in the type. Keeping the writes bound to the read that produced them is what stops a node being paired with a precondition from a different read. Recovery is a recoverFrom parameter rather than a second near-copy of the algorithm. The behaviours that were easy to lose in that collapse are each still pinned by an unchanged test: a recovering noop persists the adopted record with lastOperation "drift" at the re-read revision (guarded by dryRun), a resource that cannot be read cannot be recovered, and the drift read carries no dryRun. Orphan sweeping is aligned too. In-process destroy now sweeps, which also means destroyApp no longer runs a refresh pass first — but the Reconciler it builds must be given the registry, or the sweep silently falls back to the types of the resources still declared and skips, with a warning, exactly the orphans a destroy exists to remove. registry is optional, so nothing else would have caught that. Note the ordering flip: in-process destroy used to sweep before deleting declared resources and now sweeps after, matching the durable driver and deploy's act-then-sweep shape. Public refresh is unchanged, for standalone cleanup. The #emit/#emitStep split collapses with it: OperationLifecycleEvent is a subset of ReconcilerEvent, so one scoped emit seam serves decisions, drift and operation lifecycle alike. reconciler.ts 511 -> 361 lines, durable/operations.ts 252 -> 197, with 248 shared between them. --- .../provisioner/workflows/workflow.destroy.ts | 10 +- packages/reconciler/src/durable/operations.ts | 173 +++----- packages/reconciler/src/durable/step.ts | 2 +- packages/reconciler/src/reconcile.ts | 251 ++++++++++++ packages/reconciler/src/reconciler.ts | 368 +++++------------- .../reconciler/test/reconciler.deploy.test.ts | 53 +++ 6 files changed, 453 insertions(+), 404 deletions(-) create mode 100644 packages/reconciler/src/reconcile.ts diff --git a/packages/core/src/provisioner/workflows/workflow.destroy.ts b/packages/core/src/provisioner/workflows/workflow.destroy.ts index 813239a..d5ee58e 100644 --- a/packages/core/src/provisioner/workflows/workflow.destroy.ts +++ b/packages/core/src/provisioner/workflows/workflow.destroy.ts @@ -7,7 +7,6 @@ import { import type { StateBackend } from "@notation/state"; import { getResourceGraph } from "src/orchestrator/graph"; import { createDefaultStateBackend } from "../state-backend"; -import { refreshState } from "./workflow.refresh"; export type DestroyAppOptions = { entryPoint: string; @@ -23,11 +22,16 @@ export async function destroyApp({ emit = createLoggerReconcilerSubscriber(), }: DestroyAppOptions) { const state = stateBackend ?? createDefaultStateBackend(); - await refreshState({ entryPoint, registry, state, emit }); - const graph = await getResourceGraph(entryPoint); + + // The registry has to be threaded through: destroy sweeps orphans itself + // now, and without one the sweep falls back to the types of the resources + // still declared — so an orphan whose type the app no longer declares would + // be skipped with a warning instead of deleted. The parameter is optional, + // so nothing but this would catch it. const reconciler = new Reconciler({ state, + registry, emit, }); diff --git a/packages/reconciler/src/durable/operations.ts b/packages/reconciler/src/durable/operations.ts index b33c49c..5ccc5c0 100644 --- a/packages/reconciler/src/durable/operations.ts +++ b/packages/reconciler/src/durable/operations.ts @@ -1,21 +1,17 @@ import type { BaseResource, ResourceType } from "@notation/resource"; -import { RevConflict, type StateNode } from "@notation/state"; +import { RevConflict } from "@notation/state"; import { createMissingResourceRegistryMatchWarningEvent, createResourceRegistryFromResources, resolveResourceClass, } from "../resource-registry"; +import type { PersistState, RemoveState, StepRunner } from "../operations"; import { - createResourceOperation, - deleteResourceOperation, - readDriftOperation, - updateResourceOperation, - type PersistState, - type RemoveState, - type ResourceOperationBaseParams, -} from "../operations"; -import { decideAction, type ResourceAction } from "../plan"; -import type { DurableStateBackend } from "./state-backend"; + destroyResource, + reconcileResource as reconcile, + type EmitFromStep, + type OpenStateSession, +} from "../reconcile"; import { durableEmitter, scopeStep, type DurableStepRunner } from "./step"; import { resourceStateStore, @@ -31,71 +27,21 @@ export async function* reconcileResource( opts: DurableDeployOptions, ): AsyncGenerator { const scope = scopeStep(step, `notation:resource:${resource.id}`); - const emit = durableEmitter(scope, opts.emit); - const { stateNode, snapshot } = yield* hydrateResource(scope, resource, opts); - - // Decide the operation from desired params vs persisted state. The params - // are resolved once and then carried: deriveParams is user code and need - // not be deterministic, so an operation resolving them again could persist - // params other than the ones the decision was taken against. + // Resolved once and then carried: deriveParams is user code and need not be + // deterministic, so an operation resolving them again could persist params + // other than the ones the decision was taken against. The step also pins + // the answer across a replay. const params = yield* scope.run("params", () => resource.getParams()); - const shared = { - ...operationParams(scope, resource, opts), - resourceParams: params, - persistedOutput: stateNode?.output, - }; - let action: ResourceAction = decideAction({ resource, stateNode, params }); - - // A noop is only trusted after the remote is read back: the provider may - // have drifted from persisted state, which upgrades the decision. - if (action.decision === "noop" && (opts.driftDetection ?? true)) { - // Its own scope: the operation that follows reads the remote again, and - // the two reads must not share step keys. - const driftScope = scope.scope("drift-read"); - const driftRead = yield* readDriftOperation(driftScope, { - ...operationParams(driftScope, resource, opts), - // A dry run suppresses mutations, not reads. Threading dryRun in here - // would make the read return {} without touching the provider, which - // decideAction would then diff against the desired params and report as - // drift that is not there. - dryRun: undefined, - resourceParams: params, - persistedOutput: stateNode?.output, - }); - action = decideAction({ resource, stateNode, params, driftRead }); - } - if (action.decision === "drift-update") { - yield* emit({ - level: "info", - event: "reconciler.drift.detected", - resourceId: resource.id, - resourceType: resource.type, - diff: action.patch, - }); - } - - yield* emit({ - level: "info", - event: "reconciler.deploy.decision", - resourceId: resource.id, - resourceType: resource.type, - decision: action.decision, - }); - - if (action.decision === "noop") return; - - const persist = persistResourceState(scope, opts, resource, snapshot); - if (action.decision === "create" || action.decision === "drift-recreate") { - yield* createResourceOperation(scope, { ...shared, persist }); - return; - } - - yield* updateResourceOperation(scope, { - ...shared, - patch: action.patch, - persist, + yield* reconcile(scope, { + resource, + resourceParams: params, + openSession: durableSession(scope, opts), + emit: durableEmit(opts), + dryRun: opts.dryRun, + driftDetection: opts.driftDetection, + maxOperationAttempts: opts.maxOperationAttempts, }); } @@ -104,16 +50,15 @@ export async function* deleteResource( resource: BaseResource, opts: DurableOperationOptions, ): AsyncGenerator { - // Hydrate output from persisted state; the delete call needs the primary - // key and the state removal must be conditional on this exact snapshot. - const snapshot = yield* readSnapshot(step, opts.state, resource.id); - if (!snapshot) return; - const stateNode = toStateNode(snapshot); - resource.setOutput(stateNode.output); - - yield* deleteResourceOperation(step, { - ...operationParams(step, resource, opts), - remove: removeResourceState(step, opts, resource, snapshot), + // No recovery pass: a workflow's conditional writes are stamped with the + // step that made them, so a replay is served the recorded result rather + // than losing a race with itself. + yield* destroyResource(step, { + resource, + openSession: durableSession(step, opts), + emit: durableEmit(opts), + dryRun: opts.dryRun, + maxOperationAttempts: opts.maxOperationAttempts, }); } @@ -157,47 +102,35 @@ export async function* sweepOrphans( } } -/** - * Reads the persisted record once. The snapshot is kept so that later writes - * can be made conditional on the exact instance identity and version read - * here, and is re-served to the operations so they need no second read. - */ -async function* hydrateResource( - step: DurableStepRunner, - resource: BaseResource, - opts: DurableOperationOptions, -): AsyncGenerator< - any, - { stateNode?: StateNode; snapshot?: ResourceSnapshot }, - any -> { - const snapshot = yield* readSnapshot(step, opts.state, resource.id); - if (!snapshot) return {}; - - const stateNode = toStateNode(snapshot); - resource.setOutput(stateNode.output); - return { stateNode, snapshot }; +/** Delivery is checkpointed per scope, so the scope decides the key. */ +function durableEmit(opts: DurableOperationOptions): EmitFromStep { + return (step: StepRunner) => durableEmitter(step, opts.emit); } -function readSnapshot( - step: DurableStepRunner, - state: DurableStateBackend, - resourceId: string, -): AsyncGenerator { - return step.run("state:snapshot", () => state.snapshot(resourceId)); -} - -/** The half of the operation params every durable driver call site shares. */ -function operationParams( +/** + * Reads the persisted record once and binds the writes conditional on it. + * + * The snapshot is the precondition: it names the exact store instance and + * version the record was read at, so a write made against it cannot land on a + * record another writer has moved on. It is re-served to the operations so + * they need no second read. + */ +function durableSession( step: DurableStepRunner, - resource: BaseResource, opts: DurableOperationOptions, -): ResourceOperationBaseParams { - return { - resource, - dryRun: opts.dryRun, - emit: durableEmitter(step, opts.emit), - maxOperationAttempts: opts.maxOperationAttempts, +): OpenStateSession { + return async function* (resource: BaseResource) { + const snapshot = yield* step.run("state:snapshot", () => + opts.state.snapshot(resource.id), + ); + const persist = persistResourceState(step, opts, resource, snapshot); + if (!snapshot) return { node: undefined, persist }; + + return { + node: toStateNode(snapshot), + persist, + remove: removeResourceState(step, opts, resource, snapshot), + }; }; } diff --git a/packages/reconciler/src/durable/step.ts b/packages/reconciler/src/durable/step.ts index a3c78ae..3a37e6f 100644 --- a/packages/reconciler/src/durable/step.ts +++ b/packages/reconciler/src/durable/step.ts @@ -59,7 +59,7 @@ export function scopeStep( * event is delivered but before the checkpoint is written. */ export function durableEmitter( - step: DurableStepRunner, + step: Pick, emit: ReconcilerEventEmitter | undefined, ): EmitStep { return async function* (event) { diff --git a/packages/reconciler/src/reconcile.ts b/packages/reconciler/src/reconcile.ts new file mode 100644 index 0000000..2b125e9 --- /dev/null +++ b/packages/reconciler/src/reconcile.ts @@ -0,0 +1,251 @@ +import type { BaseResource } from "@notation/resource"; +import type { RevConflict, StateNode } from "@notation/state"; +import type { EmitStep, ReconcilerEvent } from "./events"; +import { + createResourceOperation, + deleteResourceOperation, + readDriftOperation, + updateResourceOperation, + type PersistState, + type RemoveState, + type StepRunner, +} from "./operations"; +import { decideAction } from "./plan"; + +/** + * A read of a resource's persisted record, together with the writes that are + * conditional on that exact read. + * + * The union is the point: `remove` exists only alongside a `node`, because + * removing a record that was never read is not a thing either driver can do + * safely. Keeping the writes bound to the read that produced them is what + * stops a node being combined with a precondition from a different read — a + * revision in process, a store snapshot in a workflow. + */ +export type ResourceStateSession = + | { node: undefined; persist: PersistState; remove?: never } + | { node: StateNode; persist: PersistState; remove: RemoveState }; + +/** + * Opens a session. A factory rather than an open session because recovery + * re-reads: the first pass opens once, and a recovering pass opens again + * against whatever the winning writer left behind. + */ +export type OpenStateSession = ( + resource: BaseResource, +) => AsyncGenerator; + +/** + * How a driver delivers an event from a given scope. Emission is scoped + * because a workflow checkpoints it, and two reads of the same resource in + * one execution must not share a checkpoint key. + */ +export type EmitFromStep = (step: StepRunner) => EmitStep; + +export type ReconcileResourceOptions = { + resource: BaseResource; + /** Resolved once per scheduled reconciliation, recovery included. */ + resourceParams: Record; + openSession: OpenStateSession; + emit?: EmitFromStep; + dryRun?: boolean; + driftDetection?: boolean; + maxOperationAttempts?: number; + /** + * Set when a previous attempt lost a conditional write. The remote is then + * read unconditionally and the decision retaken against it, so the attempt + * adopts what the winning writer did rather than repeating its own work. + */ + recoverFrom?: RevConflict; +}; + +/** + * Reconciles one resource: hydrate, decide, read the remote when the decision + * needs it, announce the decision, then act. + * + * Both drivers run this same generator. What stays outside it is scheduling + * (dependency levels, concurrency), the conflict retry policy, opening state + * sessions, and how a step is run — in process a plain await under a mutation + * lease, in a workflow a checkpointed step under a deployment hold. + */ +export async function* reconcileResource( + step: StepRunner, + opts: ReconcileResourceOptions, +): AsyncGenerator { + const { resource, resourceParams: params } = opts; + const recovering = opts.recoverFrom !== undefined; + + // Recovery is built on re-reading the remote, so a resource that cannot be + // read cannot be recovered: the conflict is the caller's answer. + if (recovering && !resource.read) throw opts.recoverFrom; + + const emit = opts.emit?.(step); + const session = yield* opts.openSession(resource); + if (session.node) resource.setOutput(session.node.output); + + let action = decideAction({ resource, stateNode: session.node, params }); + + // A noop is only trusted once the remote has been read back: the provider + // may have drifted from persisted state, which upgrades the decision. + // Recovery always reads, because the point of it is to see what the writer + // that won the race actually left behind. + if ( + recovering || + (action.decision === "noop" && (opts.driftDetection ?? true)) + ) { + // Its own scope: the operation that follows reads the remote again, and + // the two reads must not share step keys. + const driftStep = step.scope("drift-read"); + const driftRead = yield* readDriftOperation(driftStep, { + resource, + resourceParams: params, + persistedOutput: session.node?.output, + // Deliberately no dryRun: a dry run suppresses mutations, not reads. + // Reading is how a dry run reports drift at all. + emit: opts.emit?.(driftStep), + maxOperationAttempts: opts.maxOperationAttempts, + }); + action = decideAction({ + resource, + stateNode: session.node, + params, + driftRead, + }); + if (recovering && driftRead.kind === "present") { + resource.setOutput(driftRead.output); + } + } + + if (action.decision === "drift-update") { + yield* emitEvent(emit, { + level: "info", + event: "reconciler.drift.detected", + resourceId: resource.id, + resourceType: resource.type, + diff: action.patch, + }); + } + + yield* emitEvent(emit, { + level: "info", + event: "reconciler.deploy.decision", + resourceId: resource.id, + resourceType: resource.type, + decision: action.decision, + }); + + const shared = { + resource, + resourceParams: params, + persistedOutput: session.node?.output, + dryRun: opts.dryRun, + emit, + maxOperationAttempts: opts.maxOperationAttempts, + }; + + switch (action.decision) { + case "create": + case "drift-recreate": + yield* createResourceOperation(step, { + ...shared, + persist: session.persist, + }); + return; + case "update": + case "drift-update": + yield* updateResourceOperation(step, { + ...shared, + patch: action.patch, + persist: session.persist, + }); + return; + case "noop": + // A first-pass noop writes nothing. A recovering noop has to: it read + // the remote, found it already converged, and owes a record of that + // adoption at the revision it re-read, or the next deployment would + // reconcile against the losing writer's view. + if (!recovering || opts.dryRun) return; + yield* session.persist({ + id: resource.id, + groupId: resource.groupId, + groupType: resource.groupType, + type: resource.type, + lastOperation: "drift", + lastOperationAt: new Date().toISOString(), + config: resource.config, + params: resource.toState(params), + output: resource.toState(resource.output), + }); + } +} + +export type DestroyResourceOptions = { + resource: BaseResource; + openSession: OpenStateSession; + emit?: EmitFromStep; + dryRun?: boolean; + maxOperationAttempts?: number; + /** + * Set when a previous attempt lost its conditional removal. Carries the + * params because the recovery read needs them, and deletion is the one path + * that never resolves them otherwise. + */ + recoverFrom?: { + conflict: RevConflict; + resourceParams: Record; + }; +}; + +/** + * Deletes one resource. A resource with no persisted record was never created + * — or has already been deleted — and is skipped, which is also what makes + * the sweep of a partly-deleted deployment idempotent. + */ +export async function* destroyResource( + step: StepRunner, + opts: DestroyResourceOptions, +): AsyncGenerator { + const { resource } = opts; + const session = yield* opts.openSession(resource); + if (!session.node) return; + resource.setOutput(session.node.output); + + if (opts.recoverFrom) { + const { conflict, resourceParams } = opts.recoverFrom; + if (!resource.read) throw conflict; + + const driftStep = step.scope("drift-read"); + const remote = yield* readDriftOperation(driftStep, { + resource, + resourceParams, + persistedOutput: session.node.output, + emit: opts.emit?.(driftStep), + maxOperationAttempts: opts.maxOperationAttempts, + }); + + // Already gone remotely: the delete succeeded and only the record is + // left, so drop the record rather than calling the provider again. + if (remote.kind !== "present") { + if (!opts.dryRun) yield* session.remove(); + return; + } + resource.setOutput(remote.output); + } + + yield* deleteResourceOperation(step, { + resource, + dryRun: opts.dryRun, + emit: opts.emit?.(step), + maxOperationAttempts: opts.maxOperationAttempts, + remove: session.remove, + }); +} + +function emitEvent( + emit: EmitStep | undefined, + event: ReconcilerEvent, +): AsyncGenerator { + return emit ? emit(event) : noSteps(); +} + +async function* noSteps(): AsyncGenerator {} diff --git a/packages/reconciler/src/reconciler.ts b/packages/reconciler/src/reconciler.ts index e2b4a79..8c119cf 100644 --- a/packages/reconciler/src/reconciler.ts +++ b/packages/reconciler/src/reconciler.ts @@ -2,25 +2,14 @@ import type { BaseResource, ResourceType } from "@notation/resource"; import { RevConflict, type State, type StateNode } from "@notation/state"; import { setTimeout as sleep } from "node:timers/promises"; import { buildResourceDepthLevels } from "./dependency-graph"; +import type { Plan } from "./plan"; +import type { PersistState, RemoveState, StepRunner } from "./operations"; import { - decideAction, - getDependencyIds, - resolvePlanParams, - type DriftRead, - type Plan, - type PlanNode, - type ResourceAction, -} from "./plan"; -import { - createResourceOperation, - deleteResourceOperation, - readDriftOperation, - type OperationEventEmitter, - type PersistState, - type RemoveState, - type StepRunner, - updateResourceOperation, -} from "./operations"; + destroyResource, + reconcileResource, + type EmitFromStep, + type OpenStateSession, +} from "./reconcile"; import { createMissingResourceRegistryMatchWarningEvent, createResourceRegistryFromResources, @@ -77,7 +66,7 @@ export class Reconciler { readonly #defaultDryRun: boolean; readonly #defaultDriftDetection: boolean; readonly #emit?: ReconcilerEventEmitter; - readonly #emitStep?: OperationEventEmitter; + readonly #emitFromStep?: EmitFromStep; readonly #maxOperationAttempts?: number; readonly #mutationLeaseTtl: number; readonly #stepRunner: StepRunner; @@ -88,8 +77,10 @@ export class Reconciler { this.#defaultDryRun = opts.dryRun ?? false; this.#defaultDriftDetection = opts.driftDetection ?? true; this.#emit = opts.emit; - // Operations emit through a step seam; in process that is a plain await. - this.#emitStep = opts.emit ? toEmitStep(opts.emit) : undefined; + // Everything the shared generator emits — decisions, drift and operation + // lifecycle alike — goes through one seam; in process it is a plain + // await, and the scope it is handed is ignored because nothing is keyed. + this.#emitFromStep = opts.emit ? emitDirectly(opts.emit) : undefined; this.#maxOperationAttempts = opts.maxOperationAttempts; this.#mutationLeaseTtl = opts.mutationLeaseTtl ?? 30_000; this.#stepRunner = createStepRunner(); @@ -132,6 +123,9 @@ export class Reconciler { opts: DestroyOptions = {}, ): Promise { const dryRun = opts.dryRun ?? this.#defaultDryRun; + const resourceById = new Map( + resources.map((resource) => [resource.id, resource]), + ); const dependencyLevels = buildResourceDepthLevels(resources); for ( @@ -144,6 +138,10 @@ export class Reconciler { level.map((resource) => this.#destroyResource(resource, dryRun)), ); } + + // Then the records that were never declared, so a destroy leaves the + // deployment empty rather than leaving orphans for a later refresh. + await this.#deleteOrphans(resources, resourceById, dryRun, "destroy"); } async refresh( @@ -171,12 +169,17 @@ export class Reconciler { const params = (await resource.getParams()) as Record; await this.#retryOnRevConflict((conflict) => - this.#deployResourceOnce( - resource, - params, - dryRun, - driftDetection, - conflict, + runOperation( + reconcileResource(this.#stepRunner, { + resource, + resourceParams: params, + openSession: this.#openSession(), + emit: this.#emitFromStep, + dryRun, + driftDetection, + maxOperationAttempts: this.#maxOperationAttempts, + recoverFrom: conflict, + }), ), ); }); @@ -234,220 +237,34 @@ export class Reconciler { } } - async #deployResourceOnce( - resource: BaseResource, - params: Record, - dryRun: boolean, - driftDetection: boolean, - conflict?: RevConflict, - ) { - if (conflict) { - await this.#recoverDeployResource(resource, params, dryRun, conflict); - return; - } - - const stateNode = await this.#state.get(resource.id); - - let action: ResourceAction; - if (!stateNode) { - action = decideAction({ resource, params }); - } else { - resource.setOutput(stateNode.output); - action = decideAction({ resource, stateNode, params }); - - if (action.decision === "noop" && driftDetection) { - const driftRead = await this.#readForDrift( - resource, - params, - stateNode.output, - ); - action = decideAction({ resource, stateNode, params, driftRead }); - } - } - - if (action.decision === "drift-update") { - await this.#emit?.({ - level: "info", - event: "reconciler.drift.detected", - resourceId: resource.id, - resourceType: resource.type, - diff: action.patch, - }); - } - - await this.#emit?.({ - level: "info", - event: "reconciler.deploy.decision", - resourceId: resource.id, - resourceType: resource.type, - decision: action.decision, - }); - - switch (action.decision) { - case "create": - case "drift-recreate": - await runOperation( - createResourceOperation(this.#stepRunner, { - resource, - resourceParams: params, - persistedOutput: stateNode?.output, - dryRun, - emit: this.#emitStep, - maxOperationAttempts: this.#maxOperationAttempts, - persist: this.#persist(resource.id, stateNode?.rev ?? 0), - }), - ); - return; - case "update": - case "drift-update": - // decideAction only returns update decisions for an existing stateNode - await runOperation( - updateResourceOperation(this.#stepRunner, { - resource, - resourceParams: params, - persistedOutput: stateNode?.output, - patch: action.patch, - dryRun, - emit: this.#emitStep, - maxOperationAttempts: this.#maxOperationAttempts, - persist: this.#persist(resource.id, stateNode!.rev), - }), - ); - return; - case "noop": - return; - } - } - - async #recoverDeployResource( - resource: BaseResource, - params: Record, - dryRun: boolean, - conflict: RevConflict, - ) { - if (!resource.read) throw conflict; - - const stateNode = await this.#state.get(resource.id); - if (stateNode) resource.setOutput(stateNode.output); - - const remote = await this.#readForDrift( - resource, - params, - stateNode?.output, - ); - const action = decideAction({ - resource, - stateNode, - params, - driftRead: remote, - }); - if (remote.kind === "present") resource.setOutput(remote.output); - - // Recovery is drift adoption: the remote moved while the attempt that - // conflicted was in flight, so the same event a first-pass drift-update - // emits is owed here too. - if (action.decision === "drift-update") { - await this.#emit?.({ - level: "info", - event: "reconciler.drift.detected", - resourceId: resource.id, - resourceType: resource.type, - diff: action.patch, - }); - } - - await this.#emit?.({ - level: "info", - event: "reconciler.deploy.decision", - resourceId: resource.id, - resourceType: resource.type, - decision: action.decision, - }); - - switch (action.decision) { - case "create": - case "drift-recreate": - await runOperation( - createResourceOperation(this.#stepRunner, { - resource, - resourceParams: params, - persistedOutput: stateNode?.output, - dryRun, - emit: this.#emitStep, - maxOperationAttempts: this.#maxOperationAttempts, - persist: this.#persist(resource.id, stateNode?.rev ?? 0), - }), - ); - return; - case "update": - case "drift-update": - await runOperation( - updateResourceOperation(this.#stepRunner, { - resource, - resourceParams: params, - persistedOutput: stateNode?.output, - patch: action.patch, - dryRun, - emit: this.#emitStep, - maxOperationAttempts: this.#maxOperationAttempts, - persist: this.#persist(resource.id, stateNode?.rev ?? 0), - }), - ); - return; - case "noop": - if (dryRun) return; - await this.#state.update(resource.id, stateNode?.rev ?? 0, { - id: resource.id, - groupId: resource.groupId, - groupType: resource.groupType, - type: resource.type, - lastOperation: "drift", - lastOperationAt: new Date().toISOString(), - config: resource.config, - params: resource.toState(params), - output: resource.toState(resource.output), - }); - return; - } - } - - // In process, concurrency control is a compare-and-set against the revision - // read before the operation started; the mutation lease keeps writers apart. - #persist(resourceId: string, expectedRev: number): PersistState { - const state = this.#state; - return async function* (next) { - await state.update(resourceId, expectedRev, next); - }; - } - - #remove(resourceId: string, expectedRev: number): RemoveState { + /** + * Reads a resource's record and binds the writes that are conditional on + * that read. In process the precondition is the revision read here, which + * the mutation lease keeps other writers away from. + */ + #openSession(): OpenStateSession { const state = this.#state; - return async function* () { - await state.delete(resourceId, expectedRev); + return async function* (resource: BaseResource) { + const node = await state.get(resource.id); + const expectedRev = node?.rev ?? 0; + const persist: PersistState = async function* (next) { + await state.update(resource.id, expectedRev, next); + }; + + if (!node) return { node: undefined, persist }; + + const remove: RemoveState = async function* () { + await state.delete(resource.id, node.rev); + }; + return { node, persist, remove }; }; } - #readForDrift( - resource: BaseResource, - params: Record, - persistedOutput: Record | undefined, - ): Promise { - return runOperation( - readDriftOperation(this.#stepRunner, { - resource, - resourceParams: params, - persistedOutput, - emit: this.#emitStep, - maxOperationAttempts: this.#maxOperationAttempts, - }), - ); - } - async #deleteOrphans( resources: BaseResource[], resourceById: Map, dryRun: boolean, - workflow: "deploy" | "refresh", + workflow: "deploy" | "refresh" | "destroy", ) { await this.#withLease("reconciler:orphan-deletion", async () => { const stateNodes = await this.#state.values(); @@ -471,23 +288,17 @@ export class Reconciler { continue; } + // Built from the listing, so its config is as of that read rather + // than of the lease taken below — the session re-reads and refreshes + // output, but not this. Config reaches nothing but deriveParams on + // the delete-recovery read, so a racing write can only make that read + // one revision stale; the removal itself is still conditional on the + // session's own read. + const orphanResource = hydrateResourceFromState(Resource, stateNode); await this.#withMutationLease(stateNode.id, () => - this.#retryOnRevConflict(async (conflict) => { - const currentNode = await this.#state.get(stateNode.id); - if (!currentNode) return; - - const orphanResource = hydrateResourceFromState( - Resource, - currentNode, - ); - - await this.#deleteResourceOnce( - orphanResource, - currentNode, - dryRun, - conflict, - ); - }), + this.#retryOnRevConflict((conflict) => + this.#deleteResource(orphanResource, dryRun, conflict), + ), ); } }); @@ -495,49 +306,37 @@ export class Reconciler { async #destroyResource(resource: BaseResource, dryRun: boolean) { await this.#withMutationLease(resource.id, () => - this.#retryOnRevConflict(async (conflict) => { - const stateNode = await this.#state.get(resource.id); - if (!stateNode) { - return; - } - - resource.setOutput(stateNode.output); - await this.#deleteResourceOnce(resource, stateNode, dryRun, conflict); - }), + this.#retryOnRevConflict((conflict) => + this.#deleteResource(resource, dryRun, conflict), + ), ); } - async #deleteResourceOnce( + async #deleteResource( resource: BaseResource, - stateNode: StateNode, dryRun: boolean, conflict?: RevConflict, ) { - if (conflict) { - if (!resource.read) throw conflict; - - // Deletion never needs the desired params, so they are resolved here - // rather than for every delete: only the recovery read consumes them. - const params = (await resource.getParams()) as Record; - const remote = await this.#readForDrift( - resource, - params, - stateNode.output, - ); - if (remote.kind !== "present") { - if (!dryRun) await this.#state.delete(resource.id, stateNode.rev); - return; - } - resource.setOutput(remote.output); - } + // Deletion never needs the desired params, so they are resolved only for + // the recovery read, which is their sole consumer on this path. + const recoverFrom = conflict + ? { + conflict, + resourceParams: (await resource.getParams()) as Record< + string, + unknown + >, + } + : undefined; await runOperation( - deleteResourceOperation(this.#stepRunner, { + destroyResource(this.#stepRunner, { resource, + openSession: this.#openSession(), + emit: this.#emitFromStep, dryRun, - emit: this.#emitStep, maxOperationAttempts: this.#maxOperationAttempts, - remove: this.#remove(resource.id, stateNode.rev), + recoverFrom, }), ); } @@ -557,3 +356,12 @@ function hydrateResourceFromState( resource.setOutput(stateNode.output); return resource; } + +/** + * The in-process emit seam: a scope carries no meaning here, since nothing + * keys anything, so every scope delivers through the same emitter. + */ +function emitDirectly(emit: ReconcilerEventEmitter): EmitFromStep { + const step = toEmitStep(emit); + return () => step; +} diff --git a/packages/reconciler/test/reconciler.deploy.test.ts b/packages/reconciler/test/reconciler.deploy.test.ts index 9987a90..61c328b 100644 --- a/packages/reconciler/test/reconciler.deploy.test.ts +++ b/packages/reconciler/test/reconciler.deploy.test.ts @@ -766,6 +766,59 @@ describe("reconciler destroy + refresh", () => { expect(state.delete).toHaveBeenCalledWith("c", 1); }); + it("destroy sweeps orphans after the resources it was given", async () => { + const order: string[] = []; + const DeclaredResource = createTestResourceClass({ + type: "test/service/destroy-sweep-declared", + delete: async () => void order.push("declared"), + }); + // Its type is only in the registry: an app that stopped declaring it is + // exactly the case where falling back to the declared types would skip it. + const OrphanResource = createTestResourceClass({ + type: "test/service/destroy-sweep-orphan", + delete: async () => void order.push("orphan"), + }); + + const state = createMemoryState({ + declared: { + rev: 1, + id: "declared", + groupId: -1, + groupType: "", + type: DeclaredResource.type, + config: { name: "declared" }, + params: { name: "declared" }, + output: { name: "declared" }, + lastOperation: "create", + lastOperationAt: new Date().toISOString(), + }, + orphan: { + rev: 1, + id: "orphan", + groupId: -1, + groupType: "", + type: OrphanResource.type, + config: { name: "orphan" }, + params: { name: "orphan" }, + output: { name: "orphan" }, + lastOperation: "create", + lastOperationAt: new Date().toISOString(), + }, + }); + + const reconciler = new Reconciler({ + state, + registry: createResourceRegistry([DeclaredResource, OrphanResource]), + }); + + await reconciler.destroy([ + new DeclaredResource({ id: "declared", config: { name: "declared" } }), + ]); + + expect(order).toEqual(["declared", "orphan"]); + expect(state.store).toEqual({}); + }); + it("refresh removes orphan state entries", async () => { const deleteSpy = vi.fn(async () => undefined); const OrphanResource = createTestResourceClass({ From e065d507a211450005cc6981e3ceb4299b1ee98b Mon Sep 17 00:00:00 2001 From: djgrant <1670902+djgrant@users.noreply.github.com> Date: Tue, 28 Jul 2026 14:38:37 +0100 Subject: [PATCH 20/34] Validate the execution binding store with Valibot The hand-rolled standard-schema validator predated the convergence on Valibot for store schemas; this brings the one store defined outside the reconciler in line with the others, and lets the schema type the store state instead of a cast. --- packages/core/package.json | 1 + .../core/src/provisioner/durable-runtime.ts | 31 +++++-------------- pnpm-lock.yaml | 3 ++ 3 files changed, 11 insertions(+), 24 deletions(-) diff --git a/packages/core/package.json b/packages/core/package.json index c3e0fae..bcc145f 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -20,6 +20,7 @@ "@yieldstar/sqlite-runtime": "0.5.0", "deep-object-diff": "^1.1.9", "js-base64": "^3.7.7", + "valibot": "^1.4.2", "lodash-es": "^4.17.21", "pako": "^2.1.0", "pino": "^9.14.0", diff --git a/packages/core/src/provisioner/durable-runtime.ts b/packages/core/src/provisioner/durable-runtime.ts index 9e8e136..789da79 100644 --- a/packages/core/src/provisioner/durable-runtime.ts +++ b/packages/core/src/provisioner/durable-runtime.ts @@ -24,6 +24,7 @@ import { } from "@notation/reconciler/durable"; import { FileStateBackend, type StateNode } from "@notation/state"; import pino, { type Logger } from "pino"; +import * as v from "valibot"; import { defineStore } from "yieldstar"; export const DEFAULT_WORKFLOW_STATE_PATH = ".notation/workflows.db"; @@ -50,30 +51,12 @@ export type RunWorkflowOptions = { params?: Record; }; -type ExecutionBinding = { - deploymentId: string; - workflowId: string; -}; +const executionBindingStore = defineStore( + "notation/execution-binding", + v.object({ deploymentId: v.string(), workflowId: v.string() }), +); -const executionBindingStore = defineStore("notation/execution-binding", { - "~standard": { - version: 1 as const, - vendor: "notation", - validate(value: unknown) { - if ( - typeof value === "object" && - value !== null && - "deploymentId" in value && - typeof value.deploymentId === "string" && - "workflowId" in value && - typeof value.workflowId === "string" - ) { - return { value: value as ExecutionBinding }; - } - return { issues: [{ message: "Execution binding is invalid" }] }; - }, - }, -}); +type ExecutionBinding = v.InferOutput; /** Resident Yieldstar 0.5.0 Node runtime used by Notation application commands. */ export class NodeDurableRuntime { @@ -201,7 +184,7 @@ export class NodeDurableRuntime { id: executionId, initial: expected, }); - const existing = binding.state as ExecutionBinding; + const existing: ExecutionBinding = binding.state; if (!isDeepStrictEqual(existing, expected)) { throw new Error( `Yieldstar execution ${executionId} is bound to deployment ${existing.deploymentId} workflow ${existing.workflowId}, not deployment ${this.deploymentId} workflow ${workflowId}`, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5f1ee6d..288e8ad 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -260,6 +260,9 @@ importers: pino: specifier: ^9.14.0 version: 9.14.0 + valibot: + specifier: ^1.4.2 + version: 1.4.2(typescript@6.0.3) yieldstar: specifier: 0.5.0 version: 0.5.0 From df4ffee7f09489093bb07ad6cd98cafc44e71e00 Mon Sep 17 00:00:00 2001 From: djgrant <1670902+djgrant@users.noreply.github.com> Date: Tue, 28 Jul 2026 14:51:56 +0100 Subject: [PATCH 21/34] Fold the shared reconcile algorithm into the durable driver The two-driver seam (reconcile.ts with OpenStateSession, EmitFromStep and the recoverFrom recovery paths) outlived its second driver: the in-process reconciler was deleted when the runtime cut over, leaving the durable driver as the sole caller and the recovery machinery unreachable. Inline the algorithm into durable/operations.ts, preserving every step key, and drop the dead recovery code. The RevConflict tests still expect conflicts to propagate, which is what the durable driver always did. Also extract withRuntime so deployApp, destroyApp and planApp share one runtime acquire/close lifecycle instead of three copies. --- .../core/src/provisioner/durable-runtime.ts | 26 ++ .../provisioner/workflows/workflow.deploy.ts | 49 ++-- .../provisioner/workflows/workflow.destroy.ts | 45 ++-- .../provisioner/workflows/workflow.plan.ts | 29 +- packages/reconciler/src/durable/operations.ts | 159 ++++++++--- .../src/operations/operation.types.ts | 7 +- packages/reconciler/src/reconcile.ts | 251 ------------------ 7 files changed, 208 insertions(+), 358 deletions(-) delete mode 100644 packages/reconciler/src/reconcile.ts diff --git a/packages/core/src/provisioner/durable-runtime.ts b/packages/core/src/provisioner/durable-runtime.ts index 789da79..118c5c0 100644 --- a/packages/core/src/provisioner/durable-runtime.ts +++ b/packages/core/src/provisioner/durable-runtime.ts @@ -277,6 +277,32 @@ export class NodeDurableRuntime { } } +/** + * Runs `fn` with a Node runtime for the entry point's deployment, creating one + * when the caller did not supply a runtime and closing it again afterwards. A + * supplied runtime stays open: its lifecycle belongs to the caller. + */ +export async function withRuntime( + opts: { + entryPoint: string; + runtime?: NodeDurableRuntime; + databasePath?: string; + }, + fn: (runtime: NodeDurableRuntime) => Promise, +): Promise { + const runtime = + opts.runtime ?? + new NodeDurableRuntime({ + deploymentId: resolveDeploymentId(opts.entryPoint), + databasePath: opts.databasePath, + }); + try { + return await fn(runtime); + } finally { + if (!opts.runtime) runtime.close(); + } +} + function statesMatchIgnoringRevision(left: StateNode, right: StateNode) { const { rev: _leftRev, ...leftState } = left; const { rev: _rightRev, ...rightState } = right; diff --git a/packages/core/src/provisioner/workflows/workflow.deploy.ts b/packages/core/src/provisioner/workflows/workflow.deploy.ts index 8875070..9eba49b 100644 --- a/packages/core/src/provisioner/workflows/workflow.deploy.ts +++ b/packages/core/src/provisioner/workflows/workflow.deploy.ts @@ -6,7 +6,7 @@ import { } from "@notation/reconciler"; import { createWorkflowRouter, workflow } from "yieldstar"; import { getResourceGraph } from "src/orchestrator/graph"; -import { NodeDurableRuntime, resolveDeploymentId } from "../durable-runtime"; +import { withRuntime, type NodeDurableRuntime } from "../durable-runtime"; export type DeployAppOptions = { entryPoint: string; @@ -32,29 +32,26 @@ export async function deployApp({ emit = createLoggerReconcilerSubscriber(), }: DeployAppOptions): Promise { const graph = await getResourceGraph(entryPoint); - const deploymentId = - suppliedRuntime?.deploymentId ?? resolveDeploymentId(entryPoint); - const runtime = - suppliedRuntime ?? new NodeDurableRuntime({ deploymentId, databasePath }); - const deploy = workflow(async function* (step, event) { - yield* reconciler.deploy(step, { - deploymentId: runtime.deploymentId, - executionId: event.executionId, - resources: graph.resources, - state: runtime.state, - registry, - emit, - dryRun, - driftDetection, - maxOperationAttempts, - }); - }); - try { - await runtime.run(createWorkflowRouter({ deploy }), { - workflowId: "deploy", - executionId, - }); - } finally { - if (!suppliedRuntime) runtime.close(); - } + await withRuntime( + { entryPoint, runtime: suppliedRuntime, databasePath }, + async (runtime) => { + const deploy = workflow(async function* (step, event) { + yield* reconciler.deploy(step, { + deploymentId: runtime.deploymentId, + executionId: event.executionId, + resources: graph.resources, + state: runtime.state, + registry, + emit, + dryRun, + driftDetection, + maxOperationAttempts, + }); + }); + await runtime.run(createWorkflowRouter({ deploy }), { + workflowId: "deploy", + executionId, + }); + }, + ); } diff --git a/packages/core/src/provisioner/workflows/workflow.destroy.ts b/packages/core/src/provisioner/workflows/workflow.destroy.ts index 3f8ac8c..0c147bb 100644 --- a/packages/core/src/provisioner/workflows/workflow.destroy.ts +++ b/packages/core/src/provisioner/workflows/workflow.destroy.ts @@ -6,7 +6,7 @@ import { } from "@notation/reconciler"; import { createWorkflowRouter, workflow } from "yieldstar"; import { getResourceGraph } from "src/orchestrator/graph"; -import { NodeDurableRuntime, resolveDeploymentId } from "../durable-runtime"; +import { withRuntime, type NodeDurableRuntime } from "../durable-runtime"; export type DestroyAppOptions = { entryPoint: string; @@ -28,27 +28,24 @@ export async function destroyApp({ emit = createLoggerReconcilerSubscriber(), }: DestroyAppOptions) { const graph = await getResourceGraph(entryPoint); - const deploymentId = - suppliedRuntime?.deploymentId ?? resolveDeploymentId(entryPoint); - const runtime = - suppliedRuntime ?? new NodeDurableRuntime({ deploymentId, databasePath }); - const destroy = workflow(async function* (step, event) { - yield* reconciler.destroy(step, { - deploymentId: runtime.deploymentId, - executionId: event.executionId, - resources: graph.resources, - state: runtime.state, - registry, - emit, - maxOperationAttempts, - }); - }); - try { - await runtime.run(createWorkflowRouter({ destroy }), { - workflowId: "destroy", - executionId, - }); - } finally { - if (!suppliedRuntime) runtime.close(); - } + await withRuntime( + { entryPoint, runtime: suppliedRuntime, databasePath }, + async (runtime) => { + const destroy = workflow(async function* (step, event) { + yield* reconciler.destroy(step, { + deploymentId: runtime.deploymentId, + executionId: event.executionId, + resources: graph.resources, + state: runtime.state, + registry, + emit, + maxOperationAttempts, + }); + }); + await runtime.run(createWorkflowRouter({ destroy }), { + workflowId: "destroy", + executionId, + }); + }, + ); } diff --git a/packages/core/src/provisioner/workflows/workflow.plan.ts b/packages/core/src/provisioner/workflows/workflow.plan.ts index 5b06aba..bb7b51c 100644 --- a/packages/core/src/provisioner/workflows/workflow.plan.ts +++ b/packages/core/src/provisioner/workflows/workflow.plan.ts @@ -1,6 +1,6 @@ import { createPlan, type Plan } from "@notation/reconciler"; import { getResourceGraph } from "src/orchestrator/graph"; -import { NodeDurableRuntime, resolveDeploymentId } from "../durable-runtime"; +import { withRuntime, type NodeDurableRuntime } from "../durable-runtime"; export type { Plan, PlanNode, PlanDecision } from "@notation/reconciler"; @@ -20,19 +20,16 @@ export async function planApp({ databasePath, }: PlanAppOptions): Promise { const graph = await getResourceGraph(entryPoint); - const deploymentId = - suppliedRuntime?.deploymentId ?? resolveDeploymentId(entryPoint); - const runtime = - suppliedRuntime ?? new NodeDurableRuntime({ deploymentId, databasePath }); - try { - await runtime.initialize(); - return await createPlan({ - resources: graph.resources, - state: runtime.state, - driftDetection, - maxOperationAttempts, - }); - } finally { - if (!suppliedRuntime) runtime.close(); - } + return withRuntime( + { entryPoint, runtime: suppliedRuntime, databasePath }, + async (runtime) => { + await runtime.initialize(); + return createPlan({ + resources: graph.resources, + state: runtime.state, + driftDetection, + maxOperationAttempts, + }); + }, + ); } diff --git a/packages/reconciler/src/durable/operations.ts b/packages/reconciler/src/durable/operations.ts index 5ccc5c0..26541ec 100644 --- a/packages/reconciler/src/durable/operations.ts +++ b/packages/reconciler/src/durable/operations.ts @@ -1,17 +1,19 @@ import type { BaseResource, ResourceType } from "@notation/resource"; -import { RevConflict } from "@notation/state"; +import { RevConflict, type StateNode } from "@notation/state"; import { createMissingResourceRegistryMatchWarningEvent, createResourceRegistryFromResources, resolveResourceClass, } from "../resource-registry"; -import type { PersistState, RemoveState, StepRunner } from "../operations"; import { - destroyResource, - reconcileResource as reconcile, - type EmitFromStep, - type OpenStateSession, -} from "../reconcile"; + createResourceOperation, + deleteResourceOperation, + readDriftOperation, + updateResourceOperation, + type PersistState, + type RemoveState, +} from "../operations"; +import { decideAction } from "../plan"; import { durableEmitter, scopeStep, type DurableStepRunner } from "./step"; import { resourceStateStore, @@ -21,6 +23,23 @@ import { import type { DurableDeployOptions, DurableOperationOptions } from "./types"; import type { DurableStep } from "./yieldstar"; +/** + * A read of a resource's persisted record, together with the writes that are + * conditional on that exact read. + * + * The union is the point: `remove` exists only alongside a `node`, because + * removing a record that was never read cannot be done safely. Keeping the + * writes bound to the read that produced them is what stops a node being + * combined with a precondition from a different read. + */ +type ResourceStateSession = + | { node: undefined; persist: PersistState; remove?: never } + | { node: StateNode; persist: PersistState; remove: RemoveState }; + +/** + * Reconciles one resource: hydrate, decide, read the remote when the decision + * needs it, announce the decision, then act. + */ export async function* reconcileResource( step: DurableStep, resource: BaseResource, @@ -34,31 +53,103 @@ export async function* reconcileResource( // the answer across a replay. const params = yield* scope.run("params", () => resource.getParams()); - yield* reconcile(scope, { + const emit = durableEmitter(scope, opts.emit); + const session = yield* openStateSession(scope, opts, resource); + if (session.node) resource.setOutput(session.node.output); + + let action = decideAction({ resource, stateNode: session.node, params }); + + // A noop is only trusted once the remote has been read back: the provider + // may have drifted from persisted state, which upgrades the decision. + if (action.decision === "noop" && (opts.driftDetection ?? true)) { + // Its own scope: the operation that follows reads the remote again, and + // the two reads must not share step keys. + const driftStep = scope.scope("drift-read"); + const driftRead = yield* readDriftOperation(driftStep, { + resource, + resourceParams: params, + persistedOutput: session.node?.output, + // Deliberately no dryRun: a dry run suppresses mutations, not reads. + // Reading is how a dry run reports drift at all. + emit: durableEmitter(driftStep, opts.emit), + maxOperationAttempts: opts.maxOperationAttempts, + }); + action = decideAction({ + resource, + stateNode: session.node, + params, + driftRead, + }); + } + + if (action.decision === "drift-update") { + yield* emit({ + level: "info", + event: "reconciler.drift.detected", + resourceId: resource.id, + resourceType: resource.type, + diff: action.patch, + }); + } + + yield* emit({ + level: "info", + event: "reconciler.deploy.decision", + resourceId: resource.id, + resourceType: resource.type, + decision: action.decision, + }); + + const shared = { resource, resourceParams: params, - openSession: durableSession(scope, opts), - emit: durableEmit(opts), + persistedOutput: session.node?.output, dryRun: opts.dryRun, - driftDetection: opts.driftDetection, + emit, maxOperationAttempts: opts.maxOperationAttempts, - }); + }; + + switch (action.decision) { + case "create": + case "drift-recreate": + yield* createResourceOperation(scope, { + ...shared, + persist: session.persist, + }); + return; + case "update": + case "drift-update": + yield* updateResourceOperation(scope, { + ...shared, + patch: action.patch, + persist: session.persist, + }); + return; + case "noop": + return; + } } +/** + * Deletes one resource. A resource with no persisted record was never created + * — or has already been deleted — and is skipped, which is also what makes + * the sweep of a partly-deleted deployment idempotent. + */ export async function* deleteResource( step: DurableStepRunner, resource: BaseResource, opts: DurableOperationOptions, ): AsyncGenerator { - // No recovery pass: a workflow's conditional writes are stamped with the - // step that made them, so a replay is served the recorded result rather - // than losing a race with itself. - yield* destroyResource(step, { + const session = yield* openStateSession(step, opts, resource); + if (!session.node) return; + resource.setOutput(session.node.output); + + yield* deleteResourceOperation(step, { resource, - openSession: durableSession(step, opts), - emit: durableEmit(opts), dryRun: opts.dryRun, + emit: durableEmitter(step, opts.emit), maxOperationAttempts: opts.maxOperationAttempts, + remove: session.remove, }); } @@ -102,11 +193,6 @@ export async function* sweepOrphans( } } -/** Delivery is checkpointed per scope, so the scope decides the key. */ -function durableEmit(opts: DurableOperationOptions): EmitFromStep { - return (step: StepRunner) => durableEmitter(step, opts.emit); -} - /** * Reads the persisted record once and binds the writes conditional on it. * @@ -115,22 +201,21 @@ function durableEmit(opts: DurableOperationOptions): EmitFromStep { * record another writer has moved on. It is re-served to the operations so * they need no second read. */ -function durableSession( +async function* openStateSession( step: DurableStepRunner, opts: DurableOperationOptions, -): OpenStateSession { - return async function* (resource: BaseResource) { - const snapshot = yield* step.run("state:snapshot", () => - opts.state.snapshot(resource.id), - ); - const persist = persistResourceState(step, opts, resource, snapshot); - if (!snapshot) return { node: undefined, persist }; - - return { - node: toStateNode(snapshot), - persist, - remove: removeResourceState(step, opts, resource, snapshot), - }; + resource: BaseResource, +): AsyncGenerator { + const snapshot = yield* step.run("state:snapshot", () => + opts.state.snapshot(resource.id), + ); + const persist = persistResourceState(step, opts, resource, snapshot); + if (!snapshot) return { node: undefined, persist }; + + return { + node: toStateNode(snapshot), + persist, + remove: removeResourceState(step, opts, resource, snapshot), }; } diff --git a/packages/reconciler/src/operations/operation.types.ts b/packages/reconciler/src/operations/operation.types.ts index 492f4c9..9eeeff9 100644 --- a/packages/reconciler/src/operations/operation.types.ts +++ b/packages/reconciler/src/operations/operation.types.ts @@ -58,10 +58,9 @@ export type PersistedResourceState = Pick< }; /** - * How a driver writes state. Both are steps so that each driver can carry its - * own concurrency control: in process that is a compare-and-set against the - * revision read before the operation, and in a workflow it is a store write - * stamped with the step that made it, so a replay does not repeat it. + * How the driver writes state. Both are steps so the driver can carry its own + * concurrency control: in a workflow that is a store write stamped with the + * step that made it, so a replay does not repeat it. */ export type PersistState = ( next: PersistedResourceState, diff --git a/packages/reconciler/src/reconcile.ts b/packages/reconciler/src/reconcile.ts deleted file mode 100644 index 2b125e9..0000000 --- a/packages/reconciler/src/reconcile.ts +++ /dev/null @@ -1,251 +0,0 @@ -import type { BaseResource } from "@notation/resource"; -import type { RevConflict, StateNode } from "@notation/state"; -import type { EmitStep, ReconcilerEvent } from "./events"; -import { - createResourceOperation, - deleteResourceOperation, - readDriftOperation, - updateResourceOperation, - type PersistState, - type RemoveState, - type StepRunner, -} from "./operations"; -import { decideAction } from "./plan"; - -/** - * A read of a resource's persisted record, together with the writes that are - * conditional on that exact read. - * - * The union is the point: `remove` exists only alongside a `node`, because - * removing a record that was never read is not a thing either driver can do - * safely. Keeping the writes bound to the read that produced them is what - * stops a node being combined with a precondition from a different read — a - * revision in process, a store snapshot in a workflow. - */ -export type ResourceStateSession = - | { node: undefined; persist: PersistState; remove?: never } - | { node: StateNode; persist: PersistState; remove: RemoveState }; - -/** - * Opens a session. A factory rather than an open session because recovery - * re-reads: the first pass opens once, and a recovering pass opens again - * against whatever the winning writer left behind. - */ -export type OpenStateSession = ( - resource: BaseResource, -) => AsyncGenerator; - -/** - * How a driver delivers an event from a given scope. Emission is scoped - * because a workflow checkpoints it, and two reads of the same resource in - * one execution must not share a checkpoint key. - */ -export type EmitFromStep = (step: StepRunner) => EmitStep; - -export type ReconcileResourceOptions = { - resource: BaseResource; - /** Resolved once per scheduled reconciliation, recovery included. */ - resourceParams: Record; - openSession: OpenStateSession; - emit?: EmitFromStep; - dryRun?: boolean; - driftDetection?: boolean; - maxOperationAttempts?: number; - /** - * Set when a previous attempt lost a conditional write. The remote is then - * read unconditionally and the decision retaken against it, so the attempt - * adopts what the winning writer did rather than repeating its own work. - */ - recoverFrom?: RevConflict; -}; - -/** - * Reconciles one resource: hydrate, decide, read the remote when the decision - * needs it, announce the decision, then act. - * - * Both drivers run this same generator. What stays outside it is scheduling - * (dependency levels, concurrency), the conflict retry policy, opening state - * sessions, and how a step is run — in process a plain await under a mutation - * lease, in a workflow a checkpointed step under a deployment hold. - */ -export async function* reconcileResource( - step: StepRunner, - opts: ReconcileResourceOptions, -): AsyncGenerator { - const { resource, resourceParams: params } = opts; - const recovering = opts.recoverFrom !== undefined; - - // Recovery is built on re-reading the remote, so a resource that cannot be - // read cannot be recovered: the conflict is the caller's answer. - if (recovering && !resource.read) throw opts.recoverFrom; - - const emit = opts.emit?.(step); - const session = yield* opts.openSession(resource); - if (session.node) resource.setOutput(session.node.output); - - let action = decideAction({ resource, stateNode: session.node, params }); - - // A noop is only trusted once the remote has been read back: the provider - // may have drifted from persisted state, which upgrades the decision. - // Recovery always reads, because the point of it is to see what the writer - // that won the race actually left behind. - if ( - recovering || - (action.decision === "noop" && (opts.driftDetection ?? true)) - ) { - // Its own scope: the operation that follows reads the remote again, and - // the two reads must not share step keys. - const driftStep = step.scope("drift-read"); - const driftRead = yield* readDriftOperation(driftStep, { - resource, - resourceParams: params, - persistedOutput: session.node?.output, - // Deliberately no dryRun: a dry run suppresses mutations, not reads. - // Reading is how a dry run reports drift at all. - emit: opts.emit?.(driftStep), - maxOperationAttempts: opts.maxOperationAttempts, - }); - action = decideAction({ - resource, - stateNode: session.node, - params, - driftRead, - }); - if (recovering && driftRead.kind === "present") { - resource.setOutput(driftRead.output); - } - } - - if (action.decision === "drift-update") { - yield* emitEvent(emit, { - level: "info", - event: "reconciler.drift.detected", - resourceId: resource.id, - resourceType: resource.type, - diff: action.patch, - }); - } - - yield* emitEvent(emit, { - level: "info", - event: "reconciler.deploy.decision", - resourceId: resource.id, - resourceType: resource.type, - decision: action.decision, - }); - - const shared = { - resource, - resourceParams: params, - persistedOutput: session.node?.output, - dryRun: opts.dryRun, - emit, - maxOperationAttempts: opts.maxOperationAttempts, - }; - - switch (action.decision) { - case "create": - case "drift-recreate": - yield* createResourceOperation(step, { - ...shared, - persist: session.persist, - }); - return; - case "update": - case "drift-update": - yield* updateResourceOperation(step, { - ...shared, - patch: action.patch, - persist: session.persist, - }); - return; - case "noop": - // A first-pass noop writes nothing. A recovering noop has to: it read - // the remote, found it already converged, and owes a record of that - // adoption at the revision it re-read, or the next deployment would - // reconcile against the losing writer's view. - if (!recovering || opts.dryRun) return; - yield* session.persist({ - id: resource.id, - groupId: resource.groupId, - groupType: resource.groupType, - type: resource.type, - lastOperation: "drift", - lastOperationAt: new Date().toISOString(), - config: resource.config, - params: resource.toState(params), - output: resource.toState(resource.output), - }); - } -} - -export type DestroyResourceOptions = { - resource: BaseResource; - openSession: OpenStateSession; - emit?: EmitFromStep; - dryRun?: boolean; - maxOperationAttempts?: number; - /** - * Set when a previous attempt lost its conditional removal. Carries the - * params because the recovery read needs them, and deletion is the one path - * that never resolves them otherwise. - */ - recoverFrom?: { - conflict: RevConflict; - resourceParams: Record; - }; -}; - -/** - * Deletes one resource. A resource with no persisted record was never created - * — or has already been deleted — and is skipped, which is also what makes - * the sweep of a partly-deleted deployment idempotent. - */ -export async function* destroyResource( - step: StepRunner, - opts: DestroyResourceOptions, -): AsyncGenerator { - const { resource } = opts; - const session = yield* opts.openSession(resource); - if (!session.node) return; - resource.setOutput(session.node.output); - - if (opts.recoverFrom) { - const { conflict, resourceParams } = opts.recoverFrom; - if (!resource.read) throw conflict; - - const driftStep = step.scope("drift-read"); - const remote = yield* readDriftOperation(driftStep, { - resource, - resourceParams, - persistedOutput: session.node.output, - emit: opts.emit?.(driftStep), - maxOperationAttempts: opts.maxOperationAttempts, - }); - - // Already gone remotely: the delete succeeded and only the record is - // left, so drop the record rather than calling the provider again. - if (remote.kind !== "present") { - if (!opts.dryRun) yield* session.remove(); - return; - } - resource.setOutput(remote.output); - } - - yield* deleteResourceOperation(step, { - resource, - dryRun: opts.dryRun, - emit: opts.emit?.(step), - maxOperationAttempts: opts.maxOperationAttempts, - remove: session.remove, - }); -} - -function emitEvent( - emit: EmitStep | undefined, - event: ReconcilerEvent, -): AsyncGenerator { - return emit ? emit(event) : noSteps(); -} - -async function* noSteps(): AsyncGenerator {} From 4b31e991b86d1b39cc9c31bfe53c75669c409bfc Mon Sep 17 00:00:00 2001 From: djgrant <1670902+djgrant@users.noreply.github.com> Date: Tue, 28 Jul 2026 15:00:26 +0100 Subject: [PATCH 22/34] Drive the durable execution with a plain loop The runtime's event pump routed completion through a shared promise, a completed flag and three callbacks, with a dead branch for replay errors on executions the pump never feeds it. Replace it with driveToCompletion: run the trigger event, then loop the queue until the runner returns a result, deferring foreign tasks exactly as before. Errors now propagate by throwing instead of by rejecting a captured resolver. Also extract runDurableWorkflow so deployApp and destroyApp share the one-command-one-workflow wrapper instead of each building a router, and declare DurableStepRunner as an interface extending StepRunner rather than restating its members alongside a conditional-type assertion. --- .../core/src/provisioner/durable-runtime.ts | 184 +++++++++--------- .../provisioner/workflows/workflow.deploy.ts | 38 ++-- .../provisioner/workflows/workflow.destroy.ts | 34 ++-- packages/reconciler/src/durable/step.ts | 13 +- 4 files changed, 122 insertions(+), 147 deletions(-) diff --git a/packages/core/src/provisioner/durable-runtime.ts b/packages/core/src/provisioner/durable-runtime.ts index 118c5c0..5c27e6a 100644 --- a/packages/core/src/provisioner/durable-runtime.ts +++ b/packages/core/src/provisioner/durable-runtime.ts @@ -20,12 +20,13 @@ import { import { DurableStateBackend, resourceStateStore, + type DurableStep, type StoredResourceState, } from "@notation/reconciler/durable"; import { FileStateBackend, type StateNode } from "@notation/state"; import pino, { type Logger } from "pino"; import * as v from "valibot"; -import { defineStore } from "yieldstar"; +import { createWorkflowRouter, defineStore, workflow } from "yieldstar"; export const DEFAULT_WORKFLOW_STATE_PATH = ".notation/workflows.db"; export const DEFAULT_LEGACY_STATE_PATH = ".notation/state.json"; @@ -109,67 +110,75 @@ export class NodeDurableRuntime { ); } this.#running = true; - const executionId = opts.executionId ?? randomUUID(); - const event: WorkflowEvent = { - workflowId: opts.workflowId, - executionId, - params: opts.params ?? {}, - context: new Map(), - }; - const runner = new WorkflowRunner({ - router, - heapClient: this.#heapClient, - storeClient: this.#storeClient, - schedulerClient: this.#schedulerClient, - logger: this.#logger, - }); - - let resolveCompletion!: (value: unknown) => void; - let rejectCompletion!: (error: unknown) => void; - let completed = false; - const completion = new Promise((resolve, reject) => { - resolveCompletion = resolve; - rejectCompletion = reject; - }); - const processEvent = async (nextEvent: WorkflowEvent, logger: Logger) => { - try { - const result = await runner.run(nextEvent, logger); - if (result && nextEvent.executionId === event.executionId) { - completed = true; - resolveCompletion(result.result); - } - } catch (error) { - if (nextEvent.executionId === event.executionId) { - completed = true; - rejectCompletion(error); - return; - } - this.#logger.error({ err: error }, "Yieldstar replay failed"); - } - }; - try { + const executionId = opts.executionId ?? randomUUID(); + const runner = new WorkflowRunner({ + router, + heapClient: this.#heapClient, + storeClient: this.#storeClient, + schedulerClient: this.#schedulerClient, + logger: this.#logger, + }); + await this.initialize(); await this.#bindExecution(executionId, opts.workflowId); - await processEvent(event, this.#logger); - const eventPump = this.#processQueuedEvents( + const result = await this.#driveToCompletion(runner, { + workflowId: opts.workflowId, executionId, - processEvent, - rejectCompletion, - () => completed, - ); - try { - return await completion; - } finally { - await eventPump; - // Let the queue transaction finish before callers close the shared database. - await setImmediate(); - } + params: opts.params ?? {}, + context: new Map(), + }); + // Let the queue transaction finish before callers close the shared database. + await setImmediate(); + return result; } finally { this.#running = false; } } + /** + * Runs one execution to completion in-process: the trigger event directly, + * then every event the queue produces for it (retries, timer wake-ups), + * polling timers between rounds. Tasks queued for other executions are + * hidden for the duration and made visible again on the way out, so this + * runner never resumes an execution it was not asked to run. + */ + async #driveToCompletion( + runner: WorkflowRunner, + event: WorkflowEvent, + ): Promise { + const deferredTaskIds: number[] = []; + try { + let outcome = await runner.run(event, this.#logger); + while (!outcome) { + const task = this.#eventLoop.taskQueue.process(); + if (!task) { + this.#eventLoop.timers.processTimers(); + await new Promise((resolve) => setTimeout(resolve, 10)); + continue; + } + if (task.event.executionId !== event.executionId) { + deferredTaskIds.push(task.taskId); + continue; + } + try { + await this.#bindExecution( + task.event.executionId, + task.event.workflowId, + ); + outcome = await runner.run(task.event, this.#logger); + } finally { + this.#eventLoop.taskQueue.remove(task.taskId); + } + } + return outcome.result; + } finally { + for (const taskId of deferredTaskIds) { + this.#eventLoop.taskQueue.makeVisible(taskId); + } + } + } + async initialize(): Promise { await this.#migrateLegacyState(); } @@ -192,45 +201,6 @@ export class NodeDurableRuntime { } } - async #processQueuedEvents( - executionId: string, - processEvent: (event: WorkflowEvent, logger: Logger) => Promise, - rejectCompletion: (error: unknown) => void, - isCompleted: () => boolean, - ): Promise { - const deferredTaskIds: number[] = []; - try { - while (this.#running && !isCompleted()) { - let task = this.#eventLoop.taskQueue.process(); - while (task) { - if (task.event.executionId !== executionId) { - deferredTaskIds.push(task.taskId); - } else { - try { - await this.#bindExecution( - task.event.executionId, - task.event.workflowId, - ); - await processEvent(task.event, this.#logger); - } finally { - this.#eventLoop.taskQueue.remove(task.taskId); - } - } - if (!this.#running || isCompleted()) return; - task = this.#eventLoop.taskQueue.process(); - } - this.#eventLoop.timers.processTimers(); - await new Promise((resolve) => setTimeout(resolve, 10)); - } - } catch (error) { - rejectCompletion(error); - } finally { - for (const taskId of deferredTaskIds) { - this.#eventLoop.taskQueue.makeVisible(taskId); - } - } - } - async #migrateLegacyState(): Promise { const legacyStatePath = this.#legacyStatePath; if (!legacyStatePath) return; @@ -303,6 +273,36 @@ export async function withRuntime( } } +/** + * Wraps a reconciler generator as a single-workflow router and runs it to + * completion on the entry point's runtime. This is the cutover pattern shared + * by every mutating command: one command, one workflow, one execution. + */ +export async function runDurableWorkflow( + opts: { + entryPoint: string; + workflowId: string; + runtime?: NodeDurableRuntime; + databasePath?: string; + executionId?: string; + }, + body: ( + step: DurableStep, + executionId: string, + runtime: NodeDurableRuntime, + ) => AsyncGenerator, +): Promise { + await withRuntime(opts, async (runtime) => { + const handler = workflow(async function* (step, event) { + yield* body(step, event.executionId, runtime); + }); + await runtime.run(createWorkflowRouter({ [opts.workflowId]: handler }), { + workflowId: opts.workflowId, + executionId: opts.executionId, + }); + }); +} + function statesMatchIgnoringRevision(left: StateNode, right: StateNode) { const { rev: _leftRev, ...leftState } = left; const { rev: _rightRev, ...rightState } = right; diff --git a/packages/core/src/provisioner/workflows/workflow.deploy.ts b/packages/core/src/provisioner/workflows/workflow.deploy.ts index 9eba49b..dd89799 100644 --- a/packages/core/src/provisioner/workflows/workflow.deploy.ts +++ b/packages/core/src/provisioner/workflows/workflow.deploy.ts @@ -4,9 +4,8 @@ import { type ReconcilerEventEmitter, type ResourceRegistry, } from "@notation/reconciler"; -import { createWorkflowRouter, workflow } from "yieldstar"; import { getResourceGraph } from "src/orchestrator/graph"; -import { withRuntime, type NodeDurableRuntime } from "../durable-runtime"; +import { runDurableWorkflow, type NodeDurableRuntime } from "../durable-runtime"; export type DeployAppOptions = { entryPoint: string; @@ -26,32 +25,25 @@ export async function deployApp({ dryRun = false, maxOperationAttempts, registry, - runtime: suppliedRuntime, + runtime, executionId, databasePath, emit = createLoggerReconcilerSubscriber(), }: DeployAppOptions): Promise { const graph = await getResourceGraph(entryPoint); - await withRuntime( - { entryPoint, runtime: suppliedRuntime, databasePath }, - async (runtime) => { - const deploy = workflow(async function* (step, event) { - yield* reconciler.deploy(step, { - deploymentId: runtime.deploymentId, - executionId: event.executionId, - resources: graph.resources, - state: runtime.state, - registry, - emit, - dryRun, - driftDetection, - maxOperationAttempts, - }); - }); - await runtime.run(createWorkflowRouter({ deploy }), { - workflowId: "deploy", + await runDurableWorkflow( + { entryPoint, workflowId: "deploy", runtime, databasePath, executionId }, + (step, executionId, runtime) => + reconciler.deploy(step, { + deploymentId: runtime.deploymentId, executionId, - }); - }, + resources: graph.resources, + state: runtime.state, + registry, + emit, + dryRun, + driftDetection, + maxOperationAttempts, + }), ); } diff --git a/packages/core/src/provisioner/workflows/workflow.destroy.ts b/packages/core/src/provisioner/workflows/workflow.destroy.ts index 0c147bb..d47099b 100644 --- a/packages/core/src/provisioner/workflows/workflow.destroy.ts +++ b/packages/core/src/provisioner/workflows/workflow.destroy.ts @@ -4,9 +4,8 @@ import { type ReconcilerEventEmitter, type ResourceRegistry, } from "@notation/reconciler"; -import { createWorkflowRouter, workflow } from "yieldstar"; import { getResourceGraph } from "src/orchestrator/graph"; -import { withRuntime, type NodeDurableRuntime } from "../durable-runtime"; +import { runDurableWorkflow, type NodeDurableRuntime } from "../durable-runtime"; export type DestroyAppOptions = { entryPoint: string; @@ -22,30 +21,23 @@ export async function destroyApp({ entryPoint, maxOperationAttempts, registry, - runtime: suppliedRuntime, + runtime, executionId, databasePath, emit = createLoggerReconcilerSubscriber(), }: DestroyAppOptions) { const graph = await getResourceGraph(entryPoint); - await withRuntime( - { entryPoint, runtime: suppliedRuntime, databasePath }, - async (runtime) => { - const destroy = workflow(async function* (step, event) { - yield* reconciler.destroy(step, { - deploymentId: runtime.deploymentId, - executionId: event.executionId, - resources: graph.resources, - state: runtime.state, - registry, - emit, - maxOperationAttempts, - }); - }); - await runtime.run(createWorkflowRouter({ destroy }), { - workflowId: "destroy", + await runDurableWorkflow( + { entryPoint, workflowId: "destroy", runtime, databasePath, executionId }, + (step, executionId, runtime) => + reconciler.destroy(step, { + deploymentId: runtime.deploymentId, executionId, - }); - }, + resources: graph.resources, + state: runtime.state, + registry, + emit, + maxOperationAttempts, + }), ); } diff --git a/packages/reconciler/src/durable/step.ts b/packages/reconciler/src/durable/step.ts index 3a37e6f..f72a663 100644 --- a/packages/reconciler/src/durable/step.ts +++ b/packages/reconciler/src/durable/step.ts @@ -10,20 +10,11 @@ import type { DurableStep } from "./yieldstar"; * A durable step runner: the operation seam plus the store handle, which only * the durable driver uses. */ -export type DurableStepRunner = { - run( - key: string, - fn: () => T | Promise, - ): AsyncGenerator; - delay(key: string, ms: number): AsyncGenerator; +export interface DurableStepRunner extends StepRunner { /** Narrower than StepRunner's, so a scope keeps its store handle. */ scope(prefix: string): DurableStepRunner; store: DurableStep["store"]; -}; - -// A durable runner is one of the step runners the operations accept. -type AssertStepRunner = DurableStepRunner extends StepRunner ? true : never; -export type DurableStepRunnerIsStepRunner = AssertStepRunner; +} /** * Namespaces the step keys of `step` so an operation can be written once and From e969721370bb1684b9dedce93343803d2f42f4b3 Mon Sep 17 00:00:00 2001 From: djgrant <1670902+djgrant@users.noreply.github.com> Date: Tue, 28 Jul 2026 15:05:46 +0100 Subject: [PATCH 23/34] Trim the legacy migration and the unused state read The legacy-state migration verified every shared record twice: once from the durable listing, then again per legacy node through a fresh store read whose mismatch branch could no longer fire. Import from the ids the first pass already verified instead, dropping a store round-trip per record. DurableStateBackend.has had no callers, so remove it. --- .../core/src/provisioner/durable-runtime.ts | 30 +++++++++++-------- .../reconciler/src/durable/state-backend.ts | 4 --- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/packages/core/src/provisioner/durable-runtime.ts b/packages/core/src/provisioner/durable-runtime.ts index 5c27e6a..68f7a8a 100644 --- a/packages/core/src/provisioner/durable-runtime.ts +++ b/packages/core/src/provisioner/durable-runtime.ts @@ -213,26 +213,30 @@ export class NodeDurableRuntime { const legacyState = await new FileStateBackend(legacyStatePath).values(); const durableState = await this.state.values(); const legacyById = new Map(legacyState.map((node) => [node.id, node])); + + // Every durable record must match its legacy counterpart exactly; one + // that is missing from the legacy file, or that differs, means the two + // stores have diverged and neither can be trusted as the source. for (const current of durableState) { const legacy = legacyById.get(current.id); if (!legacy || !statesMatchIgnoringRevision(current, legacy)) { throw legacyMigrationConflict(legacyStatePath); } } + + // Import only the legacy records the durable store lacks: any shared + // record was verified identical above. + const durableIds = new Set(durableState.map((node) => node.id)); for (const node of legacyState) { - const current = await this.state.get(node.id); - if (!current) { - const { rev: _rev, ...state } = node; - await this.#storeClient.getOrCreateStore({ - definition: resourceStateStore, - id: this.state.storeId(node.id), - // Legacy records predate group metadata; -1 and "" are - // BaseResource's defaults for a resource that belongs to no group. - initial: { groupId: -1, groupType: "", ...state } as StoredResourceState, - }); - } else if (!statesMatchIgnoringRevision(current, node)) { - throw legacyMigrationConflict(legacyStatePath); - } + if (durableIds.has(node.id)) continue; + const { rev: _rev, ...state } = node; + await this.#storeClient.getOrCreateStore({ + definition: resourceStateStore, + id: this.state.storeId(node.id), + // Legacy records predate group metadata; -1 and "" are + // BaseResource's defaults for a resource that belongs to no group. + initial: { groupId: -1, groupType: "", ...state } as StoredResourceState, + }); } await rename(legacyStatePath, `${legacyStatePath}.migrated`); } diff --git a/packages/reconciler/src/durable/state-backend.ts b/packages/reconciler/src/durable/state-backend.ts index 6e1130d..55badb3 100644 --- a/packages/reconciler/src/durable/state-backend.ts +++ b/packages/reconciler/src/durable/state-backend.ts @@ -35,10 +35,6 @@ export class DurableStateBackend { return snapshot ? toStateNode(snapshot) : undefined; } - async has(id: string): Promise { - return (await this.get(id)) !== undefined; - } - snapshot(id: string): Promise { return this.#read(this.storeId(id)); } From 332e43a9371cf8cb00440ecf9191438ded1924b7 Mon Sep 17 00:00:00 2001 From: djgrant <1670902+djgrant@users.noreply.github.com> Date: Tue, 28 Jul 2026 17:20:09 +0100 Subject: [PATCH 24/34] Name the durable driver's parts for their purposes - coordination.ts -> deployment-hold.ts; acquire/releaseDeploymentCoordination -> acquire/releaseDeploymentHold. One concept, one name: the identifiers now all say hold, and the module head records that the persisted store name, step keys, and event string keep the older coordination wording. - durable/operations.ts -> reconcile.ts, so 'operations' means only resource CRUD, as it does everywhere else in the package. - DurableOperationOptions -> DurableWorkflowOptions for the same reason. - Docs: correct the store names (resource-state, deployment-coordination) that three docs still gave with a dropped notation/ prefix, point the manual's embedded example at @notation/reconciler/durable, and document the deployment-hold escape hatch (takeOverDeploymentHold) where operators will look. Drop the Yieldstar version pin from prose that would drift; it stays in package.json and the RFC's scope line. - Say why the custom event pump exists (Yieldstar's event loop is a resident server; a command needs one execution run to completion), what initialize() is for, and that store names are persisted contract like step keys. --- docs/cli/deploy.md | 2 +- docs/cli/destroy.md | 2 +- docs/internals/reconciler.md | 6 ++- docs/internals/state.md | 4 +- docs/manual/reconciler.md | 4 +- docs/rfcs/reconciler.md | 4 +- examples/reconciler/README.md | 2 +- .../core/src/provisioner/durable-runtime.ts | 13 ++++++- packages/reconciler/src/durable/deploy.ts | 4 +- .../{coordination.ts => deployment-hold.ts} | 38 ++++++++++++------- packages/reconciler/src/durable/destroy.ts | 7 ++-- packages/reconciler/src/durable/index.ts | 7 +++- .../durable/{operations.ts => reconcile.ts} | 17 ++++++--- packages/reconciler/src/durable/stores.ts | 3 ++ packages/reconciler/src/durable/types.ts | 7 ++-- 15 files changed, 77 insertions(+), 43 deletions(-) rename packages/reconciler/src/durable/{coordination.ts => deployment-hold.ts} (78%) rename packages/reconciler/src/durable/{operations.ts => reconcile.ts} (95%) diff --git a/docs/cli/deploy.md b/docs/cli/deploy.md index 4df0e43..df87124 100644 --- a/docs/cli/deploy.md +++ b/docs/cli/deploy.md @@ -4,7 +4,7 @@ notation deploy ``` -Compiles and durably deploys the stack through the resident Yieldstar 0.5.0 Node runtime. +Compiles and durably deploys the stack through the resident Yieldstar Node runtime. ```sh notation deploy infra/api.ts diff --git a/docs/cli/destroy.md b/docs/cli/destroy.md index aa1be47..7d65700 100644 --- a/docs/cli/destroy.md +++ b/docs/cli/destroy.md @@ -4,7 +4,7 @@ notation destroy ``` -Compiles the application and runs durable destroy through the resident Yieldstar 0.5.0 Node runtime. Resources are removed in reverse dependency order, then registered persisted orphans are removed. +Compiles the application and runs durable destroy through the resident Yieldstar Node runtime. Resources are removed in reverse dependency order, then registered persisted orphans are removed. ```sh notation destroy infra/api.ts diff --git a/docs/internals/reconciler.md b/docs/internals/reconciler.md index 41de5e3..8a1af53 100644 --- a/docs/internals/reconciler.md +++ b/docs/internals/reconciler.md @@ -31,9 +31,11 @@ Each attempt, delay, event, state read, state write, and coordination change has ## State and coordination -Each resource is stored under `notation/resource-state` with a deployment-scoped ID. Conditional updates and deletes compare the snapshot's UUIDv7 `instanceId` and version, so a stale execution cannot modify a deleted and recreated store. +Each resource is stored under `resource-state` with a deployment-scoped ID. Conditional updates and deletes compare the snapshot's UUIDv7 `instanceId` and version, so a stale execution cannot modify a deleted and recreated store. -Deploy and destroy share one `notation/deployment-coordination` store per deployment. `store.take` suspends a competing execution as a durable waiter and wakes it after the holder releases. Before suspending, the waiter emits `reconciler.coordination.waiting` naming the holding execution ID, so a wait behind a crashed execution is visible instead of silent. +Deploy and destroy take an exclusive hold on the deployment through one `deployment-coordination` store per deployment. `store.take` suspends a competing execution as a durable waiter and wakes it after the holder releases. Before suspending, the waiter emits `reconciler.coordination.waiting` naming the holding execution ID, so a wait behind a crashed execution is visible instead of silent. + +A failed or suspended execution keeps its hold, which is what makes resuming it safe. The hold of an execution that will never be resumed is cleared with `takeOverDeploymentHold` from `@notation/reconciler/durable` — the only supported way out of that state. ## Events diff --git a/docs/internals/state.md b/docs/internals/state.md index 77647e4..c5fc1d9 100644 --- a/docs/internals/state.md +++ b/docs/internals/state.md @@ -1,8 +1,8 @@ # State -Notation CLI deploy, destroy, plan, and dashboard use Yieldstar 0.5.0 stores in `.notation/workflows.db`. Override the database path with `NOTATION_STATE_PATH`. +Notation CLI deploy, destroy, plan, and dashboard use Yieldstar stores in `.notation/workflows.db`. Override the database path with `NOTATION_STATE_PATH`. -Each live resource is a `notation/resource-state` store scoped by deployment and resource ID. A missing store means the resource is absent. No application tombstone is written. +Each live resource is a `resource-state` store scoped by deployment and resource ID. A missing store means the resource is absent. No application tombstone is written. ```ts const state = new DurableStateBackend(storeClient, "infra/api.ts"); diff --git a/docs/manual/reconciler.md b/docs/manual/reconciler.md index 9d4435f..b7952c7 100644 --- a/docs/manual/reconciler.md +++ b/docs/manual/reconciler.md @@ -4,7 +4,7 @@ Use `deploy` and `destroy` when a Node.js application needs durable resource lif ```ts import { SqliteSchedulerClient, SqliteStoreClient, SqliteTaskQueueClient, SqliteTimersClient, createSqliteDb } from "@yieldstar/sqlite-runtime/node"; -import { DurableStateBackend, deploy as deployResources, destroy as destroyResources } from "@notation/reconciler"; +import { DurableStateBackend, deploy as deployResources, destroy as destroyResources } from "@notation/reconciler/durable"; import { workflow } from "yieldstar"; const database = createSqliteDb({ path: ".notation/workflows.db" }); @@ -38,7 +38,7 @@ The outer workflow supplies durable step execution, timers, shared stores, waiti Each live resource is one Yieldstar store. Absence is represented by no store, not a tombstone. Yieldstar's UUIDv7 store `instanceId` and version are authoritative for conditional update and delete; Notation exposes the version as the resource state's `rev`. -Operations against the same `deploymentId` are serialized through a coordination store keyed by `executionId`. Resume a crashed operation with the same execution ID; use a new globally unique execution ID for every new deploy or destroy. An execution that must wait emits a `reconciler.coordination.waiting` event naming the holder before it suspends, which also identifies a crashed holder that should be resumed instead. +Operations against the same `deploymentId` are serialized through a coordination store keyed by `executionId`. Resume a crashed operation with the same execution ID; use a new globally unique execution ID for every new deploy or destroy. An execution that must wait emits a `reconciler.coordination.waiting` event naming the holder before it suspends, which also identifies a crashed holder that should be resumed instead. If the holder is genuinely abandoned, clear its hold with `takeOverDeploymentHold`. Pass the complete desired set on every deployment. Persisted resources absent from that set are deleted through the supplied resource registry. Destroy removes current resources in reverse dependency order and then removes any persisted orphans that the registry can hydrate. diff --git a/docs/rfcs/reconciler.md b/docs/rfcs/reconciler.md index 6cc9d1c..c82b44c 100644 --- a/docs/rfcs/reconciler.md +++ b/docs/rfcs/reconciler.md @@ -13,13 +13,13 @@ Provider create, update, read, and delete calls are durable steps with stable re ## State lifecycle -`DurableStateBackend` stores one live resource per `notation/resource-state` store. The store ID is scoped by deployment and resource ID. Store absence is resource absence. +`DurableStateBackend` stores one live resource per `resource-state` store. The store ID is scoped by deployment and resource ID. Store absence is resource absence. The runtime-assigned UUIDv7 `instanceId` distinguishes a deleted store from a later store created under the same logical ID. Yieldstar's version is the concurrency token and is exposed as Notation's one-based `rev`. Workflow updates use `store.updateFrom` and deletes use `store.deleteFrom`, so both the instance and version must match the snapshot that informed the operation. ## Coordination -Each deployment has a `notation/deployment-coordination` store shared by deploy and destroy. The workflow atomically claims it with `store.take`. A concurrent execution suspends as a durable waiter and is woken when the holder releases the store. The same execution can recover an acquisition across the store-commit and heap-write crash gap through Yieldstar's applied-step ledger. +Each deployment has a `deployment-coordination` store shared by deploy and destroy. The workflow atomically claims it with `store.take`. A concurrent execution suspends as a durable waiter and is woken when the holder releases the store. The same execution can recover an acquisition across the store-commit and heap-write crash gap through Yieldstar's applied-step ledger. ## Node CLI runtime diff --git a/examples/reconciler/README.md b/examples/reconciler/README.md index 19a6ae0..6b832be 100644 --- a/examples/reconciler/README.md +++ b/examples/reconciler/README.md @@ -1,6 +1,6 @@ # Durable reconciler -This example deploys two static sites from an ordinary Node.js program using Yieldstar 0.5.0 for durable execution, state, retries, waiting, and deployment coordination. +This example deploys two static sites from an ordinary Node.js program using Yieldstar for durable execution, state, retries, waiting, and deployment coordination. [`src/index.ts`](./src/index.ts) owns the outer workflow and Node SQLite runtime. It passes Yieldstar's `step` context to `deploy`, while [`src/static-site.ts`](./src/static-site.ts) contains only the desired resources and provider lifecycle operations. diff --git a/packages/core/src/provisioner/durable-runtime.ts b/packages/core/src/provisioner/durable-runtime.ts index 68f7a8a..e8e71d6 100644 --- a/packages/core/src/provisioner/durable-runtime.ts +++ b/packages/core/src/provisioner/durable-runtime.ts @@ -59,7 +59,7 @@ const executionBindingStore = defineStore( type ExecutionBinding = v.InferOutput; -/** Resident Yieldstar 0.5.0 Node runtime used by Notation application commands. */ +/** Resident Yieldstar Node runtime used by Notation application commands. */ export class NodeDurableRuntime { readonly deploymentId: string; readonly state: DurableStateBackend; @@ -142,6 +142,11 @@ export class NodeDurableRuntime { * polling timers between rounds. Tasks queued for other executions are * hidden for the duration and made visible again on the way out, so this * runner never resumes an execution it was not asked to run. + * + * This loop exists because Yieldstar's own SqliteEventLoop is a resident + * server: it runs until stopped, serves every execution in the queue, and + * never says when one is done — none of which fits a command that must run + * exactly its own execution and then exit. */ async #driveToCompletion( runner: WorkflowRunner, @@ -179,6 +184,12 @@ export class NodeDurableRuntime { } } + /** + * Prepares the deployment database for use — today that means importing + * legacy `.notation/state.json` state on first contact. `run` calls this + * itself; read-only consumers (plan, dashboard) call it before reading so + * they see migrated state too. + */ async initialize(): Promise { await this.#migrateLegacyState(); } diff --git a/packages/reconciler/src/durable/deploy.ts b/packages/reconciler/src/durable/deploy.ts index b28c273..d9a652c 100644 --- a/packages/reconciler/src/durable/deploy.ts +++ b/packages/reconciler/src/durable/deploy.ts @@ -1,6 +1,6 @@ import { buildResourceDepthLevels } from "../dependency-graph"; -import { withDeploymentHold } from "./coordination"; -import { reconcileResource, sweepOrphans } from "./operations"; +import { withDeploymentHold } from "./deployment-hold"; +import { reconcileResource, sweepOrphans } from "./reconcile"; import { scopeStep } from "./step"; import type { DurableDeployOptions } from "./types"; import type { DurableStep } from "./yieldstar"; diff --git a/packages/reconciler/src/durable/coordination.ts b/packages/reconciler/src/durable/deployment-hold.ts similarity index 78% rename from packages/reconciler/src/durable/coordination.ts rename to packages/reconciler/src/durable/deployment-hold.ts index 094e6f3..17ba1e2 100644 --- a/packages/reconciler/src/durable/coordination.ts +++ b/packages/reconciler/src/durable/deployment-hold.ts @@ -1,9 +1,16 @@ +/** + * The deployment hold: an exclusive claim on a deployment for the length of a + * workflow execution. It lives in the `deployment-coordination` store — that + * store name, the `notation:coordination:*` step keys, and the + * `reconciler.coordination.waiting` event are persisted or published strings + * and keep the older "coordination" wording; the identifiers here do not. + */ import type { ReconcilerEventEmitter } from "../events"; import { durableEmitter, scopeStep } from "./step"; import { deploymentCoordinationStore, type CoordinationState } from "./stores"; import type { DurableStep, StoreClient, WorkflowStore } from "./yieldstar"; -type CoordinationOptions = { +type DeploymentHoldOptions = { deploymentId: string; executionId: string; emit?: ReconcilerEventEmitter; @@ -13,19 +20,22 @@ type CoordinationOptions = { * Prevents concurrent executions from mutating the same deployment. Names * the holder so an operator can resume it after a crash. */ -async function* acquireDeploymentCoordination( +async function* acquireDeploymentHold( step: DurableStep, - opts: CoordinationOptions, + opts: DeploymentHoldOptions, ): AsyncGenerator, any> { - const coordination = yield* step.store(deploymentCoordinationStore, { + const hold = yield* step.store(deploymentCoordinationStore, { id: opts.deploymentId, initial: { holder: null }, }); - const snapshot = yield* coordination.get("notation:coordination:inspect"); + const snapshot = yield* hold.get("notation:coordination:inspect"); const holder = snapshot.state.holder; if (holder !== null && holder !== opts.executionId) { - yield* durableEmitter(scopeStep(step, "notation:coordination"), opts.emit)({ + yield* durableEmitter( + scopeStep(step, "notation:coordination"), + opts.emit, + )({ level: "warn", event: "reconciler.coordination.waiting", deploymentId: opts.deploymentId, @@ -34,7 +44,7 @@ async function* acquireDeploymentCoordination( }); } - yield* coordination.take( + yield* hold.take( "notation:coordination:acquire", (state) => state.holder === null || state.holder === opts.executionId, (draft) => { @@ -42,14 +52,14 @@ async function* acquireDeploymentCoordination( }, ); - return coordination; + return hold; } -function releaseDeploymentCoordination( - coordination: WorkflowStore, +function releaseDeploymentHold( + hold: WorkflowStore, executionId: string, ) { - return coordination.update("notation:coordination:release", (draft) => { + return hold.update("notation:coordination:release", (draft) => { if (draft.holder === executionId) draft.holder = null; }); } @@ -68,12 +78,12 @@ function releaseDeploymentCoordination( */ export async function* withDeploymentHold( step: DurableStep, - opts: CoordinationOptions, + opts: DeploymentHoldOptions, body: () => AsyncGenerator, ): AsyncGenerator { - const coordination = yield* acquireDeploymentCoordination(step, opts); + const hold = yield* acquireDeploymentHold(step, opts); yield* body(); - yield* releaseDeploymentCoordination(coordination, opts.executionId); + yield* releaseDeploymentHold(hold, opts.executionId); } export type DeploymentHoldTakeover = diff --git a/packages/reconciler/src/durable/destroy.ts b/packages/reconciler/src/durable/destroy.ts index fa6158a..9e2c0c7 100644 --- a/packages/reconciler/src/durable/destroy.ts +++ b/packages/reconciler/src/durable/destroy.ts @@ -1,6 +1,6 @@ import { buildResourceDepthLevels } from "../dependency-graph"; -import { withDeploymentHold } from "./coordination"; -import { deleteResource, sweepOrphans } from "./operations"; +import { withDeploymentHold } from "./deployment-hold"; +import { deleteResource, sweepOrphans } from "./reconcile"; import { scopeStep } from "./step"; import type { DurableDestroyOptions } from "./types"; import type { DurableStep } from "./yieldstar"; @@ -12,8 +12,7 @@ export async function* destroy( ): AsyncGenerator { yield* withDeploymentHold(step, opts, async function* () { // Delete in reverse dependency order, so dependents are gone before the - // resources they depend on. Resources with no persisted state were never - // created (or are already deleted) and are skipped by deleteResource. + // resources they depend on. const levels = buildResourceDepthLevels(opts.resources); for (let index = levels.length - 1; index >= 0; index -= 1) { for (const resource of levels[index]!) { diff --git a/packages/reconciler/src/durable/index.ts b/packages/reconciler/src/durable/index.ts index 72646b2..e53b764 100644 --- a/packages/reconciler/src/durable/index.ts +++ b/packages/reconciler/src/durable/index.ts @@ -1,4 +1,7 @@ /** + * The durable reconciler driver: `deploy` and `destroy` as generators that a + * Yieldstar workflow composes, and the store-backed state they run against. + * * Step keys are a public contract: they are what a resumed execution matches * its cached work against, so changing one re-executes the work behind it. * The shapes in use are: @@ -28,7 +31,7 @@ export { destroy } from "./destroy"; export { takeOverDeploymentHold, type DeploymentHoldTakeover, -} from "./coordination"; +} from "./deployment-hold"; export { DurableStateBackend } from "./state-backend"; export { deploymentCoordinationStore, @@ -39,6 +42,6 @@ export { export { type DurableDeployOptions, type DurableDestroyOptions, - type DurableOperationOptions, + type DurableWorkflowOptions, } from "./types"; export type { DurableStep } from "./yieldstar"; diff --git a/packages/reconciler/src/durable/operations.ts b/packages/reconciler/src/durable/reconcile.ts similarity index 95% rename from packages/reconciler/src/durable/operations.ts rename to packages/reconciler/src/durable/reconcile.ts index 26541ec..def2cad 100644 --- a/packages/reconciler/src/durable/operations.ts +++ b/packages/reconciler/src/durable/reconcile.ts @@ -1,3 +1,8 @@ +/** + * Per-resource reconciliation: converging, deleting, and sweeping single + * resources, each built on one read of the resource's persisted record and + * writes conditional on that read. + */ import type { BaseResource, ResourceType } from "@notation/resource"; import { RevConflict, type StateNode } from "@notation/state"; import { @@ -20,7 +25,7 @@ import { toStateNode, type ResourceSnapshot, } from "./stores"; -import type { DurableDeployOptions, DurableOperationOptions } from "./types"; +import type { DurableDeployOptions, DurableWorkflowOptions } from "./types"; import type { DurableStep } from "./yieldstar"; /** @@ -138,7 +143,7 @@ export async function* reconcileResource( export async function* deleteResource( step: DurableStepRunner, resource: BaseResource, - opts: DurableOperationOptions, + opts: DurableWorkflowOptions, ): AsyncGenerator { const session = yield* openStateSession(step, opts, resource); if (!session.node) return; @@ -160,7 +165,7 @@ export async function* deleteResource( */ export async function* sweepOrphans( step: DurableStepRunner, - opts: DurableOperationOptions, + opts: DurableWorkflowOptions, workflow: "deploy" | "destroy", ): AsyncGenerator { const resourceById = new Map( @@ -203,7 +208,7 @@ export async function* sweepOrphans( */ async function* openStateSession( step: DurableStepRunner, - opts: DurableOperationOptions, + opts: DurableWorkflowOptions, resource: BaseResource, ): AsyncGenerator { const snapshot = yield* step.run("state:snapshot", () => @@ -227,7 +232,7 @@ async function* openStateSession( */ function persistResourceState( step: DurableStepRunner, - opts: DurableOperationOptions, + opts: DurableWorkflowOptions, resource: BaseResource, snapshot: ResourceSnapshot | undefined, ): PersistState { @@ -263,7 +268,7 @@ function persistResourceState( function removeResourceState( step: DurableStepRunner, - opts: DurableOperationOptions, + opts: DurableWorkflowOptions, resource: BaseResource, snapshot: ResourceSnapshot, ): RemoveState { diff --git a/packages/reconciler/src/durable/stores.ts b/packages/reconciler/src/durable/stores.ts index c5f03b5..890477c 100644 --- a/packages/reconciler/src/durable/stores.ts +++ b/packages/reconciler/src/durable/stores.ts @@ -2,6 +2,9 @@ import type { StateNode } from "@notation/state"; import * as v from "valibot"; import { defineStore, type StoreSnapshot } from "./yieldstar"; +// Store names are persisted identifiers, like the step keys mapped in +// index.ts: renaming one orphans every record stored under the old name. + /** * Loose on purpose: PersistedResourceState carries an index signature so a * driver can persist fields this schema does not name yet, and v.object would diff --git a/packages/reconciler/src/durable/types.ts b/packages/reconciler/src/durable/types.ts index 76f7000..f9c6022 100644 --- a/packages/reconciler/src/durable/types.ts +++ b/packages/reconciler/src/durable/types.ts @@ -3,7 +3,8 @@ import type { ReconcilerEventEmitter } from "../events"; import type { ResourceRegistry } from "../resource-registry"; import type { DurableStateBackend } from "./state-backend"; -export type DurableOperationOptions = { +/** What both durable workflows need; "operation" would mean resource CRUD here. */ +export type DurableWorkflowOptions = { deploymentId: string; executionId: string; resources: BaseResource[]; @@ -14,8 +15,8 @@ export type DurableOperationOptions = { maxOperationAttempts?: number; }; -export type DurableDeployOptions = DurableOperationOptions & { +export type DurableDeployOptions = DurableWorkflowOptions & { driftDetection?: boolean; }; -export type DurableDestroyOptions = DurableOperationOptions; +export type DurableDestroyOptions = DurableWorkflowOptions; From d6346a806a089adb31f43f5ea61662e117682994 Mon Sep 17 00:00:00 2001 From: djgrant <1670902+djgrant@users.noreply.github.com> Date: Tue, 28 Jul 2026 17:47:51 +0100 Subject: [PATCH 25/34] Give every concept one name, and delete what nothing depends on Nothing on this branch has shipped: @notation/reconciler and @notation/state were never published, and the released core@0.11.1 never wrote .notation/state.json. So the persisted strings, the legacy migration, and the file/SQLite state backends had no dependents. - Rename the deployment hold's persisted strings to match its name: the deployment-hold store, notation:hold:* step keys, and the reconciler.hold.waiting event. Delete the comment that apologised for the old coordination wording. - URI-encode resource ids inside step scopes and give the orphan sweep one scope (notation:orphans) in both workflows, making the key delimiter unambiguous instead of documenting the ambiguity. - Delete the legacy state.json migration, and with it initialize(), the legacyStatePath option, and FileStateBackend. - Delete the orphaned @notation/state-sqlite package. - Move the orphan-deletion-skipped event next to the other events and inline its single-use factory; name event types after their event strings (DeployDecisionEvent, DriftDetectedEvent, HoldWaitingEvent, OrphanDeletionSkippedEvent). - Drop the runtime's unused params option, its redundant per-task re-binding, and its Yieldstar-branded log and error wording. - Trim comments to what is and what binds: no history, no defences. --- .changeset/reconciler.md | 1 - docs/cli/dashboard.md | 2 +- docs/cli/deploy.md | 8 +- docs/cli/destroy.md | 2 +- docs/internals/reconciler.md | 14 +- docs/internals/state.md | 14 +- docs/manual/reconciler.md | 4 +- docs/rfcs/reconciler.md | 6 +- examples/reconciler/README.md | 4 +- examples/reconciler/src/index.ts | 2 +- packages/cli/src/deploy.ts | 2 +- packages/cli/src/destroy.ts | 2 +- packages/cli/src/index.ts | 1 - .../core/src/provisioner/durable-runtime.ts | 112 ++------------ .../core/src/provisioner/resource-registry.ts | 10 +- .../provisioner/workflows/workflow.plan.ts | 8 +- .../test/provisioner/durable-runtime.test.ts | 46 +----- .../provisioner/resource-registry.test.ts | 18 --- packages/reconciler/src/durable/deploy.ts | 3 +- .../reconciler/src/durable/deployment-hold.ts | 55 +++---- packages/reconciler/src/durable/destroy.ts | 11 +- packages/reconciler/src/durable/index.ts | 32 ++-- packages/reconciler/src/durable/reconcile.ts | 43 +++--- .../reconciler/src/durable/state-backend.ts | 14 +- packages/reconciler/src/durable/stores.ts | 19 +-- packages/reconciler/src/durable/types.ts | 1 - packages/reconciler/src/events.ts | 26 ++-- .../src/operations/operation.types.ts | 17 +-- packages/reconciler/src/planner.ts | 2 +- packages/reconciler/src/resource-registry.ts | 24 --- .../test/durable-reconciliation.test.ts | 20 +-- .../reconciler/test/logger-subscriber.test.ts | 4 +- .../test/operation.workflows.test.ts | 9 +- packages/state-sqlite/package.json | 20 --- packages/state-sqlite/src/index.ts | 103 ------------- packages/state-sqlite/src/node-sqlite.d.ts | 20 --- .../state-sqlite/test/state-sqlite.test.ts | 95 ------------ packages/state-sqlite/tsconfig.json | 7 - packages/state-sqlite/tsup.config.ts | 10 -- packages/state/package.json | 3 - packages/state/src/state.ts | 138 ------------------ packages/state/test/state-backend.test.ts | 44 ------ pnpm-lock.yaml | 14 -- 43 files changed, 154 insertions(+), 836 deletions(-) delete mode 100644 packages/state-sqlite/package.json delete mode 100644 packages/state-sqlite/src/index.ts delete mode 100644 packages/state-sqlite/src/node-sqlite.d.ts delete mode 100644 packages/state-sqlite/test/state-sqlite.test.ts delete mode 100644 packages/state-sqlite/tsconfig.json delete mode 100644 packages/state-sqlite/tsup.config.ts diff --git a/.changeset/reconciler.md b/.changeset/reconciler.md index f28bafd..e304772 100644 --- a/.changeset/reconciler.md +++ b/.changeset/reconciler.md @@ -6,7 +6,6 @@ "@notation/reconciler": minor "@notation/resource": minor "@notation/state": minor -"@notation/state-sqlite": minor --- Add durable Yieldstar 0.5.0 deploy and destroy workflows, a resident Node SQLite runtime for CLI execution, versioned event streams, backend-neutral dashboard state, and compiled infrastructure graphs. diff --git a/docs/cli/dashboard.md b/docs/cli/dashboard.md index 2baffaf..26780c4 100644 --- a/docs/cli/dashboard.md +++ b/docs/cli/dashboard.md @@ -4,7 +4,7 @@ notation dashboard ``` -Starts a local web dashboard for observing the deployment's Yieldstar resource stores. +Starts a local web dashboard for observing the deployment's resource state. ```sh notation dashboard infra/api.ts diff --git a/docs/cli/deploy.md b/docs/cli/deploy.md index df87124..dbc9c6e 100644 --- a/docs/cli/deploy.md +++ b/docs/cli/deploy.md @@ -4,7 +4,7 @@ notation deploy ``` -Compiles and durably deploys the stack through the resident Yieldstar Node runtime. +Compiles the stack and runs a durable deploy. ```sh notation deploy infra/api.ts @@ -20,7 +20,7 @@ notation deploy infra/api.ts --json > deploy.ndjson ## Durable execution -The command prints its Yieldstar execution ID before starting provider work. If the process crashes, resume the same durable heap with that ID: +The command prints its execution ID before starting provider work. If the process crashes, resume the same execution with that ID: ```sh notation deploy infra/api.ts --execution-id @@ -44,6 +44,4 @@ Retryable provider conditions and consistency reads suspend on durable SQLite ti 6. **Delete orphans** – persisted resources absent from the graph are deleted when their resource type is registered. -State, step results, timers, task coordination, and resource stores are persisted to `.notation/workflows.db`. Set `NOTATION_STATE_PATH` to choose another SQLite database path. - -On first use, Notation imports resource state from the legacy `.notation/state.json` file and archives it as `.notation/state.json.migrated`. If the durable database already contains conflicting resource state, Notation stops with recovery instructions instead of attempting to create resources from an empty namespace. +State, step results, timers, queued tasks, and resource stores are persisted to `.notation/workflows.db`. Set `NOTATION_STATE_PATH` to choose another SQLite database path. diff --git a/docs/cli/destroy.md b/docs/cli/destroy.md index 7d65700..17aad8c 100644 --- a/docs/cli/destroy.md +++ b/docs/cli/destroy.md @@ -4,7 +4,7 @@ notation destroy ``` -Compiles the application and runs durable destroy through the resident Yieldstar Node runtime. Resources are removed in reverse dependency order, then registered persisted orphans are removed. +Compiles the application and runs a durable destroy. Resources are removed in reverse dependency order, then registered persisted orphans are removed. ```sh notation destroy infra/api.ts diff --git a/docs/internals/reconciler.md b/docs/internals/reconciler.md index 8a1af53..eeba956 100644 --- a/docs/internals/reconciler.md +++ b/docs/internals/reconciler.md @@ -1,10 +1,10 @@ # Reconciler -The reconciler expresses deployment and destruction as Yieldstar async generators. Notation owns desired-state decisions and provider lifecycle; the caller's Yieldstar runtime owns durable execution, waiting, shared state, and coordination. +The reconciler expresses deployment and destruction as Yieldstar async generators. Notation owns desired-state decisions and provider lifecycle; the caller's Yieldstar runtime owns durable execution, waiting, and shared state. ## Deploy flow -`deploy` acquires the deployment coordination store, walks dependency levels in order, decides an action for every resource, executes provider calls as durable steps, persists the result in a resource store, and deletes registered orphans. +`deploy` takes the deployment hold, walks dependency levels in order, decides an action for every resource, executes provider calls as durable steps, persists the result in a resource store, and deletes registered orphans. | Condition | Decision | | --- | --- | @@ -19,7 +19,7 @@ Dry-run deploy performs decisions and emits lifecycle events without provider mu ## Destroy flow -`destroy` is a first-class durable operation. It acquires the same deployment coordination store as deploy, deletes desired resources in reverse dependency order, deletes hydratable persisted orphans, and conditionally removes each resource store only after the provider delete succeeds or reports that the resource is already absent. +`destroy` is a first-class durable operation. It takes the same deployment hold as deploy, deletes desired resources in reverse dependency order, deletes hydratable persisted orphans, and conditionally removes each resource store only after the provider delete succeeds or reports that the resource is already absent. Provider delete is a stable durable step, but the provider acknowledgement and Yieldstar heap checkpoint are not atomic. If the process crashes between them, replay repeats the delete, so provider create, update, and delete operations must be idempotent. Event subscribers must likewise tolerate duplicate delivery when a crash occurs before the event checkpoint. @@ -27,13 +27,13 @@ Provider delete is a stable durable step, but the provider acknowledgement and Y A resource operation throws `ResourceOperationPendingError` when it has not finished. The error gives the reconciler a delay and optional callback context. The runtime stores the context, waits without keeping the process busy, and calls the same operation again. See [Operation errors](./resource.md#operation-errors) for the complete API. -Each attempt, delay, event, state read, state write, and coordination change has a stable step key. A resumed execution must use the same execution ID. A new deploy or destroy must use a new execution ID. +Each attempt, delay, event, state read, state write, and hold change has a stable step key. A resumed execution must use the same execution ID. A new deploy or destroy must use a new execution ID. -## State and coordination +## State and the deployment hold Each resource is stored under `resource-state` with a deployment-scoped ID. Conditional updates and deletes compare the snapshot's UUIDv7 `instanceId` and version, so a stale execution cannot modify a deleted and recreated store. -Deploy and destroy take an exclusive hold on the deployment through one `deployment-coordination` store per deployment. `store.take` suspends a competing execution as a durable waiter and wakes it after the holder releases. Before suspending, the waiter emits `reconciler.coordination.waiting` naming the holding execution ID, so a wait behind a crashed execution is visible instead of silent. +Deploy and destroy take an exclusive hold on the deployment through one `deployment-hold` store per deployment. `store.take` suspends a competing execution as a durable waiter and wakes it after the holder releases. Before suspending, the waiter emits `reconciler.hold.waiting` naming the holding execution ID, so a wait behind a crashed execution is visible instead of silent. A failed or suspended execution keeps its hold, which is what makes resuming it safe. The hold of an execution that will never be resumed is cleared with `takeOverDeploymentHold` from `@notation/reconciler/durable` — the only supported way out of that state. @@ -46,7 +46,7 @@ The durable workflows emit these events: | `reconciler.deploy.decision` | After deciding what action to take for a resource | | `reconciler.drift.detected` | When drift is found between stored and actual state | | `reconciler.operation.lifecycle` | When an operation starts, finishes, skips, or fails | -| `reconciler.coordination.waiting` | When another deployment holds the coordination store | +| `reconciler.hold.waiting` | When another execution holds the deployment | | `reconciler.orphan-deletion.skipped` | When no registered class can delete an orphan | Lifecycle events cover create, read, update, and delete with `start`, `success`, `error`, `skip`, or `dry-run` status. diff --git a/docs/internals/state.md b/docs/internals/state.md index c5fc1d9..1c738e9 100644 --- a/docs/internals/state.md +++ b/docs/internals/state.md @@ -10,16 +10,8 @@ const state = new DurableStateBackend(storeClient, "infra/api.ts"); The runtime assigns a UUIDv7 `instanceId` when a store is created and increments its version on update. Conditional workflow updates and deletes compare both values, preventing a stale snapshot from modifying a deleted and recreated resource. The one-based value exposed as `StateNode.rev` is derived from the authoritative Yieldstar store version. -```ts -interface StateBackend { - get(id: string): Promise; - has(id: string): Promise; - update(id: string, expectedRev: number, patch: Partial): Promise<{ rev: number }>; - delete(id: string, expectedRev: number): Promise; - values(): Promise; -} -``` +`DurableStateBackend` is read-only: state writes happen inside the workflow, through the store handle, so each write is stamped with the step that made it and is not repeated on replay. -Coordination is not part of the state backend contract. The outer Yieldstar workflow serializes deploy and destroy through a deployment coordination store and records applied store steps for crash-safe replay. +The deployment hold is not part of resource state. The workflow serializes deploy and destroy through one `deployment-hold` store per deployment. -`MemoryStateBackend`, `FileStateBackend`, and `SqliteStateBackend` remain data adapters for tests and embedded read/write consumers. They are not CLI execution runtimes and do not provide mutation coordination. +`MemoryStateBackend` in `@notation/state` remains a read/write data adapter for tests. diff --git a/docs/manual/reconciler.md b/docs/manual/reconciler.md index b7952c7..c32bc4e 100644 --- a/docs/manual/reconciler.md +++ b/docs/manual/reconciler.md @@ -34,11 +34,11 @@ export const destroy = workflow(async function* (step, event) { }); ``` -The outer workflow supplies durable step execution, timers, shared stores, waiting, scheduling, and coordination. Checkpointed provider results are replayed from the heap after a crash, retryable provider conditions suspend on a durable timer, and conditional state writes use Yieldstar store identity and version. Provider mutations must be idempotent because a crash after provider acknowledgement but before the heap checkpoint repeats the call; event consumers must tolerate the same duplicate-delivery window. +The outer workflow supplies durable step execution, timers, shared stores, waiting, and scheduling. Checkpointed provider results are replayed from the heap after a crash, retryable provider conditions suspend on a durable timer, and conditional state writes use Yieldstar store identity and version. Provider mutations must be idempotent because a crash after provider acknowledgement but before the heap checkpoint repeats the call; event consumers must tolerate the same duplicate-delivery window. Each live resource is one Yieldstar store. Absence is represented by no store, not a tombstone. Yieldstar's UUIDv7 store `instanceId` and version are authoritative for conditional update and delete; Notation exposes the version as the resource state's `rev`. -Operations against the same `deploymentId` are serialized through a coordination store keyed by `executionId`. Resume a crashed operation with the same execution ID; use a new globally unique execution ID for every new deploy or destroy. An execution that must wait emits a `reconciler.coordination.waiting` event naming the holder before it suspends, which also identifies a crashed holder that should be resumed instead. If the holder is genuinely abandoned, clear its hold with `takeOverDeploymentHold`. +Operations against the same `deploymentId` are serialized through a deployment hold naming the holding `executionId`. Resume a crashed operation with the same execution ID; use a new globally unique execution ID for every new deploy or destroy. An execution that must wait emits a `reconciler.hold.waiting` event naming the holder before it suspends, which also identifies a crashed holder that should be resumed instead. If the holder is genuinely abandoned, clear its hold with `takeOverDeploymentHold`. Pass the complete desired set on every deployment. Persisted resources absent from that set are deleted through the supplied resource registry. Destroy removes current resources in reverse dependency order and then removes any persisted orphans that the registry can hydrate. diff --git a/docs/rfcs/reconciler.md b/docs/rfcs/reconciler.md index c82b44c..c6f3b21 100644 --- a/docs/rfcs/reconciler.md +++ b/docs/rfcs/reconciler.md @@ -3,7 +3,7 @@ **Status:** implemented **Scope:** `@notation/reconciler`, `@notation/core`, Yieldstar 0.5.0 -Notation describes reconciliation intent and resource lifecycle operations. An outer Yieldstar workflow supplies durable execution, waiting, state, and coordination by composing `deploy` or `destroy`. +Notation describes reconciliation intent and resource lifecycle operations. An outer Yieldstar workflow supplies durable execution, waiting, and state by composing `deploy` or `destroy`. ## Boundary @@ -17,9 +17,9 @@ Provider create, update, read, and delete calls are durable steps with stable re The runtime-assigned UUIDv7 `instanceId` distinguishes a deleted store from a later store created under the same logical ID. Yieldstar's version is the concurrency token and is exposed as Notation's one-based `rev`. Workflow updates use `store.updateFrom` and deletes use `store.deleteFrom`, so both the instance and version must match the snapshot that informed the operation. -## Coordination +## Deployment hold -Each deployment has a `deployment-coordination` store shared by deploy and destroy. The workflow atomically claims it with `store.take`. A concurrent execution suspends as a durable waiter and is woken when the holder releases the store. The same execution can recover an acquisition across the store-commit and heap-write crash gap through Yieldstar's applied-step ledger. +Each deployment has a `deployment-hold` store shared by deploy and destroy. The workflow atomically claims it with `store.take`. A concurrent execution suspends as a durable waiter and is woken when the holder releases the store. The same execution can recover an acquisition across the store-commit and heap-write crash gap through Yieldstar's applied-step ledger. ## Node CLI runtime diff --git a/examples/reconciler/README.md b/examples/reconciler/README.md index 6b832be..886cf4b 100644 --- a/examples/reconciler/README.md +++ b/examples/reconciler/README.md @@ -1,6 +1,6 @@ # Durable reconciler -This example deploys two static sites from an ordinary Node.js program using Yieldstar for durable execution, state, retries, waiting, and deployment coordination. +This example deploys two static sites from an ordinary Node.js program using Yieldstar for durable execution, state, retries, waiting, and the deployment hold. [`src/index.ts`](./src/index.ts) owns the outer workflow and Node SQLite runtime. It passes Yieldstar's `step` context to `deploy`, while [`src/static-site.ts`](./src/static-site.ts) contains only the desired resources and provider lifecycle operations. @@ -10,7 +10,7 @@ Run it from the repository root: pnpm --filter reconciler-example demo ``` -The generated sites are written to `sites/`, and the workflow heap, resource stores, timers, and coordination state are stored in `sites.db`. Change the resource configuration and run the command again to update the sites. Remove a resource from the array and run it again to delete that site. +The generated sites are written to `sites/`, and the workflow heap, resource stores, timers, and deployment hold are stored in `sites.db`. Change the resource configuration and run the command again to update the sites. Remove a resource from the array and run it again to delete that site. Run the integration test with: diff --git a/examples/reconciler/src/index.ts b/examples/reconciler/src/index.ts index 0f8dbe1..2a977b0 100644 --- a/examples/reconciler/src/index.ts +++ b/examples/reconciler/src/index.ts @@ -28,7 +28,7 @@ const resources = [ const deploy = workflow(async function* (step, event) { yield* reconciler.deploy(step, { - deploymentId: "static-sites", + deploymentId: runtime.deploymentId, executionId: event.executionId, resources, state: runtime.state, diff --git a/packages/cli/src/deploy.ts b/packages/cli/src/deploy.ts index 3f141c1..6c57116 100644 --- a/packages/cli/src/deploy.ts +++ b/packages/cli/src/deploy.ts @@ -26,7 +26,7 @@ export async function deploy( await compile(entryPoint, { logger }); logger.info(`Deploying ${entryPoint}`); const executionId = opts.executionId ?? randomUUID(); - logger.info(`Yieldstar execution ${executionId}`); + logger.info(`Execution ID ${executionId}`); await deployApp({ entryPoint, emit, executionId }); } diff --git a/packages/cli/src/destroy.ts b/packages/cli/src/destroy.ts index 9352f3c..177e2d7 100644 --- a/packages/cli/src/destroy.ts +++ b/packages/cli/src/destroy.ts @@ -26,7 +26,7 @@ export async function destroy( await compile(entryPoint, { logger }); logger.info(`Destroying ${entryPoint}\n`); const executionId = opts.executionId ?? randomUUID(); - logger.info(`Yieldstar execution ${executionId}`); + logger.info(`Execution ID ${executionId}`); await destroyApp({ entryPoint, emit, executionId }); } diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index 1bfade1..ee4101b 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -27,7 +27,6 @@ program const runtime = new NodeDurableRuntime({ deploymentId: resolveDeploymentId(entryPoint), }); - await runtime.initialize(); await startDashboardServer({ state: runtime.state }); }); diff --git a/packages/core/src/provisioner/durable-runtime.ts b/packages/core/src/provisioner/durable-runtime.ts index e8e71d6..f147da2 100644 --- a/packages/core/src/provisioner/durable-runtime.ts +++ b/packages/core/src/provisioner/durable-runtime.ts @@ -1,5 +1,4 @@ import { randomUUID } from "node:crypto"; -import { access, rename } from "node:fs/promises"; import path from "node:path"; import { setImmediate } from "node:timers/promises"; import { isDeepStrictEqual } from "node:util"; @@ -19,20 +18,16 @@ import { } from "@yieldstar/sqlite-runtime/node"; import { DurableStateBackend, - resourceStateStore, type DurableStep, - type StoredResourceState, } from "@notation/reconciler/durable"; -import { FileStateBackend, type StateNode } from "@notation/state"; import pino, { type Logger } from "pino"; import * as v from "valibot"; import { createWorkflowRouter, defineStore, workflow } from "yieldstar"; -export const DEFAULT_WORKFLOW_STATE_PATH = ".notation/workflows.db"; -export const DEFAULT_LEGACY_STATE_PATH = ".notation/state.json"; +const DEFAULT_DATABASE_PATH = ".notation/workflows.db"; -export function resolveWorkflowStatePath(): string { - return process.env.NOTATION_STATE_PATH ?? DEFAULT_WORKFLOW_STATE_PATH; +function resolveDatabasePath(): string { + return process.env.NOTATION_STATE_PATH ?? DEFAULT_DATABASE_PATH; } export function resolveDeploymentId(entryPoint: string): string { @@ -42,14 +37,12 @@ export function resolveDeploymentId(entryPoint: string): string { export type NodeDurableRuntimeOptions = { deploymentId: string; databasePath?: string; - legacyStatePath?: string | false; logger?: Logger; }; export type RunWorkflowOptions = { workflowId: string; executionId?: string; - params?: Record; }; const executionBindingStore = defineStore( @@ -59,7 +52,7 @@ const executionBindingStore = defineStore( type ExecutionBinding = v.InferOutput; -/** Resident Yieldstar Node runtime used by Notation application commands. */ +/** Resident durable runtime used by Notation application commands. */ export class NodeDurableRuntime { readonly deploymentId: string; readonly state: DurableStateBackend; @@ -69,22 +62,13 @@ export class NodeDurableRuntime { readonly #schedulerClient: SqliteSchedulerClient; readonly #storeClient: SqliteStoreClient; readonly #logger: Logger; - readonly #legacyStatePath: string | undefined; #running = false; constructor(opts: NodeDurableRuntimeOptions) { this.deploymentId = opts.deploymentId; this.#logger = opts.logger ?? pino({ level: "silent" }); - const databasePath = opts.databasePath ?? resolveWorkflowStatePath(); - this.#legacyStatePath = - opts.legacyStatePath === false - ? undefined - : (opts.legacyStatePath ?? - (databasePath === DEFAULT_WORKFLOW_STATE_PATH - ? DEFAULT_LEGACY_STATE_PATH - : undefined)); this.#database = createSqliteDb({ - path: databasePath, + path: opts.databasePath ?? resolveDatabasePath(), }); const taskQueueClient = new SqliteTaskQueueClient(this.#database); this.#schedulerClient = new SqliteSchedulerClient({ @@ -105,9 +89,7 @@ export class NodeDurableRuntime { opts: RunWorkflowOptions, ): Promise { if (this.#running) { - throw new Error( - "The Node Yieldstar runtime already has an active workflow", - ); + throw new Error("NodeDurableRuntime is already running a workflow"); } this.#running = true; try { @@ -120,12 +102,11 @@ export class NodeDurableRuntime { logger: this.#logger, }); - await this.initialize(); await this.#bindExecution(executionId, opts.workflowId); const result = await this.#driveToCompletion(runner, { workflowId: opts.workflowId, executionId, - params: opts.params ?? {}, + params: {}, context: new Map(), }); // Let the queue transaction finish before callers close the shared database. @@ -142,11 +123,6 @@ export class NodeDurableRuntime { * polling timers between rounds. Tasks queued for other executions are * hidden for the duration and made visible again on the way out, so this * runner never resumes an execution it was not asked to run. - * - * This loop exists because Yieldstar's own SqliteEventLoop is a resident - * server: it runs until stopped, serves every execution in the queue, and - * never says when one is done — none of which fits a command that must run - * exactly its own execution and then exit. */ async #driveToCompletion( runner: WorkflowRunner, @@ -167,10 +143,6 @@ export class NodeDurableRuntime { continue; } try { - await this.#bindExecution( - task.event.executionId, - task.event.workflowId, - ); outcome = await runner.run(task.event, this.#logger); } finally { this.#eventLoop.taskQueue.remove(task.taskId); @@ -185,15 +157,9 @@ export class NodeDurableRuntime { } /** - * Prepares the deployment database for use — today that means importing - * legacy `.notation/state.json` state on first contact. `run` calls this - * itself; read-only consumers (plan, dashboard) call it before reading so - * they see migrated state too. + * Pins an execution ID to its deployment and workflow on first use, so a + * reused ID cannot replay one workflow's cached steps inside another. */ - async initialize(): Promise { - await this.#migrateLegacyState(); - } - async #bindExecution(executionId: string, workflowId: string): Promise { const expected: ExecutionBinding = { deploymentId: this.deploymentId, @@ -207,55 +173,15 @@ export class NodeDurableRuntime { const existing: ExecutionBinding = binding.state; if (!isDeepStrictEqual(existing, expected)) { throw new Error( - `Yieldstar execution ${executionId} is bound to deployment ${existing.deploymentId} workflow ${existing.workflowId}, not deployment ${this.deploymentId} workflow ${workflowId}`, + `Execution ${executionId} is bound to deployment ${existing.deploymentId} workflow ${existing.workflowId}, not deployment ${this.deploymentId} workflow ${workflowId}`, ); } } - async #migrateLegacyState(): Promise { - const legacyStatePath = this.#legacyStatePath; - if (!legacyStatePath) return; - try { - await access(legacyStatePath); - } catch { - return; - } - - const legacyState = await new FileStateBackend(legacyStatePath).values(); - const durableState = await this.state.values(); - const legacyById = new Map(legacyState.map((node) => [node.id, node])); - - // Every durable record must match its legacy counterpart exactly; one - // that is missing from the legacy file, or that differs, means the two - // stores have diverged and neither can be trusted as the source. - for (const current of durableState) { - const legacy = legacyById.get(current.id); - if (!legacy || !statesMatchIgnoringRevision(current, legacy)) { - throw legacyMigrationConflict(legacyStatePath); - } - } - - // Import only the legacy records the durable store lacks: any shared - // record was verified identical above. - const durableIds = new Set(durableState.map((node) => node.id)); - for (const node of legacyState) { - if (durableIds.has(node.id)) continue; - const { rev: _rev, ...state } = node; - await this.#storeClient.getOrCreateStore({ - definition: resourceStateStore, - id: this.state.storeId(node.id), - // Legacy records predate group metadata; -1 and "" are - // BaseResource's defaults for a resource that belongs to no group. - initial: { groupId: -1, groupType: "", ...state } as StoredResourceState, - }); - } - await rename(legacyStatePath, `${legacyStatePath}.migrated`); - } - close(): void { if (this.#running) { throw new Error( - "Cannot close the Node Yieldstar runtime while a workflow is active", + "Cannot close NodeDurableRuntime while a workflow is running", ); } this.#database.close(); @@ -290,8 +216,8 @@ export async function withRuntime( /** * Wraps a reconciler generator as a single-workflow router and runs it to - * completion on the entry point's runtime. This is the cutover pattern shared - * by every mutating command: one command, one workflow, one execution. + * completion on the entry point's runtime: one command, one workflow, one + * execution. */ export async function runDurableWorkflow( opts: { @@ -317,15 +243,3 @@ export async function runDurableWorkflow( }); }); } - -function statesMatchIgnoringRevision(left: StateNode, right: StateNode) { - const { rev: _leftRev, ...leftState } = left; - const { rev: _rightRev, ...rightState } = right; - return isDeepStrictEqual(leftState, rightState); -} - -function legacyMigrationConflict(legacyStatePath: string) { - return new Error( - `Cannot migrate legacy state from ${legacyStatePath} because the durable database already contains different resource state. Back up both files, then remove the new durable database and retry the command to import the legacy state.`, - ); -} diff --git a/packages/core/src/provisioner/resource-registry.ts b/packages/core/src/provisioner/resource-registry.ts index 7d75f8f..932f3db 100644 --- a/packages/core/src/provisioner/resource-registry.ts +++ b/packages/core/src/provisioner/resource-registry.ts @@ -1,20 +1,12 @@ import { - createMissingResourceRegistryMatchWarningEvent, createResourceRegistry, createResourceRegistryFromResources, resolveResourceClass, - type MissingResourceRegistryMatchWarningEvent, type ResourceRegistry, } from "@notation/reconciler"; import type { BaseResource } from "src/orchestrator/resource"; -export { - createMissingResourceRegistryMatchWarningEvent, - createResourceRegistry, - resolveResourceClass, - type MissingResourceRegistryMatchWarningEvent, - type ResourceRegistry, -}; +export { createResourceRegistry, resolveResourceClass, type ResourceRegistry }; export function createResourceRegistryFromGraph( resources: BaseResource[], diff --git a/packages/core/src/provisioner/workflows/workflow.plan.ts b/packages/core/src/provisioner/workflows/workflow.plan.ts index bb7b51c..b300310 100644 --- a/packages/core/src/provisioner/workflows/workflow.plan.ts +++ b/packages/core/src/provisioner/workflows/workflow.plan.ts @@ -22,14 +22,12 @@ export async function planApp({ const graph = await getResourceGraph(entryPoint); return withRuntime( { entryPoint, runtime: suppliedRuntime, databasePath }, - async (runtime) => { - await runtime.initialize(); - return createPlan({ + (runtime) => + createPlan({ resources: graph.resources, state: runtime.state, driftDetection, maxOperationAttempts, - }); - }, + }), ); } diff --git a/packages/core/test/provisioner/durable-runtime.test.ts b/packages/core/test/provisioner/durable-runtime.test.ts index 9a072a5..80e9601 100644 --- a/packages/core/test/provisioner/durable-runtime.test.ts +++ b/packages/core/test/provisioner/durable-runtime.test.ts @@ -1,4 +1,4 @@ -import { access, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; import * as reconciler from "@notation/reconciler/durable"; @@ -156,50 +156,6 @@ describe("NodeDurableRuntime", () => { await rm(directory, { recursive: true, force: true }); }, 5_000); - it("imports and archives legacy JSON state before running", async () => { - const directory = await mkdtemp(path.join(tmpdir(), "notation-migrate-")); - const databasePath = path.join(directory, "workflows.db"); - const legacyStatePath = path.join(directory, "state.json"); - await writeFile( - legacyStatePath, - JSON.stringify({ - existing: { - rev: 7, - id: "existing", - type: "test/legacy", - config: {}, - params: {}, - output: { remoteId: "provider-123" }, - lastOperation: "create", - lastOperationAt: "2026-07-22T00:00:00.000Z", - }, - }), - ); - const runtime = new NodeDurableRuntime({ - deploymentId: "legacy-deployment", - databasePath, - legacyStatePath, - }); - const completed = workflow(async function* () {}); - - try { - await runtime.run(createWorkflowRouter({ deploy: completed }), { - workflowId: "deploy", - executionId: "migration-execution", - }); - await expect(runtime.state.get("existing")).resolves.toMatchObject({ - output: { remoteId: "provider-123" }, - }); - await expect(access(legacyStatePath)).rejects.toThrow(); - await expect( - access(`${legacyStatePath}.migrated`), - ).resolves.toBeUndefined(); - } finally { - runtime.close(); - await rm(directory, { recursive: true, force: true }); - } - }); - it("canonicalises equivalent entry-point spellings", () => { const absolute = path.resolve("infra/api.ts"); expect(resolveDeploymentId("infra/api.ts")).toBe(absolute); diff --git a/packages/core/test/provisioner/resource-registry.test.ts b/packages/core/test/provisioner/resource-registry.test.ts index e01102c..876823c 100644 --- a/packages/core/test/provisioner/resource-registry.test.ts +++ b/packages/core/test/provisioner/resource-registry.test.ts @@ -1,7 +1,6 @@ import { describe, expect, it } from "vitest"; import { resource } from "src/orchestrator/resource"; import { - createMissingResourceRegistryMatchWarningEvent, createResourceRegistry, resolveResourceClass, } from "src/provisioner/resource-registry"; @@ -21,21 +20,4 @@ describe("provisioner resource registry", () => { resolveResourceClass(registry, "test/service/unknown"), ).toBeUndefined(); }); - - it("creates a structured warning event for orphan skips", () => { - expect( - createMissingResourceRegistryMatchWarningEvent({ - workflow: "deploy", - resourceId: "orphan-id", - resourceType: "test/service/unknown", - }), - ).toEqual({ - level: "warn", - event: "reconciler.orphan-deletion.skipped", - reason: "resource-type-not-registered", - workflow: "deploy", - resourceId: "orphan-id", - resourceType: "test/service/unknown", - }); - }); }); diff --git a/packages/reconciler/src/durable/deploy.ts b/packages/reconciler/src/durable/deploy.ts index d9a652c..c0937cc 100644 --- a/packages/reconciler/src/durable/deploy.ts +++ b/packages/reconciler/src/durable/deploy.ts @@ -1,7 +1,6 @@ import { buildResourceDepthLevels } from "../dependency-graph"; import { withDeploymentHold } from "./deployment-hold"; import { reconcileResource, sweepOrphans } from "./reconcile"; -import { scopeStep } from "./step"; import type { DurableDeployOptions } from "./types"; import type { DurableStep } from "./yieldstar"; @@ -19,6 +18,6 @@ export async function* deploy( } // Then delete resources that are in state but no longer declared. - yield* sweepOrphans(scopeStep(step, "notation:orphans"), opts, "deploy"); + yield* sweepOrphans(step, opts, "deploy"); }); } diff --git a/packages/reconciler/src/durable/deployment-hold.ts b/packages/reconciler/src/durable/deployment-hold.ts index 17ba1e2..98dfdcf 100644 --- a/packages/reconciler/src/durable/deployment-hold.ts +++ b/packages/reconciler/src/durable/deployment-hold.ts @@ -1,13 +1,10 @@ /** * The deployment hold: an exclusive claim on a deployment for the length of a - * workflow execution. It lives in the `deployment-coordination` store — that - * store name, the `notation:coordination:*` step keys, and the - * `reconciler.coordination.waiting` event are persisted or published strings - * and keep the older "coordination" wording; the identifiers here do not. + * workflow execution. */ import type { ReconcilerEventEmitter } from "../events"; import { durableEmitter, scopeStep } from "./step"; -import { deploymentCoordinationStore, type CoordinationState } from "./stores"; +import { deploymentHoldStore, type DeploymentHoldState } from "./stores"; import type { DurableStep, StoreClient, WorkflowStore } from "./yieldstar"; type DeploymentHoldOptions = { @@ -23,21 +20,21 @@ type DeploymentHoldOptions = { async function* acquireDeploymentHold( step: DurableStep, opts: DeploymentHoldOptions, -): AsyncGenerator, any> { - const hold = yield* step.store(deploymentCoordinationStore, { +): AsyncGenerator, any> { + const hold = yield* step.store(deploymentHoldStore, { id: opts.deploymentId, initial: { holder: null }, }); - const snapshot = yield* hold.get("notation:coordination:inspect"); + const snapshot = yield* hold.get("notation:hold:inspect"); const holder = snapshot.state.holder; if (holder !== null && holder !== opts.executionId) { yield* durableEmitter( - scopeStep(step, "notation:coordination"), + scopeStep(step, "notation:hold"), opts.emit, )({ level: "warn", - event: "reconciler.coordination.waiting", + event: "reconciler.hold.waiting", deploymentId: opts.deploymentId, executionId: opts.executionId, holderExecutionId: holder, @@ -45,7 +42,7 @@ async function* acquireDeploymentHold( } yield* hold.take( - "notation:coordination:acquire", + "notation:hold:acquire", (state) => state.holder === null || state.holder === opts.executionId, (draft) => { draft.holder = opts.executionId; @@ -56,24 +53,20 @@ async function* acquireDeploymentHold( } function releaseDeploymentHold( - hold: WorkflowStore, + hold: WorkflowStore, executionId: string, ) { - return hold.update("notation:coordination:release", (draft) => { + return hold.update("notation:hold:release", (draft) => { if (draft.holder === executionId) draft.holder = null; }); } /** * Runs `body` while holding the deployment, releasing the hold only once - * `body` has completed. A failed or suspended execution keeps the hold, which - * is what makes it safe to resume: the resumed execution replays `take` from - * the step cache and so never re-acquires anything, so a hold released on the - * way out would leave the resumption mutating a deployment it does not hold. - * - * The cost is that an execution which will never be resumed holds its - * deployment indefinitely. That is deliberate — nothing here can tell "will - * retry" from "abandoned" — and it is resolved by an operator calling + * `body` has completed. A failed or suspended execution keeps its hold: a + * resumed execution replays `take` from the step cache without re-acquiring + * anything, so it must still be the holder. An execution that will never be + * resumed holds its deployment until an operator calls * `takeOverDeploymentHold`. */ export async function* withDeploymentHold( @@ -91,19 +84,15 @@ export type DeploymentHoldTakeover = | { taken: false; holder: string | null }; /** - * Clears a deployment hold left by an execution that will not be resumed, so - * that later deployments are not blocked behind it. Named separately from the - * workflow path because it is the only supported way out of that state: the - * hold is otherwise released solely by an execution completing. + * Clears the hold of an execution that will not be resumed, so later + * deployments are not blocked behind it. * * The write is conditional on `fromExecutionId` still being the named holder, - * so it cannot clear a hold that has since been released and re-taken by - * another execution. Confirm the holder is genuinely dead first: it may still - * be mid-flight, and taking its hold away permits a concurrent mutation of - * the same deployment. + * so it cannot clear a hold that has since moved to another execution. + * Confirm the holder is genuinely dead first: taking a live execution's hold + * away permits a concurrent mutation of the same deployment. * - * Throws if the deployment has no coordination store, i.e. if it has never - * been deployed. + * Throws if the deployment has no hold store, i.e. has never been deployed. */ export async function takeOverDeploymentHold(params: { storeClient: StoreClient; @@ -114,7 +103,7 @@ export async function takeOverDeploymentHold(params: { const { storeClient, deploymentId, fromExecutionId } = params; const read = () => storeClient.getStore({ - definition: deploymentCoordinationStore, + definition: deploymentHoldStore, id: deploymentId, }); @@ -124,7 +113,7 @@ export async function takeOverDeploymentHold(params: { } const result = await storeClient.updateStoreFrom({ - definition: deploymentCoordinationStore, + definition: deploymentHoldStore, id: deploymentId, snapshot, updater: (draft) => { diff --git a/packages/reconciler/src/durable/destroy.ts b/packages/reconciler/src/durable/destroy.ts index 9e2c0c7..7e5ee15 100644 --- a/packages/reconciler/src/durable/destroy.ts +++ b/packages/reconciler/src/durable/destroy.ts @@ -17,7 +17,10 @@ export async function* destroy( for (let index = levels.length - 1; index >= 0; index -= 1) { for (const resource of levels[index]!) { yield* deleteResource( - scopeStep(step, `notation:destroy:${resource.id}`), + scopeStep( + step, + `notation:destroy:${encodeURIComponent(resource.id)}`, + ), resource, opts, ); @@ -25,10 +28,6 @@ export async function* destroy( } // Then delete resources that are in state but no longer declared. - yield* sweepOrphans( - scopeStep(step, "notation:destroy:orphans"), - opts, - "destroy", - ); + yield* sweepOrphans(step, opts, "destroy"); }); } diff --git a/packages/reconciler/src/durable/index.ts b/packages/reconciler/src/durable/index.ts index e53b764..af12567 100644 --- a/packages/reconciler/src/durable/index.ts +++ b/packages/reconciler/src/durable/index.ts @@ -2,29 +2,23 @@ * The durable reconciler driver: `deploy` and `destroy` as generators that a * Yieldstar workflow composes, and the store-backed state they run against. * - * Step keys are a public contract: they are what a resumed execution matches - * its cached work against, so changing one re-executes the work behind it. - * The shapes in use are: + * Step keys are persisted: a resumed execution matches its cached work by + * key, so changing one re-executes the work behind it. The shapes in use are: * - * notation:resource::* per-resource reconciliation steps - * notation:destroy::* per-resource deletion steps - * notation:orphans::* orphan sweep on deploy, per record - * notation:destroy:orphans::* orphan sweep on destroy, per record - * *:remote:attempt: one provider call attempt - * *:remote:retry-delay: the wait between two attempts + * notation:resource::* per-resource reconciliation steps (deploy) + * notation:destroy::* per-resource deletion steps (destroy) + * notation:orphans::* orphan sweep, per persisted record + * *:remote:attempt: one provider call attempt + * *:remote:retry-delay: the wait between two attempts * emit:[::] event delivery checkpoint - * notation:coordination:* deployment hold: inspect/acquire/release - * state:persist: conditional write of a resource record - * state:delete: conditional removal of one + * notation:hold:* deployment hold: inspect/acquire/release + * state:persist: conditional write of a resource record + * state:delete: conditional removal of one * + * An inside a scope is URI-encoded, so the `:` delimiter is unambiguous. * The state: keys are store-handle keys and so are not scope-prefixed: a * store outlives the scope that opened it, which is why they carry the * resource id themselves. - * - * Resource ids are spliced in unescaped, so the delimiter is ambiguous: a - * resource named "orphans" sits in the same key space as the sweep's own - * segment. That predates the key map and is recorded here rather than fixed, - * since changing the composition invalidates in-flight executions. */ export { deploy } from "./deploy"; export { destroy } from "./destroy"; @@ -34,9 +28,9 @@ export { } from "./deployment-hold"; export { DurableStateBackend } from "./state-backend"; export { - deploymentCoordinationStore, + deploymentHoldStore, resourceStateStore, - type CoordinationState, + type DeploymentHoldState, type StoredResourceState, } from "./stores"; export { diff --git a/packages/reconciler/src/durable/reconcile.ts b/packages/reconciler/src/durable/reconcile.ts index def2cad..36f4897 100644 --- a/packages/reconciler/src/durable/reconcile.ts +++ b/packages/reconciler/src/durable/reconcile.ts @@ -6,7 +6,6 @@ import type { BaseResource, ResourceType } from "@notation/resource"; import { RevConflict, type StateNode } from "@notation/state"; import { - createMissingResourceRegistryMatchWarningEvent, createResourceRegistryFromResources, resolveResourceClass, } from "../resource-registry"; @@ -30,12 +29,8 @@ import type { DurableStep } from "./yieldstar"; /** * A read of a resource's persisted record, together with the writes that are - * conditional on that exact read. - * - * The union is the point: `remove` exists only alongside a `node`, because - * removing a record that was never read cannot be done safely. Keeping the - * writes bound to the read that produced them is what stops a node being - * combined with a precondition from a different read. + * conditional on that exact read. `remove` exists only alongside a `node`: + * a record that was never read cannot be removed safely. */ type ResourceStateSession = | { node: undefined; persist: PersistState; remove?: never } @@ -50,7 +45,10 @@ export async function* reconcileResource( resource: BaseResource, opts: DurableDeployOptions, ): AsyncGenerator { - const scope = scopeStep(step, `notation:resource:${resource.id}`); + const scope = scopeStep( + step, + `notation:resource:${encodeURIComponent(resource.id)}`, + ); // Resolved once and then carried: deriveParams is user code and need not be // deterministic, so an operation resolving them again could persist params @@ -74,8 +72,8 @@ export async function* reconcileResource( resource, resourceParams: params, persistedOutput: session.node?.output, - // Deliberately no dryRun: a dry run suppresses mutations, not reads. - // Reading is how a dry run reports drift at all. + // No dryRun: a dry run suppresses mutations, not reads, and reading is + // how a dry run reports drift at all. emit: durableEmitter(driftStep, opts.emit), maxOperationAttempts: opts.maxOperationAttempts, }); @@ -164,31 +162,33 @@ export async function* deleteResource( * warning, because deleting it would need a resource class we cannot resolve. */ export async function* sweepOrphans( - step: DurableStepRunner, + step: DurableStep, opts: DurableWorkflowOptions, workflow: "deploy" | "destroy", ): AsyncGenerator { + const scope = scopeStep(step, "notation:orphans"); const resourceById = new Map( opts.resources.map((resource) => [resource.id, resource]), ); - const persisted = yield* step.run("list", () => opts.state.values()); + const persisted = yield* scope.run("list", () => opts.state.values()); const registry = opts.registry ?? createResourceRegistryFromResources(opts.resources); for (const node of persisted) { if (resourceById.has(node.id)) continue; - const nodeScope = step.scope(node.id); + const nodeScope = scope.scope(encodeURIComponent(node.id)); const Resource = resolveResourceClass(registry, node.type as ResourceType); if (!Resource) { const emit = durableEmitter(nodeScope, opts.emit); - yield* emit( - createMissingResourceRegistryMatchWarningEvent({ - workflow, - resourceId: node.id, - resourceType: node.type as ResourceType, - }), - ); + yield* emit({ + level: "warn", + event: "reconciler.orphan-deletion.skipped", + reason: "resource-type-not-registered", + workflow, + resourceId: node.id, + resourceType: node.type as ResourceType, + }); continue; } @@ -203,8 +203,7 @@ export async function* sweepOrphans( * * The snapshot is the precondition: it names the exact store instance and * version the record was read at, so a write made against it cannot land on a - * record another writer has moved on. It is re-served to the operations so - * they need no second read. + * record another writer has moved on. */ async function* openStateSession( step: DurableStepRunner, diff --git a/packages/reconciler/src/durable/state-backend.ts b/packages/reconciler/src/durable/state-backend.ts index 55badb3..8b57a3f 100644 --- a/packages/reconciler/src/durable/state-backend.ts +++ b/packages/reconciler/src/durable/state-backend.ts @@ -8,12 +8,9 @@ import type { StoreClient } from "./yieldstar"; /** * Reads deployment state from outside a workflow, for the planner and for - * anything reporting on a deployment. - * - * Deliberately read-only. Writes belong to the workflow, which makes them - * through the store handle so they are stamped with the step that made them. - * There is nowhere on this interface to carry that idempotency key, so a - * write made here would be repeated on replay rather than recognised. + * anything reporting on a deployment. Read-only: state writes must be stamped + * with the workflow step that made them, and this interface has nowhere to + * carry that step key, so a write made here would repeat on replay. */ export class DurableStateBackend { readonly #client: StoreClient; @@ -51,9 +48,8 @@ export class DurableStateBackend { .map(toStateNode); } - // getStore throws for a store that does not exist rather than returning - // undefined, and the error is not distinguishable from a real failure, so - // absence is confirmed by listing. Kept here so no caller has to know that. + // getStore throws for a missing store, and the error is indistinguishable + // from a real failure, so absence is confirmed by listing. async #read(storeId: string): Promise { try { return await this.#client.getStore({ diff --git a/packages/reconciler/src/durable/stores.ts b/packages/reconciler/src/durable/stores.ts index 890477c..fc979e4 100644 --- a/packages/reconciler/src/durable/stores.ts +++ b/packages/reconciler/src/durable/stores.ts @@ -6,19 +6,16 @@ import { defineStore, type StoreSnapshot } from "./yieldstar"; // index.ts: renaming one orphans every record stored under the old name. /** - * Loose on purpose: PersistedResourceState carries an index signature so a - * driver can persist fields this schema does not name yet, and v.object would - * strip those at the store boundary rather than carry them forward. Every - * field an operation actually writes is still declared, so the schema is a - * floor on the record rather than a description of one. + * `looseObject` because PersistedResourceState carries an index signature: a + * driver may persist fields this schema does not name, and `v.object` would + * strip them at the store boundary. */ export const resourceStateStore = defineStore( "resource-state", v.looseObject({ id: v.string(), type: v.string(), - // Written by create and update. -1 and "" are BaseResource's defaults for - // a resource that belongs to no group. + // -1 and "" are BaseResource's defaults for a resource with no group. groupId: v.number(), groupType: v.string(), config: v.record(v.string(), v.unknown()), @@ -29,16 +26,16 @@ export const resourceStateStore = defineStore( }), ); -export const deploymentCoordinationStore = defineStore( - "deployment-coordination", +export const deploymentHoldStore = defineStore( + "deployment-hold", v.object({ holder: v.nullable(v.string()) }), ); export type StoredResourceState = v.InferOutput< typeof resourceStateStore.schema >; -export type CoordinationState = v.InferOutput< - typeof deploymentCoordinationStore.schema +export type DeploymentHoldState = v.InferOutput< + typeof deploymentHoldStore.schema >; /** A read of a resource record, carrying the identity a write is made against. */ diff --git a/packages/reconciler/src/durable/types.ts b/packages/reconciler/src/durable/types.ts index f9c6022..7c298fe 100644 --- a/packages/reconciler/src/durable/types.ts +++ b/packages/reconciler/src/durable/types.ts @@ -3,7 +3,6 @@ import type { ReconcilerEventEmitter } from "../events"; import type { ResourceRegistry } from "../resource-registry"; import type { DurableStateBackend } from "./state-backend"; -/** What both durable workflows need; "operation" would mean resource CRUD here. */ export type DurableWorkflowOptions = { deploymentId: string; executionId: string; diff --git a/packages/reconciler/src/events.ts b/packages/reconciler/src/events.ts index 46aacaf..e233b45 100644 --- a/packages/reconciler/src/events.ts +++ b/packages/reconciler/src/events.ts @@ -1,5 +1,4 @@ import type { ResourceType } from "@notation/resource"; -import type { MissingResourceRegistryMatchWarningEvent } from "./resource-registry"; export type OperationName = "create" | "read" | "update" | "delete"; @@ -18,7 +17,7 @@ export type OperationLifecycleEvent = { errorMessage?: string; }; -export type ReconcilerDeployEvent = { +export type DeployDecisionEvent = { level: "info"; event: "reconciler.deploy.decision"; resourceId: string; @@ -26,7 +25,7 @@ export type ReconcilerDeployEvent = { decision: "create" | "update" | "drift-update" | "drift-recreate" | "noop"; }; -export type ReconcilerDriftDetectedEvent = { +export type DriftDetectedEvent = { level: "info"; event: "reconciler.drift.detected"; resourceId: string; @@ -34,20 +33,29 @@ export type ReconcilerDriftDetectedEvent = { diff: Record; }; -export type CoordinationWaitingEvent = { +export type HoldWaitingEvent = { level: "warn"; - event: "reconciler.coordination.waiting"; + event: "reconciler.hold.waiting"; deploymentId: string; executionId: string; holderExecutionId: string; }; +export type OrphanDeletionSkippedEvent = { + level: "warn"; + event: "reconciler.orphan-deletion.skipped"; + reason: "resource-type-not-registered"; + workflow: "deploy" | "destroy"; + resourceId: string; + resourceType: ResourceType; +}; + export type ReconcilerEvent = | OperationLifecycleEvent - | CoordinationWaitingEvent - | ReconcilerDeployEvent - | ReconcilerDriftDetectedEvent - | MissingResourceRegistryMatchWarningEvent; + | DeployDecisionEvent + | DriftDetectedEvent + | HoldWaitingEvent + | OrphanDeletionSkippedEvent; export type ReconcilerEventEmitter = ( event: ReconcilerEvent, diff --git a/packages/reconciler/src/operations/operation.types.ts b/packages/reconciler/src/operations/operation.types.ts index 9eeeff9..bf92b9f 100644 --- a/packages/reconciler/src/operations/operation.types.ts +++ b/packages/reconciler/src/operations/operation.types.ts @@ -16,16 +16,9 @@ export type { export type OperationEventEmitter = EmitStep; /** - * How an operation runs a step, and how it namespaces the steps it runs. - * - * Keys are mandatory. A driver may ignore them — the in-process one does — - * but a keyless step would have to derive a key from its call site, which for - * a step reached through `scope` is the scoping wrapper's call site rather - * than the caller's, so any two keyless steps under one scope would collide. - * - * `scope` is the seam that lets one operation run at several call sites in a - * single execution: in process it is the identity, and in a workflow it - * prefixes the keys the runtime caches against. + * How an operation runs a step. Keys identify a step's cached result across a + * replay; `scope` namespaces them so one operation can run at several call + * sites in a single execution. The in-process driver ignores both. */ export type StepRunner = { run( @@ -38,8 +31,8 @@ export type StepRunner = { /** * The record an operation wants persisted; the driver owns the revision. - * Spelled out rather than derived with Omit, which would collapse against - * StateNode's index signature and widen every field to unknown. + * Not derived with Omit, which would collapse against StateNode's index + * signature and widen every field to unknown. */ export type PersistedResourceState = Pick< StateNode, diff --git a/packages/reconciler/src/planner.ts b/packages/reconciler/src/planner.ts index 8ac8b27..e759300 100644 --- a/packages/reconciler/src/planner.ts +++ b/packages/reconciler/src/planner.ts @@ -12,7 +12,7 @@ import { } from "./plan"; import { createStepRunner, runOperation } from "./step-runner"; -/** Planning reads state and never mutates it, so no lease is required. */ +/** Planning only reads state. */ export type PlannerState = Pick; export type CreatePlanOptions = { diff --git a/packages/reconciler/src/resource-registry.ts b/packages/reconciler/src/resource-registry.ts index 34dabfc..fc39437 100644 --- a/packages/reconciler/src/resource-registry.ts +++ b/packages/reconciler/src/resource-registry.ts @@ -6,15 +6,6 @@ import type { export type ResourceRegistry = Map>; -export type MissingResourceRegistryMatchWarningEvent = { - level: "warn"; - event: "reconciler.orphan-deletion.skipped"; - reason: "resource-type-not-registered"; - workflow: "deploy" | "destroy"; - resourceId: string; - resourceType: ResourceType; -}; - export function createResourceRegistry( entries: Iterable> = [], ): ResourceRegistry { @@ -48,18 +39,3 @@ export function resolveResourceClass( ): ResourceClass | undefined { return registry.get(type); } - -export function createMissingResourceRegistryMatchWarningEvent(opts: { - workflow: "deploy" | "destroy"; - resourceId: string; - resourceType: ResourceType; -}): MissingResourceRegistryMatchWarningEvent { - return { - level: "warn", - event: "reconciler.orphan-deletion.skipped", - reason: "resource-type-not-registered", - workflow: opts.workflow, - resourceId: opts.resourceId, - resourceType: opts.resourceType, - }; -} diff --git a/packages/reconciler/test/durable-reconciliation.test.ts b/packages/reconciler/test/durable-reconciliation.test.ts index 516663b..9cd8bba 100644 --- a/packages/reconciler/test/durable-reconciliation.test.ts +++ b/packages/reconciler/test/durable-reconciliation.test.ts @@ -322,7 +322,7 @@ describe("conditional state persistence", () => { }); }); -describe("deployment coordination", () => { +describe("deployment hold", () => { it("serializes concurrent deployments through durable store waiting", async () => { let unblockCreate!: () => void; const blocked = new Promise((resolve) => { @@ -375,7 +375,7 @@ describe("deployment coordination", () => { .defineOperations({ create: async () => { const snapshot = await runtime.storeClient.getStore({ - definition: durable.deploymentCoordinationStore, + definition: durable.deploymentHoldStore, id: "hold-replay", }); holders.push(snapshot.state.holder); @@ -407,7 +407,7 @@ describe("deployment coordination", () => { runtime.close(); }); - it("emits a coordination waiting event when another execution holds the deployment", async () => { + it("emits a hold waiting event when another execution holds the deployment", async () => { let unblockCreate!: () => void; const blocked = new Promise((resolve) => { unblockCreate = resolve; @@ -416,7 +416,7 @@ describe("deployment coordination", () => { const createStarted = new Promise((resolve) => { started = resolve; }); - const TestResource = resource({ type: "test/durable/coordination" }) + const TestResource = resource({ type: "test/durable/hold" }) .defineSchema({}) .defineOperations({ create: async () => { @@ -428,7 +428,7 @@ describe("deployment coordination", () => { const events: ReconcilerEvent[] = []; const runtime = createRuntime( [new TestResource({ id: "held" })], - "coordination-waiting", + "hold-waiting", { emit: (event) => void events.push(event) }, ); @@ -437,10 +437,10 @@ describe("deployment coordination", () => { await runtime.run("waiter-execution"); expect( - events.find((event) => event.event === "reconciler.coordination.waiting"), + events.find((event) => event.event === "reconciler.hold.waiting"), ).toMatchObject({ level: "warn", - deploymentId: "coordination-waiting", + deploymentId: "hold-waiting", executionId: "waiter-execution", holderExecutionId: "holder-execution", }); @@ -459,7 +459,7 @@ describe("deployment hold takeover", () => { .defineOperations({ create, delete: async () => undefined }); const runtime = createRuntime([new Resource({ id: "held" })], "takeover"); await runtime.storeClient.getOrCreateStore({ - definition: durable.deploymentCoordinationStore, + definition: durable.deploymentHoldStore, id: "takeover", initial: { holder: "abandoned-execution" }, }); @@ -482,7 +482,7 @@ describe("deployment hold takeover", () => { it("refuses to clear a hold that has moved to another execution", async () => { const runtime = createRuntime([], "takeover-race"); await runtime.storeClient.getOrCreateStore({ - definition: durable.deploymentCoordinationStore, + definition: durable.deploymentHoldStore, id: "takeover-race", initial: { holder: "current-execution" }, }); @@ -495,7 +495,7 @@ describe("deployment hold takeover", () => { expect(result).toEqual({ taken: false, holder: "current-execution" }); const snapshot = await runtime.storeClient.getStore({ - definition: durable.deploymentCoordinationStore, + definition: durable.deploymentHoldStore, id: "takeover-race", }); expect(snapshot.state.holder).toBe("current-execution"); diff --git a/packages/reconciler/test/logger-subscriber.test.ts b/packages/reconciler/test/logger-subscriber.test.ts index 721e884..0614d5b 100644 --- a/packages/reconciler/test/logger-subscriber.test.ts +++ b/packages/reconciler/test/logger-subscriber.test.ts @@ -29,7 +29,7 @@ describe("logger reconciler subscriber", () => { }); await emit({ level: "warn", - event: "reconciler.coordination.waiting", + event: "reconciler.hold.waiting", deploymentId: "deployment-1", executionId: "execution-2", holderExecutionId: "execution-1", @@ -49,7 +49,7 @@ describe("logger reconciler subscriber", () => { expect(warn).toHaveBeenCalledTimes(2); expect(warn).toHaveBeenNthCalledWith( 2, - "reconciler.coordination.waiting", + "reconciler.hold.waiting", expect.objectContaining({ level: "warn" }), ); expect(error).toHaveBeenCalledOnce(); diff --git a/packages/reconciler/test/operation.workflows.test.ts b/packages/reconciler/test/operation.workflows.test.ts index 1d4c410..2cb8cbc 100644 --- a/packages/reconciler/test/operation.workflows.test.ts +++ b/packages/reconciler/test/operation.workflows.test.ts @@ -12,6 +12,7 @@ import { type StepRunner, } from "../src/operations"; import { toEmitStep } from "../src/events"; +import { runOperation } from "../src/step-runner"; function createStepRunnerDouble() { const run = vi.fn(async function* ( @@ -38,14 +39,6 @@ function createStepRunnerDouble() { return runner; } -async function runOperation(operation: AsyncGenerator) { - let next = await operation.next(); - while (!next.done) { - next = await operation.next(); - } - return next.value; -} - describe("operation workflows", () => { it("create performs create + read-after-create + state persistence", async () => { const step = createStepRunnerDouble(); diff --git a/packages/state-sqlite/package.json b/packages/state-sqlite/package.json deleted file mode 100644 index 1ad01b7..0000000 --- a/packages/state-sqlite/package.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "type": "module", - "name": "@notation/state-sqlite", - "version": "0.1.0", - "main": "./dist/index.js", - "types": "./dist/index.d.ts", - "files": [ - "dist" - ], - "scripts": { - "build": "tsup --clean", - "dev": "tsup --watch" - }, - "dependencies": { - "@notation/state": "workspace:*" - }, - "devDependencies": { - "@types/node": "^22.13.4" - } -} diff --git a/packages/state-sqlite/src/index.ts b/packages/state-sqlite/src/index.ts deleted file mode 100644 index c01775b..0000000 --- a/packages/state-sqlite/src/index.ts +++ /dev/null @@ -1,103 +0,0 @@ -import { mkdirSync } from "node:fs"; -import { dirname } from "node:path"; -import { DatabaseSync } from "node:sqlite"; -import { - RevConflict, - type StateBackend, - type StateNode, -} from "@notation/state"; - -export class SqliteStateBackend implements StateBackend { - readonly #database: DatabaseSync; - - constructor(path: string) { - mkdirSync(dirname(path), { recursive: true }); - this.#database = new DatabaseSync(path); - this.#database.exec("PRAGMA busy_timeout = 5000"); - this.#database.exec(` - CREATE TABLE IF NOT EXISTS resources ( - id TEXT PRIMARY KEY, - rev INTEGER NOT NULL, - value TEXT NOT NULL - ) - `); - } - - close(): void { - this.#database.close(); - } - - async get(id: string): Promise { - const row = this.#database - .prepare("SELECT value FROM resources WHERE id = ?") - .get(id) as { value: string } | undefined; - return row ? (JSON.parse(row.value) as StateNode) : undefined; - } - - async has(id: string): Promise { - return Boolean( - this.#database.prepare("SELECT 1 FROM resources WHERE id = ?").get(id), - ); - } - - async update( - id: string, - expectedRev: number, - patch: Partial, - ): Promise<{ rev: number }> { - this.#database.exec("BEGIN IMMEDIATE"); - try { - const current = await this.get(id); - // A missing record counts as rev 0, so expectedRev: 0 = "must not exist". - if ((current?.rev ?? 0) !== expectedRev) { - throw new RevConflict(id, expectedRev, current?.rev); - } - - const rev = (current?.rev ?? 0) + 1; - const node = { ...current, ...patch, rev } as StateNode; - if (current) { - const result = this.#database - .prepare( - "UPDATE resources SET rev = ?, value = ? WHERE id = ? AND rev = ?", - ) - .run(rev, JSON.stringify(node), id, current.rev); - if (result.changes !== 1) { - const actual = await this.get(id); - throw new RevConflict(id, current.rev, actual?.rev); - } - } else { - this.#database - .prepare("INSERT INTO resources (id, rev, value) VALUES (?, ?, ?)") - .run(id, rev, JSON.stringify(node)); - } - this.#database.exec("COMMIT"); - return { rev }; - } catch (error) { - this.#database.exec("ROLLBACK"); - throw error; - } - } - - async delete(id: string, expectedRev: number): Promise { - const current = await this.get(id); - if ((current?.rev ?? 0) !== expectedRev) { - throw new RevConflict(id, expectedRev, current?.rev); - } - if (!current) return; - - const result = this.#database - .prepare("DELETE FROM resources WHERE id = ? AND rev = ?") - .run(id, current.rev); - if (result.changes !== 1) { - const actual = await this.get(id); - throw new RevConflict(id, current.rev, actual?.rev); - } - } - - async values(): Promise { - const rows = this.#database - .prepare("SELECT value FROM resources ORDER BY id") - .all() as { value: string }[]; - return rows.map(({ value }) => JSON.parse(value) as StateNode); - } -} diff --git a/packages/state-sqlite/src/node-sqlite.d.ts b/packages/state-sqlite/src/node-sqlite.d.ts deleted file mode 100644 index 04691f7..0000000 --- a/packages/state-sqlite/src/node-sqlite.d.ts +++ /dev/null @@ -1,20 +0,0 @@ -declare module "node:sqlite" { - export type StatementResult = { changes: number | bigint }; - export class StatementSync { - get(...values: unknown[]): unknown; - all(...values: unknown[]): unknown[]; - run(...values: unknown[]): StatementResult; - } - export class DatabaseSync { - constructor(path: string); - exec(sql: string): void; - prepare(sql: string): StatementSync; - close(): void; - } -} - -// The shared tsconfig does not load @types/node, so declare the one export -// this package uses. -declare module "node:crypto" { - export function randomUUID(): string; -} diff --git a/packages/state-sqlite/test/state-sqlite.test.ts b/packages/state-sqlite/test/state-sqlite.test.ts deleted file mode 100644 index f50456e..0000000 --- a/packages/state-sqlite/test/state-sqlite.test.ts +++ /dev/null @@ -1,95 +0,0 @@ -import { mkdtemp, rm } from "node:fs/promises"; -import { spawn } from "node:child_process"; -import { once } from "node:events"; -import { tmpdir } from "node:os"; -import path from "node:path"; -import { afterEach, describe, expect, it } from "vitest"; -import { SqliteStateBackend } from "../src"; - -const cleanups: (() => Promise)[] = []; -afterEach(async () => - Promise.all(cleanups.splice(0).map((cleanup) => cleanup())), -); - -async function createBackend() { - const directory = await mkdtemp(path.join(tmpdir(), "notation-sqlite-")); - const backend = new SqliteStateBackend(path.join(directory, "state.db")); - cleanups.push(async () => { - backend.close(); - await rm(directory, { recursive: true, force: true }); - }); - return backend; -} - -describe("SqliteStateBackend", () => { - it("persists revisions and enforces compare-and-swap", async () => { - const backend = await createBackend(); - await expect( - backend.update("service", 0, { - id: "service", - type: "test/service/main", - config: {}, - params: {}, - output: {}, - lastOperation: "create", - lastOperationAt: "2026-07-15T00:00:00.000Z", - }), - ).resolves.toEqual({ rev: 1 }); - await expect( - backend.update("service", 1, { output: { ready: true } }), - ).resolves.toEqual({ - rev: 2, - }); - await expect(backend.delete("service", 1)).rejects.toMatchObject({ - name: "RevConflict", - actualRev: 2, - }); - }); - - it("waits for a concurrent writer instead of raising database locked", async () => { - const directory = await mkdtemp( - path.join(tmpdir(), "notation-sqlite-busy-"), - ); - const databasePath = path.join(directory, "state.db"); - const backend = new SqliteStateBackend(databasePath); - cleanups.push(async () => { - backend.close(); - await rm(directory, { recursive: true, force: true }); - }); - - const blocker = spawn( - process.execPath, - [ - "--input-type=module", - "--eval", - ` - import { DatabaseSync } from "node:sqlite"; - const database = new DatabaseSync(process.argv[1]); - database.exec("BEGIN IMMEDIATE"); - process.stdout.write("locked\\n"); - setTimeout(() => { - database.exec("ROLLBACK"); - database.close(); - }, 100); - `, - databasePath, - ], - { stdio: ["ignore", "pipe", "inherit"] }, - ); - const blockerExited = once(blocker, "exit"); - await once(blocker.stdout!, "data"); - - await expect( - backend.update("service", 0, { - id: "service", - type: "test/service/main", - config: {}, - params: {}, - output: {}, - lastOperation: "create", - lastOperationAt: "2026-07-15T00:00:00.000Z", - }), - ).resolves.toEqual({ rev: 1 }); - await blockerExited; - }); -}); diff --git a/packages/state-sqlite/tsconfig.json b/packages/state-sqlite/tsconfig.json deleted file mode 100644 index 13487e3..0000000 --- a/packages/state-sqlite/tsconfig.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "extends": "tsconfig/base.json", - "compilerOptions": { - "types": ["node"] - }, - "include": ["src", "test"] -} diff --git a/packages/state-sqlite/tsup.config.ts b/packages/state-sqlite/tsup.config.ts deleted file mode 100644 index a61d2e2..0000000 --- a/packages/state-sqlite/tsup.config.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { defineConfig } from "tsup"; - -export default defineConfig({ - entry: ["src/index.ts"], - format: ["esm"], - dts: true, - sourcemap: true, - // node:sqlite only resolves with the node: prefix; don't let tsup strip it. - removeNodeProtocol: false, -}); diff --git a/packages/state/package.json b/packages/state/package.json index 8a49f6f..2c81b57 100644 --- a/packages/state/package.json +++ b/packages/state/package.json @@ -11,9 +11,6 @@ "build": "tsup --clean", "dev": "tsup --watch" }, - "dependencies": { - "@notation/utils": "workspace:*" - }, "devDependencies": { "@types/node": "^22.13.4" } diff --git a/packages/state/src/state.ts b/packages/state/src/state.ts index 971f802..45e79d8 100644 --- a/packages/state/src/state.ts +++ b/packages/state/src/state.ts @@ -1,15 +1,3 @@ -import { randomUUID } from "node:crypto"; -import { - mkdir, - readFile, - rename, - stat, - unlink, - writeFile, -} from "node:fs/promises"; -import path from "node:path"; -import { setTimeout as sleep } from "node:timers/promises"; -import { isErrorWithCode } from "@notation/utils"; import { RevConflict } from "./conflicts"; export type StateNode = { @@ -109,132 +97,6 @@ export class MemoryStateBackend implements StateBackend { } } -const FILE_LOCK_STALE_MS = 10_000; -const FILE_LOCK_TIMEOUT_MS = 5_000; -const FILE_LOCK_RETRY_MS = 25; - -export class FileStateBackend implements StateBackend { - constructor(private readonly stateFilePath: string) {} - - async get(id: string): Promise { - const state = await this.readState(); - return state[id]; - } - - async has(id: string): Promise { - const state = await this.readState(); - return id in state; - } - - async update( - id: string, - expectedRev: number, - patch: Partial, - ): Promise<{ rev: number }> { - return this.withLock(async () => { - const state = await this.readState(); - assertExpectedRev(id, state[id], expectedRev); - const rev = (state[id]?.rev ?? 0) + 1; - state[id] = { - ...state[id], - ...patch, - rev, - } as StateNode; - await this.writeState(state); - return { rev }; - }); - } - - async delete(id: string, expectedRev: number): Promise { - await this.withLock(async () => { - const state = await this.readState(); - assertExpectedRev(id, state[id], expectedRev); - delete state[id]; - await this.writeState(state); - }); - } - - async values(): Promise { - const state = await this.readState(); - return Object.values(state); - } - - private async readState(): Promise> { - try { - const file = await readFile(this.stateFilePath, "utf8"); - return JSON.parse(file) as Record; - } catch (error) { - if (isErrorWithCode(error, "ENOENT")) { - return {}; - } - - throw error; - } - } - - /** - * The read-check-write in update/delete is only safe if no other process - * interleaves, so writers hold an exclusive lock file. A lock older than - * FILE_LOCK_STALE_MS is treated as abandoned by a crashed process. - */ - private async withLock(fn: () => Promise): Promise { - const lockFilePath = `${this.stateFilePath}.lock`; - await mkdir(path.dirname(this.stateFilePath), { recursive: true }); - - const deadline = Date.now() + FILE_LOCK_TIMEOUT_MS; - for (;;) { - try { - await writeFile( - lockFilePath, - JSON.stringify({ pid: process.pid, acquiredAt: Date.now() }), - { flag: "wx" }, - ); - break; - } catch (error) { - if (!isErrorWithCode(error, "EEXIST")) throw error; - const lockStat = await stat(lockFilePath).catch(() => undefined); - if (lockStat && Date.now() - lockStat.mtimeMs > FILE_LOCK_STALE_MS) { - await unlink(lockFilePath).catch(() => undefined); - continue; - } - if (Date.now() > deadline) { - throw new Error( - `Timed out acquiring state lock at ${lockFilePath}; delete it if no other deploy is running`, - ); - } - await sleep(FILE_LOCK_RETRY_MS); - } - } - - try { - return await fn(); - } finally { - await unlink(lockFilePath).catch(() => undefined); - } - } - - private async writeState(state: Record): Promise { - const directory = path.dirname(this.stateFilePath); - await mkdir(directory, { recursive: true }); - - const tempFilePath = path.join( - directory, - `${path.basename(this.stateFilePath)}.${randomUUID()}.tmp`, - ); - - const serialized = `${JSON.stringify(state, null, 2)}\n`; - - await writeFile(tempFilePath, serialized, "utf8"); - - try { - await rename(tempFilePath, this.stateFilePath); - } catch (error) { - await unlink(tempFilePath).catch(() => undefined); - throw error; - } - } -} - // A missing record counts as rev 0, so expectedRev: 0 means "must not exist". function assertExpectedRev( id: string, diff --git a/packages/state/test/state-backend.test.ts b/packages/state/test/state-backend.test.ts index 888055a..5630b0a 100644 --- a/packages/state/test/state-backend.test.ts +++ b/packages/state/test/state-backend.test.ts @@ -1,9 +1,5 @@ -import { mkdtemp, rm } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import path from "node:path"; import { describe, expect, it } from "vitest"; import { - FileStateBackend, MemoryStateBackend, type StateBackend, type StateNode, @@ -160,51 +156,11 @@ function runStateBackendContractTests( }); } -runStateBackendContractTests("FileStateBackend", async () => { - const tempDirectory = await mkdtemp(path.join(tmpdir(), "notation-state-")); - return { - backend: new FileStateBackend(path.join(tempDirectory, "state.json")), - cleanup: () => rm(tempDirectory, { recursive: true, force: true }), - }; -}); - runStateBackendContractTests("MemoryStateBackend", async () => ({ backend: new MemoryStateBackend(), cleanup: async () => undefined, })); -describe("FileStateBackend", () => { - it("serialises concurrent CAS writers so only one wins", async () => { - const tempDirectory = await mkdtemp(path.join(tmpdir(), "notation-state-")); - const statePath = path.join(tempDirectory, "state.json"); - const first = new FileStateBackend(statePath); - const second = new FileStateBackend(statePath); - const initialNode = createStateNode("resource-a"); - - try { - await first.update(initialNode.id, 0, initialNode); - - const results = await Promise.allSettled([ - first.update(initialNode.id, 1, { output: { writer: "first" } }), - second.update(initialNode.id, 1, { output: { writer: "second" } }), - ]); - - const fulfilled = results.filter((r) => r.status === "fulfilled"); - const rejected = results.filter((r) => r.status === "rejected"); - expect(fulfilled).toHaveLength(1); - expect(rejected).toHaveLength(1); - expect((rejected[0] as PromiseRejectedResult).reason).toMatchObject({ - name: "RevConflict", - }); - await expect(first.get(initialNode.id)).resolves.toMatchObject({ - rev: 2, - }); - } finally { - await rm(tempDirectory, { recursive: true, force: true }); - } - }); -}); - describe("MemoryStateBackend", () => { it("returns values in deterministic id order", async () => { const backend = new MemoryStateBackend(); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 404775d..b3d35d6 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -426,20 +426,6 @@ importers: packages/resource: {} packages/state: - dependencies: - '@notation/utils': - specifier: workspace:* - version: link:../utils - devDependencies: - '@types/node': - specifier: ^22.13.4 - version: 22.13.4 - - packages/state-sqlite: - dependencies: - '@notation/state': - specifier: workspace:* - version: link:../state devDependencies: '@types/node': specifier: ^22.13.4 From bf18eb3504e771bfb5ec883e5469235560cf86fa Mon Sep 17 00:00:00 2001 From: djgrant <1670902+djgrant@users.noreply.github.com> Date: Tue, 28 Jul 2026 18:08:14 +0100 Subject: [PATCH 26/34] Address the simplicity review: one convention, one gate, one name - Namespace every Notation-owned store name as notation/: notation/resource-state and notation/deployment-hold join notation/execution-binding, so an application sharing a store client with these workflows cannot collide with them. Documented in the durable index key map and the manuals. - Rename NOTATION_STATE_PATH to NOTATION_DATABASE_PATH: it selects the whole workflows SQLite database (heap, timers, task queue, stores), not a state file, and the code's own constant already said so. - Split decideAction: the driftRead arms that could not fire (create on an absent remote, plain update) existed only because one function served two moments. decideDriftAction now takes over after a noop, where a state node necessarily exists and local params match it. - Gate the drift read on resource.read in the durable driver, as the planner already did: a resource with no read has no remote to compare, and its noop now stands without read skip/success lifecycle noise. A new test locks the shared behaviour in. - One name per concept: drop the State alias of StateBackend, the DurableDestroyOptions alias of DurableWorkflowOptions, and the unused StateBackend.has(). - Callers now pre-scope the step for reconcileResource as they already did for deleteResource; step keys are unchanged. - Rename the statePatch test helper to resourceStateRecord: it returns a full record, not a patch. - withRuntime rejects runtime plus databasePath instead of silently ignoring the path. --- docs/cli/dashboard.md | 2 +- docs/cli/deploy.md | 2 +- docs/internals/reconciler.md | 4 +- docs/internals/state.md | 6 +- docs/rfcs/reconciler.md | 4 +- .../core/src/provisioner/durable-runtime.ts | 7 +- packages/reconciler/src/durable/deploy.ts | 10 ++- packages/reconciler/src/durable/destroy.ts | 4 +- packages/reconciler/src/durable/index.ts | 8 +- packages/reconciler/src/durable/reconcile.ts | 43 +++++----- packages/reconciler/src/durable/stores.ts | 6 +- packages/reconciler/src/durable/types.ts | 2 - packages/reconciler/src/plan.ts | 84 +++++++++---------- packages/reconciler/src/planner.ts | 12 +-- .../test/durable-reconciliation.test.ts | 61 ++++++++++++-- packages/state/src/state.ts | 8 -- packages/state/test/state-backend.test.ts | 2 - 17 files changed, 154 insertions(+), 111 deletions(-) diff --git a/docs/cli/dashboard.md b/docs/cli/dashboard.md index 26780c4..e51fd92 100644 --- a/docs/cli/dashboard.md +++ b/docs/cli/dashboard.md @@ -10,4 +10,4 @@ Starts a local web dashboard for observing the deployment's resource state. notation dashboard infra/api.ts ``` -The dashboard reads `.notation/workflows.db`, the same database used by deploy, destroy, and plan. Set `NOTATION_STATE_PATH` to choose another SQLite database path. +The dashboard reads `.notation/workflows.db`, the same database used by deploy, destroy, and plan. Set `NOTATION_DATABASE_PATH` to choose another SQLite database path. diff --git a/docs/cli/deploy.md b/docs/cli/deploy.md index dbc9c6e..a26ddec 100644 --- a/docs/cli/deploy.md +++ b/docs/cli/deploy.md @@ -44,4 +44,4 @@ Retryable provider conditions and consistency reads suspend on durable SQLite ti 6. **Delete orphans** – persisted resources absent from the graph are deleted when their resource type is registered. -State, step results, timers, queued tasks, and resource stores are persisted to `.notation/workflows.db`. Set `NOTATION_STATE_PATH` to choose another SQLite database path. +State, step results, timers, queued tasks, and resource stores are persisted to `.notation/workflows.db`. Set `NOTATION_DATABASE_PATH` to choose another SQLite database path. diff --git a/docs/internals/reconciler.md b/docs/internals/reconciler.md index eeba956..debfd8e 100644 --- a/docs/internals/reconciler.md +++ b/docs/internals/reconciler.md @@ -31,9 +31,9 @@ Each attempt, delay, event, state read, state write, and hold change has a stabl ## State and the deployment hold -Each resource is stored under `resource-state` with a deployment-scoped ID. Conditional updates and deletes compare the snapshot's UUIDv7 `instanceId` and version, so a stale execution cannot modify a deleted and recreated store. +Each resource is stored under `notation/resource-state` with a deployment-scoped ID. Conditional updates and deletes compare the snapshot's UUIDv7 `instanceId` and version, so a stale execution cannot modify a deleted and recreated store. -Deploy and destroy take an exclusive hold on the deployment through one `deployment-hold` store per deployment. `store.take` suspends a competing execution as a durable waiter and wakes it after the holder releases. Before suspending, the waiter emits `reconciler.hold.waiting` naming the holding execution ID, so a wait behind a crashed execution is visible instead of silent. +Deploy and destroy take an exclusive hold on the deployment through one `notation/deployment-hold` store per deployment. `store.take` suspends a competing execution as a durable waiter and wakes it after the holder releases. Before suspending, the waiter emits `reconciler.hold.waiting` naming the holding execution ID, so a wait behind a crashed execution is visible instead of silent. A failed or suspended execution keeps its hold, which is what makes resuming it safe. The hold of an execution that will never be resumed is cleared with `takeOverDeploymentHold` from `@notation/reconciler/durable` — the only supported way out of that state. diff --git a/docs/internals/state.md b/docs/internals/state.md index 1c738e9..8907636 100644 --- a/docs/internals/state.md +++ b/docs/internals/state.md @@ -1,8 +1,8 @@ # State -Notation CLI deploy, destroy, plan, and dashboard use Yieldstar stores in `.notation/workflows.db`. Override the database path with `NOTATION_STATE_PATH`. +Notation CLI deploy, destroy, plan, and dashboard use Yieldstar stores in `.notation/workflows.db`. Override the database path with `NOTATION_DATABASE_PATH`. -Each live resource is a `resource-state` store scoped by deployment and resource ID. A missing store means the resource is absent. No application tombstone is written. +Each live resource is a `notation/resource-state` store scoped by deployment and resource ID. A missing store means the resource is absent. No application tombstone is written. ```ts const state = new DurableStateBackend(storeClient, "infra/api.ts"); @@ -12,6 +12,6 @@ The runtime assigns a UUIDv7 `instanceId` when a store is created and increments `DurableStateBackend` is read-only: state writes happen inside the workflow, through the store handle, so each write is stamped with the step that made it and is not repeated on replay. -The deployment hold is not part of resource state. The workflow serializes deploy and destroy through one `deployment-hold` store per deployment. +The deployment hold is not part of resource state. The workflow serializes deploy and destroy through one `notation/deployment-hold` store per deployment. `MemoryStateBackend` in `@notation/state` remains a read/write data adapter for tests. diff --git a/docs/rfcs/reconciler.md b/docs/rfcs/reconciler.md index c6f3b21..00fbca7 100644 --- a/docs/rfcs/reconciler.md +++ b/docs/rfcs/reconciler.md @@ -13,13 +13,13 @@ Provider create, update, read, and delete calls are durable steps with stable re ## State lifecycle -`DurableStateBackend` stores one live resource per `resource-state` store. The store ID is scoped by deployment and resource ID. Store absence is resource absence. +`DurableStateBackend` stores one live resource per `notation/resource-state` store. The store ID is scoped by deployment and resource ID. Store absence is resource absence. The runtime-assigned UUIDv7 `instanceId` distinguishes a deleted store from a later store created under the same logical ID. Yieldstar's version is the concurrency token and is exposed as Notation's one-based `rev`. Workflow updates use `store.updateFrom` and deletes use `store.deleteFrom`, so both the instance and version must match the snapshot that informed the operation. ## Deployment hold -Each deployment has a `deployment-hold` store shared by deploy and destroy. The workflow atomically claims it with `store.take`. A concurrent execution suspends as a durable waiter and is woken when the holder releases the store. The same execution can recover an acquisition across the store-commit and heap-write crash gap through Yieldstar's applied-step ledger. +Each deployment has a `notation/deployment-hold` store shared by deploy and destroy. The workflow atomically claims it with `store.take`. A concurrent execution suspends as a durable waiter and is woken when the holder releases the store. The same execution can recover an acquisition across the store-commit and heap-write crash gap through Yieldstar's applied-step ledger. ## Node CLI runtime diff --git a/packages/core/src/provisioner/durable-runtime.ts b/packages/core/src/provisioner/durable-runtime.ts index f147da2..5c9a027 100644 --- a/packages/core/src/provisioner/durable-runtime.ts +++ b/packages/core/src/provisioner/durable-runtime.ts @@ -27,7 +27,7 @@ import { createWorkflowRouter, defineStore, workflow } from "yieldstar"; const DEFAULT_DATABASE_PATH = ".notation/workflows.db"; function resolveDatabasePath(): string { - return process.env.NOTATION_STATE_PATH ?? DEFAULT_DATABASE_PATH; + return process.env.NOTATION_DATABASE_PATH ?? DEFAULT_DATABASE_PATH; } export function resolveDeploymentId(entryPoint: string): string { @@ -201,6 +201,11 @@ export async function withRuntime( }, fn: (runtime: NodeDurableRuntime) => Promise, ): Promise { + if (opts.runtime && opts.databasePath) { + throw new Error( + "Pass either runtime or databasePath, not both: a runtime already owns its database", + ); + } const runtime = opts.runtime ?? new NodeDurableRuntime({ diff --git a/packages/reconciler/src/durable/deploy.ts b/packages/reconciler/src/durable/deploy.ts index c0937cc..9694b10 100644 --- a/packages/reconciler/src/durable/deploy.ts +++ b/packages/reconciler/src/durable/deploy.ts @@ -1,6 +1,7 @@ import { buildResourceDepthLevels } from "../dependency-graph"; import { withDeploymentHold } from "./deployment-hold"; import { reconcileResource, sweepOrphans } from "./reconcile"; +import { scopeStep } from "./step"; import type { DurableDeployOptions } from "./types"; import type { DurableStep } from "./yieldstar"; @@ -13,7 +14,14 @@ export async function* deploy( // dependencies have converged. for (const level of buildResourceDepthLevels(opts.resources)) { for (const resource of level) { - yield* reconcileResource(step, resource, opts); + yield* reconcileResource( + scopeStep( + step, + `notation:resource:${encodeURIComponent(resource.id)}`, + ), + resource, + opts, + ); } } diff --git a/packages/reconciler/src/durable/destroy.ts b/packages/reconciler/src/durable/destroy.ts index 7e5ee15..8b0b766 100644 --- a/packages/reconciler/src/durable/destroy.ts +++ b/packages/reconciler/src/durable/destroy.ts @@ -2,13 +2,13 @@ import { buildResourceDepthLevels } from "../dependency-graph"; import { withDeploymentHold } from "./deployment-hold"; import { deleteResource, sweepOrphans } from "./reconcile"; import { scopeStep } from "./step"; -import type { DurableDestroyOptions } from "./types"; +import type { DurableWorkflowOptions } from "./types"; import type { DurableStep } from "./yieldstar"; /** Durably destroys persisted resources in reverse dependency order. */ export async function* destroy( step: DurableStep, - opts: DurableDestroyOptions, + opts: DurableWorkflowOptions, ): AsyncGenerator { yield* withDeploymentHold(step, opts, async function* () { // Delete in reverse dependency order, so dependents are gone before the diff --git a/packages/reconciler/src/durable/index.ts b/packages/reconciler/src/durable/index.ts index af12567..069215c 100644 --- a/packages/reconciler/src/durable/index.ts +++ b/packages/reconciler/src/durable/index.ts @@ -19,6 +19,13 @@ * The state: keys are store-handle keys and so are not scope-prefixed: a * store outlives the scope that opened it, which is why they carry the * resource id themselves. + * + * Store names are persisted identifiers too, and carry the "notation/" + * prefix so they cannot collide with an application's stores on a shared + * store client: + * + * notation/resource-state one record per live resource + * notation/deployment-hold one hold per deployment */ export { deploy } from "./deploy"; export { destroy } from "./destroy"; @@ -35,7 +42,6 @@ export { } from "./stores"; export { type DurableDeployOptions, - type DurableDestroyOptions, type DurableWorkflowOptions, } from "./types"; export type { DurableStep } from "./yieldstar"; diff --git a/packages/reconciler/src/durable/reconcile.ts b/packages/reconciler/src/durable/reconcile.ts index 36f4897..2bf989f 100644 --- a/packages/reconciler/src/durable/reconcile.ts +++ b/packages/reconciler/src/durable/reconcile.ts @@ -17,7 +17,7 @@ import { type PersistState, type RemoveState, } from "../operations"; -import { decideAction } from "../plan"; +import { decideAction, decideDriftAction } from "../plan"; import { durableEmitter, scopeStep, type DurableStepRunner } from "./step"; import { resourceStateStore, @@ -38,36 +38,37 @@ type ResourceStateSession = /** * Reconciles one resource: hydrate, decide, read the remote when the decision - * needs it, announce the decision, then act. + * needs it, announce the decision, then act. `step` must already be scoped to + * the resource. */ export async function* reconcileResource( - step: DurableStep, + step: DurableStepRunner, resource: BaseResource, opts: DurableDeployOptions, ): AsyncGenerator { - const scope = scopeStep( - step, - `notation:resource:${encodeURIComponent(resource.id)}`, - ); - // Resolved once and then carried: deriveParams is user code and need not be // deterministic, so an operation resolving them again could persist params // other than the ones the decision was taken against. The step also pins // the answer across a replay. - const params = yield* scope.run("params", () => resource.getParams()); + const params = yield* step.run("params", () => resource.getParams()); - const emit = durableEmitter(scope, opts.emit); - const session = yield* openStateSession(scope, opts, resource); + const emit = durableEmitter(step, opts.emit); + const session = yield* openStateSession(step, opts, resource); if (session.node) resource.setOutput(session.node.output); let action = decideAction({ resource, stateNode: session.node, params }); // A noop is only trusted once the remote has been read back: the provider - // may have drifted from persisted state, which upgrades the decision. - if (action.decision === "noop" && (opts.driftDetection ?? true)) { + // may have drifted from persisted state, which upgrades the decision. A + // resource with no read has no remote to compare, so its noop stands. + if ( + action.decision === "noop" && + (opts.driftDetection ?? true) && + resource.read + ) { // Its own scope: the operation that follows reads the remote again, and // the two reads must not share step keys. - const driftStep = scope.scope("drift-read"); + const driftStep = step.scope("drift-read"); const driftRead = yield* readDriftOperation(driftStep, { resource, resourceParams: params, @@ -77,12 +78,7 @@ export async function* reconcileResource( emit: durableEmitter(driftStep, opts.emit), maxOperationAttempts: opts.maxOperationAttempts, }); - action = decideAction({ - resource, - stateNode: session.node, - params, - driftRead, - }); + action = decideDriftAction({ resource, params, driftRead }); } if (action.decision === "drift-update") { @@ -115,14 +111,14 @@ export async function* reconcileResource( switch (action.decision) { case "create": case "drift-recreate": - yield* createResourceOperation(scope, { + yield* createResourceOperation(step, { ...shared, persist: session.persist, }); return; case "update": case "drift-update": - yield* updateResourceOperation(scope, { + yield* updateResourceOperation(step, { ...shared, patch: action.patch, persist: session.persist, @@ -136,7 +132,8 @@ export async function* reconcileResource( /** * Deletes one resource. A resource with no persisted record was never created * — or has already been deleted — and is skipped, which is also what makes - * the sweep of a partly-deleted deployment idempotent. + * the sweep of a partly-deleted deployment idempotent. `step` must already be + * scoped to the resource. */ export async function* deleteResource( step: DurableStepRunner, diff --git a/packages/reconciler/src/durable/stores.ts b/packages/reconciler/src/durable/stores.ts index fc979e4..2617a55 100644 --- a/packages/reconciler/src/durable/stores.ts +++ b/packages/reconciler/src/durable/stores.ts @@ -4,6 +4,8 @@ import { defineStore, type StoreSnapshot } from "./yieldstar"; // Store names are persisted identifiers, like the step keys mapped in // index.ts: renaming one orphans every record stored under the old name. +// Notation-owned store names carry the "notation/" prefix, because an +// application may share a store client with these workflows. /** * `looseObject` because PersistedResourceState carries an index signature: a @@ -11,7 +13,7 @@ import { defineStore, type StoreSnapshot } from "./yieldstar"; * strip them at the store boundary. */ export const resourceStateStore = defineStore( - "resource-state", + "notation/resource-state", v.looseObject({ id: v.string(), type: v.string(), @@ -27,7 +29,7 @@ export const resourceStateStore = defineStore( ); export const deploymentHoldStore = defineStore( - "deployment-hold", + "notation/deployment-hold", v.object({ holder: v.nullable(v.string()) }), ); diff --git a/packages/reconciler/src/durable/types.ts b/packages/reconciler/src/durable/types.ts index 7c298fe..d443504 100644 --- a/packages/reconciler/src/durable/types.ts +++ b/packages/reconciler/src/durable/types.ts @@ -17,5 +17,3 @@ export type DurableWorkflowOptions = { export type DurableDeployOptions = DurableWorkflowOptions & { driftDetection?: boolean; }; - -export type DurableDestroyOptions = DurableWorkflowOptions; diff --git a/packages/reconciler/src/plan.ts b/packages/reconciler/src/plan.ts index d2f8451..1b7dfb3 100644 --- a/packages/reconciler/src/plan.ts +++ b/packages/reconciler/src/plan.ts @@ -51,55 +51,20 @@ export type ResourceAction = export function decideAction(opts: { resource: BaseResource; stateNode?: StateNode; - params?: Record; - driftRead?: DriftRead; + params: Record; }): ResourceAction { - const { resource, stateNode, params, driftRead } = opts; - const desiredComparable = resource.toComparable(params ?? {}); - const previousComparable = resource.toComparable(stateNode?.params ?? {}); + const { resource, stateNode, params } = opts; + if (!stateNode) { + return { decision: "create" }; + } + + const desiredComparable = resource.toComparable(params); + const previousComparable = resource.toComparable(stateNode.params); const localPatch = diff(previousComparable, desiredComparable) as Record< string, unknown >; - if (driftRead) { - if (driftRead.kind === "absent") { - return { decision: stateNode ? "drift-recreate" : "create" }; - } - - const remoteComparable = resource.toComparable(driftRead.output); - const remotePatch = diff(remoteComparable, desiredComparable) as Record< - string, - unknown - >; - - if (Object.keys(remotePatch).length === 0) { - return { decision: "noop" }; - } - - const remoteDetailedDiff = detailedDiff( - remoteComparable, - desiredComparable, - ); - if (!stateNode || Object.keys(localPatch).length > 0) { - return { - decision: "update", - patch: remotePatch, - diff: toPlanDiff(remoteDetailedDiff), - }; - } - - return { - decision: "drift-update", - patch: remotePatch, - diff: toPlanDiff(remoteDetailedDiff), - }; - } - - if (!stateNode) { - return { decision: "create" }; - } - if (Object.keys(localPatch).length > 0) { return { decision: "update", @@ -111,6 +76,39 @@ export function decideAction(opts: { return { decision: "noop" }; } +/** + * Upgrades a noop decision with a read of the remote. Callers reach this only + * after decideAction returned noop, so a state node exists and the desired + * params match it: the remote is the only remaining source of difference. + */ +export function decideDriftAction(opts: { + resource: BaseResource; + params: Record; + driftRead: DriftRead; +}): ResourceAction { + const { resource, params, driftRead } = opts; + if (driftRead.kind === "absent") { + return { decision: "drift-recreate" }; + } + + const desiredComparable = resource.toComparable(params); + const remoteComparable = resource.toComparable(driftRead.output); + const remotePatch = diff(remoteComparable, desiredComparable) as Record< + string, + unknown + >; + + if (Object.keys(remotePatch).length === 0) { + return { decision: "noop" }; + } + + return { + decision: "drift-update", + patch: remotePatch, + diff: toPlanDiff(detailedDiff(remoteComparable, desiredComparable)), + }; +} + export async function resolvePlanParams( resource: BaseResource, ): Promise> { diff --git a/packages/reconciler/src/planner.ts b/packages/reconciler/src/planner.ts index e759300..45d1d7e 100644 --- a/packages/reconciler/src/planner.ts +++ b/packages/reconciler/src/planner.ts @@ -1,10 +1,11 @@ import type { BaseResource } from "@notation/resource"; -import type { State } from "@notation/state"; +import type { StateBackend } from "@notation/state"; import { buildResourceDepthLevels } from "./dependency-graph"; import { toEmitStep, type ReconcilerEventEmitter } from "./events"; import { readDriftOperation } from "./operations"; import { decideAction, + decideDriftAction, getDependencyIds, resolvePlanParams, type Plan, @@ -13,7 +14,7 @@ import { import { createStepRunner, runOperation } from "./step-runner"; /** Planning only reads state. */ -export type PlannerState = Pick; +export type PlannerState = Pick; export type CreatePlanOptions = { resources: BaseResource[]; @@ -54,12 +55,7 @@ export async function createPlan({ }), ); - action = decideAction({ - resource, - stateNode, - params, - driftRead, - }); + action = decideDriftAction({ resource, params, driftRead }); } nodes.push({ diff --git a/packages/reconciler/test/durable-reconciliation.test.ts b/packages/reconciler/test/durable-reconciliation.test.ts index 9cd8bba..47dc8cb 100644 --- a/packages/reconciler/test/durable-reconciliation.test.ts +++ b/packages/reconciler/test/durable-reconciliation.test.ts @@ -366,9 +366,10 @@ describe("deployment hold", () => { // The failure has to live in plain generator code. A step that fails // caches a StepError, and a cached StepError is rethrown on replay before // the step's function is reached, so no later work would ever run - // uncached. decideAction calls toComparable outside any step, after the - // resource's reads have been checkpointed. - let failBeforeSecond = true; + // uncached. For a resource with persisted state, decideAction calls + // toComparable outside any step, after the resource's reads have been + // checkpointed. + let failOnSecond = true; const holders: Array = []; const Resource = resource({ type: "test/durable/hold-replay" }) .defineSchema({}) @@ -385,22 +386,30 @@ describe("deployment hold", () => { const first = new Resource({ id: "first" }); const second = new Resource({ id: "second" }); + const third = new Resource({ id: "third" }); const toComparable = second.toComparable.bind(second); second.toComparable = (output) => { - if (failBeforeSecond) throw new Error("simulated mid-deployment failure"); + if (failOnSecond) throw new Error("simulated mid-deployment failure"); return toComparable(output); }; - const runtime = createRuntime([first, second], "hold-replay"); + const runtime = createRuntime([first, second, third], "hold-replay"); + // Persisted state for `second`, so deciding its action reaches the + // failure seam in toComparable. + await seedResourceState( + runtime.storeClient, + runtime.state.storeId("second"), + "second", + ); await expect(runtime.run("replayed-execution")).rejects.toThrow( "simulated mid-deployment failure", ); expect(holders).toEqual(["replayed-execution"]); - failBeforeSecond = false; + failOnSecond = false; await runtime.run("replayed-execution"); - // The second resource's create is the first uncached work after the + // The third resource's create is the first uncached work after the // failure, so it observes whichever hold the resumed execution is running // under. Releasing on the way out of a failure would leave it null here. expect(holders).toEqual(["replayed-execution", "replayed-execution"]); @@ -595,6 +604,40 @@ describe("drift detection and repair", () => { runtime.close(); }); + it("trusts a noop for a resource with no read instead of reading during drift detection", async () => { + const ReadlessResource = resource({ type: "test/durable/readless" }) + .defineSchema({}) + .defineOperations({ + create: async () => undefined, + update: async () => undefined, + delete: async () => undefined, + }); + const events: ReconcilerEvent[] = []; + const runtime = createRuntime( + [new ReadlessResource({ id: "readless" })], + "readless-drift", + { driftDetection: true, emit: (event) => void events.push(event) }, + ); + + await runtime.run("deploy-1"); + events.length = 0; + await runtime.run("deploy-2"); + + // With nothing to read, the drift read would only replay persisted output + // through read skip/success lifecycle events a plan never emits. + expect( + events.filter( + (event) => + event.event === "reconciler.operation.lifecycle" && + event.operation === "read", + ), + ).toEqual([]); + expect( + events.find((event) => event.event === "reconciler.deploy.decision"), + ).toMatchObject({ resourceId: "readless", decision: "noop" }); + runtime.close(); + }); + it("reads the remote during a dry run rather than reporting false drift", async () => { const remote = { name: "expected" }; const read = vi.fn(async () => remote); @@ -779,11 +822,11 @@ function seedResourceState( return storeClient.getOrCreateStore({ definition: durable.resourceStateStore, id: storeId, - initial: statePatch(resourceId), + initial: resourceStateRecord(resourceId), }); } -function statePatch(id: string) { +function resourceStateRecord(id: string) { return { id, type: "test/durable/state", diff --git a/packages/state/src/state.ts b/packages/state/src/state.ts index 45e79d8..c559a3a 100644 --- a/packages/state/src/state.ts +++ b/packages/state/src/state.ts @@ -14,7 +14,6 @@ export type StateNode = { export interface StateBackend { get(id: string): Promise; - has(id: string): Promise; /** * The stored revision must match expectedRev. A missing record counts as * revision 0, so expectedRev: 0 asserts that the record does not exist yet. @@ -28,8 +27,6 @@ export interface StateBackend { values(): Promise; } -export type State = StateBackend; - export class MemoryStateBackend implements StateBackend { #state: Record; @@ -42,11 +39,6 @@ export class MemoryStateBackend implements StateBackend { return state[id]; } - async has(id: string): Promise { - const state = await this.readState(); - return id in state; - } - async update( id: string, expectedRev: number, diff --git a/packages/state/test/state-backend.test.ts b/packages/state/test/state-backend.test.ts index 5630b0a..4ae22b9 100644 --- a/packages/state/test/state-backend.test.ts +++ b/packages/state/test/state-backend.test.ts @@ -39,7 +39,6 @@ function runStateBackendContractTests( try { await expect(fixture.backend.get("missing")).resolves.toBeUndefined(); - await expect(fixture.backend.has("missing")).resolves.toBe(false); await expect(fixture.backend.values()).resolves.toEqual([]); } finally { await fixture.cleanup(); @@ -124,7 +123,6 @@ function runStateBackendContractTests( await expect( fixture.backend.get(initialNode.id), ).resolves.toBeUndefined(); - await expect(fixture.backend.has(initialNode.id)).resolves.toBe(false); await expect(fixture.backend.values()).resolves.toEqual([]); } finally { await fixture.cleanup(); From f7af72ccb3f6fd7a9008c056af3abcd17f26b01b Mon Sep 17 00:00:00 2001 From: djgrant <1670902+djgrant@users.noreply.github.com> Date: Tue, 28 Jul 2026 18:20:41 +0100 Subject: [PATCH 27/34] Restore plan-time event logging, and prune what nothing uses Wire emit back through planApp and the CLI plan command, as on main: deploy and destroy still surface reconciler events through the same seam, and plan-time drift reads call remote providers, so their lifecycle belongs in front of the user. createPlan's emit option has a caller again. Drop the example's undeclared-in-source dependencies, state the hold inventory's and waiting event's exact bindings, and let sweepOrphans document itself once. --- docs/internals/reconciler.md | 2 +- examples/reconciler/package.json | 3 --- packages/cli/src/plan.ts | 10 +++++++++- .../core/src/provisioner/workflows/workflow.plan.ts | 10 +++++++++- packages/reconciler/src/durable/deploy.ts | 1 - packages/reconciler/src/durable/destroy.ts | 1 - packages/reconciler/src/durable/index.ts | 5 ++++- pnpm-lock.yaml | 9 --------- 8 files changed, 23 insertions(+), 18 deletions(-) diff --git a/docs/internals/reconciler.md b/docs/internals/reconciler.md index debfd8e..12f9a42 100644 --- a/docs/internals/reconciler.md +++ b/docs/internals/reconciler.md @@ -33,7 +33,7 @@ Each attempt, delay, event, state read, state write, and hold change has a stabl Each resource is stored under `notation/resource-state` with a deployment-scoped ID. Conditional updates and deletes compare the snapshot's UUIDv7 `instanceId` and version, so a stale execution cannot modify a deleted and recreated store. -Deploy and destroy take an exclusive hold on the deployment through one `notation/deployment-hold` store per deployment. `store.take` suspends a competing execution as a durable waiter and wakes it after the holder releases. Before suspending, the waiter emits `reconciler.hold.waiting` naming the holding execution ID, so a wait behind a crashed execution is visible instead of silent. +Deploy and destroy take an exclusive hold on the deployment through one `notation/deployment-hold` store per deployment. `store.take` suspends a competing execution as a durable waiter and wakes it after the holder releases. A waiter that finds the hold already taken when it inspects it emits `reconciler.hold.waiting` naming the holding execution ID, so a wait behind a crashed execution is visible instead of silent; a holder that appears only between that inspection and the `take` suspends the waiter without the event. A failed or suspended execution keeps its hold, which is what makes resuming it safe. The hold of an execution that will never be resumed is cleared with `takeOverDeploymentHold` from `@notation/reconciler/durable` — the only supported way out of that state. diff --git a/examples/reconciler/package.json b/examples/reconciler/package.json index 8a26c85..a634030 100644 --- a/examples/reconciler/package.json +++ b/examples/reconciler/package.json @@ -13,9 +13,6 @@ "@notation/reconciler": "workspace:*", "@notation/resource": "workspace:*", "@notation/utils": "workspace:*", - "@yieldstar/core": "0.5.0", - "@yieldstar/sqlite-runtime": "0.5.0", - "pino": "^9.9.0", "yieldstar": "0.5.0" }, "devDependencies": { diff --git a/packages/cli/src/plan.ts b/packages/cli/src/plan.ts index 35841cc..5880ad5 100644 --- a/packages/cli/src/plan.ts +++ b/packages/cli/src/plan.ts @@ -1,4 +1,9 @@ -import { planApp, type Plan, type PlanNode } from "@notation/core"; +import { + createLoggerReconcilerSubscriber, + planApp, + type Plan, + type PlanNode, +} from "@notation/core"; import { compile } from "./compile"; import { defaultLogger, type Logger } from "./logger"; import { redirectStdoutToStderr } from "./stdio"; @@ -19,6 +24,7 @@ const decisionSymbols: Record = { export async function plan(entryPoint: string, opts: PlanCommandOptions = {}) { const logger = opts.logger ?? defaultLogger; + const emit = createLoggerReconcilerSubscriber({ logger }); if (opts.json) { let result: Plan; const { restore } = redirectStdoutToStderr(); @@ -26,6 +32,7 @@ export async function plan(entryPoint: string, opts: PlanCommandOptions = {}) { await compile(entryPoint, { logger }); result = await planApp({ entryPoint, + emit, }); } finally { restore(); @@ -38,6 +45,7 @@ export async function plan(entryPoint: string, opts: PlanCommandOptions = {}) { logger.info(`Planning ${entryPoint}\n`); const result = await planApp({ entryPoint, + emit, }); printPlanSummary(result, logger); } diff --git a/packages/core/src/provisioner/workflows/workflow.plan.ts b/packages/core/src/provisioner/workflows/workflow.plan.ts index b300310..057cf2a 100644 --- a/packages/core/src/provisioner/workflows/workflow.plan.ts +++ b/packages/core/src/provisioner/workflows/workflow.plan.ts @@ -1,4 +1,9 @@ -import { createPlan, type Plan } from "@notation/reconciler"; +import { + createLoggerReconcilerSubscriber, + createPlan, + type Plan, + type ReconcilerEventEmitter, +} from "@notation/reconciler"; import { getResourceGraph } from "src/orchestrator/graph"; import { withRuntime, type NodeDurableRuntime } from "../durable-runtime"; @@ -10,6 +15,7 @@ export type PlanAppOptions = { maxOperationAttempts?: number; runtime?: NodeDurableRuntime; databasePath?: string; + emit?: ReconcilerEventEmitter; }; export async function planApp({ @@ -18,6 +24,7 @@ export async function planApp({ maxOperationAttempts, runtime: suppliedRuntime, databasePath, + emit = createLoggerReconcilerSubscriber(), }: PlanAppOptions): Promise { const graph = await getResourceGraph(entryPoint); return withRuntime( @@ -27,6 +34,7 @@ export async function planApp({ resources: graph.resources, state: runtime.state, driftDetection, + emit, maxOperationAttempts, }), ); diff --git a/packages/reconciler/src/durable/deploy.ts b/packages/reconciler/src/durable/deploy.ts index 9694b10..8e5239a 100644 --- a/packages/reconciler/src/durable/deploy.ts +++ b/packages/reconciler/src/durable/deploy.ts @@ -25,7 +25,6 @@ export async function* deploy( } } - // Then delete resources that are in state but no longer declared. yield* sweepOrphans(step, opts, "deploy"); }); } diff --git a/packages/reconciler/src/durable/destroy.ts b/packages/reconciler/src/durable/destroy.ts index 8b0b766..91df953 100644 --- a/packages/reconciler/src/durable/destroy.ts +++ b/packages/reconciler/src/durable/destroy.ts @@ -27,7 +27,6 @@ export async function* destroy( } } - // Then delete resources that are in state but no longer declared. yield* sweepOrphans(step, opts, "destroy"); }); } diff --git a/packages/reconciler/src/durable/index.ts b/packages/reconciler/src/durable/index.ts index 069215c..d98dd2e 100644 --- a/packages/reconciler/src/durable/index.ts +++ b/packages/reconciler/src/durable/index.ts @@ -22,10 +22,13 @@ * * Store names are persisted identifiers too, and carry the "notation/" * prefix so they cannot collide with an application's stores on a shared - * store client: + * store client. This driver owns: * * notation/resource-state one record per live resource * notation/deployment-hold one hold per deployment + * + * @notation/core's durable runtime persists one more name under the same + * prefix, notation/execution-binding (see its durable-runtime module). */ export { deploy } from "./deploy"; export { destroy } from "./destroy"; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b3d35d6..344636a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -163,15 +163,6 @@ importers: '@notation/utils': specifier: workspace:* version: link:../../packages/utils - '@yieldstar/core': - specifier: 0.5.0 - version: 0.5.0 - '@yieldstar/sqlite-runtime': - specifier: 0.5.0 - version: 0.5.0 - pino: - specifier: ^9.9.0 - version: 9.14.0 yieldstar: specifier: 0.5.0 version: 0.5.0 From d7ab352bae0c0c327e806b73c2a78a50765bb668 Mon Sep 17 00:00:00 2001 From: djgrant <1670902+djgrant@users.noreply.github.com> Date: Tue, 28 Jul 2026 18:37:52 +0100 Subject: [PATCH 28/34] Give each reconciler concept one owner MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit takeOverDeploymentHold clears the hold, full stop: the unused toExecutionId transfer parameter is gone. Deployment identity now lives only on DurableStateBackend, which already carried it in its store prefix; DurableWorkflowOptions no longer repeats it and the deployment hold keys off the backend. The durable module is imported as `durable` everywhere instead of shadow-naming the reconciler package. Emitter absence is absorbed in one layer — the adapters (toEmitStep, durableEmitter) — so operations always hold an emit step and neither the planner nor emitLifecycleEvent guards again. Docs drop the sample's duplicated deploymentId and the sentences describing what state is not. --- docs/internals/state.md | 4 +--- docs/manual/reconciler.md | 4 +--- examples/reconciler/src/index.ts | 5 ++--- .../core/src/provisioner/workflows/workflow.deploy.ts | 5 ++--- .../src/provisioner/workflows/workflow.destroy.ts | 5 ++--- .../core/test/provisioner/durable-runtime.test.ts | 5 ++--- .../core/test/provisioner/operation.create.test.ts | 2 ++ packages/reconciler/src/durable/deployment-hold.ts | 11 ++++++----- packages/reconciler/src/durable/state-backend.ts | 4 ++++ packages/reconciler/src/durable/step.ts | 5 +++-- packages/reconciler/src/durable/types.ts | 1 - packages/reconciler/src/events.ts | 8 ++++++-- packages/reconciler/src/operations/operation.types.ts | 6 +++--- packages/reconciler/src/planner.ts | 2 +- .../reconciler/test/durable-reconciliation.test.ts | 2 -- packages/reconciler/test/operation.workflows.test.ts | 4 ++++ 16 files changed, 39 insertions(+), 34 deletions(-) diff --git a/docs/internals/state.md b/docs/internals/state.md index 8907636..1c0fa03 100644 --- a/docs/internals/state.md +++ b/docs/internals/state.md @@ -12,6 +12,4 @@ The runtime assigns a UUIDv7 `instanceId` when a store is created and increments `DurableStateBackend` is read-only: state writes happen inside the workflow, through the store handle, so each write is stamped with the step that made it and is not repeated on replay. -The deployment hold is not part of resource state. The workflow serializes deploy and destroy through one `notation/deployment-hold` store per deployment. - -`MemoryStateBackend` in `@notation/state` remains a read/write data adapter for tests. +The workflow serializes deploy and destroy through one `notation/deployment-hold` store per deployment. diff --git a/docs/manual/reconciler.md b/docs/manual/reconciler.md index c32bc4e..e98d61e 100644 --- a/docs/manual/reconciler.md +++ b/docs/manual/reconciler.md @@ -17,7 +17,6 @@ const state = new DurableStateBackend(storeClient, "my-application"); export const deploy = workflow(async function* (step, event) { yield* deployResources(step, { - deploymentId: "my-application", executionId: event.executionId, resources, state, @@ -26,7 +25,6 @@ export const deploy = workflow(async function* (step, event) { export const destroy = workflow(async function* (step, event) { yield* destroyResources(step, { - deploymentId: "my-application", executionId: event.executionId, resources, state, @@ -38,7 +36,7 @@ The outer workflow supplies durable step execution, timers, shared stores, waiti Each live resource is one Yieldstar store. Absence is represented by no store, not a tombstone. Yieldstar's UUIDv7 store `instanceId` and version are authoritative for conditional update and delete; Notation exposes the version as the resource state's `rev`. -Operations against the same `deploymentId` are serialized through a deployment hold naming the holding `executionId`. Resume a crashed operation with the same execution ID; use a new globally unique execution ID for every new deploy or destroy. An execution that must wait emits a `reconciler.hold.waiting` event naming the holder before it suspends, which also identifies a crashed holder that should be resumed instead. If the holder is genuinely abandoned, clear its hold with `takeOverDeploymentHold`. +Operations against the same deployment — the `deploymentId` the `DurableStateBackend` is constructed with — are serialized through a deployment hold naming the holding `executionId`. Resume a crashed operation with the same execution ID; use a new globally unique execution ID for every new deploy or destroy. An execution that must wait emits a `reconciler.hold.waiting` event naming the holder before it suspends, which also identifies a crashed holder that should be resumed instead. If the holder is genuinely abandoned, clear its hold with `takeOverDeploymentHold`. Pass the complete desired set on every deployment. Persisted resources absent from that set are deleted through the supplied resource registry. Destroy removes current resources in reverse dependency order and then removes any persisted orphans that the registry can hydrate. diff --git a/examples/reconciler/src/index.ts b/examples/reconciler/src/index.ts index 2a977b0..2efaede 100644 --- a/examples/reconciler/src/index.ts +++ b/examples/reconciler/src/index.ts @@ -1,6 +1,6 @@ import { NodeDurableRuntime } from "@notation/core"; import { createResourceRegistry } from "@notation/reconciler"; -import * as reconciler from "@notation/reconciler/durable"; +import * as durable from "@notation/reconciler/durable"; import { createWorkflowRouter, workflow } from "yieldstar"; import { StaticSite } from "./static-site"; @@ -27,8 +27,7 @@ const resources = [ ]; const deploy = workflow(async function* (step, event) { - yield* reconciler.deploy(step, { - deploymentId: runtime.deploymentId, + yield* durable.deploy(step, { executionId: event.executionId, resources, state: runtime.state, diff --git a/packages/core/src/provisioner/workflows/workflow.deploy.ts b/packages/core/src/provisioner/workflows/workflow.deploy.ts index dd89799..a970089 100644 --- a/packages/core/src/provisioner/workflows/workflow.deploy.ts +++ b/packages/core/src/provisioner/workflows/workflow.deploy.ts @@ -1,4 +1,4 @@ -import * as reconciler from "@notation/reconciler/durable"; +import * as durable from "@notation/reconciler/durable"; import { createLoggerReconcilerSubscriber, type ReconcilerEventEmitter, @@ -34,8 +34,7 @@ export async function deployApp({ await runDurableWorkflow( { entryPoint, workflowId: "deploy", runtime, databasePath, executionId }, (step, executionId, runtime) => - reconciler.deploy(step, { - deploymentId: runtime.deploymentId, + durable.deploy(step, { executionId, resources: graph.resources, state: runtime.state, diff --git a/packages/core/src/provisioner/workflows/workflow.destroy.ts b/packages/core/src/provisioner/workflows/workflow.destroy.ts index d47099b..aa31b8b 100644 --- a/packages/core/src/provisioner/workflows/workflow.destroy.ts +++ b/packages/core/src/provisioner/workflows/workflow.destroy.ts @@ -1,4 +1,4 @@ -import * as reconciler from "@notation/reconciler/durable"; +import * as durable from "@notation/reconciler/durable"; import { createLoggerReconcilerSubscriber, type ReconcilerEventEmitter, @@ -30,8 +30,7 @@ export async function destroyApp({ await runDurableWorkflow( { entryPoint, workflowId: "destroy", runtime, databasePath, executionId }, (step, executionId, runtime) => - reconciler.destroy(step, { - deploymentId: runtime.deploymentId, + durable.destroy(step, { executionId, resources: graph.resources, state: runtime.state, diff --git a/packages/core/test/provisioner/durable-runtime.test.ts b/packages/core/test/provisioner/durable-runtime.test.ts index 80e9601..1c0ad39 100644 --- a/packages/core/test/provisioner/durable-runtime.test.ts +++ b/packages/core/test/provisioner/durable-runtime.test.ts @@ -1,7 +1,7 @@ import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; -import * as reconciler from "@notation/reconciler/durable"; +import * as durable from "@notation/reconciler/durable"; import { ResourceOperationPendingError, resource, @@ -41,8 +41,7 @@ describe("NodeDurableRuntime", () => { }); const resources = [new PendingResource({ id: "pending" })]; const deploy = workflow(async function* (step, event) { - yield* reconciler.deploy(step, { - deploymentId: runtime.deploymentId, + yield* durable.deploy(step, { executionId: event.executionId, resources, state: runtime.state, diff --git a/packages/core/test/provisioner/operation.create.test.ts b/packages/core/test/provisioner/operation.create.test.ts index 84ed0e3..3a8785b 100644 --- a/packages/core/test/provisioner/operation.create.test.ts +++ b/packages/core/test/provisioner/operation.create.test.ts @@ -3,6 +3,7 @@ import { createResourceOperation, createStepRunner, runOperation, + toEmitStep, } from "@notation/reconciler"; import { MemoryStateBackend } from "@notation/state"; import { @@ -38,6 +39,7 @@ describe("resource creation", () => { persist: async function* (next) { await stateBackend.update(testResource.id, 0, next); }, + emit: toEmitStep(), }), ); diff --git a/packages/reconciler/src/durable/deployment-hold.ts b/packages/reconciler/src/durable/deployment-hold.ts index 98dfdcf..126a61a 100644 --- a/packages/reconciler/src/durable/deployment-hold.ts +++ b/packages/reconciler/src/durable/deployment-hold.ts @@ -3,12 +3,14 @@ * workflow execution. */ import type { ReconcilerEventEmitter } from "../events"; +import type { DurableStateBackend } from "./state-backend"; import { durableEmitter, scopeStep } from "./step"; import { deploymentHoldStore, type DeploymentHoldState } from "./stores"; import type { DurableStep, StoreClient, WorkflowStore } from "./yieldstar"; type DeploymentHoldOptions = { - deploymentId: string; + /** Deployment identity comes from the state backend. */ + state: Pick; executionId: string; emit?: ReconcilerEventEmitter; }; @@ -22,7 +24,7 @@ async function* acquireDeploymentHold( opts: DeploymentHoldOptions, ): AsyncGenerator, any> { const hold = yield* step.store(deploymentHoldStore, { - id: opts.deploymentId, + id: opts.state.deploymentId, initial: { holder: null }, }); @@ -35,7 +37,7 @@ async function* acquireDeploymentHold( )({ level: "warn", event: "reconciler.hold.waiting", - deploymentId: opts.deploymentId, + deploymentId: opts.state.deploymentId, executionId: opts.executionId, holderExecutionId: holder, }); @@ -98,7 +100,6 @@ export async function takeOverDeploymentHold(params: { storeClient: StoreClient; deploymentId: string; fromExecutionId: string; - toExecutionId?: string | null; }): Promise { const { storeClient, deploymentId, fromExecutionId } = params; const read = () => @@ -117,7 +118,7 @@ export async function takeOverDeploymentHold(params: { id: deploymentId, snapshot, updater: (draft) => { - draft.holder = params.toExecutionId ?? null; + draft.holder = null; }, }); diff --git a/packages/reconciler/src/durable/state-backend.ts b/packages/reconciler/src/durable/state-backend.ts index 8b57a3f..092ef85 100644 --- a/packages/reconciler/src/durable/state-backend.ts +++ b/packages/reconciler/src/durable/state-backend.ts @@ -13,10 +13,14 @@ import type { StoreClient } from "./yieldstar"; * carry that step key, so a write made here would repeat on replay. */ export class DurableStateBackend { + /** The deployment this backend belongs to; workflows and the deployment + * hold key off this rather than carrying the ID separately. */ + readonly deploymentId: string; readonly #client: StoreClient; readonly #prefix: string; constructor(client: StoreClient, deploymentId: string) { + this.deploymentId = deploymentId; this.#client = client; // Keep deployment prefixes disjoint so orphan cleanup cannot delete // another deployment's stores. diff --git a/packages/reconciler/src/durable/step.ts b/packages/reconciler/src/durable/step.ts index f72a663..b3ec055 100644 --- a/packages/reconciler/src/durable/step.ts +++ b/packages/reconciler/src/durable/step.ts @@ -7,8 +7,9 @@ import type { StepRunner } from "../operations"; import type { DurableStep } from "./yieldstar"; /** - * A durable step runner: the operation seam plus the store handle, which only - * the durable driver uses. + * The operation seam (`StepRunner`) plus the store handle, which only the + * durable driver uses. `DurableStep` is yieldstar's raw step primitive; + * `scopeStep` wraps one of those into one of these. */ export interface DurableStepRunner extends StepRunner { /** Narrower than StepRunner's, so a scope keeps its store handle. */ diff --git a/packages/reconciler/src/durable/types.ts b/packages/reconciler/src/durable/types.ts index d443504..3679ca8 100644 --- a/packages/reconciler/src/durable/types.ts +++ b/packages/reconciler/src/durable/types.ts @@ -4,7 +4,6 @@ import type { ResourceRegistry } from "../resource-registry"; import type { DurableStateBackend } from "./state-backend"; export type DurableWorkflowOptions = { - deploymentId: string; executionId: string; resources: BaseResource[]; state: DurableStateBackend; diff --git a/packages/reconciler/src/events.ts b/packages/reconciler/src/events.ts index e233b45..00081ea 100644 --- a/packages/reconciler/src/events.ts +++ b/packages/reconciler/src/events.ts @@ -71,9 +71,13 @@ export type EmitStep = ( event: TEvent, ) => AsyncGenerator; -/** Adapts a plain emitter to the driver seam, for drivers that just await. */ +/** + * Adapts a plain emitter to the driver seam, for drivers that just await. + * Absorbs an absent emitter: the returned step then delivers nothing, so + * downstream code always has an emit step and never guards. + */ export function toEmitStep( - emit: ((event: TEvent) => void | Promise) | undefined, + emit?: (event: TEvent) => void | Promise, ): EmitStep { return async function* (event) { await emit?.(event); diff --git a/packages/reconciler/src/operations/operation.types.ts b/packages/reconciler/src/operations/operation.types.ts index bf92b9f..4c98443 100644 --- a/packages/reconciler/src/operations/operation.types.ts +++ b/packages/reconciler/src/operations/operation.types.ts @@ -64,7 +64,9 @@ export type RemoveState = () => AsyncGenerator; export type ResourceOperationBaseParams = { resource: BaseResource; dryRun?: boolean; - emit?: OperationEventEmitter; + /** Always present: an absent emitter is absorbed where the step is made + * (`toEmitStep`, `durableEmitter`), not guarded here. */ + emit: OperationEventEmitter; maxOperationAttempts?: number; }; @@ -115,8 +117,6 @@ export async function* emitLifecycleEvent( status: OperationLifecycleStatus, extra: Partial = {}, ): AsyncGenerator { - if (!params.emit) return; - yield* params.emit({ level: status === "error" ? "error" : "info", event: "reconciler.operation.lifecycle", diff --git a/packages/reconciler/src/planner.ts b/packages/reconciler/src/planner.ts index 45d1d7e..70409bc 100644 --- a/packages/reconciler/src/planner.ts +++ b/packages/reconciler/src/planner.ts @@ -34,7 +34,7 @@ export async function createPlan({ const resourceById = new Map( resources.map((resource) => [resource.id, resource]), ); - const emitStep = emit ? toEmitStep(emit) : undefined; + const emitStep = toEmitStep(emit); const nodes: PlanNode[] = []; for (const level of buildResourceDepthLevels(resources)) { diff --git a/packages/reconciler/test/durable-reconciliation.test.ts b/packages/reconciler/test/durable-reconciliation.test.ts index 47dc8cb..35de461 100644 --- a/packages/reconciler/test/durable-reconciliation.test.ts +++ b/packages/reconciler/test/durable-reconciliation.test.ts @@ -710,7 +710,6 @@ function createRuntime( const state = new durable.DurableStateBackend(storeClient, deploymentId); const deploy = workflow(async function* (step, event) { yield* durable.deploy(step, { - deploymentId, executionId: event.executionId, resources, state, @@ -724,7 +723,6 @@ function createRuntime( }); const destroy = workflow(async function* (step, event) { yield* durable.destroy(step, { - deploymentId, executionId: event.executionId, resources, state, diff --git a/packages/reconciler/test/operation.workflows.test.ts b/packages/reconciler/test/operation.workflows.test.ts index 2cb8cbc..08ca0db 100644 --- a/packages/reconciler/test/operation.workflows.test.ts +++ b/packages/reconciler/test/operation.workflows.test.ts @@ -143,6 +143,7 @@ describe("operation workflows", () => { readResourceOperation(step, { resource: testResource, resourceParams: await testResource.getParams(), + emit: toEmitStep(), }), ); @@ -180,6 +181,7 @@ describe("operation workflows", () => { readResourceOperation(step, { resource: new TestResource({ id: "pending-limit" }), resourceParams: {}, + emit: toEmitStep(), maxOperationAttempts: 2, }), ), @@ -207,6 +209,7 @@ describe("operation workflows", () => { resource: new TestResource({ id: "eventually-visible" }), resourceParams: {}, persist, + emit: toEmitStep(), }), ), ).rejects.toThrowError("resource is absent"); @@ -263,6 +266,7 @@ describe("operation workflows", () => { deleteResourceOperation(step, { resource: testResource, remove, + emit: toEmitStep(), }), ), ).rejects.toMatchObject({ From 1b72b995413f8157301db2462702779fee5d0256 Mon Sep 17 00:00:00 2001 From: djgrant <1670902+djgrant@users.noreply.github.com> Date: Tue, 28 Jul 2026 18:53:34 +0100 Subject: [PATCH 29/34] Remove dead vocabulary and make the persisted-key map exact - lastOperation persists only what is written: create and update. Drift repair persists update, and delete removes the store, so the drift and delete members were unreachable; the package is unpublished, so no persisted record carries them. - Drop the unconsumed ResourceApi, StateApi and DeepObjectDiffApi exports. - StateBackend is now the read interface its consumers use; the compare-and-swap update/delete half existed only for test seeding, which MemoryStateBackend's constructor already covers. - Cover notation:orphans:list in the durable key map and give the emit entry the *: scope prefix its keys always carry. - Say "registered" where the code says resource-type-not-registered, keep "load the persisted record" distinct from the drift read, name toEmitStep/durableEmitter/createStepRunner instead of "the in-process driver", align the decision table with delete-orphan, and state the no-tombstone fact once, in docs/internals/state.md. --- docs/internals/reconciler.md | 4 +- docs/manual/reconciler.md | 4 +- .../test/provisioner/operation.create.test.ts | 8 +- packages/dashboard/server/server.test.ts | 10 +- packages/reconciler/src/durable/index.ts | 3 +- packages/reconciler/src/durable/reconcile.ts | 6 +- packages/reconciler/src/durable/stores.ts | 4 +- packages/reconciler/src/events.ts | 6 +- packages/reconciler/src/index.ts | 4 - .../src/operations/operation.types.ts | 2 +- packages/reconciler/test/planner.test.ts | 80 ++++---- packages/state/src/state.ts | 59 +----- packages/state/test/state-backend.test.ts | 174 +++--------------- 13 files changed, 105 insertions(+), 259 deletions(-) diff --git a/docs/internals/reconciler.md b/docs/internals/reconciler.md index 12f9a42..d4845df 100644 --- a/docs/internals/reconciler.md +++ b/docs/internals/reconciler.md @@ -13,13 +13,13 @@ The reconciler expresses deployment and destruction as Yieldstar async generator | In state, params unchanged, no drift | **noop** | | In state, but deleted from the provider | **drift-recreate** | | In state, provider state differs from stored state | **drift-update** | -| In state, not in graph | **delete** | +| In state, not in graph | **delete-orphan** | Dry-run deploy performs decisions and emits lifecycle events without provider mutations or state mutations. When drift detection is enabled, it can still call provider read operations to decide whether a nominal noop has drifted. ## Destroy flow -`destroy` is a first-class durable operation. It takes the same deployment hold as deploy, deletes desired resources in reverse dependency order, deletes hydratable persisted orphans, and conditionally removes each resource store only after the provider delete succeeds or reports that the resource is already absent. +`destroy` is a first-class durable operation. It takes the same deployment hold as deploy, deletes desired resources in reverse dependency order, deletes registered persisted orphans, and conditionally removes each resource store only after the provider delete succeeds or reports that the resource is already absent. Provider delete is a stable durable step, but the provider acknowledgement and Yieldstar heap checkpoint are not atomic. If the process crashes between them, replay repeats the delete, so provider create, update, and delete operations must be idempotent. Event subscribers must likewise tolerate duplicate delivery when a crash occurs before the event checkpoint. diff --git a/docs/manual/reconciler.md b/docs/manual/reconciler.md index e98d61e..a404ec9 100644 --- a/docs/manual/reconciler.md +++ b/docs/manual/reconciler.md @@ -34,10 +34,10 @@ export const destroy = workflow(async function* (step, event) { The outer workflow supplies durable step execution, timers, shared stores, waiting, and scheduling. Checkpointed provider results are replayed from the heap after a crash, retryable provider conditions suspend on a durable timer, and conditional state writes use Yieldstar store identity and version. Provider mutations must be idempotent because a crash after provider acknowledgement but before the heap checkpoint repeats the call; event consumers must tolerate the same duplicate-delivery window. -Each live resource is one Yieldstar store. Absence is represented by no store, not a tombstone. Yieldstar's UUIDv7 store `instanceId` and version are authoritative for conditional update and delete; Notation exposes the version as the resource state's `rev`. +Each live resource is one Yieldstar store. Yieldstar's UUIDv7 store `instanceId` and version are authoritative for conditional update and delete; Notation exposes the version as the resource state's `rev`. Operations against the same deployment — the `deploymentId` the `DurableStateBackend` is constructed with — are serialized through a deployment hold naming the holding `executionId`. Resume a crashed operation with the same execution ID; use a new globally unique execution ID for every new deploy or destroy. An execution that must wait emits a `reconciler.hold.waiting` event naming the holder before it suspends, which also identifies a crashed holder that should be resumed instead. If the holder is genuinely abandoned, clear its hold with `takeOverDeploymentHold`. -Pass the complete desired set on every deployment. Persisted resources absent from that set are deleted through the supplied resource registry. Destroy removes current resources in reverse dependency order and then removes any persisted orphans that the registry can hydrate. +Pass the complete desired set on every deployment. Persisted resources absent from that set are deleted through the supplied resource registry. Destroy removes current resources in reverse dependency order and then removes any persisted orphans whose resource type is registered. The runnable Node SQLite composition is in `examples/reconciler`. diff --git a/packages/core/test/provisioner/operation.create.test.ts b/packages/core/test/provisioner/operation.create.test.ts index 3a8785b..04f4a18 100644 --- a/packages/core/test/provisioner/operation.create.test.ts +++ b/packages/core/test/provisioner/operation.create.test.ts @@ -4,8 +4,8 @@ import { createStepRunner, runOperation, toEmitStep, + type PersistedResourceState, } from "@notation/reconciler"; -import { MemoryStateBackend } from "@notation/state"; import { TestResourceSchema, testResourceConfig, @@ -15,7 +15,7 @@ import { describe("resource creation", () => { it("passes computed input to resource.create", async () => { - const stateBackend = new MemoryStateBackend(); + let persisted: PersistedResourceState | undefined; const readResult = { ...testResourceOutput, volatileComputed: "123" }; const createMock = vi.fn(async () => ({ primaryKey: "" })); const readMock = vi.fn(async () => readResult); @@ -37,7 +37,7 @@ describe("resource creation", () => { resource: testResource, resourceParams: await testResource.getParams(), persist: async function* (next) { - await stateBackend.update(testResource.id, 0, next); + persisted = next; }, emit: toEmitStep(), }), @@ -47,7 +47,7 @@ describe("resource creation", () => { const persistedOutput = testResource.toState(readResult); expect(createMock.mock.calls[0]).toEqual([params, undefined]); - await expect(stateBackend.get(testResource.id)).resolves.toMatchObject({ + expect(persisted).toMatchObject({ id: testResource.id, output: persistedOutput, lastOperation: "create", diff --git a/packages/dashboard/server/server.test.ts b/packages/dashboard/server/server.test.ts index e9cf2b3..059f469 100644 --- a/packages/dashboard/server/server.test.ts +++ b/packages/dashboard/server/server.test.ts @@ -4,11 +4,9 @@ import { readStateSnapshot } from "./server"; describe("dashboard state", () => { it("reads state through the backend contract", async () => { - const state = new MemoryStateBackend(); - await state.update( - "service", - 0, - { + const state = new MemoryStateBackend({ + service: { + rev: 1, id: "service", type: "test/service/main", config: {}, @@ -17,7 +15,7 @@ describe("dashboard state", () => { lastOperation: "create", lastOperationAt: "2026-07-18T00:00:00.000Z", }, - ); + }); await expect(readStateSnapshot(state)).resolves.toMatchObject({ service: { diff --git a/packages/reconciler/src/durable/index.ts b/packages/reconciler/src/durable/index.ts index d98dd2e..a1327da 100644 --- a/packages/reconciler/src/durable/index.ts +++ b/packages/reconciler/src/durable/index.ts @@ -7,10 +7,11 @@ * * notation:resource::* per-resource reconciliation steps (deploy) * notation:destroy::* per-resource deletion steps (destroy) + * notation:orphans:list the orphan sweep's one read of persisted state * notation:orphans::* orphan sweep, per persisted record * *:remote:attempt: one provider call attempt * *:remote:retry-delay: the wait between two attempts - * emit:[::] event delivery checkpoint + * *:emit:[::] event delivery checkpoint * notation:hold:* deployment hold: inspect/acquire/release * state:persist: conditional write of a resource record * state:delete: conditional removal of one diff --git a/packages/reconciler/src/durable/reconcile.ts b/packages/reconciler/src/durable/reconcile.ts index 2bf989f..35762bf 100644 --- a/packages/reconciler/src/durable/reconcile.ts +++ b/packages/reconciler/src/durable/reconcile.ts @@ -37,9 +37,9 @@ type ResourceStateSession = | { node: StateNode; persist: PersistState; remove: RemoveState }; /** - * Reconciles one resource: hydrate, decide, read the remote when the decision - * needs it, announce the decision, then act. `step` must already be scoped to - * the resource. + * Reconciles one resource: load the persisted record, decide, read the + * remote when the decision needs it, announce the decision, then act. `step` + * must already be scoped to the resource. */ export async function* reconcileResource( step: DurableStepRunner, diff --git a/packages/reconciler/src/durable/stores.ts b/packages/reconciler/src/durable/stores.ts index 2617a55..322d99e 100644 --- a/packages/reconciler/src/durable/stores.ts +++ b/packages/reconciler/src/durable/stores.ts @@ -23,7 +23,9 @@ export const resourceStateStore = defineStore( config: v.record(v.string(), v.unknown()), params: v.record(v.string(), v.unknown()), output: v.record(v.string(), v.unknown()), - lastOperation: v.picklist(["drift", "create", "update", "delete"]), + // Only the operations that leave a record behind: delete removes the + // store, and drift repair persists as "update". + lastOperation: v.picklist(["create", "update"]), lastOperationAt: v.string(), }), ); diff --git a/packages/reconciler/src/events.ts b/packages/reconciler/src/events.ts index 00081ea..fbbd201 100644 --- a/packages/reconciler/src/events.ts +++ b/packages/reconciler/src/events.ts @@ -63,9 +63,9 @@ export type ReconcilerEventEmitter = ( /** * The seam a driver fills in to deliver an event. Emission is a step so that - * each driver decides how it is recorded: the in-process driver simply awaits - * the emitter, while the durable driver checkpoints it so that replaying a - * workflow does not re-emit events it has already delivered. + * each driver decides how it is recorded: `toEmitStep` simply awaits the + * emitter, while the durable driver's `durableEmitter` checkpoints it so that + * replaying a workflow does not re-emit events it has already delivered. */ export type EmitStep = ( event: TEvent, diff --git a/packages/reconciler/src/index.ts b/packages/reconciler/src/index.ts index 6e4bb33..b0b90e5 100644 --- a/packages/reconciler/src/index.ts +++ b/packages/reconciler/src/index.ts @@ -1,7 +1,3 @@ -export type ResourceApi = typeof import("@notation/resource"); -export type StateApi = typeof import("@notation/state"); -export type DeepObjectDiffApi = typeof import("deep-object-diff"); - export * from "./resource-registry"; export * from "./operations"; export * from "./dependency-graph"; diff --git a/packages/reconciler/src/operations/operation.types.ts b/packages/reconciler/src/operations/operation.types.ts index 4c98443..aacca62 100644 --- a/packages/reconciler/src/operations/operation.types.ts +++ b/packages/reconciler/src/operations/operation.types.ts @@ -18,7 +18,7 @@ export type OperationEventEmitter = EmitStep; /** * How an operation runs a step. Keys identify a step's cached result across a * replay; `scope` namespaces them so one operation can run at several call - * sites in a single execution. The in-process driver ignores both. + * sites in a single execution. `createStepRunner` ignores both. */ export type StepRunner = { run( diff --git a/packages/reconciler/test/planner.test.ts b/packages/reconciler/test/planner.test.ts index 8066578..40d4972 100644 --- a/packages/reconciler/test/planner.test.ts +++ b/packages/reconciler/test/planner.test.ts @@ -15,15 +15,17 @@ describe("createPlan", () => { create: async () => undefined, delete: async () => undefined, }); - const state = new MemoryStateBackend(); - await state.update("orphan", 0, { - id: "orphan", - type: TestResource.type, - config: {}, - params: {}, - output: {}, - lastOperation: "create", - lastOperationAt: "2026-07-22T00:00:00.000Z", + const state = new MemoryStateBackend({ + orphan: { + rev: 1, + id: "orphan", + type: TestResource.type, + config: {}, + params: {}, + output: {}, + lastOperation: "create", + lastOperationAt: "2026-07-22T00:00:00.000Z", + }, }); const plan = await createPlan({ @@ -48,15 +50,17 @@ describe("createPlan", () => { }, delete: async () => undefined, }); - const state = new MemoryStateBackend(); - await state.update("existing", 0, { - id: "existing", - type: TestResource.type, - config: {}, - params: {}, - output: {}, - lastOperation: "create", - lastOperationAt: "2026-07-22T00:00:00.000Z", + const state = new MemoryStateBackend({ + existing: { + rev: 1, + id: "existing", + type: TestResource.type, + config: {}, + params: {}, + output: {}, + lastOperation: "create", + lastOperationAt: "2026-07-22T00:00:00.000Z", + }, }); await expect( @@ -77,15 +81,17 @@ describe("createPlan", () => { }, delete: async () => undefined, }); - const state = new MemoryStateBackend(); - await state.update("existing", 0, { - id: "existing", - type: TestResource.type, - config: {}, - params: {}, - output: {}, - lastOperation: "create", - lastOperationAt: "2026-07-22T00:00:00.000Z", + const state = new MemoryStateBackend({ + existing: { + rev: 1, + id: "existing", + type: TestResource.type, + config: {}, + params: {}, + output: {}, + lastOperation: "create", + lastOperationAt: "2026-07-22T00:00:00.000Z", + }, }); const plan = await createPlan({ @@ -117,15 +123,17 @@ describe("createPlan", () => { }, delete: async () => undefined, }); - const state = new MemoryStateBackend(); - await state.update("existing", 0, { - id: "existing", - type: TestResource.type, - config: {}, - params: {}, - output: {}, - lastOperation: "create", - lastOperationAt: "2026-07-22T00:00:00.000Z", + const state = new MemoryStateBackend({ + existing: { + rev: 1, + id: "existing", + type: TestResource.type, + config: {}, + params: {}, + output: {}, + lastOperation: "create", + lastOperationAt: "2026-07-22T00:00:00.000Z", + }, }); const plan = await createPlan({ diff --git a/packages/state/src/state.ts b/packages/state/src/state.ts index c559a3a..c1fc04e 100644 --- a/packages/state/src/state.ts +++ b/packages/state/src/state.ts @@ -1,5 +1,3 @@ -import { RevConflict } from "./conflicts"; - export type StateNode = { rev: number; id: string; @@ -7,23 +5,17 @@ export type StateNode = { config: Record; params: Record; output: Record; - lastOperation: "drift" | "create" | "update" | "delete"; + lastOperation: "create" | "update"; lastOperationAt: string; [key: string]: unknown; }; +/** + * Read-only: state writes happen inside the durable workflow, through the + * store handle, so each write is stamped with the step that made it. + */ export interface StateBackend { get(id: string): Promise; - /** - * The stored revision must match expectedRev. A missing record counts as - * revision 0, so expectedRev: 0 asserts that the record does not exist yet. - */ - update( - id: string, - expectedRev: number, - patch: Partial, - ): Promise<{ rev: number }>; - delete(id: string, expectedRev: number): Promise; values(): Promise; } @@ -39,30 +31,6 @@ export class MemoryStateBackend implements StateBackend { return state[id]; } - async update( - id: string, - expectedRev: number, - patch: Partial, - ): Promise<{ rev: number }> { - const state = await this.readState(); - assertExpectedRev(id, state[id], expectedRev); - const rev = (state[id]?.rev ?? 0) + 1; - state[id] = { - ...state[id], - ...patch, - rev, - } as StateNode; - await this.writeState(state); - return { rev }; - } - - async delete(id: string, expectedRev: number): Promise { - const state = await this.readState(); - assertExpectedRev(id, state[id], expectedRev); - delete state[id]; - await this.writeState(state); - } - async values(): Promise { const state = await this.readState(); return Object.entries(state) @@ -83,23 +51,10 @@ export class MemoryStateBackend implements StateBackend { private async readState(): Promise> { return cloneAsPersistedState(this.#state); } - - private async writeState(state: Record): Promise { - this.#state = cloneAsPersistedState(state); - } -} - -// A missing record counts as rev 0, so expectedRev: 0 means "must not exist". -function assertExpectedRev( - id: string, - node: StateNode | undefined, - expectedRev: number, -): void { - if ((node?.rev ?? 0) !== expectedRev) { - throw new RevConflict(id, expectedRev, node?.rev); - } } +// Seeds and reads pass through JSON, so callers see what a persisted backend +// would return and cannot mutate the backend through a shared reference. function cloneAsPersistedState( state: Record, ): Record { diff --git a/packages/state/test/state-backend.test.ts b/packages/state/test/state-backend.test.ts index 4ae22b9..0f5b1ea 100644 --- a/packages/state/test/state-backend.test.ts +++ b/packages/state/test/state-backend.test.ts @@ -1,21 +1,12 @@ import { describe, expect, it } from "vitest"; -import { - MemoryStateBackend, - type StateBackend, - type StateNode, -} from "src/state"; - -type BackendFixture = { - backend: StateBackend; - cleanup: () => Promise; -}; +import { MemoryStateBackend, type StateNode } from "src/state"; function createStateNode( id: string, overrides: Partial = {}, ): StateNode { return { - rev: 0, + rev: 1, id, groupId: 1, groupType: "stack", @@ -29,148 +20,43 @@ function createStateNode( }; } -function runStateBackendContractTests( - label: string, - createBackend: () => Promise, -) { - describe(label, () => { - it("starts with empty state", async () => { - const fixture = await createBackend(); - - try { - await expect(fixture.backend.get("missing")).resolves.toBeUndefined(); - await expect(fixture.backend.values()).resolves.toEqual([]); - } finally { - await fixture.cleanup(); - } - }); - - it("merges patches on update", async () => { - const fixture = await createBackend(); - const initialNode = createStateNode("resource-a"); - - try { - await fixture.backend.update(initialNode.id, 0, initialNode); - await fixture.backend.update(initialNode.id, 1, { - output: { status: "ready" }, - lastOperation: "update", - }); - - await expect(fixture.backend.get(initialNode.id)).resolves.toEqual({ - ...initialNode, - rev: 2, - output: { status: "ready" }, - lastOperation: "update", - }); - } finally { - await fixture.cleanup(); - } - }); - - it("rejects stale updates and deletes", async () => { - const fixture = await createBackend(); - const initialNode = createStateNode("resource-a"); - - try { - await expect( - fixture.backend.update(initialNode.id, 0, initialNode), - ).resolves.toEqual({ rev: 1 }); - await expect( - fixture.backend.update(initialNode.id, 0, { output: {} }), - ).rejects.toMatchObject({ - name: "RevConflict", - expectedRev: 0, - actualRev: 1, - }); - await expect( - fixture.backend.delete(initialNode.id, 0), - ).rejects.toMatchObject({ - name: "RevConflict", - }); - } finally { - await fixture.cleanup(); - } - }); - - it("treats expectedRev 0 as an expect-absent assertion", async () => { - const fixture = await createBackend(); - const initialNode = createStateNode("resource-a"); - - try { - await expect( - fixture.backend.update(initialNode.id, 0, initialNode), - ).resolves.toEqual({ rev: 1 }); - await expect( - fixture.backend.update(initialNode.id, 0, initialNode), - ).rejects.toMatchObject({ - name: "RevConflict", - expectedRev: 0, - actualRev: 1, - }); - } finally { - await fixture.cleanup(); - } - }); - - it("deletes values", async () => { - const fixture = await createBackend(); - const initialNode = createStateNode("resource-a"); - - try { - await fixture.backend.update(initialNode.id, 0, initialNode); - await fixture.backend.delete(initialNode.id, 1); - - await expect( - fixture.backend.get(initialNode.id), - ).resolves.toBeUndefined(); - await expect(fixture.backend.values()).resolves.toEqual([]); - } finally { - await fixture.cleanup(); - } - }); - - it("returns all values", async () => { - const fixture = await createBackend(); - const firstNode = createStateNode("resource-a"); - const secondNode = createStateNode("resource-b"); +describe("MemoryStateBackend", () => { + it("starts with empty state", async () => { + const backend = new MemoryStateBackend(); - try { - await fixture.backend.update(firstNode.id, 0, firstNode); - await fixture.backend.update(secondNode.id, 0, secondNode); + await expect(backend.get("missing")).resolves.toBeUndefined(); + await expect(backend.values()).resolves.toEqual([]); + }); - const values = await fixture.backend.values(); + it("returns seeded nodes by id", async () => { + const node = createStateNode("resource-a"); + const backend = new MemoryStateBackend({ [node.id]: node }); - expect(values).toHaveLength(2); - expect(values).toEqual( - expect.arrayContaining([ - { ...firstNode, rev: 1 }, - { ...secondNode, rev: 1 }, - ]), - ); - } finally { - await fixture.cleanup(); - } - }); + await expect(backend.get(node.id)).resolves.toEqual(node); + await expect(backend.get("missing")).resolves.toBeUndefined(); }); -} - -runStateBackendContractTests("MemoryStateBackend", async () => ({ - backend: new MemoryStateBackend(), - cleanup: async () => undefined, -})); -describe("MemoryStateBackend", () => { it("returns values in deterministic id order", async () => { - const backend = new MemoryStateBackend(); const laterNode = createStateNode("resource-z"); const earlierNode = createStateNode("resource-a"); + const backend = new MemoryStateBackend({ + [laterNode.id]: laterNode, + [earlierNode.id]: earlierNode, + }); - await backend.update(laterNode.id, 0, laterNode); - await backend.update(earlierNode.id, 0, earlierNode); + await expect(backend.values()).resolves.toEqual([earlierNode, laterNode]); + }); - await expect(backend.values()).resolves.toEqual([ - { ...earlierNode, rev: 1 }, - { ...laterNode, rev: 1 }, - ]); + it("isolates reads from the seed object and from each other", async () => { + const node = createStateNode("resource-a"); + const backend = new MemoryStateBackend({ [node.id]: node }); + + node.output["name"] = "mutated-seed"; + const read = await backend.get(node.id); + read!.output["name"] = "mutated-read"; + + await expect(backend.get(node.id)).resolves.toMatchObject({ + output: { name: "resource-a-output" }, + }); }); }); From 4f7ea1b9bf698d389dec140ad085747554e3531e Mon Sep 17 00:00:00 2001 From: djgrant <1670902+djgrant@users.noreply.github.com> Date: Tue, 28 Jul 2026 19:09:11 +0100 Subject: [PATCH 30/34] Give each remaining concept one name - Type store snapshots as PersistedResourceState and drop StoredResourceState; a satisfies assertion pins the valibot schema's output to it. - Delete the PlannerState alias; createPlan takes StateBackend. - Expose the store version directly as StateNode.version, removing the one-based rev shim; RevConflict becomes VersionConflict, its unread fields now constructor arguments that only feed the message. - Inline the vestigial resourceParams aliases in the create/update/read operations. - Rename ReadResourceParams to ResolvedResourceParams: the neutral base create and update extend, and what read takes exactly. - Rename runWithCliErrorHandling to runWithErrorHandling, matching its file name. --- docs/internals/state.md | 2 +- docs/manual/reconciler.md | 2 +- docs/rfcs/reconciler.md | 2 +- packages/cli/src/index.ts | 4 ++-- packages/cli/src/run-with-error-handling.ts | 2 +- .../cli/test/run-with-error-handling.test.ts | 6 +++--- packages/dashboard/server/server.test.ts | 4 ++-- packages/reconciler/src/durable/index.ts | 1 - packages/reconciler/src/durable/reconcile.ts | 10 +++++----- packages/reconciler/src/durable/stores.ts | 13 ++++++++----- .../src/operations/operation.create.ts | 10 ++++------ .../reconciler/src/operations/operation.read.ts | 14 ++++++-------- .../src/operations/operation.types.ts | 17 +++++++++-------- .../src/operations/operation.update.ts | 10 ++++------ packages/reconciler/src/planner.ts | 5 +---- .../test/durable-reconciliation.test.ts | 12 ++++++------ packages/reconciler/test/planner.test.ts | 8 ++++---- packages/state/src/conflicts.ts | 17 +++++++++++------ packages/state/src/state.ts | 3 ++- packages/state/test/state-backend.test.ts | 2 +- 20 files changed, 72 insertions(+), 72 deletions(-) diff --git a/docs/internals/state.md b/docs/internals/state.md index 1c0fa03..c8a2d51 100644 --- a/docs/internals/state.md +++ b/docs/internals/state.md @@ -8,7 +8,7 @@ Each live resource is a `notation/resource-state` store scoped by deployment and const state = new DurableStateBackend(storeClient, "infra/api.ts"); ``` -The runtime assigns a UUIDv7 `instanceId` when a store is created and increments its version on update. Conditional workflow updates and deletes compare both values, preventing a stale snapshot from modifying a deleted and recreated resource. The one-based value exposed as `StateNode.rev` is derived from the authoritative Yieldstar store version. +The runtime assigns a UUIDv7 `instanceId` when a store is created and increments its version on update. Conditional workflow updates and deletes compare both values, preventing a stale snapshot from modifying a deleted and recreated resource. The store version is exposed unchanged as `StateNode.version`. `DurableStateBackend` is read-only: state writes happen inside the workflow, through the store handle, so each write is stamped with the step that made it and is not repeated on replay. diff --git a/docs/manual/reconciler.md b/docs/manual/reconciler.md index a404ec9..2b95384 100644 --- a/docs/manual/reconciler.md +++ b/docs/manual/reconciler.md @@ -34,7 +34,7 @@ export const destroy = workflow(async function* (step, event) { The outer workflow supplies durable step execution, timers, shared stores, waiting, and scheduling. Checkpointed provider results are replayed from the heap after a crash, retryable provider conditions suspend on a durable timer, and conditional state writes use Yieldstar store identity and version. Provider mutations must be idempotent because a crash after provider acknowledgement but before the heap checkpoint repeats the call; event consumers must tolerate the same duplicate-delivery window. -Each live resource is one Yieldstar store. Yieldstar's UUIDv7 store `instanceId` and version are authoritative for conditional update and delete; Notation exposes the version as the resource state's `rev`. +Each live resource is one Yieldstar store. Yieldstar's UUIDv7 store `instanceId` and version are authoritative for conditional update and delete; Notation exposes the version unchanged as the resource state's `version`. Operations against the same deployment — the `deploymentId` the `DurableStateBackend` is constructed with — are serialized through a deployment hold naming the holding `executionId`. Resume a crashed operation with the same execution ID; use a new globally unique execution ID for every new deploy or destroy. An execution that must wait emits a `reconciler.hold.waiting` event naming the holder before it suspends, which also identifies a crashed holder that should be resumed instead. If the holder is genuinely abandoned, clear its hold with `takeOverDeploymentHold`. diff --git a/docs/rfcs/reconciler.md b/docs/rfcs/reconciler.md index 00fbca7..7820d73 100644 --- a/docs/rfcs/reconciler.md +++ b/docs/rfcs/reconciler.md @@ -15,7 +15,7 @@ Provider create, update, read, and delete calls are durable steps with stable re `DurableStateBackend` stores one live resource per `notation/resource-state` store. The store ID is scoped by deployment and resource ID. Store absence is resource absence. -The runtime-assigned UUIDv7 `instanceId` distinguishes a deleted store from a later store created under the same logical ID. Yieldstar's version is the concurrency token and is exposed as Notation's one-based `rev`. Workflow updates use `store.updateFrom` and deletes use `store.deleteFrom`, so both the instance and version must match the snapshot that informed the operation. +The runtime-assigned UUIDv7 `instanceId` distinguishes a deleted store from a later store created under the same logical ID. Yieldstar's version is the concurrency token, exposed unchanged as `StateNode.version`. Workflow updates use `store.updateFrom` and deletes use `store.deleteFrom`, so both the instance and version must match the snapshot that informed the operation. ## Deployment hold diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index ee4101b..f2d7fd1 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -5,7 +5,7 @@ import { deploy } from "./deploy"; import { destroy } from "./destroy"; import { plan } from "./plan"; import { defaultLogger } from "./logger"; -import { runWithCliErrorHandling } from "./run-with-error-handling"; +import { runWithErrorHandling } from "./run-with-error-handling"; import { visualise } from "./visualise"; import { watch } from "./watch"; import { startDashboardServer } from "@notation/dashboard"; @@ -81,7 +81,7 @@ program await watch(entryPoint); }); -process.exitCode = await runWithCliErrorHandling( +process.exitCode = await runWithErrorHandling( () => program.parseAsync(process.argv), { logger: defaultLogger, command: process.argv[2] ?? program.name() }, ); diff --git a/packages/cli/src/run-with-error-handling.ts b/packages/cli/src/run-with-error-handling.ts index 85f8f78..552f788 100644 --- a/packages/cli/src/run-with-error-handling.ts +++ b/packages/cli/src/run-with-error-handling.ts @@ -1,6 +1,6 @@ import type { Logger } from "./logger"; -export async function runWithCliErrorHandling( +export async function runWithErrorHandling( fn: () => Promise, opts: { logger: Logger; command: string }, ): Promise<0 | 1> { diff --git a/packages/cli/test/run-with-error-handling.test.ts b/packages/cli/test/run-with-error-handling.test.ts index 2ca64e9..343c1e1 100644 --- a/packages/cli/test/run-with-error-handling.test.ts +++ b/packages/cli/test/run-with-error-handling.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it, vi } from "vitest"; -import { runWithCliErrorHandling } from "../src/run-with-error-handling"; +import { runWithErrorHandling } from "../src/run-with-error-handling"; describe("CLI error handling", () => { it("reports credential failures with command-specific guidance", async () => { @@ -7,7 +7,7 @@ describe("CLI error handling", () => { error.name = "CredentialsProviderError"; const logger = { info: vi.fn(), warn: vi.fn(), error: vi.fn() }; - const exitCode = await runWithCliErrorHandling( + const exitCode = await runWithErrorHandling( async () => { throw error; }, @@ -27,7 +27,7 @@ describe("CLI error handling", () => { const error = new Error("deploy failed"); const logger = { info: vi.fn(), warn: vi.fn(), error: vi.fn() }; - const exitCode = await runWithCliErrorHandling( + const exitCode = await runWithErrorHandling( async () => { throw error; }, diff --git a/packages/dashboard/server/server.test.ts b/packages/dashboard/server/server.test.ts index 059f469..8cfb300 100644 --- a/packages/dashboard/server/server.test.ts +++ b/packages/dashboard/server/server.test.ts @@ -6,7 +6,7 @@ describe("dashboard state", () => { it("reads state through the backend contract", async () => { const state = new MemoryStateBackend({ service: { - rev: 1, + version: 1, id: "service", type: "test/service/main", config: {}, @@ -20,7 +20,7 @@ describe("dashboard state", () => { await expect(readStateSnapshot(state)).resolves.toMatchObject({ service: { id: "service", - rev: 1, + version: 1, output: { ready: true }, }, }); diff --git a/packages/reconciler/src/durable/index.ts b/packages/reconciler/src/durable/index.ts index a1327da..71f54a0 100644 --- a/packages/reconciler/src/durable/index.ts +++ b/packages/reconciler/src/durable/index.ts @@ -42,7 +42,6 @@ export { deploymentHoldStore, resourceStateStore, type DeploymentHoldState, - type StoredResourceState, } from "./stores"; export { type DurableDeployOptions, diff --git a/packages/reconciler/src/durable/reconcile.ts b/packages/reconciler/src/durable/reconcile.ts index 35762bf..929cbe0 100644 --- a/packages/reconciler/src/durable/reconcile.ts +++ b/packages/reconciler/src/durable/reconcile.ts @@ -4,7 +4,7 @@ * writes conditional on that read. */ import type { BaseResource, ResourceType } from "@notation/resource"; -import { RevConflict, type StateNode } from "@notation/state"; +import { VersionConflict, type StateNode } from "@notation/state"; import { createResourceRegistryFromResources, resolveResourceClass, @@ -253,10 +253,10 @@ function persistResourceState( () => next, ); if (!result.updated) { - throw new RevConflict( + throw new VersionConflict( resource.id, - snapshot.version + 1, - result.actualVersion + 1, + snapshot.version, + result.actualVersion, ); } }; @@ -277,7 +277,7 @@ function removeResourceState( snapshot, ); if (!result.deleted) { - throw new RevConflict(resource.id, snapshot.version + 1, undefined); + throw new VersionConflict(resource.id, snapshot.version, undefined); } }; } diff --git a/packages/reconciler/src/durable/stores.ts b/packages/reconciler/src/durable/stores.ts index 322d99e..7ec3e8a 100644 --- a/packages/reconciler/src/durable/stores.ts +++ b/packages/reconciler/src/durable/stores.ts @@ -1,5 +1,6 @@ import type { StateNode } from "@notation/state"; import * as v from "valibot"; +import type { PersistedResourceState } from "../operations"; import { defineStore, type StoreSnapshot } from "./yieldstar"; // Store names are persisted identifiers, like the step keys mapped in @@ -35,17 +36,19 @@ export const deploymentHoldStore = defineStore( v.object({ holder: v.nullable(v.string()) }), ); -export type StoredResourceState = v.InferOutput< +// The schema validates exactly the record operations persist; drift between +// the two is a type error here. +({}) as v.InferOutput< typeof resourceStateStore.schema ->; +> satisfies PersistedResourceState; + export type DeploymentHoldState = v.InferOutput< typeof deploymentHoldStore.schema >; /** A read of a resource record, carrying the identity a write is made against. */ -export type ResourceSnapshot = StoreSnapshot; +export type ResourceSnapshot = StoreSnapshot; -/** Store versions count from zero, state revisions from one. */ export function toStateNode(snapshot: ResourceSnapshot): StateNode { - return { ...snapshot.state, rev: snapshot.version + 1 }; + return { ...snapshot.state, version: snapshot.version }; } diff --git a/packages/reconciler/src/operations/operation.create.ts b/packages/reconciler/src/operations/operation.create.ts index d76d06f..2aa60b4 100644 --- a/packages/reconciler/src/operations/operation.create.ts +++ b/packages/reconciler/src/operations/operation.create.ts @@ -19,16 +19,14 @@ export async function* createResourceOperation( } try { - const resourceParams = params.resourceParams; - const computedPrimaryKey = yield* runPendingOperation( step, "create:remote", - (context) => params.resource.create(resourceParams, context), + (context) => params.resource.create(params.resourceParams, context), params.maxOperationAttempts, ); - params.resource.setOutput(resourceParams); + params.resource.setOutput(params.resourceParams); if (computedPrimaryKey) { params.resource.setOutput({ ...computedPrimaryKey, @@ -38,7 +36,7 @@ export async function* createResourceOperation( const readResult = yield* readResourceOperation(step, { resource: params.resource, - resourceParams, + resourceParams: params.resourceParams, persistedOutput: params.persistedOutput, emit: params.emit, maxOperationAttempts: params.maxOperationAttempts, @@ -57,7 +55,7 @@ export async function* createResourceOperation( lastOperation: "create", lastOperationAt: new Date().toISOString(), config: params.resource.config, - params: params.resource.toState(resourceParams), + params: params.resource.toState(params.resourceParams), output: params.resource.toState(params.resource.output), }); diff --git a/packages/reconciler/src/operations/operation.read.ts b/packages/reconciler/src/operations/operation.read.ts index 76e02da..2d246ee 100644 --- a/packages/reconciler/src/operations/operation.read.ts +++ b/packages/reconciler/src/operations/operation.read.ts @@ -1,7 +1,7 @@ import { ResourceNotFoundError } from "@notation/resource"; import type { DriftRead } from "../plan"; import { - type ReadResourceParams, + type ResolvedResourceParams, type StepRunner, emitLifecycleEvent, getErrorDetails, @@ -10,7 +10,7 @@ import { runPendingOperation } from "./operation.pending"; export async function* readResourceOperation( step: StepRunner, - params: ReadResourceParams, + params: ResolvedResourceParams, ): AsyncGenerator, unknown> { yield* emitLifecycleEvent(params, "read", "start"); @@ -20,12 +20,10 @@ export async function* readResourceOperation( } try { - const resourceParams = params.resourceParams; - if (!params.resource.read) { const merged = params.persistedOutput - ? { ...params.persistedOutput, ...resourceParams } - : resourceParams; + ? { ...params.persistedOutput, ...params.resourceParams } + : params.resourceParams; yield* emitLifecycleEvent(params, "read", "skip", { reason: "read-not-implemented", @@ -42,7 +40,7 @@ export async function* readResourceOperation( ); const mergedOutput = { - ...resourceParams, + ...params.resourceParams, ...remote, }; @@ -61,7 +59,7 @@ export async function* readResourceOperation( */ export async function* readDriftOperation( step: StepRunner, - params: ReadResourceParams, + params: ResolvedResourceParams, ): AsyncGenerator { try { const output = yield* readResourceOperation(step, params); diff --git a/packages/reconciler/src/operations/operation.types.ts b/packages/reconciler/src/operations/operation.types.ts index aacca62..e87d1f5 100644 --- a/packages/reconciler/src/operations/operation.types.ts +++ b/packages/reconciler/src/operations/operation.types.ts @@ -30,7 +30,7 @@ export type StepRunner = { }; /** - * The record an operation wants persisted; the driver owns the revision. + * The record an operation wants persisted; the driver owns the version. * Not derived with Omit, which would collapse against StateNode's index * signature and widen every field to unknown. */ @@ -71,21 +71,22 @@ export type ResourceOperationBaseParams = { }; /** - * Everything a read needs is resolved before the operation starts: the - * desired params, and — for a resource with no read operation — the output - * the last write persisted. An operation that resolved either itself could - * see a different answer from the one the decision was taken against. + * Inputs resolved before an operation starts: the desired params, and — for + * a resource with no read operation — the output the last write persisted. + * An operation that resolved either itself could see a different answer from + * the one the decision was taken against. Read takes exactly this; create + * and update add their write. */ -export type ReadResourceParams = ResourceOperationBaseParams & { +export type ResolvedResourceParams = ResourceOperationBaseParams & { resourceParams: Record; persistedOutput?: Record; }; -export type CreateResourceParams = ReadResourceParams & { +export type CreateResourceParams = ResolvedResourceParams & { persist: PersistState; }; -export type UpdateResourceParams = ReadResourceParams & { +export type UpdateResourceParams = ResolvedResourceParams & { patch: Record; persist: PersistState; }; diff --git a/packages/reconciler/src/operations/operation.update.ts b/packages/reconciler/src/operations/operation.update.ts index 838161e..ca51010 100644 --- a/packages/reconciler/src/operations/operation.update.ts +++ b/packages/reconciler/src/operations/operation.update.ts @@ -27,8 +27,6 @@ export async function* updateResourceOperation( } try { - const resourceParams = params.resourceParams; - yield* runPendingOperation( step, "update:remote", @@ -36,7 +34,7 @@ export async function* updateResourceOperation( params.resource.update!( params.resource.key, params.patch, - resourceParams, + params.resourceParams, params.resource.toState(params.resource.output), context, ), @@ -45,12 +43,12 @@ export async function* updateResourceOperation( params.resource.setOutput({ ...params.resource.key, - ...resourceParams, + ...params.resourceParams, }); const readResult = yield* readResourceOperation(step, { resource: params.resource, - resourceParams, + resourceParams: params.resourceParams, persistedOutput: params.persistedOutput, emit: params.emit, maxOperationAttempts: params.maxOperationAttempts, @@ -69,7 +67,7 @@ export async function* updateResourceOperation( lastOperation: "update", lastOperationAt: new Date().toISOString(), config: params.resource.config, - params: params.resource.toState(resourceParams), + params: params.resource.toState(params.resourceParams), output: params.resource.toState(params.resource.output), }); diff --git a/packages/reconciler/src/planner.ts b/packages/reconciler/src/planner.ts index 70409bc..c032c50 100644 --- a/packages/reconciler/src/planner.ts +++ b/packages/reconciler/src/planner.ts @@ -13,12 +13,9 @@ import { } from "./plan"; import { createStepRunner, runOperation } from "./step-runner"; -/** Planning only reads state. */ -export type PlannerState = Pick; - export type CreatePlanOptions = { resources: BaseResource[]; - state: PlannerState; + state: StateBackend; driftDetection?: boolean; emit?: ReconcilerEventEmitter; maxOperationAttempts?: number; diff --git a/packages/reconciler/test/durable-reconciliation.test.ts b/packages/reconciler/test/durable-reconciliation.test.ts index 35de461..b05f661 100644 --- a/packages/reconciler/test/durable-reconciliation.test.ts +++ b/packages/reconciler/test/durable-reconciliation.test.ts @@ -71,7 +71,7 @@ describe("durable execution and replay", () => { expect(await runtime.state.get("pending")).toMatchObject({ id: "pending", lastOperation: "create", - rev: 1, + version: 0, }); runtime.close(); }); @@ -95,7 +95,7 @@ describe("durable execution and replay", () => { await runtime.run("resume-execution"); expect(create).toHaveBeenCalledOnce(); - expect(await runtime.state.get("resume")).toMatchObject({ rev: 1 }); + expect(await runtime.state.get("resume")).toMatchObject({ version: 0 }); runtime.close(); }); @@ -190,7 +190,7 @@ describe("durable execution and replay", () => { await runtime.run("post-write-read-execution"); expect(reads).toBe(2); expect(await runtime.state.get("eventually-readable")).toMatchObject({ - rev: 1, + version: 0, }); runtime.close(); }); @@ -282,11 +282,11 @@ describe("conditional state persistence", () => { resources[0] = new RaceResource({ id: "raced", config: { name: "after" } }); await expect(runtime.run("deploy-2")).rejects.toMatchObject({ - name: "RevConflict", + name: "VersionConflict", }); // The losing write left the other writer's record intact. expect(await runtime.state.get("raced")).toMatchObject({ - rev: 2, + version: 1, lastOperationAt: "1999-01-01T00:00:00.000Z", }); runtime.close(); @@ -314,7 +314,7 @@ describe("conditional state persistence", () => { await runtime.run("deploy-1"); await expect(runtime.destroy("destroy-1")).rejects.toMatchObject({ - name: "RevConflict", + name: "VersionConflict", }); // State survives a removal that could not be proven safe. expect(await runtime.state.get("delete-raced")).toBeDefined(); diff --git a/packages/reconciler/test/planner.test.ts b/packages/reconciler/test/planner.test.ts index 40d4972..b4b0c05 100644 --- a/packages/reconciler/test/planner.test.ts +++ b/packages/reconciler/test/planner.test.ts @@ -17,7 +17,7 @@ describe("createPlan", () => { }); const state = new MemoryStateBackend({ orphan: { - rev: 1, + version: 1, id: "orphan", type: TestResource.type, config: {}, @@ -52,7 +52,7 @@ describe("createPlan", () => { }); const state = new MemoryStateBackend({ existing: { - rev: 1, + version: 1, id: "existing", type: TestResource.type, config: {}, @@ -83,7 +83,7 @@ describe("createPlan", () => { }); const state = new MemoryStateBackend({ existing: { - rev: 1, + version: 1, id: "existing", type: TestResource.type, config: {}, @@ -125,7 +125,7 @@ describe("createPlan", () => { }); const state = new MemoryStateBackend({ existing: { - rev: 1, + version: 1, id: "existing", type: TestResource.type, config: {}, diff --git a/packages/state/src/conflicts.ts b/packages/state/src/conflicts.ts index a1e666c..23494ad 100644 --- a/packages/state/src/conflicts.ts +++ b/packages/state/src/conflicts.ts @@ -1,13 +1,18 @@ -export class RevConflict extends Error { - readonly name = "RevConflict"; +/** + * A conditional state write or removal found the record moved past the + * version it was read at. Nothing catches this: it fails the workflow, and + * the constructor arguments exist to name the losing write in the message. + */ +export class VersionConflict extends Error { + readonly name = "VersionConflict"; constructor( - readonly id: string, - readonly expectedRev: number, - readonly actualRev: number | undefined, + id: string, + expectedVersion: number, + actualVersion: number | undefined, ) { super( - `State revision conflict for ${id}: expected ${expectedRev}, got ${actualRev ?? "missing"}`, + `State version conflict for ${id}: expected ${expectedVersion}, got ${actualVersion ?? "missing"}`, ); } } diff --git a/packages/state/src/state.ts b/packages/state/src/state.ts index c1fc04e..dca02cf 100644 --- a/packages/state/src/state.ts +++ b/packages/state/src/state.ts @@ -1,5 +1,6 @@ export type StateNode = { - rev: number; + /** The backing store's version of the record, counted from zero. */ + version: number; id: string; type: string; config: Record; diff --git a/packages/state/test/state-backend.test.ts b/packages/state/test/state-backend.test.ts index 0f5b1ea..ac1a198 100644 --- a/packages/state/test/state-backend.test.ts +++ b/packages/state/test/state-backend.test.ts @@ -6,7 +6,7 @@ function createStateNode( overrides: Partial = {}, ): StateNode { return { - rev: 1, + version: 1, id, groupId: 1, groupType: "stack", From d115da9d3aaf9e101eb9be31a7cc1d3df2f24dca Mon Sep 17 00:00:00 2001 From: djgrant <1670902+djgrant@users.noreply.github.com> Date: Tue, 28 Jul 2026 19:20:02 +0100 Subject: [PATCH 31/34] Report the true delete-conflict fact, unify event types, and give the crash-window contract one home per audience - removeResourceState now distinguishes a version conflict (naming the actual version) from a genuinely absent record, instead of reporting every failed deleteFrom as missing; the delete-race test locks the distinction in. - resourceType is ResourceType on all four resource-bearing reconciler events instead of string on two of them. - The crash-window/idempotency contract is stated once for CLI readers (docs/cli/deploy.md) and once for internals readers (docs/internals/reconciler.md, moved to Waiting and replay where it applies to all provider calls); destroy.md, the manual, and the RFC now reference those homes instead of restating it. --- docs/cli/destroy.md | 2 +- docs/internals/reconciler.md | 4 ++-- docs/manual/reconciler.md | 2 +- docs/rfcs/reconciler.md | 2 +- packages/reconciler/src/durable/reconcile.ts | 8 +++++++- packages/reconciler/src/events.ts | 4 ++-- packages/reconciler/test/durable-reconciliation.test.ts | 3 +++ 7 files changed, 17 insertions(+), 8 deletions(-) diff --git a/docs/cli/destroy.md b/docs/cli/destroy.md index 17aad8c..36e2209 100644 --- a/docs/cli/destroy.md +++ b/docs/cli/destroy.md @@ -24,4 +24,4 @@ notation destroy infra/api.ts --execution-id Retryable deletes suspend on durable SQLite timers. Resource state is removed only after the provider delete succeeds or reports that the resource is already absent. -The provider acknowledgement and heap checkpoint are not atomic. A crash between them repeats the delete, so provider delete operations must be idempotent and event consumers must tolerate duplicate delivery. +The crash-window contract described under [notation deploy](./deploy.md#durable-execution) applies to deletes as well. diff --git a/docs/internals/reconciler.md b/docs/internals/reconciler.md index d4845df..5ed071b 100644 --- a/docs/internals/reconciler.md +++ b/docs/internals/reconciler.md @@ -21,10 +21,10 @@ Dry-run deploy performs decisions and emits lifecycle events without provider mu `destroy` is a first-class durable operation. It takes the same deployment hold as deploy, deletes desired resources in reverse dependency order, deletes registered persisted orphans, and conditionally removes each resource store only after the provider delete succeeds or reports that the resource is already absent. -Provider delete is a stable durable step, but the provider acknowledgement and Yieldstar heap checkpoint are not atomic. If the process crashes between them, replay repeats the delete, so provider create, update, and delete operations must be idempotent. Event subscribers must likewise tolerate duplicate delivery when a crash occurs before the event checkpoint. - ## Waiting and replay +Provider calls are stable durable steps, but provider acknowledgement and the Yieldstar heap checkpoint are not atomic. If the process crashes between them, replay repeats the call, so provider create, update, and delete operations must be idempotent. Event subscribers must likewise tolerate duplicate delivery when a crash occurs before the event checkpoint. + A resource operation throws `ResourceOperationPendingError` when it has not finished. The error gives the reconciler a delay and optional callback context. The runtime stores the context, waits without keeping the process busy, and calls the same operation again. See [Operation errors](./resource.md#operation-errors) for the complete API. Each attempt, delay, event, state read, state write, and hold change has a stable step key. A resumed execution must use the same execution ID. A new deploy or destroy must use a new execution ID. diff --git a/docs/manual/reconciler.md b/docs/manual/reconciler.md index 2b95384..e851df4 100644 --- a/docs/manual/reconciler.md +++ b/docs/manual/reconciler.md @@ -32,7 +32,7 @@ export const destroy = workflow(async function* (step, event) { }); ``` -The outer workflow supplies durable step execution, timers, shared stores, waiting, and scheduling. Checkpointed provider results are replayed from the heap after a crash, retryable provider conditions suspend on a durable timer, and conditional state writes use Yieldstar store identity and version. Provider mutations must be idempotent because a crash after provider acknowledgement but before the heap checkpoint repeats the call; event consumers must tolerate the same duplicate-delivery window. +The outer workflow supplies durable step execution, timers, shared stores, waiting, and scheduling. Checkpointed provider results are replayed from the heap after a crash, retryable provider conditions suspend on a durable timer, and conditional state writes use Yieldstar store identity and version. Provider operations and event consumers are bound by the crash-window contract stated in [the reconciler internals](../internals/reconciler.md#waiting-and-replay). Each live resource is one Yieldstar store. Yieldstar's UUIDv7 store `instanceId` and version are authoritative for conditional update and delete; Notation exposes the version unchanged as the resource state's `version`. diff --git a/docs/rfcs/reconciler.md b/docs/rfcs/reconciler.md index 7820d73..a91a637 100644 --- a/docs/rfcs/reconciler.md +++ b/docs/rfcs/reconciler.md @@ -9,7 +9,7 @@ Notation describes reconciliation intent and resource lifecycle operations. An o Live resource objects remain in the workflow process. They are not serialized into workflow parameters. This keeps provider clients and operation closures under Notation's lifecycle control while Yieldstar persists step results and shared state. -Provider create, update, read, and delete calls are durable steps with stable resource-scoped keys. Once a result reaches the heap checkpoint, replay uses the cached result and continues at state persistence. Provider mutations must be idempotent because a crash after provider acknowledgement but before that checkpoint repeats the call. Retryable provider conditions become Yieldstar delays, allowing the process to wait without polling the provider continuously. +Provider create, update, read, and delete calls are durable steps with stable resource-scoped keys. Once a result reaches the heap checkpoint, replay uses the cached result and continues at state persistence. The resulting crash-window contract is stated in [the reconciler internals](../internals/reconciler.md#waiting-and-replay). Retryable provider conditions become Yieldstar delays, allowing the process to wait without polling the provider continuously. ## State lifecycle diff --git a/packages/reconciler/src/durable/reconcile.ts b/packages/reconciler/src/durable/reconcile.ts index 929cbe0..0fd4d9b 100644 --- a/packages/reconciler/src/durable/reconcile.ts +++ b/packages/reconciler/src/durable/reconcile.ts @@ -277,7 +277,13 @@ function removeResourceState( snapshot, ); if (!result.deleted) { - throw new VersionConflict(resource.id, snapshot.version, undefined); + // "conflict" carries the version the record moved to; "not-found" means + // the record is genuinely gone, which the message reports as "missing". + throw new VersionConflict( + resource.id, + snapshot.version, + result.reason === "conflict" ? result.actualVersion : undefined, + ); } }; } diff --git a/packages/reconciler/src/events.ts b/packages/reconciler/src/events.ts index fbbd201..df93c66 100644 --- a/packages/reconciler/src/events.ts +++ b/packages/reconciler/src/events.ts @@ -21,7 +21,7 @@ export type DeployDecisionEvent = { level: "info"; event: "reconciler.deploy.decision"; resourceId: string; - resourceType: string; + resourceType: ResourceType; decision: "create" | "update" | "drift-update" | "drift-recreate" | "noop"; }; @@ -29,7 +29,7 @@ export type DriftDetectedEvent = { level: "info"; event: "reconciler.drift.detected"; resourceId: string; - resourceType: string; + resourceType: ResourceType; diff: Record; }; diff --git a/packages/reconciler/test/durable-reconciliation.test.ts b/packages/reconciler/test/durable-reconciliation.test.ts index b05f661..330ba6b 100644 --- a/packages/reconciler/test/durable-reconciliation.test.ts +++ b/packages/reconciler/test/durable-reconciliation.test.ts @@ -315,6 +315,9 @@ describe("conditional state persistence", () => { await runtime.run("deploy-1"); await expect(runtime.destroy("destroy-1")).rejects.toMatchObject({ name: "VersionConflict", + // A conflict names the version the record moved to; only a genuinely + // absent record may be reported as "missing". + message: expect.stringMatching(/expected 0, got 1$/), }); // State survives a removal that could not be proven safe. expect(await runtime.state.get("delete-raced")).toBeDefined(); From e45b6b6b9ef7506447d7491972ebd34b2b8fe17e Mon Sep 17 00:00:00 2001 From: djgrant <1670902+djgrant@users.noreply.github.com> Date: Tue, 28 Jul 2026 19:32:22 +0100 Subject: [PATCH 32/34] Remove the last stale references and inert config --- docs/internals/resource.md | 2 +- docs/internals/state.md | 2 +- packages/reconciler/src/durable/index.ts | 6 +----- packages/reconciler/test/operation.workflows.test.ts | 2 +- pnpm-workspace.yaml | 4 ---- 5 files changed, 4 insertions(+), 12 deletions(-) diff --git a/docs/internals/resource.md b/docs/internals/resource.md index 851be55..fff3520 100644 --- a/docs/internals/resource.md +++ b/docs/internals/resource.md @@ -173,7 +173,7 @@ new ResourceOperationPendingError(message: string, { | Handler result | Meaning | What the reconciler does | | -------------- | ------- | ------------------------ | | Return normally | The operation finished. | Continues the deployment. | -| `throw new ResourceNotFoundError(message, { cause })` | `read` found no resource for the given key. | Treats the resource as absent during planning and refresh. A read after create or update fails because that operation claimed to have finished. | +| `throw new ResourceNotFoundError(message, { cause })` | `read` found no resource for the given key. | Treats the resource as absent during planning and drift detection. A read after create or update fails because that operation claimed to have finished. | | `throw new ResourceOperationPendingError(message, { retryAfterMs, callbackContext })` | The operation has not finished. | Waits for `retryAfterMs`, then calls the same handler again. It passes `callbackContext` as the handler's final argument. | | Throw any other error | The operation failed. | Stops the deployment. | diff --git a/docs/internals/state.md b/docs/internals/state.md index c8a2d51..cb0bd8e 100644 --- a/docs/internals/state.md +++ b/docs/internals/state.md @@ -2,7 +2,7 @@ Notation CLI deploy, destroy, plan, and dashboard use Yieldstar stores in `.notation/workflows.db`. Override the database path with `NOTATION_DATABASE_PATH`. -Each live resource is a `notation/resource-state` store scoped by deployment and resource ID. A missing store means the resource is absent. No application tombstone is written. +Each live resource is a `notation/resource-state` store scoped by deployment and resource ID. A missing store means the resource is absent. ```ts const state = new DurableStateBackend(storeClient, "infra/api.ts"); diff --git a/packages/reconciler/src/durable/index.ts b/packages/reconciler/src/durable/index.ts index 71f54a0..2430ac3 100644 --- a/packages/reconciler/src/durable/index.ts +++ b/packages/reconciler/src/durable/index.ts @@ -38,11 +38,7 @@ export { type DeploymentHoldTakeover, } from "./deployment-hold"; export { DurableStateBackend } from "./state-backend"; -export { - deploymentHoldStore, - resourceStateStore, - type DeploymentHoldState, -} from "./stores"; +export { deploymentHoldStore, resourceStateStore } from "./stores"; export { type DurableDeployOptions, type DurableWorkflowOptions, diff --git a/packages/reconciler/test/operation.workflows.test.ts b/packages/reconciler/test/operation.workflows.test.ts index 08ca0db..d04ba14 100644 --- a/packages/reconciler/test/operation.workflows.test.ts +++ b/packages/reconciler/test/operation.workflows.test.ts @@ -239,7 +239,7 @@ describe("operation workflows", () => { ); // State is removed only after the provider delete resolves; which record - // and revision that targets is the driver's concern, not the operation's. + // and version that targets is the driver's concern, not the operation's. expect(remove).toHaveBeenCalledOnce(); expect(events.map((event) => event.status)).toEqual(["start", "success"]); }); diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index d76313e..fcbc84a 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -11,10 +11,6 @@ allowBuilds: # The docs framework ships no dist/; its prepare script builds it after clone. # Matching by repo URL rather than resolved commit needs pnpm 11.11+. "@notation/docs@git+ssh://git@github.com/djgrant/docs.git": true -minimumReleaseAgeExclude: - - "@yieldstar/core@0.5.0" - - "@yieldstar/sqlite-runtime@0.5.0" - - yieldstar@0.5.0 overrides: lodash-es@<4.18.1: ^4.18.1 js-yaml@3: 3.15.0 From 6bfdae31870677b69d8c84ae622718d141b45b7a Mon Sep 17 00:00:00 2001 From: djgrant <1670902+djgrant@users.noreply.github.com> Date: Tue, 28 Jul 2026 19:49:35 +0100 Subject: [PATCH 33/34] Give each concept one name and the drift gate one home - Inline OperationEventEmitter as EmitStep, so "emitter" names only the plain callback and the step seam is named as a step. - Rename the deploy scope key notation:resource: to notation:deploy:, symmetric with notation:destroy:, and drop the key map's parentheticals. - Rename the scoped step key state:snapshot to persisted-record, so the state: prefix means only store-handle keys, as the key map says. - Extract applyDriftDetection: the noop-plus-drift-read gate and its driftDetection default now live once, used by both reconcileResource and createPlan; the redundant outer defaults in deployApp and planApp are gone too. - Require executionId through runtime.run, runDurableWorkflow, and deployApp/destroyApp: the outermost caller generates the resume handle and reports it, so no layer silently defaults an ID the caller can never learn. - Reorder deploy.md's "What happens": ordering precedes reconciliation. --- docs/cli/deploy.md | 4 +- examples/reconciler/src/index.ts | 7 ++++ packages/cli/src/watch.ts | 4 ++ .../core/src/provisioner/durable-runtime.ts | 13 ++++-- .../provisioner/workflows/workflow.deploy.ts | 6 ++- .../provisioner/workflows/workflow.destroy.ts | 3 +- .../provisioner/workflows/workflow.plan.ts | 3 +- packages/reconciler/src/durable/deploy.ts | 2 +- packages/reconciler/src/durable/index.ts | 4 +- packages/reconciler/src/durable/reconcile.ts | 42 ++++++++----------- .../src/operations/operation.read.ts | 33 ++++++++++++++- .../src/operations/operation.types.ts | 4 +- packages/reconciler/src/planner.ts | 31 ++++++-------- .../test/durable-reconciliation.test.ts | 2 +- 14 files changed, 97 insertions(+), 61 deletions(-) diff --git a/docs/cli/deploy.md b/docs/cli/deploy.md index a26ddec..6af74db 100644 --- a/docs/cli/deploy.md +++ b/docs/cli/deploy.md @@ -36,9 +36,9 @@ Retryable provider conditions and consistency reads suspend on durable SQLite ti 2. **Build resource graph** – the worker imports the compiled output and collects declared resources. -3. **Reconcile** – Notation compares desired resources with Yieldstar stores, then creates, updates, recreates, or leaves each resource unchanged. +3. **Order dependencies** – dependency levels run in topological order. -4. **Order dependencies** – dependency levels run in topological order. +4. **Reconcile** – Notation compares desired resources with Yieldstar stores, then creates, updates, recreates, or leaves each resource unchanged. 5. **Detect drift** – unchanged resources are read from the provider and repaired when their remote state differs. diff --git a/examples/reconciler/src/index.ts b/examples/reconciler/src/index.ts index 2efaede..8898a7b 100644 --- a/examples/reconciler/src/index.ts +++ b/examples/reconciler/src/index.ts @@ -1,3 +1,4 @@ +import { randomUUID } from "node:crypto"; import { NodeDurableRuntime } from "@notation/core"; import { createResourceRegistry } from "@notation/reconciler"; import * as durable from "@notation/reconciler/durable"; @@ -35,9 +36,15 @@ const deploy = workflow(async function* (step, event) { }); }); +// The resume handle for this run: rerunning with the same ID replays +// checkpointed work instead of repeating it. +const executionId = randomUUID(); +console.log(`Execution ID ${executionId}`); + try { await runtime.run(createWorkflowRouter({ deploy }), { workflowId: "deploy", + executionId, }); } finally { runtime.close(); diff --git a/packages/cli/src/watch.ts b/packages/cli/src/watch.ts index bc2b6b2..e87e389 100644 --- a/packages/cli/src/watch.ts +++ b/packages/cli/src/watch.ts @@ -1,4 +1,5 @@ import chokidar from "chokidar"; +import { randomUUID } from "node:crypto"; import { createLoggerReconcilerSubscriber, deployApp } from "@notation/core"; import { compile } from "./compile"; import { defaultLogger, type Logger } from "./logger"; @@ -40,8 +41,11 @@ export async function watch( isDeploying = true; + const executionId = randomUUID(); + logger.info(`Execution ID ${executionId}`); deployApp({ entryPoint, + executionId, driftDetection: false, emit: createLoggerReconcilerSubscriber({ logger }), }) diff --git a/packages/core/src/provisioner/durable-runtime.ts b/packages/core/src/provisioner/durable-runtime.ts index 5c9a027..48385da 100644 --- a/packages/core/src/provisioner/durable-runtime.ts +++ b/packages/core/src/provisioner/durable-runtime.ts @@ -1,4 +1,3 @@ -import { randomUUID } from "node:crypto"; import path from "node:path"; import { setImmediate } from "node:timers/promises"; import { isDeepStrictEqual } from "node:util"; @@ -40,9 +39,15 @@ export type NodeDurableRuntimeOptions = { logger?: Logger; }; +/** + * The execution ID is required rather than defaulted: it is the handle for + * resuming a crashed execution, so the caller that starts a run must already + * hold it. Generation belongs to the outermost caller (e.g. the CLI, which + * prints the ID before any provider work). + */ export type RunWorkflowOptions = { workflowId: string; - executionId?: string; + executionId: string; }; const executionBindingStore = defineStore( @@ -93,7 +98,7 @@ export class NodeDurableRuntime { } this.#running = true; try { - const executionId = opts.executionId ?? randomUUID(); + const { executionId } = opts; const runner = new WorkflowRunner({ router, heapClient: this.#heapClient, @@ -230,7 +235,7 @@ export async function runDurableWorkflow( workflowId: string; runtime?: NodeDurableRuntime; databasePath?: string; - executionId?: string; + executionId: string; }, body: ( step: DurableStep, diff --git a/packages/core/src/provisioner/workflows/workflow.deploy.ts b/packages/core/src/provisioner/workflows/workflow.deploy.ts index a970089..cd102f8 100644 --- a/packages/core/src/provisioner/workflows/workflow.deploy.ts +++ b/packages/core/src/provisioner/workflows/workflow.deploy.ts @@ -14,14 +14,16 @@ export type DeployAppOptions = { maxOperationAttempts?: number; registry?: ResourceRegistry; runtime?: NodeDurableRuntime; - executionId?: string; + /** Required: the resume handle for this run, generated by the caller. */ + executionId: string; databasePath?: string; emit?: ReconcilerEventEmitter; }; export async function deployApp({ entryPoint, - driftDetection = true, + // Defaulted in one place: the reconciler's drift gate treats absent as on. + driftDetection, dryRun = false, maxOperationAttempts, registry, diff --git a/packages/core/src/provisioner/workflows/workflow.destroy.ts b/packages/core/src/provisioner/workflows/workflow.destroy.ts index aa31b8b..b7633c7 100644 --- a/packages/core/src/provisioner/workflows/workflow.destroy.ts +++ b/packages/core/src/provisioner/workflows/workflow.destroy.ts @@ -12,7 +12,8 @@ export type DestroyAppOptions = { maxOperationAttempts?: number; registry?: ResourceRegistry; runtime?: NodeDurableRuntime; - executionId?: string; + /** Required: the resume handle for this run, generated by the caller. */ + executionId: string; databasePath?: string; emit?: ReconcilerEventEmitter; }; diff --git a/packages/core/src/provisioner/workflows/workflow.plan.ts b/packages/core/src/provisioner/workflows/workflow.plan.ts index 057cf2a..f0ceb4e 100644 --- a/packages/core/src/provisioner/workflows/workflow.plan.ts +++ b/packages/core/src/provisioner/workflows/workflow.plan.ts @@ -20,7 +20,8 @@ export type PlanAppOptions = { export async function planApp({ entryPoint, - driftDetection = true, + // Defaulted in one place: the reconciler's drift gate treats absent as on. + driftDetection, maxOperationAttempts, runtime: suppliedRuntime, databasePath, diff --git a/packages/reconciler/src/durable/deploy.ts b/packages/reconciler/src/durable/deploy.ts index 8e5239a..fb76450 100644 --- a/packages/reconciler/src/durable/deploy.ts +++ b/packages/reconciler/src/durable/deploy.ts @@ -17,7 +17,7 @@ export async function* deploy( yield* reconcileResource( scopeStep( step, - `notation:resource:${encodeURIComponent(resource.id)}`, + `notation:deploy:${encodeURIComponent(resource.id)}`, ), resource, opts, diff --git a/packages/reconciler/src/durable/index.ts b/packages/reconciler/src/durable/index.ts index 2430ac3..5993def 100644 --- a/packages/reconciler/src/durable/index.ts +++ b/packages/reconciler/src/durable/index.ts @@ -5,8 +5,8 @@ * Step keys are persisted: a resumed execution matches its cached work by * key, so changing one re-executes the work behind it. The shapes in use are: * - * notation:resource::* per-resource reconciliation steps (deploy) - * notation:destroy::* per-resource deletion steps (destroy) + * notation:deploy::* per-resource reconciliation steps + * notation:destroy::* per-resource deletion steps * notation:orphans:list the orphan sweep's one read of persisted state * notation:orphans::* orphan sweep, per persisted record * *:remote:attempt: one provider call attempt diff --git a/packages/reconciler/src/durable/reconcile.ts b/packages/reconciler/src/durable/reconcile.ts index 0fd4d9b..62afa70 100644 --- a/packages/reconciler/src/durable/reconcile.ts +++ b/packages/reconciler/src/durable/reconcile.ts @@ -10,14 +10,14 @@ import { resolveResourceClass, } from "../resource-registry"; import { + applyDriftDetection, createResourceOperation, deleteResourceOperation, - readDriftOperation, updateResourceOperation, type PersistState, type RemoveState, } from "../operations"; -import { decideAction, decideDriftAction } from "../plan"; +import { decideAction } from "../plan"; import { durableEmitter, scopeStep, type DurableStepRunner } from "./step"; import { resourceStateStore, @@ -58,28 +58,20 @@ export async function* reconcileResource( let action = decideAction({ resource, stateNode: session.node, params }); - // A noop is only trusted once the remote has been read back: the provider - // may have drifted from persisted state, which upgrades the decision. A - // resource with no read has no remote to compare, so its noop stands. - if ( - action.decision === "noop" && - (opts.driftDetection ?? true) && - resource.read - ) { - // Its own scope: the operation that follows reads the remote again, and - // the two reads must not share step keys. - const driftStep = step.scope("drift-read"); - const driftRead = yield* readDriftOperation(driftStep, { - resource, - resourceParams: params, - persistedOutput: session.node?.output, - // No dryRun: a dry run suppresses mutations, not reads, and reading is - // how a dry run reports drift at all. - emit: durableEmitter(driftStep, opts.emit), - maxOperationAttempts: opts.maxOperationAttempts, - }); - action = decideDriftAction({ resource, params, driftRead }); - } + // Its own scope: when the gate fires, the operation that follows the drift + // read reads the remote again, and the two reads must not share step keys. + const driftStep = step.scope("drift-read"); + action = yield* applyDriftDetection(driftStep, { + action, + driftDetection: opts.driftDetection, + resource, + resourceParams: params, + persistedOutput: session.node?.output, + // No dryRun: a dry run suppresses mutations, not reads, and reading is + // how a dry run reports drift at all. + emit: durableEmitter(driftStep, opts.emit), + maxOperationAttempts: opts.maxOperationAttempts, + }); if (action.decision === "drift-update") { yield* emit({ @@ -207,7 +199,7 @@ async function* openStateSession( opts: DurableWorkflowOptions, resource: BaseResource, ): AsyncGenerator { - const snapshot = yield* step.run("state:snapshot", () => + const snapshot = yield* step.run("persisted-record", () => opts.state.snapshot(resource.id), ); const persist = persistResourceState(step, opts, resource, snapshot); diff --git a/packages/reconciler/src/operations/operation.read.ts b/packages/reconciler/src/operations/operation.read.ts index 2d246ee..72705be 100644 --- a/packages/reconciler/src/operations/operation.read.ts +++ b/packages/reconciler/src/operations/operation.read.ts @@ -1,5 +1,5 @@ import { ResourceNotFoundError } from "@notation/resource"; -import type { DriftRead } from "../plan"; +import { decideDriftAction, type DriftRead, type ResourceAction } from "../plan"; import { type ResolvedResourceParams, type StepRunner, @@ -69,3 +69,34 @@ export async function* readDriftOperation( throw error; } } + +/** + * The drift gate, shared by every driver: a noop is only trusted once the + * remote has been read back, because the provider may have drifted from + * persisted state, which upgrades the decision. A resource with no read has + * no remote to compare, so its noop stands. `driftDetection` defaults to on + * here and nowhere else. Any other decision passes through untouched. + */ +export async function* applyDriftDetection( + step: StepRunner, + params: ResolvedResourceParams & { + action: ResourceAction; + driftDetection?: boolean; + }, +): AsyncGenerator { + const { action, driftDetection, ...readParams } = params; + if ( + action.decision !== "noop" || + !(driftDetection ?? true) || + !readParams.resource.read + ) { + return action; + } + + const driftRead = yield* readDriftOperation(step, readParams); + return decideDriftAction({ + resource: readParams.resource, + params: readParams.resourceParams, + driftRead, + }); +} diff --git a/packages/reconciler/src/operations/operation.types.ts b/packages/reconciler/src/operations/operation.types.ts index e87d1f5..9f9c8ef 100644 --- a/packages/reconciler/src/operations/operation.types.ts +++ b/packages/reconciler/src/operations/operation.types.ts @@ -13,8 +13,6 @@ export type { OperationName, } from "../events"; -export type OperationEventEmitter = EmitStep; - /** * How an operation runs a step. Keys identify a step's cached result across a * replay; `scope` namespaces them so one operation can run at several call @@ -66,7 +64,7 @@ export type ResourceOperationBaseParams = { dryRun?: boolean; /** Always present: an absent emitter is absorbed where the step is made * (`toEmitStep`, `durableEmitter`), not guarded here. */ - emit: OperationEventEmitter; + emit: EmitStep; maxOperationAttempts?: number; }; diff --git a/packages/reconciler/src/planner.ts b/packages/reconciler/src/planner.ts index c032c50..441d249 100644 --- a/packages/reconciler/src/planner.ts +++ b/packages/reconciler/src/planner.ts @@ -2,10 +2,9 @@ import type { BaseResource } from "@notation/resource"; import type { StateBackend } from "@notation/state"; import { buildResourceDepthLevels } from "./dependency-graph"; import { toEmitStep, type ReconcilerEventEmitter } from "./events"; -import { readDriftOperation } from "./operations"; +import { applyDriftDetection } from "./operations"; import { decideAction, - decideDriftAction, getDependencyIds, resolvePlanParams, type Plan, @@ -24,7 +23,7 @@ export type CreatePlanOptions = { export async function createPlan({ resources, state, - driftDetection = true, + driftDetection, emit, maxOperationAttempts, }: CreatePlanOptions): Promise { @@ -39,21 +38,17 @@ export async function createPlan({ const stateNode = await state.get(resource.id); if (stateNode) resource.setOutput(stateNode.output); const params = await resolvePlanParams(resource); - let action = decideAction({ resource, stateNode, params }); - - if (action.decision === "noop" && driftDetection && resource.read) { - const driftRead = await runOperation( - readDriftOperation(createStepRunner(), { - resource, - resourceParams: params, - persistedOutput: stateNode?.output, - emit: emitStep, - maxOperationAttempts, - }), - ); - - action = decideDriftAction({ resource, params, driftRead }); - } + const action = await runOperation( + applyDriftDetection(createStepRunner(), { + action: decideAction({ resource, stateNode, params }), + driftDetection, + resource, + resourceParams: params, + persistedOutput: stateNode?.output, + emit: emitStep, + maxOperationAttempts, + }), + ); nodes.push({ id: resource.id, diff --git a/packages/reconciler/test/durable-reconciliation.test.ts b/packages/reconciler/test/durable-reconciliation.test.ts index 330ba6b..64ecba1 100644 --- a/packages/reconciler/test/durable-reconciliation.test.ts +++ b/packages/reconciler/test/durable-reconciliation.test.ts @@ -84,7 +84,7 @@ describe("durable execution and replay", () => { const runtime = createRuntime( [new TestResource({ id: "resume" })], "crash-resume", - { crashAfterStep: "notation:resource:resume:create:remote:attempt:0" }, + { crashAfterStep: "notation:deploy:resume:create:remote:attempt:0" }, ); await expect(runtime.run("resume-execution")).rejects.toThrow( From 42b4314c108d8aa334122cace0473d8153c3c33c Mon Sep 17 00:00:00 2001 From: djgrant <1670902+djgrant@users.noreply.github.com> Date: Tue, 28 Jul 2026 20:04:30 +0100 Subject: [PATCH 34/34] Name the hold clearance for what it does, and make delete absent-tolerant - Rename takeOverDeploymentHold to clearDeploymentHold (DeploymentHoldTakeover to DeploymentHoldClearance, taken to cleared): the operation clears a hold, it does not transfer one, and "take" already means acquire in this file. - deleteResourceOperation now treats ResourceNotFoundError from a delete handler as success: absence is delete's goal state, which makes crash-window replayed deletes idempotent as the docs already claimed. Test added. - docs/cli/deploy.md: the compiled output is imported in-process by the CLI; the worker does not exist yet. --- docs/cli/deploy.md | 2 +- docs/internals/reconciler.md | 2 +- docs/internals/resource.md | 4 +-- docs/manual/reconciler.md | 2 +- .../reconciler/src/durable/deployment-hold.ts | 22 ++++++------- packages/reconciler/src/durable/index.ts | 4 +-- .../src/operations/operation.delete.ts | 30 +++++++++++------- .../test/durable-reconciliation.test.ts | 10 +++--- .../test/operation.workflows.test.ts | 31 +++++++++++++++++++ 9 files changed, 73 insertions(+), 34 deletions(-) diff --git a/docs/cli/deploy.md b/docs/cli/deploy.md index 6af74db..8020225 100644 --- a/docs/cli/deploy.md +++ b/docs/cli/deploy.md @@ -34,7 +34,7 @@ Retryable provider conditions and consistency reads suspend on durable SQLite ti 1. **Compile** – esbuild compiles infrastructure and runtime modules to `dist/`. -2. **Build resource graph** – the worker imports the compiled output and collects declared resources. +2. **Build resource graph** – the CLI imports the compiled output in-process and collects declared resources. 3. **Order dependencies** – dependency levels run in topological order. diff --git a/docs/internals/reconciler.md b/docs/internals/reconciler.md index 5ed071b..2d3b1ca 100644 --- a/docs/internals/reconciler.md +++ b/docs/internals/reconciler.md @@ -35,7 +35,7 @@ Each resource is stored under `notation/resource-state` with a deployment-scoped Deploy and destroy take an exclusive hold on the deployment through one `notation/deployment-hold` store per deployment. `store.take` suspends a competing execution as a durable waiter and wakes it after the holder releases. A waiter that finds the hold already taken when it inspects it emits `reconciler.hold.waiting` naming the holding execution ID, so a wait behind a crashed execution is visible instead of silent; a holder that appears only between that inspection and the `take` suspends the waiter without the event. -A failed or suspended execution keeps its hold, which is what makes resuming it safe. The hold of an execution that will never be resumed is cleared with `takeOverDeploymentHold` from `@notation/reconciler/durable` — the only supported way out of that state. +A failed or suspended execution keeps its hold, which is what makes resuming it safe. The hold of an execution that will never be resumed is cleared with `clearDeploymentHold` from `@notation/reconciler/durable` — the only supported way out of that state. ## Events diff --git a/docs/internals/resource.md b/docs/internals/resource.md index fff3520..dc2a26f 100644 --- a/docs/internals/resource.md +++ b/docs/internals/resource.md @@ -173,11 +173,11 @@ new ResourceOperationPendingError(message: string, { | Handler result | Meaning | What the reconciler does | | -------------- | ------- | ------------------------ | | Return normally | The operation finished. | Continues the deployment. | -| `throw new ResourceNotFoundError(message, { cause })` | `read` found no resource for the given key. | Treats the resource as absent during planning and drift detection. A read after create or update fails because that operation claimed to have finished. | +| `throw new ResourceNotFoundError(message, { cause })` | `read` or `delete` found no resource for the given key. | From `read`, treats the resource as absent during planning and drift detection; a read after create or update fails because that operation claimed to have finished. From `delete`, treats the delete as complete, since absence is its goal state. | | `throw new ResourceOperationPendingError(message, { retryAfterMs, callbackContext })` | The operation has not finished. | Waits for `retryAfterMs`, then calls the same handler again. It passes `callbackContext` as the handler's final argument. | | Throw any other error | The operation failed. | Stops the deployment. | -`ResourceNotFoundError` is for `read`. A `delete` handler must catch the provider's missing-resource error and return normally. +`ResourceNotFoundError` means the resource is absent wherever it is thrown. A `delete` handler may either catch the provider's missing-resource error and return normally, or translate it to `ResourceNotFoundError`; both count as success. `ResourceOperationPendingError` may be thrown by `create`, `read`, `update`, or `delete`. Its options are: diff --git a/docs/manual/reconciler.md b/docs/manual/reconciler.md index e851df4..b9ca129 100644 --- a/docs/manual/reconciler.md +++ b/docs/manual/reconciler.md @@ -36,7 +36,7 @@ The outer workflow supplies durable step execution, timers, shared stores, waiti Each live resource is one Yieldstar store. Yieldstar's UUIDv7 store `instanceId` and version are authoritative for conditional update and delete; Notation exposes the version unchanged as the resource state's `version`. -Operations against the same deployment — the `deploymentId` the `DurableStateBackend` is constructed with — are serialized through a deployment hold naming the holding `executionId`. Resume a crashed operation with the same execution ID; use a new globally unique execution ID for every new deploy or destroy. An execution that must wait emits a `reconciler.hold.waiting` event naming the holder before it suspends, which also identifies a crashed holder that should be resumed instead. If the holder is genuinely abandoned, clear its hold with `takeOverDeploymentHold`. +Operations against the same deployment — the `deploymentId` the `DurableStateBackend` is constructed with — are serialized through a deployment hold naming the holding `executionId`. Resume a crashed operation with the same execution ID; use a new globally unique execution ID for every new deploy or destroy. An execution that must wait emits a `reconciler.hold.waiting` event naming the holder before it suspends, which also identifies a crashed holder that should be resumed instead. If the holder is genuinely abandoned, clear its hold with `clearDeploymentHold`. Pass the complete desired set on every deployment. Persisted resources absent from that set are deleted through the supplied resource registry. Destroy removes current resources in reverse dependency order and then removes any persisted orphans whose resource type is registered. diff --git a/packages/reconciler/src/durable/deployment-hold.ts b/packages/reconciler/src/durable/deployment-hold.ts index 126a61a..770f6e4 100644 --- a/packages/reconciler/src/durable/deployment-hold.ts +++ b/packages/reconciler/src/durable/deployment-hold.ts @@ -69,7 +69,7 @@ function releaseDeploymentHold( * resumed execution replays `take` from the step cache without re-acquiring * anything, so it must still be the holder. An execution that will never be * resumed holds its deployment until an operator calls - * `takeOverDeploymentHold`. + * `clearDeploymentHold`. */ export async function* withDeploymentHold( step: DurableStep, @@ -81,9 +81,9 @@ export async function* withDeploymentHold( yield* releaseDeploymentHold(hold, opts.executionId); } -export type DeploymentHoldTakeover = - | { taken: true; previousHolder: string } - | { taken: false; holder: string | null }; +export type DeploymentHoldClearance = + | { cleared: true; previousHolder: string } + | { cleared: false; holder: string | null }; /** * Clears the hold of an execution that will not be resumed, so later @@ -91,16 +91,16 @@ export type DeploymentHoldTakeover = * * The write is conditional on `fromExecutionId` still being the named holder, * so it cannot clear a hold that has since moved to another execution. - * Confirm the holder is genuinely dead first: taking a live execution's hold - * away permits a concurrent mutation of the same deployment. + * Confirm the holder is genuinely dead first: clearing a live execution's + * hold permits a concurrent mutation of the same deployment. * * Throws if the deployment has no hold store, i.e. has never been deployed. */ -export async function takeOverDeploymentHold(params: { +export async function clearDeploymentHold(params: { storeClient: StoreClient; deploymentId: string; fromExecutionId: string; -}): Promise { +}): Promise { const { storeClient, deploymentId, fromExecutionId } = params; const read = () => storeClient.getStore({ @@ -110,7 +110,7 @@ export async function takeOverDeploymentHold(params: { const snapshot = await read(); if (snapshot.state.holder !== fromExecutionId) { - return { taken: false, holder: snapshot.state.holder }; + return { cleared: false, holder: snapshot.state.holder }; } const result = await storeClient.updateStoreFrom({ @@ -123,8 +123,8 @@ export async function takeOverDeploymentHold(params: { }); if (!result.updated) { - return { taken: false, holder: (await read()).state.holder }; + return { cleared: false, holder: (await read()).state.holder }; } - return { taken: true, previousHolder: fromExecutionId }; + return { cleared: true, previousHolder: fromExecutionId }; } diff --git a/packages/reconciler/src/durable/index.ts b/packages/reconciler/src/durable/index.ts index 5993def..cd46d8a 100644 --- a/packages/reconciler/src/durable/index.ts +++ b/packages/reconciler/src/durable/index.ts @@ -34,8 +34,8 @@ export { deploy } from "./deploy"; export { destroy } from "./destroy"; export { - takeOverDeploymentHold, - type DeploymentHoldTakeover, + clearDeploymentHold, + type DeploymentHoldClearance, } from "./deployment-hold"; export { DurableStateBackend } from "./state-backend"; export { deploymentHoldStore, resourceStateStore } from "./stores"; diff --git a/packages/reconciler/src/operations/operation.delete.ts b/packages/reconciler/src/operations/operation.delete.ts index 3bb4e8b..44f6558 100644 --- a/packages/reconciler/src/operations/operation.delete.ts +++ b/packages/reconciler/src/operations/operation.delete.ts @@ -1,3 +1,4 @@ +import { ResourceNotFoundError } from "@notation/resource"; import { type DeleteResourceParams, type StepRunner, @@ -18,17 +19,24 @@ export async function* deleteResourceOperation( } try { - yield* runPendingOperation( - step, - "delete:remote", - (context) => - params.resource.delete( - params.resource.key, - params.resource.toState(params.resource.output), - context, - ), - params.maxOperationAttempts, - ); + try { + yield* runPendingOperation( + step, + "delete:remote", + (context) => + params.resource.delete( + params.resource.key, + params.resource.toState(params.resource.output), + context, + ), + params.maxOperationAttempts, + ); + } catch (error) { + // Absence is delete's goal state, so a delete that finds the resource + // already gone — a crash-window replay, or an out-of-band removal — + // has succeeded. + if (!ResourceNotFoundError.is(error)) throw error; + } yield* params.remove(); diff --git a/packages/reconciler/test/durable-reconciliation.test.ts b/packages/reconciler/test/durable-reconciliation.test.ts index 64ecba1..9c482d9 100644 --- a/packages/reconciler/test/durable-reconciliation.test.ts +++ b/packages/reconciler/test/durable-reconciliation.test.ts @@ -463,7 +463,7 @@ describe("deployment hold", () => { }); }); -describe("deployment hold takeover", () => { +describe("deployment hold clearing", () => { it("clears a hold its named holder still has, and unblocks the deployment", async () => { const create = vi.fn(async () => undefined); const Resource = resource({ type: "test/durable/takeover" }) @@ -476,14 +476,14 @@ describe("deployment hold takeover", () => { initial: { holder: "abandoned-execution" }, }); - const result = await durable.takeOverDeploymentHold({ + const result = await durable.clearDeploymentHold({ storeClient: runtime.storeClient, deploymentId: "takeover", fromExecutionId: "abandoned-execution", }); expect(result).toEqual({ - taken: true, + cleared: true, previousHolder: "abandoned-execution", }); await runtime.run("later-execution"); @@ -499,13 +499,13 @@ describe("deployment hold takeover", () => { initial: { holder: "current-execution" }, }); - const result = await durable.takeOverDeploymentHold({ + const result = await durable.clearDeploymentHold({ storeClient: runtime.storeClient, deploymentId: "takeover-race", fromExecutionId: "abandoned-execution", }); - expect(result).toEqual({ taken: false, holder: "current-execution" }); + expect(result).toEqual({ cleared: false, holder: "current-execution" }); const snapshot = await runtime.storeClient.getStore({ definition: durable.deploymentHoldStore, id: "takeover-race", diff --git a/packages/reconciler/test/operation.workflows.test.ts b/packages/reconciler/test/operation.workflows.test.ts index d04ba14..fafb595 100644 --- a/packages/reconciler/test/operation.workflows.test.ts +++ b/packages/reconciler/test/operation.workflows.test.ts @@ -244,6 +244,37 @@ describe("operation workflows", () => { expect(events.map((event) => event.status)).toEqual(["start", "success"]); }); + it("delete treats ResourceNotFoundError as the resource already being absent", async () => { + const step = createStepRunnerDouble(); + const events: OperationLifecycleEvent[] = []; + const remove = vi.fn(async function* () {}); + + const TestResource = resource({ type: "test/service/delete-absent" }) + .defineSchema({}) + .defineOperations({ + create: (async () => ({})) as any, + delete: async () => { + throw new ResourceNotFoundError("resource is already gone"); + }, + }); + + const testResource = new TestResource({ id: "test-delete-absent" }); + + await runOperation( + deleteResourceOperation(step, { + resource: testResource, + remove, + emit: toEmitStep((event) => void events.push(event)), + }), + ); + + // Absence is delete's goal state: state is removed and the operation + // reports success, which is what makes a crash-window replayed delete + // idempotent even when the handler surfaces the provider's missing error. + expect(remove).toHaveBeenCalledOnce(); + expect(events.map((event) => event.status)).toEqual(["start", "success"]); + }); + it("delete rethrows an unclassified resource error", async () => { const step = createStepRunnerDouble(); const remove = vi.fn(async function* () {});