From 0a23b079ab453275acdfd5cbff195c7e25ae963d Mon Sep 17 00:00:00 2001 From: djgrant <1670902+djgrant@users.noreply.github.com> Date: Wed, 22 Jul 2026 07:59:56 +0100 Subject: [PATCH 1/2] Harden durable YieldStar reconciliation - Emit reconciler.drift.detected as a durable step before a drift-update decision, restoring event parity with the synchronous reconciler. - Emit a durable update skip lifecycle event when a resource has no update operation instead of returning silently. - Remove an unreachable dryRun check after the provider mutation. - Read store snapshots in one round trip in YieldStarStateBackend and treat a store deleted mid-read as resource absence, instead of listing every store before each get, update, and delete. - Return the committed store version from YieldStarStateBackend.update rather than a locally computed revision. - Test orphan deletion through the registry and drift repair with events on the durable path. - Document resuming a crashed deployment with the same execution ID to release deployment coordination through replay. --- docs/manual/reconciler.md | 2 + packages/reconciler/src/yieldstar.ts | 94 ++++++++++-------- .../test/yieldstar.integration.test.ts | 95 +++++++++++++++++-- 3 files changed, 145 insertions(+), 46 deletions(-) diff --git a/docs/manual/reconciler.md b/docs/manual/reconciler.md index 2dd96fb..cf10677 100644 --- a/docs/manual/reconciler.md +++ b/docs/manual/reconciler.md @@ -29,6 +29,8 @@ The outer workflow supplies durable step execution, timers, shared stores, waiti Each live resource is one YieldStar store. Absence is represented by no store, not a tombstone. YieldStar's UUIDv7 store `instanceId` and version are authoritative for conditional update and delete; Notation exposes the version as the resource state's `rev` for its existing state contract. +Deployments against the same `deploymentId` are serialized through a coordination store keyed by `executionId`. If a deployment crashes while holding the coordination store, resume it by running the same execution ID again: replay reclaims the acquisition through YieldStar's applied-step ledger and releases it on completion. A different execution ID waits durably until the holder releases. + Pass the complete desired set on every invocation. Persisted resources absent from that set are deleted through the supplied resource registry. The runnable Node SQLite version is in `examples/reconciler`. diff --git a/packages/reconciler/src/yieldstar.ts b/packages/reconciler/src/yieldstar.ts index 24375ba..bf59769 100644 --- a/packages/reconciler/src/yieldstar.ts +++ b/packages/reconciler/src/yieldstar.ts @@ -177,13 +177,21 @@ async function* reconcileResource( resource, stateNode: stateNode ?? undefined, params, - driftRead: - remote.status === "not-found" - ? remote - : { status: "found", output: remote.output }, + driftRead: remote, }); } + if (action.decision === "drift-update") { + const diff = action.patch; + yield* emitDurably(step, `${prefix}:drift-detected`, opts.emit, () => ({ + level: "info", + event: "reconciler.drift.detected", + resourceId: resource.id, + resourceType: resource.type, + diff, + })); + } + yield* emitDurably(step, `${prefix}:decision`, opts.emit, () => ({ level: "info", event: "reconciler.deploy.decision", @@ -206,7 +214,18 @@ async function* reconcileResource( resource.setOutput(params); if (primaryKey) resource.setOutput({ ...primaryKey, ...resource.output }); } else { - if (!resource.update) return; + if (!resource.update) { + yield* emitDurably(step, `${prefix}:update-skip`, opts.emit, () => ({ + level: "info", + event: "reconciler.operation.lifecycle", + operation: "update", + status: "skip", + resourceId: resource.id, + resourceType: resource.type, + reason: "update-not-implemented", + })); + return; + } yield* runProviderCall( step, `${prefix}:update`, @@ -231,7 +250,6 @@ async function* reconcileResource( ); if (read.status === "found") resource.setOutput({ ...resource.output, ...read.output }); - if (opts.dryRun) return; const operation = action.decision === "create" || action.decision === "drift-recreate" @@ -406,15 +424,31 @@ export class YieldStarStateBackend { } async get(id: string): Promise { - const storeId = this.storeId(id); - const ids = await this.#client.listStores(yieldStarResourceStateStore); - if (!ids.includes(storeId)) return undefined; - return toStateNode( - await this.#client.getStore({ + const snapshot = await this.#tryGetSnapshot(this.storeId(id)); + return snapshot ? toStateNode(snapshot) : undefined; + } + + /** + * Reads a store snapshot in one round trip. A missing store is resource + * absence, so a read failure is re-checked against the store listing before + * it is allowed to propagate. + */ + async #tryGetSnapshot( + storeId: string, + ): Promise< + | { state: StoredResourceState; instanceId: string; version: number } + | undefined + > { + try { + return await this.#client.getStore({ definition: yieldStarResourceStateStore, id: storeId, - }), - ); + }); + } catch (error) { + const ids = await this.#client.listStores(yieldStarResourceStateStore); + if (!ids.includes(storeId)) return undefined; + throw error; + } } async has(id: string): Promise { @@ -427,8 +461,8 @@ export class YieldStarStateBackend { patch: Partial, ): Promise<{ rev: number }> { const storeId = this.storeId(id); - const ids = await this.#client.listStores(yieldStarResourceStateStore); - if (!ids.includes(storeId)) { + const snapshot = await this.#tryGetSnapshot(storeId); + if (!snapshot) { if (expectedRev !== 0) throw new RevConflict(id, expectedRev, undefined); const initial = { ...patch, id } as StoredResourceState; const created = await this.#client.getOrCreateStore({ @@ -439,14 +473,9 @@ export class YieldStarStateBackend { return { rev: created.version + 1 }; } - const snapshot = await this.#client.getStore({ - definition: yieldStarResourceStateStore, - id: storeId, - }); const actualRev = snapshot.version + 1; if (actualRev !== expectedRev) throw new RevConflict(id, expectedRev, actualRev); - const rev = actualRev + 1; const result = await this.#client.updateStoreFrom({ definition: yieldStarResourceStateStore, id: storeId, @@ -456,20 +485,16 @@ export class YieldStarStateBackend { }, }); if (!result.updated) throw new RevConflict(id, expectedRev, undefined); - return { rev }; + return { rev: result.version + 1 }; } async delete(id: string, expectedRev: number): Promise { const storeId = this.storeId(id); - const ids = await this.#client.listStores(yieldStarResourceStateStore); - if (!ids.includes(storeId)) { + const snapshot = await this.#tryGetSnapshot(storeId); + if (!snapshot) { if (expectedRev !== 0) throw new RevConflict(id, expectedRev, undefined); return; } - const snapshot = await this.#client.getStore({ - definition: yieldStarResourceStateStore, - id: storeId, - }); const actualRev = snapshot.version + 1; if (actualRev !== expectedRev) throw new RevConflict(id, expectedRev, actualRev); @@ -484,19 +509,14 @@ export class YieldStarStateBackend { async values(): Promise { const prefix = `${this.#deploymentId}:`; const ids = await this.#client.listStores(yieldStarResourceStateStore); - const nodes = await Promise.all( + const snapshots = await Promise.all( ids .filter((id) => id.startsWith(prefix)) - .map(async (id) => - toStateNode( - await this.#client.getStore({ - definition: yieldStarResourceStateStore, - id, - }), - ), - ), + .map((id) => this.#tryGetSnapshot(id)), ); - return nodes; + return snapshots + .filter((snapshot) => snapshot !== undefined) + .map(toStateNode); } snapshot(id: string) { diff --git a/packages/reconciler/test/yieldstar.integration.test.ts b/packages/reconciler/test/yieldstar.integration.test.ts index 0ddbc5d..5e1b570 100644 --- a/packages/reconciler/test/yieldstar.integration.test.ts +++ b/packages/reconciler/test/yieldstar.integration.test.ts @@ -17,6 +17,11 @@ import { reconcileWithYieldStar, yieldStarResourceStateStore, } from "../src/yieldstar"; +import type { ReconcilerEvent } from "../src/reconciler"; +import { + createResourceRegistry, + type ResourceRegistry, +} from "../src/resource-registry"; const logger = pino({ level: "silent" }); @@ -42,7 +47,7 @@ describe("YieldStar reconciliation", () => { const runtime = createRuntime( [new PendingResource({ id: "pending" })], "durable-wait", - { maxAttempts: 3, retryInterval: 1 }, + { retryOptions: { maxAttempts: 3, retryInterval: 1 } }, ); await runtime.run("wait-execution"); @@ -67,8 +72,7 @@ describe("YieldStar reconciliation", () => { const runtime = createRuntime( [new TestResource({ id: "resume" })], "crash-resume", - undefined, - "notation:resource:resume:create", + { crashAfterStep: "notation:resource:resume:create" }, ); await expect(runtime.run("resume-execution")).rejects.toThrow( @@ -160,19 +164,90 @@ describe("YieldStar reconciliation", () => { expect(await runtime.state.values()).toHaveLength(1); runtime.close(); }); + + it("deletes orphaned resources through the registry on a later deployment", async () => { + const deleteSpy = vi.fn(async () => undefined); + const OrphanResource = resource({ type: "test/yieldstar/orphan" }) + .defineSchema({}) + .defineOperations({ create: async () => undefined, delete: deleteSpy }); + const resources: BaseResource[] = [new OrphanResource({ id: "orphan" })]; + const runtime = createRuntime(resources, "orphan-deletion", { + registry: createResourceRegistry([OrphanResource]), + }); + + await runtime.run("deploy-1"); + expect(await runtime.state.values()).toHaveLength(1); + + resources.length = 0; + await runtime.run("deploy-2"); + + expect(deleteSpy).toHaveBeenCalledOnce(); + expect(await runtime.state.values()).toHaveLength(0); + expect(await runtime.state.get("orphan")).toBeUndefined(); + runtime.close(); + }); + + it("emits drift detection and repairs remote drift with update", async () => { + let remote = { name: "expected" }; + const updateSpy = vi.fn(async () => { + remote = { name: "expected" }; + }); + const DriftResource = resource({ type: "test/yieldstar/drift" }) + .defineSchema({ + name: { + presence: "required", + propertyType: "param", + valueType: "string" as any, + }, + }) + .defineOperations({ + create: async () => remote, + read: async () => remote, + update: updateSpy, + delete: async () => undefined, + }); + const events: ReconcilerEvent[] = []; + const runtime = createRuntime( + [new DriftResource({ id: "drifted", config: { name: "expected" } })], + "drift-repair", + { driftDetection: true, emit: (event) => void events.push(event) }, + ); + + await runtime.run("deploy-1"); + remote = { name: "drifted" }; + await runtime.run("deploy-2"); + + expect(updateSpy).toHaveBeenCalledOnce(); + expect( + events.find((event) => event.event === "reconciler.drift.detected"), + ).toMatchObject({ resourceId: "drifted", diff: { name: "expected" } }); + expect( + events.filter( + (event) => + event.event === "reconciler.deploy.decision" && + event.decision === "drift-update", + ), + ).toHaveLength(1); + runtime.close(); + }); }); function createRuntime( resources: BaseResource[], deploymentId: string, - retryOptions?: { maxAttempts: number; retryInterval: number }, - crashAfterStep?: string, + options: { + retryOptions?: { maxAttempts: number; retryInterval: number }; + crashAfterStep?: string; + registry?: ResourceRegistry; + driftDetection?: boolean; + emit?: (event: ReconcilerEvent) => void; + } = {}, ) { const database = createSqliteDb({ path: ":memory:" }); const scheduler = new TestScheduler(); const sqliteHeap = new SqliteHeapClient(database); - const heap = crashAfterStep - ? new CrashAfterWriteHeap(sqliteHeap, crashAfterStep) + const heap = options.crashAfterStep + ? new CrashAfterWriteHeap(sqliteHeap, options.crashAfterStep) : sqliteHeap; const storeClient = new SqliteStoreClient({ db: database, @@ -185,8 +260,10 @@ function createRuntime( executionId: event.executionId, resources, state, - driftDetection: false, - retryOptions, + registry: options.registry, + driftDetection: options.driftDetection ?? false, + emit: options.emit, + retryOptions: options.retryOptions, }); }); const router = createWorkflowRouter({ deploy }); From c193eecafd7f642a7fc65d5d9480abfa5d7b9ac1 Mon Sep 17 00:00:00 2001 From: djgrant <1670902+djgrant@users.noreply.github.com> Date: Wed, 22 Jul 2026 08:26:15 +0100 Subject: [PATCH 2/2] Run CLI workflows on YieldStar --- .changeset/reconciler.md | 2 +- docs/cli/dashboard.md | 16 +- docs/cli/deploy.md | 38 +- docs/cli/destroy.md | 10 +- docs/internals/reconciler.md | 128 +-- docs/internals/state.md | 130 +-- docs/manual/reconciler.md | 25 +- docs/rfcs/reconciler.md | 22 +- examples/reconciler/README.md | 2 +- examples/reconciler/src/index.ts | 4 +- packages/cli/src/deploy.ts | 5 + packages/cli/src/destroy.ts | 6 +- packages/cli/src/index.ts | 20 +- packages/cli/src/plan.ts | 10 +- packages/core/package.json | 8 +- packages/core/src/provisioner/index.ts | 2 +- .../core/src/provisioner/state-backend.ts | 16 - .../core/src/provisioner/workflows/index.ts | 1 - .../provisioner/workflows/workflow.deploy.ts | 46 +- .../provisioner/workflows/workflow.destroy.ts | 43 +- .../provisioner/workflows/workflow.plan.ts | 41 +- .../provisioner/workflows/workflow.refresh.ts | 39 - .../core/src/provisioner/yieldstar-runtime.ts | 145 ++++ .../test/provisioner/operation.create.test.ts | 50 -- .../test/provisioner/state-backend.test.ts | 59 -- .../provisioner/yieldstar-runtime.test.ts | 60 ++ packages/reconciler/src/events.ts | 44 + packages/reconciler/src/index.ts | 5 +- packages/reconciler/src/logger-subscriber.ts | 2 +- packages/reconciler/src/operation-support.ts | 71 ++ packages/reconciler/src/operations/index.ts | 5 - .../src/operations/operation.create.ts | 90 -- .../src/operations/operation.delete.ts | 69 -- .../src/operations/operation.read.ts | 101 --- .../src/operations/operation.types.ts | 143 --- .../src/operations/operation.update.ts | 100 --- packages/reconciler/src/planner.ts | 81 ++ packages/reconciler/src/protocol.ts | 2 +- packages/reconciler/src/reconciler.ts | 642 -------------- packages/reconciler/src/resource-registry.ts | 10 +- packages/reconciler/src/yieldstar.ts | 405 +++++++-- .../test/operation.workflows.test.ts | 315 ------- packages/reconciler/test/planner.test.ts | 36 + .../reconciler/test/reconciler.deploy.test.ts | 811 ------------------ .../reconciler/test/reconciler.plan.test.ts | 402 --------- .../test/yieldstar.integration.test.ts | 118 ++- packages/state-sqlite/src/index.ts | 96 +-- .../state-sqlite/test/state-sqlite.test.ts | 27 - packages/state/src/conflicts.ts | 11 - packages/state/src/state.ts | 130 +-- packages/state/test/state-backend.test.ts | 21 - pnpm-lock.yaml | 18 +- pnpm-workspace.yaml | 4 +- 53 files changed, 1103 insertions(+), 3584 deletions(-) delete mode 100644 packages/core/src/provisioner/state-backend.ts delete mode 100644 packages/core/src/provisioner/workflows/workflow.refresh.ts create mode 100644 packages/core/src/provisioner/yieldstar-runtime.ts delete mode 100644 packages/core/test/provisioner/operation.create.test.ts delete mode 100644 packages/core/test/provisioner/state-backend.test.ts create mode 100644 packages/core/test/provisioner/yieldstar-runtime.test.ts create mode 100644 packages/reconciler/src/events.ts create mode 100644 packages/reconciler/src/operation-support.ts delete mode 100644 packages/reconciler/src/operations/index.ts delete mode 100644 packages/reconciler/src/operations/operation.create.ts delete mode 100644 packages/reconciler/src/operations/operation.delete.ts delete mode 100644 packages/reconciler/src/operations/operation.read.ts delete mode 100644 packages/reconciler/src/operations/operation.types.ts delete mode 100644 packages/reconciler/src/operations/operation.update.ts create mode 100644 packages/reconciler/src/planner.ts delete mode 100644 packages/reconciler/src/reconciler.ts delete mode 100644 packages/reconciler/test/operation.workflows.test.ts create mode 100644 packages/reconciler/test/planner.test.ts delete mode 100644 packages/reconciler/test/reconciler.deploy.test.ts delete mode 100644 packages/reconciler/test/reconciler.plan.test.ts diff --git a/.changeset/reconciler.md b/.changeset/reconciler.md index 3ddd372..98300a3 100644 --- a/.changeset/reconciler.md +++ b/.changeset/reconciler.md @@ -9,4 +9,4 @@ "@notation/state-sqlite": minor --- -Add the reconciler API, versioned event streams, renewable mutation leases, SQLite state, backend-neutral dashboard state, compiled infrastructure graphs, and durable YieldStar 0.5.0 reconciliation for Node.js runtimes. +Add durable YieldStar 0.5.0 deploy and destroy workflows, a resident Node SQLite runtime for CLI execution, versioned event streams, backend-neutral dashboard state, and compiled infrastructure graphs. diff --git a/docs/cli/dashboard.md b/docs/cli/dashboard.md index f630ad7..d30ab74 100644 --- a/docs/cli/dashboard.md +++ b/docs/cli/dashboard.md @@ -1,21 +1,13 @@ # notation dashboard ```sh -notation dashboard +notation dashboard ``` -Starts a local web dashboard for observing deployment state. +Starts a local web dashboard for observing the deployment's YieldStar resource stores. ```sh -notation dashboard +notation dashboard infra/api.ts ``` -The dashboard uses the same state backend as deploy and destroy. Set -`NOTATION_STATE_PATH` to select SQLite: - -```sh -NOTATION_STATE_PATH=.notation/state.db notation dashboard -``` - -The server reads through `StateBackend`, so file and SQLite state produce the same -dashboard payload. +The dashboard reads `.notation/workflows.db`, the same database used by deploy, destroy, and plan. Set `NOTATION_STATE_PATH` to choose another SQLite database path. diff --git a/docs/cli/deploy.md b/docs/cli/deploy.md index 940cf7e..97b8d70 100644 --- a/docs/cli/deploy.md +++ b/docs/cli/deploy.md @@ -4,7 +4,7 @@ notation deploy ``` -Compiles and deploys the stack to AWS. +Compiles and durably deploys the stack through the resident YieldStar 0.5.0 Node runtime. ```sh notation deploy infra/api.ts @@ -12,32 +12,36 @@ notation deploy infra/api.ts ## Event stream -`--json` writes versioned reconciler events to stdout as newline-delimited JSON. Build -output and diagnostics move to stderr. +`--json` writes versioned reconciler events to stdout as newline-delimited JSON. Build output, the execution ID, and diagnostics move to stderr. ```sh notation deploy infra/api.ts --json > deploy.ndjson ``` +## Durable execution + +The command prints its YieldStar execution ID before starting provider work. If the process crashes, resume the same durable heap with that ID: + +```sh +notation deploy infra/api.ts --execution-id +``` + +Do not reuse a completed execution ID for a new deploy or for destroy. + +Retryable provider conditions and consistency reads suspend on durable SQLite timers. The CLI stays resident until the scheduler wakes the execution and the workflow completes; completed provider calls are replayed from the heap rather than repeated. + ## What happens -1. **Compile** – esbuild compiles infra and runtime modules to `dist/`. +1. **Compile** – esbuild compiles infrastructure and runtime modules to `dist/`. -2. **Build resource graph** – imports the compiled output and collects the declared resources. +2. **Build resource graph** – the worker imports the compiled output and collects declared resources. -3. **Reconcile** – the reconciler compares desired state (graph) against current state (`.notation/state.json`): - - New resources → **create** - - Changed params → **update** - - No changes → **noop** - - Orphaned resources (in state but not in graph) → **delete** +3. **Reconcile** – Notation compares desired resources with YieldStar stores, then creates, updates, recreates, or leaves each resource unchanged. -4. **Topological deployment** – resources deploy in dependency order (levels). Resources at the same level deploy concurrently. +4. **Order dependencies** – dependency levels run in topological order. -5. **Drift detection** – enabled by default. Reads actual AWS state and compares against stored state. If drifted, Notation updates to match your definition. +5. **Detect drift** – unchanged resources are read from the provider and repaired when their remote state differs. -State is persisted to `.notation/state.json` after each operation. Set -`NOTATION_STATE_PATH` to a path ending in `.db` or `.sqlite` to use SQLite: +6. **Delete orphans** – persisted resources absent from the graph are deleted when their resource type is registered. -```sh -NOTATION_STATE_PATH=.notation/state.db notation deploy infra/api.ts -``` +State, step results, timers, task coordination, and resource stores are persisted to `.notation/workflows.db`. Set `NOTATION_STATE_PATH` to choose another SQLite database path. diff --git a/docs/cli/destroy.md b/docs/cli/destroy.md index a74ff30..624bd3b 100644 --- a/docs/cli/destroy.md +++ b/docs/cli/destroy.md @@ -4,7 +4,7 @@ notation destroy ``` -Removes all resources in the stack. Tears down runs in reverse dependency order, so routes are removed before APIs and Lambdas before IAM roles etc. +Compiles the application and runs durable destroy through the resident YieldStar 0.5.0 Node runtime. Resources are removed in reverse dependency order, then registered persisted orphans are removed. ```sh notation destroy infra/api.ts @@ -15,3 +15,11 @@ notation destroy infra/api.ts ```sh notation destroy infra/api.ts --json > destroy.ndjson ``` + +The command prints its execution ID. Resume a crashed destroy with the same ID so a provider delete that already completed is replayed instead of repeated: + +```sh +notation destroy infra/api.ts --execution-id +``` + +Retryable deletes suspend on durable SQLite timers. Resource state is removed only after the provider delete succeeds or reports that the resource is already absent. diff --git a/docs/internals/reconciler.md b/docs/internals/reconciler.md index dc0e93c..022622c 100644 --- a/docs/internals/reconciler.md +++ b/docs/internals/reconciler.md @@ -1,126 +1,40 @@ # Reconciler -The reconciler runs deployment operations to transition infrastructure from its current state to the state defined in the project. - -Source: `@notation/reconciler` - -## Durable workflow boundary - -`reconcileWithYieldStar` is the durable Node.js integration. It is an async generator intended to be composed inside an application-owned YieldStar workflow: - -```ts -const deploy = workflow(async function* (step, event) { - yield* reconcileWithYieldStar(step, { - deploymentId: "production", - executionId: event.executionId, - resources, - state, - }); -}); -``` - -The host owns runtime wiring and scheduling. Notation owns graph ordering, decisions, provider calls, drift reads, state persistence, and orphan lifecycle. Provider calls and state mutations are YieldStar steps, so completed calls are replayed instead of repeated after a process crash. - -The synchronous `Reconciler` described below remains the CLI path for this release. +The reconciler expresses deployment and destruction as YieldStar async generators. Notation owns desired-state decisions and provider lifecycle; the caller's YieldStar runtime owns durable execution, waiting, shared state, and coordination. ## Deploy flow -```ts [packages/reconciler/src/index.ts] -const reconciler = new Reconciler({ state, registry, emit }); -await reconciler.deploy(resources, { dryRun, driftDetection }); -``` +`deployWithYieldStar` acquires the deployment coordination store, walks dependency levels in order, decides an action for every resource, executes provider calls as durable steps, persists the result in a resource store, and deletes registered orphans. -The reconciler walks the resource graph and, for each resource, determines an action: +| Condition | Decision | +| --- | --- | +| Not in state | **create** | +| In state, params changed | **update** | +| In state, params unchanged, no drift | **noop** | +| In state, but deleted from the provider | **drift-recreate** | +| In state, provider state differs from stored state | **drift-update** | +| In state, not in graph | **delete** | -| Condition | Decision | -| --------------------------------------------- | ------------------ | -| Not in state | **create** | -| In state, params changed | **update** | -| In state, params unchanged, no drift | **noop** | -| In state, but deleted from AWS | **drift-recreate** | -| In state, AWS state differs from stored state | **drift-update** | -| In state, not in graph (orphan) | **delete** | +Dry-run deploy performs decisions and emits lifecycle events without calling providers or mutating state. -The `dryRun` flag runs the full diffing pipeline without executing any operations, so you can preview what a deploy would do. +## Destroy flow -## Topological ordering +`destroyWithYieldStar` is a first-class durable operation. It acquires the same deployment coordination store as deploy, deletes desired resources in reverse dependency order, deletes hydratable persisted orphans, and conditionally removes each resource store only after the provider delete succeeds or reports that the resource is already absent. -Resources are deployed in dependency order using `buildResourceDepthLevels()`. This function partitions the resource graph into levels – each level contains resources whose dependencies have all been satisfied by previous levels. +Provider delete is a stable durable step. If the process crashes after the provider acknowledges deletion but before state removal, replay uses the cached delete result and continues at the conditional store delete. -``` -Level 0: IAM Role, CloudWatch LogGroup -Level 1: Lambda Function (depends on Role, LogGroup) -Level 2: API Gateway Integration (depends on Lambda) -Level 3: API Gateway Route (depends on Integration) -``` +## Waiting and replay -Resources within a level deploy concurrently, so independent resources like the IAM Role and LogGroup above are provisioned in parallel. Dependent resources wait for their dependencies. +Retryable provider errors become YieldStar `RetryableError` delays. The resident Node runtime can remain idle until the SQLite timer queues a wake-up, then rebuild the resource graph and replay completed heap steps. Reads that wait for provider consistency use the same mechanism. -Destroy operates in reverse order with dependents getting removed before their dependencies. +Every provider call, event emission, state read, state write, and coordination transition has a stable step key. A resumed execution must use the same execution ID. A new deploy or destroy must use a new execution ID so its heap does not alias an earlier operation. -### Cycle detection +## State and coordination -Cycle detection is built in. If resources form a circular dependency, the build fails with: +Each resource is stored under `notation/resource-state` with a deployment-scoped ID. Conditional updates and deletes compare the snapshot's UUIDv7 `instanceId` and version, so a stale execution cannot modify a deleted and recreated store. -``` -Resource dependency cycle detected -``` - -This catches configuration errors before any cloud operations are attempted. - -## Drift detection - -Drift detection is enabled by default. After confirming no local changes to a resource, the reconciler reads the resource's current state from AWS (via the resource's `read()` operation) and diffs it against stored state. - -If AWS has drifted (e.g. someone changed a Lambda timeout in the console, or an IAM policy was modified by another tool), Notation updates the resource to match the canoncial definition in the source code. - -Properties marked as `volatile` in the schema (like `LastModified` timestamps) are excluded from drift comparison. +Deploy and destroy share one `notation/deployment-coordination` store per deployment. `store.take` suspends a competing execution as a durable waiter and wakes it after the holder releases. ## Events -The reconciler emits events at each step of an operation's lifecycle. The default `createConsoleReconcilerSubscriber()` logs these to the console with formatted output. - -| Event | When | -| ------------------------------------ | --------------------------------------------------- | -| `reconciler.deploy.decision` | After deciding what action to take for a resource | -| `reconciler.drift.detected` | When drift is found between stored and actual state | -| `reconciler.operation.lifecycle` | When an operation starts, finishes, skips, or fails | -| `reconciler.orphan-deletion.skipped` | When no registered class can delete an orphan | - -Lifecycle events contain the operation (`create`, `read`, `update`, or `delete`) and its -status (`start`, `success`, `error`, `skip`, or `dry-run`). Events carry the resource ID, -type, and relevant diff or error details. - -## Operations - -Each CRUD operation is implemented as an async generator with retry support: - -- **`createResourceOperation`** – creates the resource, reads back its state, persists to state backend -- **`updateResourceOperation`** – applies the update, reads back new state, persists to state backend -- **`deleteResourceOperation`** – deletes the resource, removes the entry from state backend -- **`readResourceOperation`** – reads current state from the cloud provider (used for drift detection) - -### Retry and polling - -Operations support polling for eventual consistency: - -```ts [packages/reconciler/src/index.ts] -{ - maxAttempts: 10, - retryInterval: 2000, -} -``` - -This handles AWS services that return success before the resource is fully available. For example, after creating an IAM Role, a Lambda function may briefly fail to deploy until the role propagates. The retry loop handles cases like this. - -### Operation lifecycle - -Each operation follows the following pattern: - -1. Emit `started` event -2. Execute the cloud operation (with retries) -3. Read back the resource state -4. Persist to state backend -5. Emit `completed` event (or `failed` on error) - -State is updated after the provider operation and read-back complete. +The durable workflows emit `reconciler.deploy.decision`, `reconciler.drift.detected`, `reconciler.operation.lifecycle`, and `reconciler.orphan-deletion.skipped`. Lifecycle events cover create, read, update, and delete with `start`, `success`, `error`, `skip`, or `dry-run` status. diff --git a/docs/internals/state.md b/docs/internals/state.md index 9109ac0..0fd06c0 100644 --- a/docs/internals/state.md +++ b/docs/internals/state.md @@ -1,139 +1,25 @@ # State -Notation tracks deployed resources in a state backend. State is the bridge between what is defined and what actually exists in the cloud. +Notation CLI deploy, destroy, plan, and dashboard use YieldStar 0.5.0 stores in `.notation/workflows.db`. Override the database path with `NOTATION_STATE_PATH`. -Source: `@notation/state` - -## State file - -Default location: `.notation/state.json`. Override with the `NOTATION_STATE_PATH` environment variable. - -Each resource entry records everything needed to diff, update, or delete the resource: - -```json -{ - "my-api-lambda-getTodos": { - "rev": 3, - "id": "my-api-lambda-getTodos", - "type": "aws/lambda/LambdaFunction", - "config": { - "service": "aws/lambda", - "timeout": 5, - "memory": 64 - }, - "params": { - "FunctionName": "my-api-getTodos", - "Runtime": "nodejs18.x", - "Handler": "index.getTodos", - "MemorySize": 64, - "Timeout": 5 - }, - "output": { - "FunctionArn": "arn:aws:lambda:us-east-1:123456789:function:my-api-getTodos", - "FunctionUrl": "https://xyz.lambda-url.us-east-1.on.aws/" - }, - "lastOperation": "create", - "lastOperationAt": "2027-01-15T10:30:00.000Z" - } -} -``` - -Key fields: - -- **`id`** – unique identifier derived from the resource's position in the graph -- **`rev`** – monotonically increasing revision used for compare-and-swap writes -- **`type`** – the resource type string (e.g., `aws/lambda/LambdaFunction`) -- **`config`** – user-facing configuration values -- **`params`** – the full set of parameters sent to the cloud provider -- **`output`** – computed values returned by the provider after creation -- **`lastOperation`** – what the reconciler last did (`create`, `update`, `delete`) -- **`lastOperationAt`** – ISO timestamp of the last operation - -## Backends - -Three built-in backends: - -### `FileStateBackend` (default) - -Reads and writes JSON to disk. Uses atomic writes – writes to a temporary file first, then renames – to prevent corruption if the process is interrupted mid-write. - -```ts [packages/state/src/file.ts] -const state = new FileStateBackend(".notation/state.json"); -``` - -### `MemoryStateBackend` - -In-memory backend used for testing. Deep-clones on read and write to simulate persistence semantics (mutations to returned objects don't affect stored data). - -```ts [packages/state/src/memory.ts] -const state = new MemoryStateBackend(); -``` - -### `SqliteStateBackend` - -Stores state and leases in SQLite. Select it in the CLI by setting -`NOTATION_STATE_PATH` to a path ending in `.db` or `.sqlite`. +Each live resource is a `notation/resource-state` store scoped by deployment and resource ID. A missing store means the resource is absent. No application tombstone is written. ```ts -const state = new SqliteStateBackend(".notation/state.db"); +const state = new YieldStarStateBackend(storeClient, "infra/api.ts"); ``` -### `YieldStarStateBackend` - -The durable workflow integration stores resources in YieldStar 0.5.0 stores and runs on the Node SQLite runtime. Each resource is a live store; a missing store means a missing resource. +The runtime assigns a UUIDv7 `instanceId` when a store is created and increments its version on update. Conditional workflow updates and deletes compare both values, preventing a stale snapshot from modifying a deleted and recreated resource. The one-based value exposed as `StateNode.rev` is derived from the authoritative YieldStar store version. ```ts -const state = new YieldStarStateBackend(storeClient, "production"); -``` - -YieldStar assigns a UUIDv7 `instanceId` when the store is created and increments its version on update. Conditional workflow updates and deletes compare both values, preventing a stale snapshot from modifying a deleted and recreated resource. The one-based value exposed as `StateNode.rev` is derived from the authoritative YieldStar store version. - -### `StateBackend` interface - -All backends implement the same interface: - -```ts [@notation/state/src/backend.ts] interface StateBackend { get(id: string): Promise; has(id: string): Promise; - update( - id: string, - patch: Partial, - expectedRev?: number, - ): Promise<{ rev: number }>; - delete(id: string, expectedRev?: number): Promise; + update(id: string, expectedRev: number, patch: Partial): Promise<{ rev: number }>; + delete(id: string, expectedRev: number): Promise; values(): Promise; - lease(scope: string, ttl: number): Promise; } ``` -Every backend provides compare-and-swap writes and renewable exclusive leases. The -reconciler holds a per-resource lease across the provider operation and state write, so -concurrent deploys cannot both perform the same create or update. It renews long-running -leases until the mutation finishes. Orphan deletion additionally holds a snapshot lease -while it decides which state records no longer appear in the desired graph. - -## How state is used - -### Deploy - -The reconciler reads state to diff against the desired resource graph: - -1. For each resource in the graph, check if it exists in state -2. If it exists, compare `params` to detect changes -3. Execute the appropriate operation (create, update, noop) -4. After each operation, update the state entry with new params and output - -### Destroy - -The reconciler reads state to find resources to delete: - -1. Load all state entries -2. Delete resources in reverse dependency order -3. Remove each entry from state after successful deletion - -### Orphan detection - -The reconciler checks for orphaned resources – resources that exist in state but are no longer present in the resource graph. This happens when you remove a function export or delete a `.fn.ts` file. +Coordination is not part of the state backend contract. The outer YieldStar workflow serializes deploy and destroy through a deployment coordination store and records applied store steps for crash-safe replay. -Orphaned resources are deleted from AWS and removed from state. +`MemoryStateBackend`, `FileStateBackend`, and `SqliteStateBackend` remain data adapters for tests and embedded read/write consumers. They are not CLI execution runtimes and do not provide mutation coordination. diff --git a/docs/manual/reconciler.md b/docs/manual/reconciler.md index cf10677..60c1861 100644 --- a/docs/manual/reconciler.md +++ b/docs/manual/reconciler.md @@ -1,10 +1,10 @@ # Reconciler -Use `reconcileWithYieldStar` when a Node.js application needs durable resource reconciliation without starting the Notation CLI. Notation supplies reconciliation decisions and resource lifecycle operations; the application owns the outer YieldStar workflow and chooses the runtime. +Use `deployWithYieldStar` and `destroyWithYieldStar` when a Node.js application needs durable resource lifecycle operations without starting the Notation CLI. Notation owns reconciliation intent, graph ordering, provider calls, and resource state; the application owns the outer YieldStar workflow and runtime. ```ts import { SqliteSchedulerClient, SqliteStoreClient, SqliteTaskQueueClient, SqliteTimersClient, createSqliteDb } from "@yieldstar/sqlite-runtime/node"; -import { YieldStarStateBackend, reconcileWithYieldStar } from "@notation/reconciler"; +import { YieldStarStateBackend, deployWithYieldStar, destroyWithYieldStar } from "@notation/reconciler"; import { workflow } from "yieldstar"; const database = createSqliteDb({ path: ".notation/workflows.db" }); @@ -16,7 +16,16 @@ const storeClient = new SqliteStoreClient({ db: database, schedulerClient }); const state = new YieldStarStateBackend(storeClient, "my-application"); export const deploy = workflow(async function* (step, event) { - yield* reconcileWithYieldStar(step, { + yield* deployWithYieldStar(step, { + deploymentId: "my-application", + executionId: event.executionId, + resources, + state, + }); +}); + +export const destroy = workflow(async function* (step, event) { + yield* destroyWithYieldStar(step, { deploymentId: "my-application", executionId: event.executionId, resources, @@ -25,12 +34,12 @@ export const deploy = workflow(async function* (step, event) { }); ``` -The outer workflow supplies durable step execution, timers, shared stores, waiting, and scheduling. `reconcileWithYieldStar` uses those primitives to cache completed provider calls, retry provider waiting without holding a process, persist state conditionally, delete state conditionally, and serialize deployments with `store.take`. +The outer workflow supplies durable step execution, timers, shared stores, waiting, scheduling, and coordination. Completed provider calls are replayed from the heap after a crash, retryable provider conditions suspend on a durable timer, and conditional state writes use YieldStar store identity and version. -Each live resource is one YieldStar store. Absence is represented by no store, not a tombstone. YieldStar's UUIDv7 store `instanceId` and version are authoritative for conditional update and delete; Notation exposes the version as the resource state's `rev` for its existing state contract. +Each live resource is one YieldStar store. Absence is represented by no store, not a tombstone. YieldStar's UUIDv7 store `instanceId` and version are authoritative for conditional update and delete; Notation exposes the version as the resource state's `rev`. -Deployments against the same `deploymentId` are serialized through a coordination store keyed by `executionId`. If a deployment crashes while holding the coordination store, resume it by running the same execution ID again: replay reclaims the acquisition through YieldStar's applied-step ledger and releases it on completion. A different execution ID waits durably until the holder releases. +Operations against the same `deploymentId` are serialized through a coordination store keyed by `executionId`. Resume a crashed operation with the same execution ID; use a new globally unique execution ID for every new deploy or destroy. -Pass the complete desired set on every invocation. Persisted resources absent from that set are deleted through the supplied resource registry. +Pass the complete desired set on every deployment. Persisted resources absent from that set are deleted through the supplied resource registry. Destroy removes current resources in reverse dependency order and then removes any persisted orphans that the registry can hydrate. -The runnable Node SQLite version is in `examples/reconciler`. +The runnable Node SQLite composition is in `examples/reconciler`. diff --git a/docs/rfcs/reconciler.md b/docs/rfcs/reconciler.md index 87056a7..eabaa70 100644 --- a/docs/rfcs/reconciler.md +++ b/docs/rfcs/reconciler.md @@ -1,30 +1,28 @@ # RFC: Durable YieldStar reconciliation -**Status:** implemented release slice -**Scope:** `@notation/reconciler`, YieldStar 0.5.0 +**Status:** implemented +**Scope:** `@notation/reconciler`, `@notation/core`, YieldStar 0.5.0 -Notation describes reconciliation and resource lifecycle operations. A host-owned YieldStar workflow supplies durable execution, waiting, state, and coordination by calling `yield* reconcileWithYieldStar(step, options)`. +Notation describes reconciliation intent and resource lifecycle operations. An outer YieldStar workflow supplies durable execution, waiting, state, and coordination by composing `deployWithYieldStar` or `destroyWithYieldStar`. ## Boundary -Live resource objects remain in the workflow worker. They are not serialized into workflow parameters. This keeps provider clients and operation closures under application control while YieldStar persists step results and shared state. +Live resource objects remain in the workflow process. They are not serialized into workflow parameters. This keeps provider clients and operation closures under Notation's lifecycle control while YieldStar persists step results and shared state. -Provider create, update, read, and delete calls are durable steps with stable resource-scoped keys. A process crash after a completed create replays the cached result and continues at state persistence instead of creating the provider resource again. Retryable provider conditions become YieldStar delays, allowing the process to stop until the scheduler wakes the execution. +Provider create, update, read, and delete calls are durable steps with stable resource-scoped keys. A process crash after a completed provider call replays the cached result and continues at state persistence instead of repeating the call. Retryable provider conditions become YieldStar delays, allowing the process to wait without polling the provider continuously. ## State lifecycle -`YieldStarStateBackend` stores one live resource per `notation/resource-state` store. The store ID is scoped by deployment and resource ID. Store absence is resource absence; no application tombstone is created. +`YieldStarStateBackend` stores one live resource per `notation/resource-state` store. The store ID is scoped by deployment and resource ID. Store absence is resource absence. The runtime-assigned UUIDv7 `instanceId` distinguishes a deleted store from a later store created under the same logical ID. YieldStar's version is the concurrency token and is exposed as Notation's one-based `rev`. Workflow updates use `store.updateFrom` and deletes use `store.deleteFrom`, so both the instance and version must match the snapshot that informed the operation. -`values` uses YieldStar 0.5.0's merged `listStores` lifecycle API, and administrative cleanup uses `deleteStore`. - ## Coordination -Each deployment has a `notation/deployment-coordination` store. The workflow atomically claims it with `store.take`. A concurrent execution suspends as a durable waiter and is woken when the holder releases the store. The same execution can recover an acquisition across the store-commit/heap-write crash gap through YieldStar's applied-step ledger. +Each deployment has a `notation/deployment-coordination` store shared by deploy and destroy. The workflow atomically claims it with `store.take`. A concurrent execution suspends as a durable waiter and is woken when the holder releases the store. The same execution can recover an acquisition across the store-commit and heap-write crash gap through YieldStar's applied-step ledger. -## Release boundary +## Node CLI runtime -This slice delivers durable deploy reconciliation, drift handling, orphan deletion, Node SQLite execution, external state access, conditional persistence, and concurrent deployment serialization. The existing synchronous `Reconciler` remains the CLI execution path in this release. +`NodeYieldStarRuntime` wires `WorkflowRunner`, `SqliteHeapClient`, `SqliteStoreClient`, `SqliteSchedulerClient`, and `SqliteEventLoop` against one Node SQLite database. CLI deploy and destroy run through this resident runtime and wait for a workflow result across timer and store wake-ups. -The next stacked phase will move CLI deploy and destroy onto a resident workflow runtime, add durable destroy as a first-class workflow operation, and fan independent dependency-level resources into coordinated child executions. +The CLI prints a new execution ID for each operation. Re-running with `--execution-id ` resumes that operation from its durable heap after a process crash. diff --git a/examples/reconciler/README.md b/examples/reconciler/README.md index 2af91ca..0dfdf4c 100644 --- a/examples/reconciler/README.md +++ b/examples/reconciler/README.md @@ -2,7 +2,7 @@ This example deploys two static sites from an ordinary Node.js program using YieldStar 0.5.0 for durable execution, state, retries, waiting, and deployment coordination. -[`src/index.ts`](./src/index.ts) owns the outer workflow and Node SQLite runtime. It passes YieldStar's `step` context to `reconcileWithYieldStar`, while [`src/static-site.ts`](./src/static-site.ts) contains only the desired resources and provider lifecycle operations. +[`src/index.ts`](./src/index.ts) owns the outer workflow and Node SQLite runtime. It passes YieldStar's `step` context to `deployWithYieldStar`, while [`src/static-site.ts`](./src/static-site.ts) contains only the desired resources and provider lifecycle operations. Run it from the repository root: diff --git a/examples/reconciler/src/index.ts b/examples/reconciler/src/index.ts index b17146e..edf2b93 100644 --- a/examples/reconciler/src/index.ts +++ b/examples/reconciler/src/index.ts @@ -10,7 +10,7 @@ import { import { YieldStarStateBackend, createResourceRegistry, - reconcileWithYieldStar, + deployWithYieldStar, } from "@notation/reconciler"; import pino from "pino"; import { createWorkflowRouter, workflow } from "yieldstar"; @@ -44,7 +44,7 @@ const resources = [ ]; const deploy = workflow(async function* (step, event) { - yield* reconcileWithYieldStar(step, { + yield* deployWithYieldStar(step, { deploymentId: "static-sites", executionId: event.executionId, resources, diff --git a/packages/cli/src/deploy.ts b/packages/cli/src/deploy.ts index 07fdad1..ce85607 100644 --- a/packages/cli/src/deploy.ts +++ b/packages/cli/src/deploy.ts @@ -3,12 +3,14 @@ import { createNdjsonEventEmitter, deployApp, } from "@notation/core"; +import { randomUUID } from "node:crypto"; import { compile } from "./compile"; import { defaultLogger, type Logger } from "./logger"; import { redirectStdoutToStderr } from "./stdio"; export type DeployCommandOptions = { json?: boolean; + executionId?: string; logger?: Logger; }; @@ -25,11 +27,14 @@ export async function deploy( await compile(entryPoint, { logger }); logger.info(`Deploying ${entryPoint}`); + const executionId = opts.executionId ?? randomUUID(); + logger.info(`YieldStar execution ${executionId}`); try { await deployApp({ entryPoint, emit, + executionId, }); } catch (err: any) { if (err.name === "CredentialsProviderError") { diff --git a/packages/cli/src/destroy.ts b/packages/cli/src/destroy.ts index acdc5b0..543c3e6 100644 --- a/packages/cli/src/destroy.ts +++ b/packages/cli/src/destroy.ts @@ -3,12 +3,14 @@ import { createNdjsonEventEmitter, destroyApp, } from "@notation/core"; +import { randomUUID } from "node:crypto"; import { compile } from "./compile"; import { defaultLogger, type Logger } from "./logger"; import { redirectStdoutToStderr } from "./stdio"; export type DestroyCommandOptions = { json?: boolean; + executionId?: string; logger?: Logger; }; @@ -23,5 +25,7 @@ export async function destroy( await compile(entryPoint, { logger }); logger.info(`Destroying ${entryPoint}\n`); - await destroyApp({ entryPoint, emit }); + const executionId = opts.executionId ?? randomUUID(); + logger.info(`YieldStar execution ${executionId}`); + await destroyApp({ entryPoint, emit, executionId }); } diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index 3ef90b7..f437e6b 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -7,7 +7,7 @@ import { plan } from "./plan"; import { visualise } from "./visualise"; import { watch } from "./watch"; import { startDashboardServer } from "@notation/dashboard"; -import { createDefaultStateBackend } from "@notation/core"; +import { NodeYieldStarRuntime } from "@notation/core"; program .command("compile") @@ -19,9 +19,11 @@ program program .command("dashboard") + .argument("", "entryPoint") .description("Start Notation Dashboard") - .action(async () => { - await startDashboardServer({ state: createDefaultStateBackend() }); + .action(async (entryPoint) => { + const runtime = new NodeYieldStarRuntime({ deploymentId: entryPoint }); + await startDashboardServer({ state: runtime.state }); }); program @@ -29,8 +31,12 @@ program .argument("", "entryPoint") .description("Deploy Notation App") .option("--json", "stream reconciler events as NDJSON") + .option("--execution-id ", "resume a durable execution") .action(async (entryPoint, options) => { - await deploy(entryPoint, { json: options.json }); + await deploy(entryPoint, { + json: options.json, + executionId: options.executionId, + }); }); program @@ -38,8 +44,12 @@ program .argument("", "entryPoint") .description("Destroy Notation App") .option("--json", "stream reconciler events as NDJSON") + .option("--execution-id ", "resume a durable execution") .action(async (entryPoint, options) => { - await destroy(entryPoint, { json: options.json }); + await destroy(entryPoint, { + json: options.json, + executionId: options.executionId, + }); }); program diff --git a/packages/cli/src/plan.ts b/packages/cli/src/plan.ts index ef52ec8..960f2f3 100644 --- a/packages/cli/src/plan.ts +++ b/packages/cli/src/plan.ts @@ -1,9 +1,4 @@ -import { - createLoggerReconcilerSubscriber, - planApp, - type Plan, - type PlanNode, -} from "@notation/core"; +import { planApp, type Plan, type PlanNode } from "@notation/core"; import { compile } from "./compile"; import { defaultLogger, type Logger } from "./logger"; import { redirectStdoutToStderr } from "./stdio"; @@ -24,7 +19,6 @@ const decisionSymbols: Record = { export async function plan(entryPoint: string, opts: PlanCommandOptions = {}) { const logger = opts.logger ?? defaultLogger; - const emit = createLoggerReconcilerSubscriber({ logger }); try { if (opts.json) { let result: Plan; @@ -33,7 +27,6 @@ export async function plan(entryPoint: string, opts: PlanCommandOptions = {}) { await compile(entryPoint, { logger }); result = await planApp({ entryPoint, - emit, }); } finally { restore(); @@ -46,7 +39,6 @@ export async function plan(entryPoint: string, opts: PlanCommandOptions = {}) { logger.info(`Planning ${entryPoint}\n`); const result = await planApp({ entryPoint, - emit, }); printPlanSummary(result, logger); } catch (err: any) { diff --git a/packages/core/package.json b/packages/core/package.json index 542912a..cc55f70 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -15,12 +15,14 @@ "dependencies": { "@notation/reconciler": "workspace:*", "@notation/resource": "workspace:*", - "@notation/state": "workspace:*", - "@notation/state-sqlite": "workspace:*", + "@yieldstar/core": "0.5.0", + "@yieldstar/sqlite-runtime": "0.5.0", "deep-object-diff": "^1.1.9", "js-base64": "^3.7.7", "lodash-es": "^4.17.21", - "pako": "^2.1.0" + "pako": "^2.1.0", + "pino": "^9.14.0", + "yieldstar": "0.5.0" }, "devDependencies": { "@types/common-tags": "^1.8.4", diff --git a/packages/core/src/provisioner/index.ts b/packages/core/src/provisioner/index.ts index 89bf6e7..5f70072 100644 --- a/packages/core/src/provisioner/index.ts +++ b/packages/core/src/provisioner/index.ts @@ -1,3 +1,3 @@ export * from "./workflows"; export * from "./resource-registry"; -export * from "./state-backend"; +export * from "./yieldstar-runtime"; diff --git a/packages/core/src/provisioner/state-backend.ts b/packages/core/src/provisioner/state-backend.ts deleted file mode 100644 index 437e56c..0000000 --- a/packages/core/src/provisioner/state-backend.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { FileStateBackend, type StateBackend } from "@notation/state"; -import { SqliteStateBackend } from "@notation/state-sqlite"; - -export const DEFAULT_STATE_PATH = "./.notation/state.json"; - -export function resolveStatePath(): string { - return process.env.NOTATION_STATE_PATH ?? DEFAULT_STATE_PATH; -} - -export function createDefaultStateBackend(): StateBackend { - const statePath = resolveStatePath(); - if (statePath.endsWith(".db") || statePath.endsWith(".sqlite")) { - return new SqliteStateBackend(statePath); - } - return new FileStateBackend(statePath); -} diff --git a/packages/core/src/provisioner/workflows/index.ts b/packages/core/src/provisioner/workflows/index.ts index 9dd1579..835d222 100644 --- a/packages/core/src/provisioner/workflows/index.ts +++ b/packages/core/src/provisioner/workflows/index.ts @@ -7,4 +7,3 @@ export { export * from "./workflow.deploy"; export * from "./workflow.destroy"; export * from "./workflow.plan"; -export * from "./workflow.refresh"; diff --git a/packages/core/src/provisioner/workflows/workflow.deploy.ts b/packages/core/src/provisioner/workflows/workflow.deploy.ts index 3eaa88e..afe3de9 100644 --- a/packages/core/src/provisioner/workflows/workflow.deploy.ts +++ b/packages/core/src/provisioner/workflows/workflow.deploy.ts @@ -1,19 +1,21 @@ import { - Reconciler, + deployWithYieldStar, createLoggerReconcilerSubscriber, type ReconcilerEventEmitter, type ResourceRegistry, } from "@notation/reconciler"; -import type { StateBackend } from "@notation/state"; +import { createWorkflowRouter, workflow } from "yieldstar"; import { getResourceGraph } from "src/orchestrator/graph"; -import { createDefaultStateBackend } from "../state-backend"; +import { NodeYieldStarRuntime } from "../yieldstar-runtime"; export type DeployAppOptions = { entryPoint: string; driftDetection?: boolean; dryRun?: boolean; registry?: ResourceRegistry; - state?: StateBackend; + runtime?: NodeYieldStarRuntime; + executionId?: string; + databasePath?: string; emit?: ReconcilerEventEmitter; }; @@ -22,19 +24,33 @@ export async function deployApp({ driftDetection = true, dryRun = false, registry, - state: stateBackend, + runtime: suppliedRuntime, + executionId, + databasePath, emit = createLoggerReconcilerSubscriber(), }: DeployAppOptions): Promise { const graph = await getResourceGraph(entryPoint); - const state = stateBackend ?? createDefaultStateBackend(); - const reconciler = new Reconciler({ - state, - registry, - emit, - }); - - await reconciler.deploy(graph.resources, { - dryRun, - driftDetection, + const runtime = + suppliedRuntime ?? + new NodeYieldStarRuntime({ deploymentId: entryPoint, databasePath }); + const deploy = workflow(async function* (step, event) { + yield* deployWithYieldStar(step, { + deploymentId: runtime.deploymentId, + executionId: event.executionId, + resources: graph.resources, + state: runtime.state, + registry, + emit, + dryRun, + driftDetection, + }); }); + try { + await runtime.run(createWorkflowRouter({ deploy }), { + workflowId: "deploy", + executionId, + }); + } finally { + if (!suppliedRuntime) runtime.close(); + } } diff --git a/packages/core/src/provisioner/workflows/workflow.destroy.ts b/packages/core/src/provisioner/workflows/workflow.destroy.ts index 813239a..36ee0e3 100644 --- a/packages/core/src/provisioner/workflows/workflow.destroy.ts +++ b/packages/core/src/provisioner/workflows/workflow.destroy.ts @@ -1,35 +1,50 @@ import { - Reconciler, + destroyWithYieldStar, createLoggerReconcilerSubscriber, type ReconcilerEventEmitter, type ResourceRegistry, } from "@notation/reconciler"; -import type { StateBackend } from "@notation/state"; +import { createWorkflowRouter, workflow } from "yieldstar"; import { getResourceGraph } from "src/orchestrator/graph"; -import { createDefaultStateBackend } from "../state-backend"; -import { refreshState } from "./workflow.refresh"; +import { NodeYieldStarRuntime } from "../yieldstar-runtime"; export type DestroyAppOptions = { entryPoint: string; registry?: ResourceRegistry; - state?: StateBackend; + runtime?: NodeYieldStarRuntime; + executionId?: string; + databasePath?: string; emit?: ReconcilerEventEmitter; }; export async function destroyApp({ entryPoint, registry, - state: stateBackend, + runtime: suppliedRuntime, + executionId, + databasePath, emit = createLoggerReconcilerSubscriber(), }: DestroyAppOptions) { - const state = stateBackend ?? createDefaultStateBackend(); - await refreshState({ entryPoint, registry, state, emit }); - const graph = await getResourceGraph(entryPoint); - const reconciler = new Reconciler({ - state, - emit, + const runtime = + suppliedRuntime ?? + new NodeYieldStarRuntime({ deploymentId: entryPoint, databasePath }); + const destroy = workflow(async function* (step, event) { + yield* destroyWithYieldStar(step, { + deploymentId: runtime.deploymentId, + executionId: event.executionId, + resources: graph.resources, + state: runtime.state, + registry, + emit, + }); }); - - await reconciler.destroy(graph.resources); + try { + await runtime.run(createWorkflowRouter({ destroy }), { + workflowId: "destroy", + executionId, + }); + } finally { + if (!suppliedRuntime) runtime.close(); + } } diff --git a/packages/core/src/provisioner/workflows/workflow.plan.ts b/packages/core/src/provisioner/workflows/workflow.plan.ts index 2e51cb8..39679c2 100644 --- a/packages/core/src/provisioner/workflows/workflow.plan.ts +++ b/packages/core/src/provisioner/workflows/workflow.plan.ts @@ -1,38 +1,33 @@ -import { - Reconciler, - createLoggerReconcilerSubscriber, - type Plan, - type ReconcilerEventEmitter, - type ResourceRegistry, -} from "@notation/reconciler"; -import type { StateBackend } from "@notation/state"; +import { createPlan, type Plan } from "@notation/reconciler"; import { getResourceGraph } from "src/orchestrator/graph"; -import { createDefaultStateBackend } from "../state-backend"; +import { NodeYieldStarRuntime } from "../yieldstar-runtime"; export type { Plan, PlanNode, PlanDecision } from "@notation/reconciler"; export type PlanAppOptions = { entryPoint: string; driftDetection?: boolean; - registry?: ResourceRegistry; - state?: StateBackend; - emit?: ReconcilerEventEmitter; + runtime?: NodeYieldStarRuntime; + databasePath?: string; }; export async function planApp({ entryPoint, driftDetection = true, - registry, - state: stateBackend, - emit = createLoggerReconcilerSubscriber(), + runtime: suppliedRuntime, + databasePath, }: PlanAppOptions): Promise { const graph = await getResourceGraph(entryPoint); - const state = stateBackend ?? createDefaultStateBackend(); - const reconciler = new Reconciler({ - state, - registry, - emit, - }); - - return reconciler.plan(graph.resources, { driftDetection }); + const runtime = + suppliedRuntime ?? + new NodeYieldStarRuntime({ deploymentId: entryPoint, databasePath }); + try { + return await createPlan({ + resources: graph.resources, + state: runtime.state, + driftDetection, + }); + } finally { + if (!suppliedRuntime) runtime.close(); + } } diff --git a/packages/core/src/provisioner/workflows/workflow.refresh.ts b/packages/core/src/provisioner/workflows/workflow.refresh.ts deleted file mode 100644 index b463a87..0000000 --- a/packages/core/src/provisioner/workflows/workflow.refresh.ts +++ /dev/null @@ -1,39 +0,0 @@ -import { - Reconciler, - createLoggerReconcilerSubscriber, - type ReconcilerEventEmitter, - type ResourceRegistry, -} from "@notation/reconciler"; -import type { StateBackend } from "@notation/state"; -import { getResourceGraph } from "src/orchestrator/graph"; -import { createDefaultStateBackend } from "../state-backend"; - -/** - * @description Destroy resources that are in state but not in the orchestration graph - */ -export type RefreshStateOptions = { - entryPoint: string; - dryRun?: boolean; - registry?: ResourceRegistry; - state?: StateBackend; - emit?: ReconcilerEventEmitter; -}; - -export async function refreshState({ - entryPoint, - dryRun = false, - registry, - state: stateBackend, - emit = createLoggerReconcilerSubscriber(), -}: RefreshStateOptions): Promise { - const graph = await getResourceGraph(entryPoint); - const state = stateBackend ?? createDefaultStateBackend(); - - const reconciler = new Reconciler({ - state, - registry, - emit, - }); - - await reconciler.refresh(graph.resources, { dryRun }); -} diff --git a/packages/core/src/provisioner/yieldstar-runtime.ts b/packages/core/src/provisioner/yieldstar-runtime.ts new file mode 100644 index 0000000..70ef18d --- /dev/null +++ b/packages/core/src/provisioner/yieldstar-runtime.ts @@ -0,0 +1,145 @@ +import { randomUUID } from "node:crypto"; +import { setImmediate } from "node:timers/promises"; +import { + WorkflowRunner, + type WorkflowEvent, + type WorkflowRouter, +} from "@yieldstar/core"; +import { + SqliteEventLoop, + SqliteHeapClient, + SqliteSchedulerClient, + SqliteStoreClient, + SqliteTaskQueueClient, + SqliteTimersClient, + createSqliteDb, +} from "@yieldstar/sqlite-runtime/node"; +import { YieldStarStateBackend } from "@notation/reconciler"; +import pino, { type Logger } from "pino"; + +export const DEFAULT_WORKFLOW_STATE_PATH = ".notation/workflows.db"; + +export function resolveWorkflowStatePath(): string { + return process.env.NOTATION_STATE_PATH ?? DEFAULT_WORKFLOW_STATE_PATH; +} + +export type NodeYieldStarRuntimeOptions = { + deploymentId: string; + databasePath?: string; + logger?: Logger; +}; + +export type RunWorkflowOptions = { + workflowId: string; + executionId?: string; + params?: Record; +}; + +/** Resident YieldStar 0.5.0 Node runtime used by Notation application commands. */ +export class NodeYieldStarRuntime { + readonly deploymentId: string; + readonly state: YieldStarStateBackend; + readonly #database: ReturnType; + readonly #eventLoop: SqliteEventLoop; + readonly #heapClient: SqliteHeapClient; + readonly #schedulerClient: SqliteSchedulerClient; + readonly #storeClient: SqliteStoreClient; + readonly #logger: Logger; + #running = false; + + constructor(opts: NodeYieldStarRuntimeOptions) { + this.deploymentId = opts.deploymentId; + this.#logger = opts.logger ?? pino({ level: "silent" }); + this.#database = createSqliteDb({ + path: opts.databasePath ?? resolveWorkflowStatePath(), + }); + const taskQueueClient = new SqliteTaskQueueClient(this.#database); + this.#schedulerClient = new SqliteSchedulerClient({ + taskQueueClient, + timersClient: new SqliteTimersClient(this.#database), + }); + this.#storeClient = new SqliteStoreClient({ + db: this.#database, + schedulerClient: this.#schedulerClient, + }); + this.#heapClient = new SqliteHeapClient(this.#database); + this.#eventLoop = new SqliteEventLoop(this.#database); + this.state = new YieldStarStateBackend( + this.#storeClient, + this.deploymentId, + ); + } + + async run( + router: WorkflowRouter, + opts: RunWorkflowOptions, + ): Promise { + if (this.#running) { + throw new Error( + "The Node YieldStar runtime already has an active workflow", + ); + } + this.#running = true; + const event: WorkflowEvent = { + workflowId: opts.workflowId, + executionId: opts.executionId ?? randomUUID(), + params: opts.params ?? {}, + context: new Map(), + }; + const runner = new WorkflowRunner({ + router, + heapClient: this.#heapClient, + storeClient: this.#storeClient, + schedulerClient: this.#schedulerClient, + logger: this.#logger, + }); + + let resolveCompletion!: (value: unknown) => void; + let rejectCompletion!: (error: unknown) => void; + const completion = new Promise((resolve, reject) => { + resolveCompletion = resolve; + rejectCompletion = reject; + }); + const processEvent = async (nextEvent: WorkflowEvent, logger: Logger) => { + try { + const result = await runner.run(nextEvent, logger); + if (result && nextEvent.executionId === event.executionId) { + this.#eventLoop.stop(); + resolveCompletion(result.result); + } + } catch (error) { + if (nextEvent.executionId === event.executionId) { + this.#eventLoop.stop(); + rejectCompletion(error); + return; + } + this.#logger.error({ err: error }, "YieldStar replay failed"); + } + }; + + try { + await processEvent(event, this.#logger); + this.#eventLoop.start({ onNewEvent: processEvent, logger: this.#logger }); + try { + return await completion; + } finally { + // Let SqliteEventLoop remove the completed queue item before callers + // close the shared database. + await setImmediate(); + } + } finally { + this.#eventLoop.stop(); + this.#running = false; + } + } + + close(): void { + if (this.#running) { + throw new Error( + "Cannot close the Node YieldStar runtime while a workflow is active", + ); + } + this.#eventLoop.stop(); + this.#database.close(); + } +} diff --git a/packages/core/test/provisioner/operation.create.test.ts b/packages/core/test/provisioner/operation.create.test.ts deleted file mode 100644 index 2de482d..0000000 --- a/packages/core/test/provisioner/operation.create.test.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { describe, expect, it, vi } from "vitest"; -import { createResourceOperation, createStepRunner, runOperation } from "@notation/reconciler"; -import { MemoryStateBackend } from "@notation/state"; -import { - TestResourceSchema, - testResourceConfig, - testOperations, - testResourceOutput, -} from "test/orchestrator/resource.doubles"; - -describe("resource creation", () => { - it("passes computed input to resource.create", async () => { - const stateBackend = new MemoryStateBackend(); - const readResult = { ...testResourceOutput, volatileComputed: "123" }; - const createMock = vi.fn(async () => ({ primaryKey: "" })); - const readMock = vi.fn(async () => readResult); - - const TestResource = TestResourceSchema.defineOperations({ - ...testOperations, - create: createMock, - read: readMock, - }); - - const testResource = new TestResource({ - id: "test-resource", - config: testResourceConfig, - }); - const step = createStepRunner(); - - await runOperation( - createResourceOperation(step, { - resource: testResource, - state: stateBackend, - expectedRev: 0, - }), - ); - - const params = await testResource.getParams(); - const persistedOutput = testResource.toState(readResult); - - expect(createMock.mock.calls[0]).toEqual([params]); - await expect(stateBackend.get(testResource.id)).resolves.toMatchObject({ - id: testResource.id, - output: persistedOutput, - lastOperation: "create", - }); - expect(testResource.output).not.toEqual(testResourceOutput); - expect(testResource.output).toEqual(readResult); - }); -}); diff --git a/packages/core/test/provisioner/state-backend.test.ts b/packages/core/test/provisioner/state-backend.test.ts deleted file mode 100644 index 5ac04a4..0000000 --- a/packages/core/test/provisioner/state-backend.test.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { mkdtempSync, rmSync } from "node:fs"; -import { tmpdir } from "node:os"; -import path from "node:path"; -import { afterEach, beforeEach, describe, expect, it } from "vitest"; -import { FileStateBackend } from "@notation/state"; -import { SqliteStateBackend } from "@notation/state-sqlite"; -import { createDefaultStateBackend } from "src/provisioner/state-backend"; - -describe("createDefaultStateBackend", () => { - let directory: string; - const originalStatePath = process.env.NOTATION_STATE_PATH; - - beforeEach(() => { - directory = mkdtempSync(path.join(tmpdir(), "notation-state-")); - }); - - afterEach(() => { - if (originalStatePath === undefined) { - delete process.env.NOTATION_STATE_PATH; - } else { - process.env.NOTATION_STATE_PATH = originalStatePath; - } - rmSync(directory, { recursive: true, force: true }); - }); - - it("uses the file backend for the default JSON path", () => { - delete process.env.NOTATION_STATE_PATH; - - expect(createDefaultStateBackend()).toBeInstanceOf(FileStateBackend); - }); - - it("uses the sqlite backend for .db paths", () => { - process.env.NOTATION_STATE_PATH = path.join(directory, "state.db"); - - const backend = createDefaultStateBackend(); - expect(backend).toBeInstanceOf(SqliteStateBackend); - (backend as SqliteStateBackend).close(); - }); - - it("uses the sqlite backend for .sqlite paths", () => { - process.env.NOTATION_STATE_PATH = path.join(directory, "state.sqlite"); - - const backend = createDefaultStateBackend(); - expect(backend).toBeInstanceOf(SqliteStateBackend); - (backend as SqliteStateBackend).close(); - }); - - it("creates missing parent directories for sqlite paths", () => { - process.env.NOTATION_STATE_PATH = path.join( - directory, - ".notation", - "state.db", - ); - - const backend = createDefaultStateBackend(); - expect(backend).toBeInstanceOf(SqliteStateBackend); - (backend as SqliteStateBackend).close(); - }); -}); diff --git a/packages/core/test/provisioner/yieldstar-runtime.test.ts b/packages/core/test/provisioner/yieldstar-runtime.test.ts new file mode 100644 index 0000000..0037219 --- /dev/null +++ b/packages/core/test/provisioner/yieldstar-runtime.test.ts @@ -0,0 +1,60 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { deployWithYieldStar } from "@notation/reconciler"; +import { resource } from "@notation/resource"; +import { createWorkflowRouter, workflow } from "yieldstar"; +import { describe, expect, it } from "vitest"; +import { NodeYieldStarRuntime } from "src/provisioner/yieldstar-runtime"; + +describe("NodeYieldStarRuntime", () => { + it("stays resident across a provider delay and resumes from the SQLite event loop", async () => { + const directory = await mkdtemp(path.join(tmpdir(), "notation-runtime-")); + const runtime = new NodeYieldStarRuntime({ + deploymentId: "resident-wait", + databasePath: path.join(directory, "workflows.db"), + }); + let attempts = 0; + const PendingResource = resource({ type: "test/runtime/pending" }) + .defineSchema({}) + .defineOperations({ + create: async () => { + attempts += 1; + if (attempts === 1) { + const error = new Error("provider is pending"); + error.name = "ProviderPending"; + throw error; + } + }, + delete: async () => undefined, + retryLaterOnError: [ + { name: "ProviderPending", reason: "provider is pending" }, + ], + }); + const resources = [new PendingResource({ id: "pending" })]; + const deploy = workflow(async function* (step, event) { + yield* deployWithYieldStar(step, { + deploymentId: runtime.deploymentId, + executionId: event.executionId, + resources, + state: runtime.state, + driftDetection: false, + retryOptions: { maxAttempts: 3, retryInterval: 10 }, + }); + }); + + try { + await runtime.run(createWorkflowRouter({ deploy }), { + workflowId: "deploy", + executionId: "resident-execution", + }); + expect(attempts).toBe(2); + await expect(runtime.state.get("pending")).resolves.toMatchObject({ + lastOperation: "create", + }); + } finally { + runtime.close(); + await rm(directory, { recursive: true, force: true }); + } + }, 5_000); +}); diff --git a/packages/reconciler/src/events.ts b/packages/reconciler/src/events.ts new file mode 100644 index 0000000..7b883ac --- /dev/null +++ b/packages/reconciler/src/events.ts @@ -0,0 +1,44 @@ +import type { ResourceType } from "@notation/resource"; + +export type OperationName = "create" | "read" | "update" | "delete"; + +export type OperationLifecycleStatus = + "start" | "success" | "error" | "skip" | "dry-run"; + +export type OperationLifecycleEvent = { + level: "info" | "error"; + event: "reconciler.operation.lifecycle"; + operation: OperationName; + status: OperationLifecycleStatus; + resourceId: string; + resourceType: ResourceType; + reason?: string; + errorName?: string; + errorMessage?: string; +}; + +export type ReconcilerDeployEvent = { + level: "info"; + event: "reconciler.deploy.decision"; + resourceId: string; + resourceType: string; + decision: "create" | "update" | "drift-update" | "drift-recreate" | "noop"; +}; + +export type ReconcilerDriftDetectedEvent = { + level: "info"; + event: "reconciler.drift.detected"; + resourceId: string; + resourceType: string; + diff: Record; +}; + +export type ReconcilerEvent = + | OperationLifecycleEvent + | ReconcilerDeployEvent + | ReconcilerDriftDetectedEvent + | import("./resource-registry").MissingResourceRegistryMatchWarningEvent; + +export type ReconcilerEventEmitter = ( + event: ReconcilerEvent, +) => void | Promise; diff --git a/packages/reconciler/src/index.ts b/packages/reconciler/src/index.ts index 8274b60..ca2edca 100644 --- a/packages/reconciler/src/index.ts +++ b/packages/reconciler/src/index.ts @@ -4,10 +4,11 @@ export type DeepObjectDiffApi = typeof import("deep-object-diff"); export type YieldStarApi = typeof import("yieldstar"); export * from "./resource-registry"; -export * from "./operations"; export * from "./dependency-graph"; export * from "./plan"; -export * from "./reconciler"; +export * from "./planner"; +export * from "./events"; +export * from "./operation-support"; export * from "./logger-subscriber"; export * from "./protocol"; export * from "./yieldstar"; diff --git a/packages/reconciler/src/logger-subscriber.ts b/packages/reconciler/src/logger-subscriber.ts index 318a4cb..02ca5b0 100644 --- a/packages/reconciler/src/logger-subscriber.ts +++ b/packages/reconciler/src/logger-subscriber.ts @@ -1,4 +1,4 @@ -import type { ReconcilerEvent, ReconcilerEventEmitter } from "./reconciler"; +import type { ReconcilerEvent, ReconcilerEventEmitter } from "./events"; export type Logger = Pick; diff --git a/packages/reconciler/src/operation-support.ts b/packages/reconciler/src/operation-support.ts new file mode 100644 index 0000000..1d4042f --- /dev/null +++ b/packages/reconciler/src/operation-support.ts @@ -0,0 +1,71 @@ +import type { ErrorMatcher } from "@notation/resource"; +import type { + OperationLifecycleEvent, + OperationLifecycleStatus, + OperationName, +} from "./events"; + +export type PollOptions = { + maxAttempts: number; + retryInterval: number; +}; + +export const DEFAULT_RETRY_OPTIONS: PollOptions = { + maxAttempts: 10, + retryInterval: 1_000, +}; + +export const DEFAULT_READ_POLL_OPTIONS: PollOptions = { + maxAttempts: 30, + retryInterval: 1_000, +}; + +export function matchError( + error: unknown, + matchers: ErrorMatcher[] | undefined, +): ErrorMatcher | undefined { + if (!matchers || matchers.length === 0) return undefined; + + const name = + typeof error === "object" && error && "name" in error + ? String((error as { name?: unknown }).name) + : undefined; + const message = + typeof error === "object" && error && "message" in error + ? String((error as { message?: unknown }).message) + : undefined; + + return matchers.find((matcher) => { + if (matcher.name !== name) return false; + if (matcher.message && matcher.message !== message) return false; + return true; + }); +} + +export function createLifecycleEvent(params: { + operation: OperationName; + status: OperationLifecycleStatus; + resourceId: string; + resourceType: OperationLifecycleEvent["resourceType"]; + reason?: string; + error?: unknown; +}): OperationLifecycleEvent { + const error = params.error; + const details = + error === undefined + ? {} + : error instanceof Error + ? { errorName: error.name, errorMessage: error.message } + : { errorName: "UnknownError", errorMessage: String(error) }; + + return { + level: params.status === "error" ? "error" : "info", + event: "reconciler.operation.lifecycle", + operation: params.operation, + status: params.status, + resourceId: params.resourceId, + resourceType: params.resourceType, + ...(params.reason ? { reason: params.reason } : {}), + ...details, + }; +} diff --git a/packages/reconciler/src/operations/index.ts b/packages/reconciler/src/operations/index.ts deleted file mode 100644 index 04c66b6..0000000 --- a/packages/reconciler/src/operations/index.ts +++ /dev/null @@ -1,5 +0,0 @@ -export * from "./operation.types"; -export * from "./operation.create"; -export * from "./operation.read"; -export * from "./operation.update"; -export * from "./operation.delete"; diff --git a/packages/reconciler/src/operations/operation.create.ts b/packages/reconciler/src/operations/operation.create.ts deleted file mode 100644 index 98eac89..0000000 --- a/packages/reconciler/src/operations/operation.create.ts +++ /dev/null @@ -1,90 +0,0 @@ -import { RetryableError, createWorkflow } from "yieldstar"; -import { - DEFAULT_RETRY_OPTIONS, - type CreateResourceParams, - type StepRunner, - emitLifecycleEvent, - getErrorDetails, - matchError, -} from "./operation.types"; -import { readResourceOperation } from "./operation.read"; - -export async function* createResourceOperation( - step: StepRunner, - params: CreateResourceParams, -): AsyncGenerator { - await emitLifecycleEvent(params, "create", "start"); - - if (params.dryRun) { - await emitLifecycleEvent(params, "create", "dry-run"); - return; - } - - try { - const resourceParams = yield* step.run("create:get-params", () => - params.resource.getParams(), - ); - - const computedPrimaryKey = yield* step.run("create:remote", async () => { - try { - return await params.resource.create(resourceParams); - } catch (err) { - const matcher = matchError(err, params.resource.retryLaterOnError); - if (matcher) { - throw new RetryableError(matcher.reason, { - ...(params.retryOptions ?? DEFAULT_RETRY_OPTIONS), - }); - } - throw err; - } - }); - - params.resource.setOutput(resourceParams); - if (computedPrimaryKey) { - params.resource.setOutput({ - ...computedPrimaryKey, - ...params.resource.output, - }); - } - - const readResult = yield* readResourceOperation(step, { - resource: params.resource, - state: params.state, - emit: params.emit, - readPollOptions: params.readPollOptions, - }); - - params.resource.setOutput({ - ...params.resource.output, - ...readResult, - }); - - yield* step.run("create:persist-state", async () => { - await params.state.update(params.resource.id, params.expectedRev, { - id: params.resource.id, - groupId: params.resource.groupId, - groupType: params.resource.groupType, - type: params.resource.type, - lastOperation: "create", - lastOperationAt: new Date().toISOString(), - config: params.resource.config, - params: params.resource.toState(resourceParams), - output: params.resource.toState(params.resource.output), - }); - }); - - await emitLifecycleEvent(params, "create", "success"); - } catch (err) { - await emitLifecycleEvent(params, "create", "error", getErrorDetails(err)); - throw err; - } -} - -export const createResourceWorkflow: unknown = createWorkflow( - async function* (step, event) { - return yield* createResourceOperation( - step as StepRunner, - event.params as CreateResourceParams, - ); - }, -); diff --git a/packages/reconciler/src/operations/operation.delete.ts b/packages/reconciler/src/operations/operation.delete.ts deleted file mode 100644 index ffa5de4..0000000 --- a/packages/reconciler/src/operations/operation.delete.ts +++ /dev/null @@ -1,69 +0,0 @@ -import { RetryableError, createWorkflow } from "yieldstar"; -import { - DEFAULT_RETRY_OPTIONS, - type DeleteResourceParams, - type StepRunner, - emitLifecycleEvent, - getErrorDetails, - matchError, -} from "./operation.types"; - -export async function* deleteResourceOperation( - step: StepRunner, - params: DeleteResourceParams, -): AsyncGenerator { - await emitLifecycleEvent(params, "delete", "start"); - - if (params.dryRun) { - await emitLifecycleEvent(params, "delete", "dry-run"); - return; - } - - try { - try { - yield* step.run("delete:remote", async () => { - try { - await params.resource.delete( - params.resource.key, - params.resource.toState(params.resource.output), - ); - } catch (err) { - const matcher = matchError(err, params.resource.retryLaterOnError); - if (matcher) { - throw new RetryableError(matcher.reason, { - ...(params.retryOptions ?? DEFAULT_RETRY_OPTIONS), - }); - } - throw err; - } - }); - } catch (err) { - const matcher = matchError(err, params.resource.notFoundOnError); - if (matcher) { - await emitLifecycleEvent(params, "delete", "skip", { - reason: matcher.reason, - }); - } else { - throw err; - } - } - - yield* step.run("delete:persist-state", () => - params.state.delete(params.resource.id, params.expectedRev), - ); - - await emitLifecycleEvent(params, "delete", "success"); - } catch (err) { - await emitLifecycleEvent(params, "delete", "error", getErrorDetails(err)); - throw err; - } -} - -export const deleteResourceWorkflow: unknown = createWorkflow( - async function* (step, event) { - return yield* deleteResourceOperation( - step as StepRunner, - event.params as DeleteResourceParams, - ); - }, -); diff --git a/packages/reconciler/src/operations/operation.read.ts b/packages/reconciler/src/operations/operation.read.ts deleted file mode 100644 index 79db1de..0000000 --- a/packages/reconciler/src/operations/operation.read.ts +++ /dev/null @@ -1,101 +0,0 @@ -import { createWorkflow } from "yieldstar"; -import { - DEFAULT_READ_POLL_OPTIONS, - type ReadResourceParams, - type StepRunner, - emitLifecycleEvent, - getErrorDetails, -} from "./operation.types"; - -type ReadRetryCondition = { - key: string; - reason: string; - value?: unknown; -}; - -function needsReadRetry( - readResult: Record, - retryConditions: ReadonlyArray, -) { - return retryConditions.find((condition) => { - const resultValue = readResult[condition.key]; - if (condition.value !== undefined) { - return resultValue !== condition.value; - } - - return !resultValue; - }); -} - -export async function* readResourceOperation( - step: StepRunner, - params: ReadResourceParams, -): AsyncGenerator, unknown> { - await emitLifecycleEvent(params, "read", "start"); - - if (params.dryRun) { - await emitLifecycleEvent(params, "read", "dry-run"); - return {}; - } - - try { - const resourceParams = yield* step.run("read:get-params", () => - params.resource.getParams(), - ); - - if (!params.resource.read) { - const stateNode = yield* step.run("read:get-state-node", () => - params.state.get(params.resource.id), - ); - const merged = stateNode - ? { ...stateNode.output, ...resourceParams } - : resourceParams; - - await emitLifecycleEvent(params, "read", "skip", { - reason: "read-not-implemented", - }); - await emitLifecycleEvent(params, "read", "success"); - return merged as Record; - } - - let remoteOutput: Record = {}; - const retryConditions = (params.resource.retryReadOnCondition ?? []).filter( - Boolean, - ) as ReadRetryCondition[]; - - if (retryConditions.length > 0) { - yield* step.poll( - "read:poll-until-settled", - params.readPollOptions ?? DEFAULT_READ_POLL_OPTIONS, - async () => { - remoteOutput = await params.resource.read!(params.resource.key); - return !needsReadRetry(remoteOutput, retryConditions); - }, - ); - } else { - remoteOutput = yield* step.run("read:remote", () => - params.resource.read!(params.resource.key), - ); - } - - const mergedOutput = { - ...resourceParams, - ...remoteOutput, - }; - - await emitLifecycleEvent(params, "read", "success"); - return mergedOutput; - } catch (err) { - await emitLifecycleEvent(params, "read", "error", getErrorDetails(err)); - throw err; - } -} - -export const readResourceWorkflow: unknown = createWorkflow( - async function* (step, event) { - return yield* readResourceOperation( - step as StepRunner, - event.params as ReadResourceParams, - ); - }, -); diff --git a/packages/reconciler/src/operations/operation.types.ts b/packages/reconciler/src/operations/operation.types.ts deleted file mode 100644 index 3def119..0000000 --- a/packages/reconciler/src/operations/operation.types.ts +++ /dev/null @@ -1,143 +0,0 @@ -import type { - BaseResource, - ErrorMatcher, - ResourceType, -} from "@notation/resource"; -import type { State } from "@notation/state"; - -export type OperationName = "create" | "read" | "update" | "delete"; - -export type OperationLifecycleStatus = - "start" | "success" | "error" | "skip" | "dry-run"; - -export type OperationLifecycleEvent = { - level: "info" | "error"; - event: "reconciler.operation.lifecycle"; - operation: OperationName; - status: OperationLifecycleStatus; - resourceId: string; - resourceType: ResourceType; - reason?: string; - errorName?: string; - errorMessage?: string; -}; - -export type OperationEventEmitter = ( - event: OperationLifecycleEvent, -) => void | Promise; - -export type PollOptions = { - maxAttempts: number; - retryInterval: number; -}; - -export type StepRunner = { - run(fn: () => T | Promise): AsyncGenerator; - run( - key: string, - fn: () => T | Promise, - ): AsyncGenerator; - poll( - opts: PollOptions, - predicate: () => boolean | Promise, - ): AsyncGenerator; - poll( - key: string, - opts: PollOptions, - predicate: () => boolean | Promise, - ): AsyncGenerator; - delay(ms: number): AsyncGenerator; - delay(key: string, ms: number): AsyncGenerator; -}; - -export type ResourceOperationBaseParams = { - resource: BaseResource; - state: Pick; - dryRun?: boolean; - emit?: OperationEventEmitter; - retryOptions?: PollOptions; - readPollOptions?: PollOptions; -}; - -export type CreateResourceParams = ResourceOperationBaseParams & { - expectedRev: number; -}; - -export type ReadResourceParams = ResourceOperationBaseParams; - -export type UpdateResourceParams = ResourceOperationBaseParams & { - patch: Record; - expectedRev: number; -}; - -export type DeleteResourceParams = ResourceOperationBaseParams & { - expectedRev: number; -}; - -export const DEFAULT_RETRY_OPTIONS: PollOptions = { - maxAttempts: 10, - retryInterval: 1_000, -}; - -export const DEFAULT_READ_POLL_OPTIONS: PollOptions = { - maxAttempts: 30, - retryInterval: 1_000, -}; - -export function matchError( - err: unknown, - matchers: ErrorMatcher[] | undefined, -): ErrorMatcher | undefined { - if (!matchers || matchers.length === 0) return undefined; - - const name = - typeof err === "object" && err && "name" in err - ? String((err as { name?: unknown }).name) - : undefined; - const message = - typeof err === "object" && err && "message" in err - ? String((err as { message?: unknown }).message) - : undefined; - - return matchers.find((matcher) => { - if (matcher.name !== name) return false; - if (matcher.message && matcher.message !== message) return false; - return true; - }); -} - -export function getErrorDetails(err: unknown): { - errorName: string; - errorMessage: string; -} { - if (err instanceof Error) { - return { - errorName: err.name, - errorMessage: err.message, - }; - } - - return { - errorName: "UnknownError", - errorMessage: String(err), - }; -} - -export async function emitLifecycleEvent( - params: ResourceOperationBaseParams, - operation: OperationName, - status: OperationLifecycleStatus, - extra: Partial = {}, -) { - if (!params.emit) return; - - await params.emit({ - level: status === "error" ? "error" : "info", - event: "reconciler.operation.lifecycle", - operation, - status, - resourceId: params.resource.id, - resourceType: params.resource.type, - ...extra, - }); -} diff --git a/packages/reconciler/src/operations/operation.update.ts b/packages/reconciler/src/operations/operation.update.ts deleted file mode 100644 index 55dcba1..0000000 --- a/packages/reconciler/src/operations/operation.update.ts +++ /dev/null @@ -1,100 +0,0 @@ -import { RetryableError, createWorkflow } from "yieldstar"; -import { - DEFAULT_RETRY_OPTIONS, - type StepRunner, - type UpdateResourceParams, - emitLifecycleEvent, - getErrorDetails, - matchError, -} from "./operation.types"; -import { readResourceOperation } from "./operation.read"; - -export async function* updateResourceOperation( - step: StepRunner, - params: UpdateResourceParams, -): AsyncGenerator { - await emitLifecycleEvent(params, "update", "start"); - - if (params.dryRun) { - await emitLifecycleEvent(params, "update", "dry-run"); - return; - } - - if (!params.resource.update) { - await emitLifecycleEvent(params, "update", "skip", { - reason: "update-not-implemented", - }); - await emitLifecycleEvent(params, "update", "success"); - return; - } - - try { - const resourceParams = yield* step.run("update:get-params", () => - params.resource.getParams(), - ); - - yield* step.run("update:remote", async () => { - try { - await params.resource.update!( - params.resource.key, - params.patch, - resourceParams, - params.resource.toState(params.resource.output), - ); - } catch (err) { - const matcher = matchError(err, params.resource.retryLaterOnError); - if (matcher) { - throw new RetryableError(matcher.reason, { - ...(params.retryOptions ?? DEFAULT_RETRY_OPTIONS), - }); - } - throw err; - } - }); - - params.resource.setOutput({ - ...params.resource.key, - ...resourceParams, - }); - - const readResult = yield* readResourceOperation(step, { - resource: params.resource, - state: params.state, - emit: params.emit, - readPollOptions: params.readPollOptions, - }); - - params.resource.setOutput({ - ...params.resource.output, - ...readResult, - }); - - yield* step.run("update:persist-state", async () => { - await params.state.update(params.resource.id, params.expectedRev, { - id: params.resource.id, - groupId: params.resource.groupId, - groupType: params.resource.groupType, - type: params.resource.type, - lastOperation: "update", - lastOperationAt: new Date().toISOString(), - config: params.resource.config, - params: params.resource.toState(resourceParams), - output: params.resource.toState(params.resource.output), - }); - }); - - await emitLifecycleEvent(params, "update", "success"); - } catch (err) { - await emitLifecycleEvent(params, "update", "error", getErrorDetails(err)); - throw err; - } -} - -export const updateResourceWorkflow: unknown = createWorkflow( - async function* (step, event) { - return yield* updateResourceOperation( - step as StepRunner, - event.params as UpdateResourceParams, - ); - }, -); diff --git a/packages/reconciler/src/planner.ts b/packages/reconciler/src/planner.ts new file mode 100644 index 0000000..6e90b56 --- /dev/null +++ b/packages/reconciler/src/planner.ts @@ -0,0 +1,81 @@ +import type { BaseResource } from "@notation/resource"; +import type { StateBackend } from "@notation/state"; +import { buildResourceDepthLevels } from "./dependency-graph"; +import { + decideAction, + getDependencyIds, + resolvePlanParams, + type Plan, + type PlanNode, +} from "./plan"; + +export type CreatePlanOptions = { + resources: BaseResource[]; + state: StateBackend; + driftDetection?: boolean; +}; + +export async function createPlan({ + resources, + state, + driftDetection = true, +}: CreatePlanOptions): Promise { + const resourceById = new Map( + resources.map((resource) => [resource.id, resource]), + ); + const nodes: PlanNode[] = []; + + for (const level of buildResourceDepthLevels(resources)) { + for (const resource of level) { + const stateNode = await state.get(resource.id); + if (stateNode) resource.setOutput(stateNode.output); + const params = await resolvePlanParams(resource); + let action = decideAction({ resource, stateNode, params }); + + if (action.decision === "noop" && driftDetection && resource.read) { + try { + const output = await resource.read(resource.key); + action = decideAction({ + resource, + stateNode, + params, + driftRead: { status: "found", output }, + }); + } catch (error) { + const notFound = resource.notFoundOnError?.some( + (matcher) => matcher.name === (error as Error)?.name, + ); + if (!notFound) throw error; + action = decideAction({ + resource, + stateNode, + params, + driftRead: { status: "not-found" }, + }); + } + } + + nodes.push({ + id: resource.id, + type: resource.type, + decision: action.decision, + ...("diff" in action ? { diff: action.diff } : {}), + params, + dependsOn: getDependencyIds(resource), + }); + } + } + + for (const stateNode of await state.values()) { + if (resourceById.has(stateNode.id)) continue; + nodes.push({ + id: stateNode.id, + type: stateNode.type, + decision: "delete-orphan", + params: stateNode.params, + dependsOn: [], + }); + } + + return { createdAt: new Date().toISOString(), nodes }; +} diff --git a/packages/reconciler/src/protocol.ts b/packages/reconciler/src/protocol.ts index bf3706e..a8ab407 100644 --- a/packages/reconciler/src/protocol.ts +++ b/packages/reconciler/src/protocol.ts @@ -1,4 +1,4 @@ -import type { ReconcilerEvent, ReconcilerEventEmitter } from "./reconciler"; +import type { ReconcilerEvent, ReconcilerEventEmitter } from "./events"; export const EVENT_STREAM_VERSION = 1 as const; diff --git a/packages/reconciler/src/reconciler.ts b/packages/reconciler/src/reconciler.ts deleted file mode 100644 index 2db4628..0000000 --- a/packages/reconciler/src/reconciler.ts +++ /dev/null @@ -1,642 +0,0 @@ -import type { BaseResource, ResourceType } from "@notation/resource"; -import { RevConflict, type State, type StateNode } from "@notation/state"; -import { RetryableError } from "yieldstar"; -import { setTimeout as sleep } from "node:timers/promises"; -import { buildResourceDepthLevels } from "./dependency-graph"; -import { - decideAction, - getDependencyIds, - resolvePlanParams, - type DriftRead, - type Plan, - type PlanNode, - type ResourceAction, -} from "./plan"; -import { - createResourceOperation, - deleteResourceOperation, - matchError, - readResourceOperation, - type OperationLifecycleEvent, - type PollOptions, - type StepRunner, - updateResourceOperation, -} from "./operations"; -import { - createMissingResourceRegistryMatchWarningEvent, - createResourceRegistryFromResources, - resolveResourceClass, - type MissingResourceRegistryMatchWarningEvent, - type ResourceRegistry, -} from "./resource-registry"; - -export type ReconcilerDeployEvent = { - level: "info"; - event: "reconciler.deploy.decision"; - resourceId: string; - resourceType: string; - decision: "create" | "update" | "drift-update" | "drift-recreate" | "noop"; -}; - -export type ReconcilerDriftDetectedEvent = { - level: "info"; - event: "reconciler.drift.detected"; - resourceId: string; - resourceType: string; - diff: Record; -}; - -export type ReconcilerEvent = - | OperationLifecycleEvent - | ReconcilerDeployEvent - | ReconcilerDriftDetectedEvent - | MissingResourceRegistryMatchWarningEvent; - -export type ReconcilerEventEmitter = ( - event: ReconcilerEvent, -) => void | Promise; - -export type ReconcilerState = Pick< - State, - "get" | "update" | "delete" | "values" | "lease" ->; - -export type ReconcilerOptions = { - state: ReconcilerState; - registry?: ResourceRegistry; - dryRun?: boolean; - driftDetection?: boolean; - emit?: ReconcilerEventEmitter; - retryOptions?: PollOptions; - readPollOptions?: PollOptions; - mutationLeaseTtl?: number; -}; - -export type DeployOptions = { - dryRun?: boolean; - driftDetection?: boolean; -}; - -export type DestroyOptions = { - dryRun?: boolean; -}; - -export type RefreshOptions = { - dryRun?: boolean; -}; - -export type PlanOptions = { - driftDetection?: boolean; -}; - -export class Reconciler { - readonly #state: ReconcilerState; - readonly #registry?: ResourceRegistry; - readonly #defaultDryRun: boolean; - readonly #defaultDriftDetection: boolean; - readonly #emit?: ReconcilerEventEmitter; - readonly #retryOptions?: PollOptions; - readonly #readPollOptions?: PollOptions; - readonly #mutationLeaseTtl: number; - readonly #stepRunner: StepRunner; - - constructor(opts: ReconcilerOptions) { - this.#state = opts.state; - this.#registry = opts.registry; - this.#defaultDryRun = opts.dryRun ?? false; - this.#defaultDriftDetection = opts.driftDetection ?? true; - this.#emit = opts.emit; - this.#retryOptions = opts.retryOptions; - this.#readPollOptions = opts.readPollOptions; - this.#mutationLeaseTtl = opts.mutationLeaseTtl ?? 30_000; - this.#stepRunner = createStepRunner(); - } - - async deploy( - resources: BaseResource[], - opts: DeployOptions = {}, - ): Promise { - const dryRun = opts.dryRun ?? this.#defaultDryRun; - const driftDetection = opts.driftDetection ?? this.#defaultDriftDetection; - const resourceById = new Map( - resources.map((resource) => [resource.id, resource]), - ); - - const dependencyLevels = buildResourceDepthLevels(resources); - for (const level of dependencyLevels) { - await Promise.all( - level.map((resource) => - this.#deployResource(resource, dryRun, driftDetection), - ), - ); - } - - await this.#deleteOrphans(resources, resourceById, dryRun, "deploy"); - } - - async plan(resources: BaseResource[], opts: PlanOptions = {}): Promise { - const driftDetection = opts.driftDetection ?? this.#defaultDriftDetection; - const resourceById = new Map( - resources.map((resource) => [resource.id, resource]), - ); - const nodes: PlanNode[] = []; - - const dependencyLevels = buildResourceDepthLevels(resources); - for (const level of dependencyLevels) { - for (const resource of level) { - nodes.push(await this.#planResource(resource, driftDetection)); - } - } - - const stateNodes = await this.#state.values(); - for (const stateNode of stateNodes) { - if (resourceById.has(stateNode.id)) continue; - - nodes.push({ - id: stateNode.id, - type: stateNode.type, - decision: "delete-orphan", - params: stateNode.params, - dependsOn: [], - }); - } - - return { - createdAt: new Date().toISOString(), - nodes, - }; - } - - async destroy( - resources: BaseResource[], - opts: DestroyOptions = {}, - ): Promise { - const dryRun = opts.dryRun ?? this.#defaultDryRun; - const dependencyLevels = buildResourceDepthLevels(resources); - - for ( - let levelIndex = dependencyLevels.length - 1; - levelIndex >= 0; - levelIndex -= 1 - ) { - const level = dependencyLevels[levelIndex]!; - await Promise.all( - level.map((resource) => this.#destroyResource(resource, dryRun)), - ); - } - } - - async refresh( - resources: BaseResource[], - opts: RefreshOptions = {}, - ): Promise { - const dryRun = opts.dryRun ?? this.#defaultDryRun; - const resourceById = new Map( - resources.map((resource) => [resource.id, resource]), - ); - - await this.#deleteOrphans(resources, resourceById, dryRun, "refresh"); - } - - async #deployResource( - resource: BaseResource, - dryRun: boolean, - driftDetection: boolean, - ) { - await this.#withMutationLease(resource.id, () => - this.#retryOnRevConflict((conflict) => - this.#deployResourceOnce(resource, dryRun, driftDetection, conflict), - ), - ); - } - - async #withMutationLease(resourceId: string, fn: () => Promise) { - return this.#withLease(`reconciler:resource:${resourceId}`, fn); - } - - async #withLease(scope: string, fn: () => Promise): Promise { - const lease = await this.#state.lease(scope, this.#mutationLeaseTtl); - const controller = new AbortController(); - let renewalError: unknown; - const heartbeat = (async () => { - try { - while (!controller.signal.aborted) { - await sleep( - Math.max(1, Math.floor(this.#mutationLeaseTtl / 3)), - undefined, - { - signal: controller.signal, - }, - ); - await lease.renew(this.#mutationLeaseTtl); - } - } catch (error) { - if (!controller.signal.aborted) renewalError = error; - } - })(); - - try { - const result = await fn(); - if (renewalError) throw renewalError; - return result; - } finally { - controller.abort(); - await heartbeat; - await lease.release(); - } - } - - async #retryOnRevConflict(fn: (conflict?: RevConflict) => Promise) { - let conflict: RevConflict | undefined; - for (let attempt = 0; attempt < 3; attempt += 1) { - try { - await fn(conflict); - return; - } catch (error) { - if (!(error instanceof RevConflict) || attempt === 2) throw error; - // Re-throwing the conflict supplied for recovery means the resource - // cannot be recovered safely (for example, it has no read operation). - if (error === conflict) throw error; - conflict = error; - } - } - } - - async #deployResourceOnce( - resource: BaseResource, - dryRun: boolean, - driftDetection: boolean, - conflict?: RevConflict, - ) { - if (conflict) { - await this.#recoverDeployResource(resource, dryRun, conflict); - return; - } - - const stateNode = await this.#state.get(resource.id); - - let action: ResourceAction; - if (!stateNode) { - action = decideAction({ resource }); - } else { - resource.setOutput(stateNode.output); - const params = await resource.getParams(); - action = decideAction({ resource, stateNode, params }); - - if (action.decision === "noop" && driftDetection) { - const driftRead = await this.#readForDrift(resource); - action = decideAction({ resource, stateNode, params, driftRead }); - } - } - - if (action.decision === "drift-update") { - await this.#emit?.({ - level: "info", - event: "reconciler.drift.detected", - resourceId: resource.id, - resourceType: resource.type, - diff: action.patch, - }); - } - - await this.#emit?.({ - level: "info", - event: "reconciler.deploy.decision", - resourceId: resource.id, - resourceType: resource.type, - decision: action.decision, - }); - - switch (action.decision) { - case "create": - case "drift-recreate": - await runOperation( - createResourceOperation(this.#stepRunner, { - resource, - state: this.#state, - dryRun, - emit: this.#emit, - retryOptions: this.#retryOptions, - readPollOptions: this.#readPollOptions, - expectedRev: stateNode?.rev ?? 0, - }), - ); - return; - case "update": - case "drift-update": - // decideAction only returns update decisions for an existing stateNode - await runOperation( - updateResourceOperation(this.#stepRunner, { - resource, - state: this.#state, - patch: action.patch, - dryRun, - emit: this.#emit, - retryOptions: this.#retryOptions, - readPollOptions: this.#readPollOptions, - expectedRev: stateNode!.rev, - }), - ); - return; - case "noop": - return; - } - } - - async #recoverDeployResource( - resource: BaseResource, - dryRun: boolean, - conflict: RevConflict, - ) { - if (!resource.read) throw conflict; - - const stateNode = await this.#state.get(resource.id); - if (stateNode) resource.setOutput(stateNode.output); - - const params = await resource.getParams(); - const remote = await this.#readForDrift(resource); - const action = decideAction({ - resource, - stateNode, - params, - driftRead: remote, - }); - if (remote.status === "found") resource.setOutput(remote.output); - - await this.#emit?.({ - level: "info", - event: "reconciler.deploy.decision", - resourceId: resource.id, - resourceType: resource.type, - decision: action.decision, - }); - - switch (action.decision) { - case "create": - case "drift-recreate": - await runOperation( - createResourceOperation(this.#stepRunner, { - resource, - state: this.#state, - dryRun, - emit: this.#emit, - retryOptions: this.#retryOptions, - readPollOptions: this.#readPollOptions, - expectedRev: stateNode?.rev ?? 0, - }), - ); - return; - case "update": - case "drift-update": - await runOperation( - updateResourceOperation(this.#stepRunner, { - resource, - state: this.#state, - patch: action.patch, - dryRun, - emit: this.#emit, - retryOptions: this.#retryOptions, - readPollOptions: this.#readPollOptions, - expectedRev: stateNode?.rev ?? 0, - }), - ); - return; - case "noop": - if (dryRun) return; - await this.#state.update(resource.id, stateNode?.rev ?? 0, { - id: resource.id, - groupId: resource.groupId, - groupType: resource.groupType, - type: resource.type, - lastOperation: "drift", - lastOperationAt: new Date().toISOString(), - config: resource.config, - params: resource.toState(params), - output: resource.toState(resource.output), - }); - return; - } - } - - async #planResource( - resource: BaseResource, - driftDetection: boolean, - ): Promise { - const stateNode = await this.#state.get(resource.id); - if (stateNode) { - resource.setOutput(stateNode.output); - } - - const params = await resolvePlanParams(resource); - let action = decideAction({ resource, stateNode, params }); - - if (action.decision === "noop" && driftDetection) { - const driftRead = await this.#readForDrift(resource); - action = decideAction({ resource, stateNode, params, driftRead }); - } - - return { - id: resource.id, - type: resource.type, - decision: action.decision, - ...("diff" in action ? { diff: action.diff } : {}), - params, - dependsOn: getDependencyIds(resource), - }; - } - - async #readForDrift(resource: BaseResource): Promise { - try { - const output = await runOperation( - readResourceOperation(this.#stepRunner, { - resource, - state: this.#state, - emit: this.#emit, - readPollOptions: this.#readPollOptions, - }), - ); - return { status: "found", output }; - } catch (err) { - const matcher = matchError(err, resource.notFoundOnError); - if (!matcher) throw err; - return { status: "not-found" }; - } - } - - async #deleteOrphans( - resources: BaseResource[], - resourceById: Map, - dryRun: boolean, - workflow: "deploy" | "refresh", - ) { - await this.#withLease("reconciler:orphan-deletion", async () => { - const stateNodes = await this.#state.values(); - const registry = - this.#registry ?? createResourceRegistryFromResources(resources); - - for (const stateNode of stateNodes) { - if (resourceById.has(stateNode.id)) continue; - - const stateNodeResourceType = stateNode.type as ResourceType; - - const Resource = resolveResourceClass(registry, stateNodeResourceType); - if (!Resource) { - await this.#emit?.( - createMissingResourceRegistryMatchWarningEvent({ - workflow, - resourceId: stateNode.id, - resourceType: stateNodeResourceType, - }), - ); - continue; - } - - await this.#withMutationLease(stateNode.id, () => - this.#retryOnRevConflict(async (conflict) => { - const currentNode = await this.#state.get(stateNode.id); - if (!currentNode) return; - - const orphanResource = hydrateResourceFromState( - Resource, - currentNode, - ); - - await this.#deleteResourceOnce( - orphanResource, - currentNode, - dryRun, - conflict, - ); - }), - ); - } - }); - } - - async #destroyResource(resource: BaseResource, dryRun: boolean) { - await this.#withMutationLease(resource.id, () => - this.#retryOnRevConflict(async (conflict) => { - const stateNode = await this.#state.get(resource.id); - if (!stateNode) { - return; - } - - resource.setOutput(stateNode.output); - await this.#deleteResourceOnce(resource, stateNode, dryRun, conflict); - }), - ); - } - - async #deleteResourceOnce( - resource: BaseResource, - stateNode: StateNode, - dryRun: boolean, - conflict?: RevConflict, - ) { - if (conflict) { - if (!resource.read) throw conflict; - - const remote = await this.#readForDrift(resource); - if (remote.status === "not-found") { - if (!dryRun) await this.#state.delete(resource.id, stateNode.rev); - return; - } - resource.setOutput(remote.output); - } - - await runOperation( - deleteResourceOperation(this.#stepRunner, { - resource, - state: this.#state, - dryRun, - emit: this.#emit, - retryOptions: this.#retryOptions, - expectedRev: stateNode.rev, - }), - ); - } -} - -export async function runOperation( - operation: AsyncGenerator, -) { - let next = await operation.next(); - while (!next.done) { - next = await operation.next(); - } - return next.value; -} - -function hydrateResourceFromState( - Resource: new (opts: { - id: string; - config: Record; - }) => BaseResource, - stateNode: StateNode, -): BaseResource { - const resource = new Resource({ - id: stateNode.id, - config: stateNode.config, - }); - resource.setOutput(stateNode.output); - return resource; -} - -export function createStepRunner(): StepRunner { - return { - async *run( - arg1: string | (() => T | Promise), - arg2?: () => T | Promise, - ): AsyncGenerator { - const fn = (typeof arg1 === "string" ? arg2 : arg1) as - (() => T | Promise) | undefined; - - if (!fn) { - throw new Error("Missing run function"); - } - - while (true) { - try { - return await fn(); - } catch (err) { - if (!(err instanceof RetryableError)) { - throw err; - } - } - } - }, - async *poll( - arg1: string | PollOptions, - arg2: PollOptions | (() => boolean | Promise), - arg3?: () => boolean | Promise, - ): AsyncGenerator { - const opts = (typeof arg1 === "string" ? arg2 : arg1) as PollOptions; - const predicate = (typeof arg1 === "string" ? arg3 : arg2) as - (() => boolean | Promise) | undefined; - - if (!predicate) { - throw new Error("Missing poll predicate"); - } - - for (let attempt = 0; attempt < opts.maxAttempts; attempt += 1) { - if (await predicate()) return; - } - - throw new RetryableError("Polling reached max retries", { - maxAttempts: opts.maxAttempts, - retryInterval: opts.retryInterval, - }); - }, - async *delay( - arg1: string | number, - arg2?: number, - ): AsyncGenerator { - const ms = typeof arg1 === "number" ? arg1 : arg2; - if (ms === undefined) { - throw new Error("Missing delay duration"); - } - - await new Promise((resolve) => setTimeout(resolve, ms)); - }, - }; -} diff --git a/packages/reconciler/src/resource-registry.ts b/packages/reconciler/src/resource-registry.ts index 916c0ab..34dabfc 100644 --- a/packages/reconciler/src/resource-registry.ts +++ b/packages/reconciler/src/resource-registry.ts @@ -1,4 +1,8 @@ -import type { BaseResource, ResourceClass, ResourceType } from "@notation/resource"; +import type { + BaseResource, + ResourceClass, + ResourceType, +} from "@notation/resource"; export type ResourceRegistry = Map>; @@ -6,7 +10,7 @@ export type MissingResourceRegistryMatchWarningEvent = { level: "warn"; event: "reconciler.orphan-deletion.skipped"; reason: "resource-type-not-registered"; - workflow: "deploy" | "refresh"; + workflow: "deploy" | "destroy"; resourceId: string; resourceType: ResourceType; }; @@ -46,7 +50,7 @@ export function resolveResourceClass( } export function createMissingResourceRegistryMatchWarningEvent(opts: { - workflow: "deploy" | "refresh"; + workflow: "deploy" | "destroy"; resourceId: string; resourceType: ResourceType; }): MissingResourceRegistryMatchWarningEvent { diff --git a/packages/reconciler/src/yieldstar.ts b/packages/reconciler/src/yieldstar.ts index bf59769..6c249ce 100644 --- a/packages/reconciler/src/yieldstar.ts +++ b/packages/reconciler/src/yieldstar.ts @@ -8,20 +8,21 @@ import { type WorkflowStore, } from "yieldstar"; import { buildResourceDepthLevels } from "./dependency-graph"; +import type { OperationName, ReconcilerEventEmitter } from "./events"; import { decideAction, type ResourceAction } from "./plan"; import { DEFAULT_READ_POLL_OPTIONS, DEFAULT_RETRY_OPTIONS, + createLifecycleEvent, matchError, type PollOptions, -} from "./operations"; +} from "./operation-support"; import { createMissingResourceRegistryMatchWarningEvent, createResourceRegistryFromResources, resolveResourceClass, type ResourceRegistry, } from "./resource-registry"; -import type { ReconcilerEventEmitter } from "./reconciler"; type StoredResourceState = Omit; type CoordinationState = { holder: string | null }; @@ -54,26 +55,31 @@ export const yieldStarDeploymentCoordinationStore = defineStore( type YieldStarStep = Parameters>[0]; -export type YieldStarReconciliationOptions = { +export type YieldStarOperationOptions = { deploymentId: string; executionId: string; resources: BaseResource[]; state: YieldStarStateBackend; registry?: ResourceRegistry; dryRun?: boolean; - driftDetection?: boolean; emit?: ReconcilerEventEmitter; retryOptions?: PollOptions; readPollOptions?: PollOptions; }; +export type YieldStarDeployOptions = YieldStarOperationOptions & { + driftDetection?: boolean; +}; + +export type YieldStarDestroyOptions = YieldStarOperationOptions; + /** * Reconciles resources as a custom YieldStar step. The caller owns the outer * workflow and runtime; Notation owns resource decisions and lifecycle calls. */ -export async function* reconcileWithYieldStar( +export async function* deployWithYieldStar( step: YieldStarStep, - opts: YieldStarReconciliationOptions, + opts: YieldStarDeployOptions, ): AsyncGenerator { const coordination = yield* step.store(yieldStarDeploymentCoordinationStore, { id: opts.deploymentId, @@ -138,10 +144,84 @@ export async function* reconcileWithYieldStar( } } +/** Durably destroys persisted resources in reverse dependency order. */ +export async function* destroyWithYieldStar( + step: YieldStarStep, + opts: YieldStarDestroyOptions, +): AsyncGenerator { + const coordination = yield* step.store(yieldStarDeploymentCoordinationStore, { + id: opts.deploymentId, + initial: { holder: null }, + }); + + yield* coordination.take( + "notation:coordination:acquire", + (state) => state.holder === null || state.holder === opts.executionId, + (draft) => { + draft.holder = opts.executionId; + }, + ); + + try { + const resourceById = new Map( + opts.resources.map((resource) => [resource.id, resource]), + ); + const levels = buildResourceDepthLevels(opts.resources); + + for (let index = levels.length - 1; index >= 0; index -= 1) { + for (const resource of levels[index]!) { + const stateNode = yield* step.run( + `notation:destroy:${resource.id}:state:lookup`, + () => opts.state.get(resource.id), + ); + if (!stateNode) continue; + resource.setOutput(stateNode.output); + yield* deleteResource(step, resource, opts, "destroy"); + } + } + + const persisted = yield* step.run("notation:destroy:orphans:list", () => + opts.state.values(), + ); + const registry = + opts.registry ?? createResourceRegistryFromResources(opts.resources); + + for (const node of persisted) { + if (resourceById.has(node.id)) continue; + const Resource = resolveResourceClass( + registry, + node.type as ResourceType, + ); + if (!Resource) { + yield* emitDurably( + step, + `notation:destroy:orphan:${node.id}:warning`, + opts.emit, + () => + createMissingResourceRegistryMatchWarningEvent({ + workflow: "destroy", + resourceId: node.id, + resourceType: node.type as ResourceType, + }), + ); + continue; + } + + const resource = new Resource({ id: node.id, config: node.config }); + resource.setOutput(node.output); + yield* deleteResource(step, resource, opts, "destroy-orphan"); + } + } finally { + yield* coordination.update("notation:coordination:release", (draft) => { + if (draft.holder === opts.executionId) draft.holder = null; + }); + } +} + async function* reconcileResource( step: YieldStarStep, resource: BaseResource, - opts: YieldStarReconciliationOptions, + opts: YieldStarDeployOptions, ): AsyncGenerator { const prefix = `notation:resource:${resource.id}`; let stateNode = yield* step.run(`${prefix}:state:lookup`, () => @@ -201,97 +281,137 @@ async function* reconcileResource( })); if (action.decision === "noop") return; - if (opts.dryRun) return; - - if (action.decision === "create" || action.decision === "drift-recreate") { - const primaryKey = yield* runProviderCall( + const operation = + action.decision === "create" || action.decision === "drift-recreate" + ? "create" + : "update"; + const patch = "patch" in action ? action.patch : {}; + yield* emitOperationLifecycle( + step, + `${prefix}:${operation}:start`, + opts.emit, + resource, + operation, + "start", + ); + if (opts.dryRun) { + yield* emitOperationLifecycle( step, - `${prefix}:create`, - () => resource.create(params), + `${prefix}:${operation}:dry-run`, + opts.emit, resource, - opts.retryOptions, + operation, + "dry-run", ); - resource.setOutput(params); - if (primaryKey) resource.setOutput({ ...primaryKey, ...resource.output }); - } else { - if (!resource.update) { - yield* emitDurably(step, `${prefix}:update-skip`, opts.emit, () => ({ - level: "info", - event: "reconciler.operation.lifecycle", - operation: "update", - status: "skip", - resourceId: resource.id, - resourceType: resource.type, - reason: "update-not-implemented", - })); - return; + return; + } + + try { + if (operation === "create") { + const primaryKey = yield* runProviderCall( + step, + `${prefix}:create`, + () => resource.create(params), + resource, + opts.retryOptions, + ); + resource.setOutput(params); + if (primaryKey) resource.setOutput({ ...primaryKey, ...resource.output }); + } else { + if (!resource.update) { + yield* emitOperationLifecycle( + step, + `${prefix}:update:skip`, + opts.emit, + resource, + "update", + "skip", + { reason: "update-not-implemented" }, + ); + return; + } + yield* runProviderCall( + step, + `${prefix}:update`, + () => + resource.update!( + resource.key, + patch, + params, + resource.toState(resource.output), + ), + resource, + opts.retryOptions, + ); + resource.setOutput({ ...resource.key, ...params }); } - yield* runProviderCall( + + const read = yield* readRemote( step, - `${prefix}:update`, - () => - resource.update!( - resource.key, - action.patch, - params, - resource.toState(resource.output), - ), resource, - opts.retryOptions, + opts, + `${prefix}:read-after-write`, ); - resource.setOutput({ ...resource.key, ...params }); - } - - const read = yield* readRemote( - step, - resource, - opts, - `${prefix}:read-after-write`, - ); - if (read.status === "found") - resource.setOutput({ ...resource.output, ...read.output }); - - const operation = - action.decision === "create" || action.decision === "drift-recreate" - ? "create" - : "update"; - const nextState: StoredResourceState = { - id: resource.id, - groupId: resource.groupId, - groupType: resource.groupType, - type: resource.type, - lastOperation: operation, - lastOperationAt: new Date().toISOString(), - config: resource.config, - params: resource.toState(params), - output: resource.toState(resource.output), - }; + if (read.status === "found") + resource.setOutput({ ...resource.output, ...read.output }); + + const nextState: StoredResourceState = { + id: resource.id, + groupId: resource.groupId, + groupType: resource.groupType, + type: resource.type, + lastOperation: operation, + lastOperationAt: new Date().toISOString(), + config: resource.config, + params: resource.toState(params), + output: resource.toState(resource.output), + }; - if (!stateStore || !snapshot) { - yield* step.store(yieldStarResourceStateStore, { - id: opts.state.storeId(resource.id), - initial: nextState, - }); - return; - } + if (!stateStore || !snapshot) { + yield* step.store(yieldStarResourceStateStore, { + id: opts.state.storeId(resource.id), + initial: nextState, + }); + } else { + const result = yield* stateStore.updateFrom( + `${prefix}:state:persist`, + snapshot, + () => nextState, + ); + if (!result.updated) + throw new RevConflict( + resource.id, + stateNode?.rev ?? 0, + result.actualVersion + 1, + ); + } - const result = yield* stateStore.updateFrom( - `${prefix}:state:persist`, - snapshot, - () => nextState, - ); - if (!result.updated) - throw new RevConflict( - resource.id, - stateNode?.rev ?? 0, - result.actualVersion + 1, + yield* emitOperationLifecycle( + step, + `${prefix}:${operation}:success`, + opts.emit, + resource, + operation, + "success", ); + } catch (error) { + yield* emitOperationLifecycle( + step, + `${prefix}:${operation}:error`, + opts.emit, + resource, + operation, + "error", + { error }, + ); + throw error; + } } async function* deleteResource( step: YieldStarStep, resource: BaseResource, - opts: YieldStarReconciliationOptions, + opts: YieldStarOperationOptions, suffix: string, ): AsyncGenerator { const prefix = `notation:${suffix}:${resource.id}`; @@ -300,7 +420,28 @@ async function* deleteResource( const stateNode = toStateNode(snapshot); resource.setOutput(stateNode.output); - if (!opts.dryRun) { + yield* emitOperationLifecycle( + step, + `${prefix}:delete:start`, + opts.emit, + resource, + "delete", + "start", + ); + + if (opts.dryRun) { + yield* emitOperationLifecycle( + step, + `${prefix}:delete:dry-run`, + opts.emit, + resource, + "delete", + "dry-run", + ); + return; + } + + try { try { yield* runProviderCall( step, @@ -311,6 +452,15 @@ async function* deleteResource( ); } catch (error) { if (!matchError(error, resource.notFoundOnError)) throw error; + yield* emitOperationLifecycle( + step, + `${prefix}:delete:not-found`, + opts.emit, + resource, + "delete", + "skip", + { reason: "resource-not-found" }, + ); } const deleted = yield* stateStore.deleteFrom( @@ -319,13 +469,32 @@ async function* deleteResource( ); if (!deleted.deleted) throw new RevConflict(resource.id, stateNode.rev, undefined); + yield* emitOperationLifecycle( + step, + `${prefix}:delete:success`, + opts.emit, + resource, + "delete", + "success", + ); + } catch (error) { + yield* emitOperationLifecycle( + step, + `${prefix}:delete:error`, + opts.emit, + resource, + "delete", + "error", + { error }, + ); + throw error; } } async function* readRemote( step: YieldStarStep, resource: BaseResource, - opts: YieldStarReconciliationOptions, + opts: YieldStarOperationOptions, key: string, ): AsyncGenerator< any, @@ -334,12 +503,29 @@ async function* readRemote( any > { if (!resource.read) { + yield* emitOperationLifecycle( + step, + `${key}:skip`, + opts.emit, + resource, + "read", + "skip", + { reason: "read-not-implemented" }, + ); return { status: "found", output: { ...(await resource.getParams()), ...resource.output }, }; } + yield* emitOperationLifecycle( + step, + `${key}:start`, + opts.emit, + resource, + "read", + "start", + ); try { const output = yield* step.run(key, async () => { const value = await resource.read!(resource.key); @@ -358,10 +544,37 @@ async function* readRemote( } return value; }); + yield* emitOperationLifecycle( + step, + `${key}:success`, + opts.emit, + resource, + "read", + "success", + ); return { status: "found", output }; } catch (error) { - if (matchError(error, resource.notFoundOnError)) + if (matchError(error, resource.notFoundOnError)) { + yield* emitOperationLifecycle( + step, + `${key}:not-found`, + opts.emit, + resource, + "read", + "skip", + { reason: "resource-not-found" }, + ); return { status: "not-found" }; + } + yield* emitOperationLifecycle( + step, + `${key}:error`, + opts.emit, + resource, + "read", + "error", + { error }, + ); throw error; } } @@ -409,6 +622,26 @@ function emitDurably( }); } +function emitOperationLifecycle( + step: YieldStarStep, + key: string, + emit: ReconcilerEventEmitter | undefined, + resource: BaseResource, + operation: OperationName, + status: "start" | "success" | "error" | "skip" | "dry-run", + extra: { reason?: string; error?: unknown } = {}, +) { + return emitDurably(step, key, emit, () => + createLifecycleEvent({ + operation, + status, + resourceId: resource.id, + resourceType: resource.type, + ...extra, + }), + ); +} + /** A Notation state backend backed by YieldStar 0.5 durable stores. */ export class YieldStarStateBackend { readonly #client: StoreClient; diff --git a/packages/reconciler/test/operation.workflows.test.ts b/packages/reconciler/test/operation.workflows.test.ts deleted file mode 100644 index 8709086..0000000 --- a/packages/reconciler/test/operation.workflows.test.ts +++ /dev/null @@ -1,315 +0,0 @@ -import { describe, expect, it, vi } from "vitest"; -import { RetryableError } from "yieldstar"; -import { resource } from "@notation/resource"; -import { - createResourceOperation, - deleteResourceOperation, - readResourceOperation, - type OperationLifecycleEvent, - type PollOptions, - type StepRunner, -} from "../src/operations"; - -function createStepRunnerDouble(): StepRunner { - const run = vi.fn(async function* ( - arg1: string | (() => T | Promise), - arg2?: () => T | Promise, - ): AsyncGenerator { - const fn = (typeof arg1 === "string" ? arg2 : arg1) as () => T | Promise; - if (!fn) { - throw new Error("Missing run function"); - } - - while (true) { - try { - return await fn(); - } catch (err) { - if (!(err instanceof RetryableError)) { - throw err; - } - } - } - }); - - const poll = vi.fn(async function* ( - arg1: string | PollOptions, - arg2: PollOptions | (() => boolean | Promise), - arg3?: () => boolean | Promise, - ): AsyncGenerator { - const opts = (typeof arg1 === "string" ? arg2 : arg1) as PollOptions; - const predicate = (typeof arg1 === "string" ? arg3 : arg2) as - | (() => boolean | Promise) - | undefined; - - if (!predicate) { - throw new Error("Missing poll predicate"); - } - - for (let attempt = 0; attempt < opts.maxAttempts; attempt++) { - if (await predicate()) { - return; - } - } - - throw new RetryableError("Polling reached max retries", { - maxAttempts: opts.maxAttempts, - retryInterval: opts.retryInterval, - }); - }); - - const delay = vi.fn(async function* (): AsyncGenerator { - return; - }); - - return { - run, - poll, - delay, - }; -} - -async function runOperation(operation: AsyncGenerator) { - let next = await operation.next(); - while (!next.done) { - next = await operation.next(); - } - return next.value; -} - -describe("operation workflows", () => { - it("create performs create + read-after-create + state persistence", async () => { - const step = createStepRunnerDouble(); - const events: OperationLifecycleEvent[] = []; - const state = { - get: vi.fn(async () => undefined), - update: vi.fn(async () => undefined), - delete: vi.fn(async () => undefined), - }; - - let createAttempts = 0; - const createMock = vi.fn(async () => { - createAttempts += 1; - if (createAttempts === 1) { - const err = new Error("eventual consistency"); - err.name = "RetryCreate"; - throw err; - } - return { remoteId: "abc" }; - }); - - const TestResource = resource({ type: "test/service/create" }) - .defineSchema({}) - .defineOperations({ - create: createMock, - read: async () => ({ remoteId: "abc", status: "ready" }), - delete: async () => undefined, - retryLaterOnError: [{ name: "RetryCreate", reason: "retry create" }], - }); - - const testResource = new TestResource({ id: "test-create" }); - - await runOperation( - createResourceOperation(step, { - resource: testResource, - state, - expectedRev: 0, - emit: async (event) => { - events.push(event); - }, - }), - ); - - expect(createAttempts).toBe(2); - expect(state.update).toHaveBeenCalledOnce(); - expect(createMock).toHaveBeenCalledWith(await testResource.getParams()); - expect(testResource.output).toEqual({ remoteId: "abc", status: "ready" }); - expect(events.map((event) => `${event.operation}:${event.status}`)).toEqual([ - "create:start", - "read:start", - "read:success", - "create:success", - ]); - expect(events[0]).toMatchObject({ - resourceId: "test-create", - resourceType: TestResource.type, - event: "reconciler.operation.lifecycle", - }); - }); - - it("read uses durable polling semantics for retryReadOnCondition", async () => { - const step = createStepRunnerDouble(); - const state = { - get: vi.fn(async () => undefined), - update: vi.fn(async () => undefined), - delete: vi.fn(async () => undefined), - }; - - let readAttempts = 0; - const TestResource = resource({ type: "test/service/read" }) - .defineSchema({}) - .defineOperations({ - create: async () => ({}), - read: async () => { - readAttempts += 1; - if (readAttempts < 3) { - return { status: "pending" }; - } - return { status: "ready" }; - }, - delete: async () => undefined, - retryReadOnCondition: [ - { - key: "status", - value: "ready", - reason: "resource is not ready", - }, - ], - }); - - const testResource = new TestResource({ id: "test-read" }); - - const result = await runOperation( - readResourceOperation(step, { - resource: testResource, - state, - }), - ); - - expect(readAttempts).toBe(3); - expect((step.poll as any).mock.calls.length).toBe(1); - expect(result).toEqual({ status: "ready" }); - }); - - it("delete treats only resource.notFoundOnError matchers as skip", async () => { - const step = createStepRunnerDouble(); - const events: OperationLifecycleEvent[] = []; - const state = { - get: vi.fn(async () => undefined), - update: vi.fn(async () => undefined), - delete: vi.fn(async () => undefined), - }; - - const TestResource = resource({ type: "test/service/delete" }) - .defineSchema({}) - .defineOperations({ - create: async () => ({}), - delete: async () => { - const err = new Error("gone"); - err.name = "RemoteMissing"; - throw err; - }, - notFoundOnError: [ - { - name: "RemoteMissing", - reason: "already deleted remotely", - }, - ], - }); - - const testResource = new TestResource({ id: "test-delete" }); - - await runOperation( - deleteResourceOperation(step, { - resource: testResource, - state, - expectedRev: 1, - emit: async (event) => { - events.push(event); - }, - }), - ); - - expect(state.delete).toHaveBeenCalledWith("test-delete", 1); - expect(events.map((event) => event.status)).toEqual([ - "start", - "skip", - "success", - ]); - }); - - it("delete rethrows when error does not match notFoundOnError", async () => { - const step = createStepRunnerDouble(); - const state = { - get: vi.fn(async () => undefined), - update: vi.fn(async () => undefined), - delete: vi.fn(async () => undefined), - }; - - const TestResource = resource({ type: "test/service/delete-miss" }) - .defineSchema({}) - .defineOperations({ - create: async () => ({}), - delete: async () => { - const err = new Error("still exists"); - err.name = "DifferentError"; - throw err; - }, - notFoundOnError: [ - { - name: "RemoteMissing", - reason: "already deleted remotely", - }, - ], - }); - - const testResource = new TestResource({ id: "test-delete-miss" }); - - await expect( - runOperation( - deleteResourceOperation(step, { - resource: testResource, - state, - expectedRev: 1, - }), - ), - ).rejects.toMatchObject({ name: "DifferentError", message: "still exists" }); - - expect(state.delete).not.toHaveBeenCalled(); - }); - - it("emits structured error details on operation failure", async () => { - const step = createStepRunnerDouble(); - const events: OperationLifecycleEvent[] = []; - const state = { - get: vi.fn(async () => undefined), - update: vi.fn(async () => undefined), - delete: vi.fn(async () => undefined), - }; - - const TestResource = resource({ type: "test/service/create-error" }) - .defineSchema({}) - .defineOperations({ - create: async () => { - const err = new Error("boom"); - err.name = "CreateFailed"; - throw err; - }, - delete: async () => undefined, - }); - - const testResource = new TestResource({ id: "test-create-error" }); - - await expect( - runOperation( - createResourceOperation(step, { - resource: testResource, - state, - expectedRev: 0, - emit: async (event) => { - events.push(event); - }, - }), - ), - ).rejects.toMatchObject({ name: "CreateFailed", message: "boom" }); - - expect(events.map((event) => event.status)).toEqual(["start", "error"]); - expect(events[1]).toMatchObject({ - operation: "create", - status: "error", - resourceId: "test-create-error", - resourceType: TestResource.type, - errorName: "CreateFailed", - errorMessage: "boom", - }); - }); -}); diff --git a/packages/reconciler/test/planner.test.ts b/packages/reconciler/test/planner.test.ts new file mode 100644 index 0000000..a215e82 --- /dev/null +++ b/packages/reconciler/test/planner.test.ts @@ -0,0 +1,36 @@ +import { resource } from "@notation/resource"; +import { MemoryStateBackend } from "@notation/state"; +import { describe, expect, it } from "vitest"; +import { createPlan } from "../src/planner"; + +describe("createPlan", () => { + it("plans desired creates and persisted orphans without mutation execution", async () => { + const TestResource = resource({ type: "test/planner/resource" }) + .defineSchema({}) + .defineOperations({ + create: async () => undefined, + delete: async () => undefined, + }); + const state = new MemoryStateBackend(); + await state.update("orphan", 0, { + id: "orphan", + type: TestResource.type, + config: {}, + params: {}, + output: {}, + lastOperation: "create", + lastOperationAt: "2026-07-22T00:00:00.000Z", + }); + + const plan = await createPlan({ + resources: [new TestResource({ id: "desired" })], + state, + driftDetection: false, + }); + + expect(plan.nodes).toEqual([ + expect.objectContaining({ id: "desired", decision: "create" }), + expect.objectContaining({ id: "orphan", decision: "delete-orphan" }), + ]); + }); +}); diff --git a/packages/reconciler/test/reconciler.deploy.test.ts b/packages/reconciler/test/reconciler.deploy.test.ts deleted file mode 100644 index 7220a65..0000000 --- a/packages/reconciler/test/reconciler.deploy.test.ts +++ /dev/null @@ -1,811 +0,0 @@ -import { describe, expect, it, vi } from "vitest"; -import { resource, type ErrorMatcher } from "@notation/resource"; -import { - LeaseConflict, - MemoryStateBackend, - RevConflict, - type StateNode, -} from "@notation/state"; -import { Reconciler, createResourceRegistry } from "../src"; - -function createMemoryState(initial: Record = {}) { - const store: Record = { ...initial }; - - return { - store, - get: vi.fn(async (id: string) => store[id]), - update: vi.fn( - async (id: string, expectedRev: number, patch: Partial) => { - const actualRev = store[id]?.rev ?? 0; - if (actualRev !== expectedRev) { - throw new RevConflict(id, expectedRev, store[id]?.rev); - } - const rev = actualRev + 1; - store[id] = { - ...(store[id] ?? {}), - ...patch, - rev, - } as StateNode; - return { rev }; - }, - ), - delete: vi.fn(async (id: string, expectedRev: number) => { - const actualRev = store[id]?.rev ?? 0; - if (actualRev !== expectedRev) { - throw new RevConflict(id, expectedRev, store[id]?.rev); - } - delete store[id]; - }), - values: vi.fn(async () => Object.values(store)), - lease: vi.fn(async (scope: string, ttl: number) => { - let expiresAt = new Date(Date.now() + ttl).toISOString(); - return { - scope, - get expiresAt() { - return expiresAt; - }, - renew: vi.fn(async (nextTtl: number) => { - expiresAt = new Date(Date.now() + nextTtl).toISOString(); - return expiresAt; - }), - release: vi.fn(async () => undefined), - }; - }), - }; -} - -function createTestResourceClass(opts: { - type: `${string}/${string}/${string}`; - create?: ( - params: Record, - ) => Promise | void>; - read?: (key: Record) => Promise>; - update?: ( - key: Record, - patch: Record, - params: Record, - state: Record, - ) => Promise; - delete?: ( - key: Record, - state: Record, - ) => Promise; - notFoundOnError?: ErrorMatcher[]; -}) { - return resource({ type: opts.type }) - .defineSchema({ - name: { - presence: "required", - propertyType: "param", - valueType: "string" as any, - }, - }) - .defineOperations({ - create: opts.create ?? (async () => ({})), - read: opts.read, - update: opts.update, - delete: opts.delete ?? (async () => undefined), - notFoundOnError: opts.notFoundOnError, - }); -} - -const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); - -describe("reconciler deploy", () => { - it("chooses create vs update from desired params vs state", async () => { - const createSpy = vi.fn(async () => ({ name: "new" })); - const updateSpy = vi.fn(async () => undefined); - - const CreateResource = createTestResourceClass({ - type: "test/service/create-choice", - create: createSpy, - read: async () => ({ name: "new" }), - }); - const UpdateResource = createTestResourceClass({ - type: "test/service/update-choice", - update: updateSpy, - read: async () => ({ name: "new" }), - }); - - const state = createMemoryState({ - existing: { - rev: 1, - id: "existing", - groupId: -1, - groupType: "", - type: UpdateResource.type, - config: { name: "old" }, - params: { name: "old" }, - output: { name: "old" }, - lastOperation: "create", - lastOperationAt: new Date().toISOString(), - }, - }); - - const events: string[] = []; - const reconciler = new Reconciler({ - state, - driftDetection: false, - emit: async (event) => { - if ("operation" in event) { - events.push(`${event.operation}:${event.status}:${event.resourceId}`); - } - }, - }); - - await reconciler.deploy([ - new CreateResource({ id: "new", config: { name: "new" } }), - new UpdateResource({ id: "existing", config: { name: "new" } }), - ]); - - expect(createSpy).toHaveBeenCalledOnce(); - expect(updateSpy).toHaveBeenCalledOnce(); - expect(updateSpy.mock.calls[0]?.[1]).toEqual({ name: "new" }); - expect(events).toContain("create:success:new"); - expect(events).toContain("update:success:existing"); - }); - - it("persists first-time creates with an expect-absent revision", async () => { - const CreateResource = createTestResourceClass({ - type: "test/service/first-create", - create: async () => ({ name: "new" }), - read: async () => ({ name: "new" }), - }); - const state = createMemoryState(); - const reconciler = new Reconciler({ state, driftDetection: false }); - - await reconciler.deploy([ - new CreateResource({ id: "new", config: { name: "new" } }), - ]); - - expect(state.update).toHaveBeenCalledWith("new", 0, expect.any(Object)); - }); - - it("leases a resource before remote create so concurrent deploys cannot duplicate it", async () => { - let signalCreateStarted!: () => void; - const createStarted = new Promise((resolve) => { - signalCreateStarted = resolve; - }); - let allowCreateToFinish!: () => void; - const createCanFinish = new Promise((resolve) => { - allowCreateToFinish = resolve; - }); - const createSpy = vi.fn(async () => { - signalCreateStarted(); - await createCanFinish; - return { name: "new" }; - }); - const CreateResource = createTestResourceClass({ - type: "test/service/concurrent-create", - create: createSpy, - read: async () => ({ name: "new" }), - }); - const state = new MemoryStateBackend(); - const first = new Reconciler({ state, driftDetection: false }); - const second = new Reconciler({ state, driftDetection: false }); - - const firstDeploy = first.deploy([ - new CreateResource({ id: "new", config: { name: "new" } }), - ]); - await createStarted; - - await expect( - second.deploy([ - new CreateResource({ id: "new", config: { name: "new" } }), - ]), - ).rejects.toBeInstanceOf(LeaseConflict); - - allowCreateToFinish(); - await firstDeploy; - expect(createSpy).toHaveBeenCalledOnce(); - }); - - it("reads remote state after an update conflict instead of repeating the update", async () => { - let remoteName = "old"; - const readSpy = vi.fn(async () => ({ name: remoteName })); - const updateSpy = vi.fn(async (_key, _patch, params) => { - remoteName = params.name as string; - }); - const UpdateResource = createTestResourceClass({ - type: "test/service/update-conflict", - read: readSpy, - update: updateSpy, - }); - const state = createMemoryState({ - existing: { - rev: 1, - id: "existing", - groupId: -1, - groupType: "", - type: UpdateResource.type, - config: { name: "old" }, - params: { name: "old" }, - output: { name: "old" }, - lastOperation: "create", - lastOperationAt: new Date().toISOString(), - }, - }); - const updateState = state.update.getMockImplementation()!; - state.update - .mockImplementationOnce(async () => { - state.store.existing = { - ...state.store.existing!, - rev: 2, - config: { name: "concurrent" }, - params: { name: "concurrent" }, - output: { name: "concurrent" }, - }; - throw new RevConflict("existing", 1, 2); - }) - .mockImplementation(updateState); - - const reconciler = new Reconciler({ state, driftDetection: false }); - await reconciler.deploy([ - new UpdateResource({ id: "existing", config: { name: "new" } }), - ]); - - expect(updateSpy).toHaveBeenCalledOnce(); - expect(readSpy).toHaveBeenCalledTimes(2); - expect(state.update).toHaveBeenLastCalledWith( - "existing", - 2, - expect.objectContaining({ - params: { name: "new" }, - output: { name: "new" }, - lastOperation: "drift", - }), - ); - expect(state.store.existing).toMatchObject({ - rev: 3, - params: { name: "new" }, - output: { name: "new" }, - }); - }); - - it("reads remote state after a create conflict instead of creating twice", async () => { - let remoteName: string | undefined; - const createSpy = vi.fn(async (params) => { - remoteName = params.name as string; - return { name: remoteName }; - }); - const readSpy = vi.fn(async () => ({ name: remoteName! })); - const CreateResource = createTestResourceClass({ - type: "test/service/create-conflict", - create: createSpy, - read: readSpy, - }); - const state = createMemoryState(); - const updateState = state.update.getMockImplementation()!; - state.update - .mockImplementationOnce(async () => { - state.store.new = { - rev: 1, - id: "new", - groupId: -1, - groupType: "", - type: CreateResource.type, - config: { name: "concurrent" }, - params: { name: "concurrent" }, - output: { name: "concurrent" }, - lastOperation: "create", - lastOperationAt: new Date().toISOString(), - }; - throw new RevConflict("new", 0, 1); - }) - .mockImplementation(updateState); - - const reconciler = new Reconciler({ state, driftDetection: false }); - await reconciler.deploy([ - new CreateResource({ id: "new", config: { name: "new" } }), - ]); - - expect(createSpy).toHaveBeenCalledOnce(); - expect(readSpy).toHaveBeenCalledTimes(2); - expect(state.store.new).toMatchObject({ - rev: 2, - params: { name: "new" }, - output: { name: "new" }, - lastOperation: "drift", - }); - }); - - it("does not blindly retry a conflicted mutation without a read operation", async () => { - const updateSpy = vi.fn(async () => undefined); - const UpdateResource = createTestResourceClass({ - type: "test/service/unreadable-conflict", - update: updateSpy, - }); - const state = createMemoryState({ - existing: { - rev: 1, - id: "existing", - groupId: -1, - groupType: "", - type: UpdateResource.type, - config: { name: "old" }, - params: { name: "old" }, - output: { name: "old" }, - lastOperation: "create", - lastOperationAt: new Date().toISOString(), - }, - }); - state.update.mockImplementationOnce(async () => { - state.store.existing = { ...state.store.existing!, rev: 2 }; - throw new RevConflict("existing", 1, 2); - }); - - const reconciler = new Reconciler({ state, driftDetection: false }); - await expect( - reconciler.deploy([ - new UpdateResource({ id: "existing", config: { name: "new" } }), - ]), - ).rejects.toMatchObject({ - id: "existing", - expectedRev: 1, - actualRev: 2, - }); - - expect(updateSpy).toHaveBeenCalledOnce(); - expect(state.update).toHaveBeenCalledOnce(); - }); - - it("runs independent resources concurrently per dependency depth", async () => { - const marks: Record = {}; - - const AResource = createTestResourceClass({ - type: "test/service/a", - create: async () => { - marks.aStart = Date.now(); - await sleep(60); - marks.aEnd = Date.now(); - return { name: "a" }; - }, - read: async () => ({ name: "a" }), - }); - const CResource = createTestResourceClass({ - type: "test/service/c", - create: async () => { - marks.cStart = Date.now(); - await sleep(60); - marks.cEnd = Date.now(); - return { name: "c" }; - }, - read: async () => ({ name: "c" }), - }); - const BResource = createTestResourceClass({ - type: "test/service/b", - create: async () => { - marks.bStart = Date.now(); - return { name: "b" }; - }, - read: async () => ({ name: "b" }), - }); - - const state = createMemoryState(); - const resourceA = new AResource({ id: "a", config: { name: "a" } }); - const resourceB = new BResource({ - id: "b", - config: { name: "b" }, - dependencies: { a: resourceA }, - }); - const resourceC = new CResource({ id: "c", config: { name: "c" } }); - - const reconciler = new Reconciler({ state, driftDetection: false }); - await reconciler.deploy([resourceA, resourceB, resourceC]); - - expect(Math.abs(marks.aStart - marks.cStart)).toBeLessThan(40); - expect(marks.bStart).toBeGreaterThanOrEqual(marks.aEnd); - }); - - it("detects drift using live read output and converges with update", async () => { - const updateSpy = vi.fn(async () => undefined); - const events: Array> = []; - const TestResource = createTestResourceClass({ - type: "test/service/drift", - read: async () => ({ name: "drifted" }), - update: updateSpy, - }); - - const state = createMemoryState({ - resource: { - rev: 1, - id: "resource", - groupId: -1, - groupType: "", - type: TestResource.type, - config: { name: "desired" }, - params: { name: "desired" }, - output: { name: "desired" }, - lastOperation: "create", - lastOperationAt: new Date().toISOString(), - }, - }); - - const reconciler = new Reconciler({ - state, - driftDetection: true, - emit: async (event) => { - events.push(event as unknown as Record); - }, - }); - await reconciler.deploy([ - new TestResource({ id: "resource", config: { name: "desired" } }), - ]); - - expect(updateSpy).toHaveBeenCalledOnce(); - expect(updateSpy.mock.calls[0]?.[1]).toEqual({ name: "desired" }); - expect(events).toContainEqual({ - level: "info", - event: "reconciler.drift.detected", - resourceId: "resource", - resourceType: TestResource.type, - diff: { name: "desired" }, - }); - }); - - it("deletes orphaned state entries by reconstructing from registry", async () => { - const deleteSpy = vi.fn(async () => undefined); - const OrphanResource = createTestResourceClass({ - type: "test/service/orphan", - delete: deleteSpy, - }); - - const state = createMemoryState({ - orphan: { - rev: 1, - id: "orphan", - groupId: -1, - groupType: "", - type: OrphanResource.type, - config: { name: "from-state" }, - params: { name: "from-state" }, - output: { name: "from-state" }, - lastOperation: "create", - lastOperationAt: new Date().toISOString(), - }, - }); - - const reconciler = new Reconciler({ - state, - registry: createResourceRegistry([OrphanResource]), - driftDetection: false, - }); - - await reconciler.deploy([]); - - expect(deleteSpy).toHaveBeenCalledOnce(); - expect(state.delete).toHaveBeenCalledWith("orphan", 1); - }); - - it("dryRun emits operation intent without applying side effects", async () => { - const createSpy = vi.fn(async () => ({ name: "new" })); - const deleteSpy = vi.fn(async () => undefined); - - const CreateResource = createTestResourceClass({ - type: "test/service/dry-run-create", - create: createSpy, - read: async () => ({ name: "new" }), - }); - const OrphanResource = createTestResourceClass({ - type: "test/service/dry-run-orphan", - delete: deleteSpy, - }); - - const state = createMemoryState({ - orphan: { - rev: 1, - id: "orphan", - groupId: -1, - groupType: "", - type: OrphanResource.type, - config: { name: "orphan" }, - params: { name: "orphan" }, - output: { name: "orphan" }, - lastOperation: "create", - lastOperationAt: new Date().toISOString(), - }, - }); - - const operationEvents: string[] = []; - const reconciler = new Reconciler({ - state, - registry: createResourceRegistry([OrphanResource]), - dryRun: true, - driftDetection: false, - emit: async (event) => { - if ("operation" in event) { - operationEvents.push( - `${event.operation}:${event.status}:${event.resourceId}`, - ); - } - }, - }); - - await reconciler.deploy([ - new CreateResource({ id: "new", config: { name: "new" } }), - ]); - - expect(createSpy).not.toHaveBeenCalled(); - expect(deleteSpy).not.toHaveBeenCalled(); - expect(state.update).not.toHaveBeenCalled(); - expect(state.delete).not.toHaveBeenCalled(); - expect(operationEvents).toContain("create:dry-run:new"); - expect(operationEvents).toContain("delete:dry-run:orphan"); - }); -}); - -describe("reconciler destroy + refresh", () => { - it("reads remote state after a delete conflict instead of deleting twice", async () => { - let remoteExists = true; - const deleteSpy = vi.fn(async () => { - remoteExists = false; - }); - const readSpy = vi.fn(async () => { - if (!remoteExists) { - const error = new Error("gone"); - error.name = "RemoteMissing"; - throw error; - } - return { name: "doomed" }; - }); - const DestroyResource = createTestResourceClass({ - type: "test/service/destroy-retry", - read: readSpy, - delete: deleteSpy, - notFoundOnError: [{ name: "RemoteMissing", reason: "deleted" }], - }); - const state = createMemoryState({ - doomed: { - rev: 1, - id: "doomed", - groupId: -1, - groupType: "", - type: DestroyResource.type, - config: { name: "doomed" }, - params: { name: "doomed" }, - output: { name: "doomed" }, - lastOperation: "create", - lastOperationAt: new Date().toISOString(), - }, - }); - const deleteState = state.delete.getMockImplementation()!; - state.delete - .mockImplementationOnce(async () => { - state.store.doomed = { ...state.store.doomed!, rev: 2 }; - throw new RevConflict("doomed", 1, 2); - }) - .mockImplementation(deleteState); - - const reconciler = new Reconciler({ state }); - await reconciler.destroy([ - new DestroyResource({ id: "doomed", config: { name: "doomed" } }), - ]); - - expect(deleteSpy).toHaveBeenCalledOnce(); - expect(readSpy).toHaveBeenCalledOnce(); - expect(state.delete).toHaveBeenCalledTimes(2); - expect(state.store.doomed).toBeUndefined(); - }); - - it("holds a backend lease for the orphan snapshot", async () => { - const state = createMemoryState(); - const release = vi.fn(async () => undefined); - const lease = vi.fn(async () => ({ - scope: "reconciler:orphan-deletion", - expiresAt: new Date(Date.now() + 10_000).toISOString(), - renew: vi.fn(async () => new Date(Date.now() + 10_000).toISOString()), - release, - })); - const reconciler = new Reconciler({ - state: { ...state, lease }, - mutationLeaseTtl: 10_000, - }); - - await reconciler.refresh([]); - - expect(lease).toHaveBeenCalledWith("reconciler:orphan-deletion", 10_000); - expect(state.values).toHaveBeenCalledOnce(); - expect(release).toHaveBeenCalledOnce(); - }); - - it("destroys resources in reverse dependency order", async () => { - const destroyOrder: string[] = []; - const deleteA = vi.fn(async () => { - destroyOrder.push("a"); - }); - const deleteB = vi.fn(async () => { - destroyOrder.push("b"); - }); - const deleteC = vi.fn(async () => { - destroyOrder.push("c"); - }); - - const AResource = createTestResourceClass({ - type: "test/service/destroy-a", - delete: deleteA, - }); - const BResource = createTestResourceClass({ - type: "test/service/destroy-b", - delete: deleteB, - }); - const CResource = createTestResourceClass({ - type: "test/service/destroy-c", - delete: deleteC, - }); - - const resourceA = new AResource({ id: "a", config: { name: "a" } }); - const resourceB = new BResource({ - id: "b", - config: { name: "b" }, - dependencies: { a: resourceA }, - }); - const resourceC = new CResource({ - id: "c", - config: { name: "c" }, - dependencies: { b: resourceB }, - }); - - const state = createMemoryState({ - a: { - rev: 1, - id: "a", - groupId: -1, - groupType: "", - type: AResource.type, - config: { name: "a" }, - params: { name: "a" }, - output: { name: "a" }, - lastOperation: "create", - lastOperationAt: new Date().toISOString(), - }, - b: { - rev: 1, - id: "b", - groupId: -1, - groupType: "", - type: BResource.type, - config: { name: "b" }, - params: { name: "b" }, - output: { name: "b" }, - lastOperation: "create", - lastOperationAt: new Date().toISOString(), - }, - c: { - rev: 1, - id: "c", - groupId: -1, - groupType: "", - type: CResource.type, - config: { name: "c" }, - params: { name: "c" }, - output: { name: "c" }, - lastOperation: "create", - lastOperationAt: new Date().toISOString(), - }, - }); - - const reconciler = new Reconciler({ state }); - await reconciler.destroy([resourceA, resourceB, resourceC]); - - expect(destroyOrder).toEqual(["c", "b", "a"]); - expect(state.delete).toHaveBeenCalledWith("a", 1); - expect(state.delete).toHaveBeenCalledWith("b", 1); - expect(state.delete).toHaveBeenCalledWith("c", 1); - }); - - it("refresh removes orphan state entries", async () => { - const deleteSpy = vi.fn(async () => undefined); - const OrphanResource = createTestResourceClass({ - type: "test/service/refresh-orphan", - delete: deleteSpy, - }); - const KeepResource = createTestResourceClass({ - type: "test/service/refresh-keep", - }); - - const keep = new KeepResource({ id: "keep", config: { name: "keep" } }); - const state = createMemoryState({ - keep: { - rev: 1, - id: "keep", - groupId: -1, - groupType: "", - type: KeepResource.type, - config: { name: "keep" }, - params: { name: "keep" }, - output: { name: "keep" }, - lastOperation: "create", - lastOperationAt: new Date().toISOString(), - }, - orphan: { - rev: 1, - id: "orphan", - groupId: -1, - groupType: "", - type: OrphanResource.type, - config: { name: "orphan" }, - params: { name: "orphan" }, - output: { name: "orphan" }, - lastOperation: "create", - lastOperationAt: new Date().toISOString(), - }, - }); - - const reconciler = new Reconciler({ - state, - registry: createResourceRegistry([OrphanResource]), - }); - - await reconciler.refresh([keep]); - - expect(deleteSpy).toHaveBeenCalledOnce(); - expect(state.delete).toHaveBeenCalledWith("orphan", 1); - expect(state.delete).not.toHaveBeenCalledWith("keep", expect.anything()); - }); - - it("destroy and refresh dryRun emit operation events without side effects", async () => { - const deleteSpy = vi.fn(async () => undefined); - const DestroyResource = createTestResourceClass({ - type: "test/service/dry-run-destroy", - delete: deleteSpy, - }); - const OrphanResource = createTestResourceClass({ - type: "test/service/dry-run-refresh", - delete: deleteSpy, - }); - - const destroyResource = new DestroyResource({ - id: "destroy-me", - config: { name: "destroy-me" }, - }); - - const state = createMemoryState({ - "destroy-me": { - rev: 1, - id: "destroy-me", - groupId: -1, - groupType: "", - type: DestroyResource.type, - config: { name: "destroy-me" }, - params: { name: "destroy-me" }, - output: { name: "destroy-me" }, - lastOperation: "create", - lastOperationAt: new Date().toISOString(), - }, - orphan: { - rev: 1, - id: "orphan", - groupId: -1, - groupType: "", - type: OrphanResource.type, - config: { name: "orphan" }, - params: { name: "orphan" }, - output: { name: "orphan" }, - lastOperation: "create", - lastOperationAt: new Date().toISOString(), - }, - }); - - const operationEvents: string[] = []; - const reconciler = new Reconciler({ - state, - dryRun: true, - registry: createResourceRegistry([OrphanResource]), - emit: async (event) => { - if ("operation" in event) { - operationEvents.push( - `${event.operation}:${event.status}:${event.resourceId}`, - ); - } - }, - }); - - await reconciler.destroy([destroyResource]); - await reconciler.refresh([destroyResource]); - - expect(deleteSpy).not.toHaveBeenCalled(); - expect(state.delete).not.toHaveBeenCalled(); - expect(operationEvents).toContain("delete:dry-run:destroy-me"); - expect(operationEvents).toContain("delete:dry-run:orphan"); - }); -}); diff --git a/packages/reconciler/test/reconciler.plan.test.ts b/packages/reconciler/test/reconciler.plan.test.ts deleted file mode 100644 index 88b8c91..0000000 --- a/packages/reconciler/test/reconciler.plan.test.ts +++ /dev/null @@ -1,402 +0,0 @@ -import { describe, expect, it, vi } from "vitest"; -import { resource, type BaseResource } from "@notation/resource"; -import type { StateNode } from "@notation/state"; -import { Reconciler, UNKNOWN_AFTER_APPLY } from "../src"; - -function createMemoryState(initial: Record = {}) { - const store: Record = { ...initial }; - - return { - store, - get: vi.fn(async (id: string) => store[id]), - update: vi.fn( - async (id: string, expectedRev: number, patch: Partial) => { - store[id] = { - ...(store[id] ?? {}), - ...patch, - } as StateNode; - }, - ), - delete: vi.fn(async (id: string) => { - delete store[id]; - }), - values: vi.fn(async () => Object.values(store)), - lease: vi.fn(async (scope: string, ttl: number) => ({ - scope, - expiresAt: new Date(Date.now() + ttl).toISOString(), - renew: vi.fn(async (nextTtl: number) => - new Date(Date.now() + nextTtl).toISOString(), - ), - release: vi.fn(async () => undefined), - })), - }; -} - -function createTestResourceClass(opts: { - type: `${string}/${string}/${string}`; - create?: ( - params: Record, - ) => Promise | void>; - read?: (key: Record) => Promise>; - update?: ( - key: Record, - patch: Record, - params: Record, - state: Record, - ) => Promise; - delete?: ( - key: Record, - state: Record, - ) => Promise; - notFoundOnError?: { name: string; reason: string }[]; -}) { - return resource({ type: opts.type }) - .defineSchema({ - name: { - presence: "required", - propertyType: "param", - valueType: "string" as any, - }, - tag: { - presence: "optional", - propertyType: "param", - valueType: "string" as any, - }, - }) - .defineOperations({ - create: opts.create ?? (async () => ({})), - read: opts.read, - update: opts.update, - delete: opts.delete ?? (async () => undefined), - notFoundOnError: opts.notFoundOnError, - }); -} - -function createStateNode( - id: string, - type: string, - params: Record, - output: Record = params, -): StateNode { - return { - id, - groupId: -1, - groupType: "", - type, - config: params, - params, - output, - lastOperation: "create", - lastOperationAt: new Date().toISOString(), - }; -} - -describe("reconciler plan", () => { - it("plans create for resources without state", async () => { - const TestResource = createTestResourceClass({ - type: "test/service/plan-create", - }); - - const state = createMemoryState(); - const reconciler = new Reconciler({ state, driftDetection: false }); - - const plan = await reconciler.plan([ - new TestResource({ id: "new", config: { name: "new" } }), - ]); - - expect(plan.nodes).toEqual([ - { - id: "new", - type: TestResource.type, - decision: "create", - params: { name: "new" }, - dependsOn: [], - }, - ]); - }); - - it("plans update with the detailed diff that justified it", async () => { - const TestResource = createTestResourceClass({ - type: "test/service/plan-update", - }); - - const state = createMemoryState({ - existing: createStateNode("existing", "test/service/plan-update", { - name: "old", - tag: "keep", - }), - }); - const reconciler = new Reconciler({ state, driftDetection: false }); - - const plan = await reconciler.plan([ - new TestResource({ id: "existing", config: { name: "new" } }), - ]); - - expect(plan.nodes).toEqual([ - { - id: "existing", - type: TestResource.type, - decision: "update", - diff: { - added: {}, - deleted: { tag: null }, - updated: { name: "new" }, - }, - params: { name: "new" }, - dependsOn: [], - }, - ]); - }); - - it("plans noop when params match state", async () => { - const TestResource = createTestResourceClass({ - type: "test/service/plan-noop", - }); - - const state = createMemoryState({ - unchanged: createStateNode("unchanged", "test/service/plan-noop", { - name: "same", - }), - }); - const reconciler = new Reconciler({ state, driftDetection: false }); - - const plan = await reconciler.plan([ - new TestResource({ id: "unchanged", config: { name: "same" } }), - ]); - - expect(plan.nodes[0]).toMatchObject({ id: "unchanged", decision: "noop" }); - }); - - it("plans drift-update from live read output when drift detection is on", async () => { - const readSpy = vi.fn(async () => ({ name: "drifted" })); - const TestResource = createTestResourceClass({ - type: "test/service/plan-drift-update", - read: readSpy, - }); - - const state = createMemoryState({ - resource: createStateNode("resource", "test/service/plan-drift-update", { - name: "desired", - }), - }); - const reconciler = new Reconciler({ state, driftDetection: true }); - - const plan = await reconciler.plan([ - new TestResource({ id: "resource", config: { name: "desired" } }), - ]); - - expect(readSpy).toHaveBeenCalledOnce(); - expect(plan.nodes[0]).toEqual({ - id: "resource", - type: TestResource.type, - decision: "drift-update", - diff: { - added: {}, - deleted: {}, - updated: { name: "desired" }, - }, - params: { name: "desired" }, - dependsOn: [], - }); - }); - - it("plans drift-recreate when the remote resource is gone", async () => { - const TestResource = createTestResourceClass({ - type: "test/service/plan-drift-recreate", - read: async () => { - const err = new Error("gone"); - err.name = "NotFoundException"; - throw err; - }, - notFoundOnError: [ - { name: "NotFoundException", reason: "deleted remotely" }, - ], - }); - - const state = createMemoryState({ - resource: createStateNode( - "resource", - "test/service/plan-drift-recreate", - { name: "desired" }, - ), - }); - const reconciler = new Reconciler({ state, driftDetection: true }); - - const plan = await reconciler.plan([ - new TestResource({ id: "resource", config: { name: "desired" } }), - ]); - - expect(plan.nodes[0]).toMatchObject({ - id: "resource", - decision: "drift-recreate", - }); - }); - - it("skips remote reads when drift detection is off", async () => { - const readSpy = vi.fn(async () => ({ name: "drifted" })); - const TestResource = createTestResourceClass({ - type: "test/service/plan-no-read", - read: readSpy, - }); - - const state = createMemoryState({ - resource: createStateNode("resource", "test/service/plan-no-read", { - name: "desired", - }), - }); - const reconciler = new Reconciler({ state, driftDetection: false }); - - await reconciler.plan([ - new TestResource({ id: "resource", config: { name: "desired" } }), - ]); - - expect(readSpy).not.toHaveBeenCalled(); - }); - - it("plans delete-orphan for state nodes without a matching resource", async () => { - const state = createMemoryState({ - orphan: createStateNode("orphan", "test/service/plan-orphan", { - name: "orphan", - }), - }); - const reconciler = new Reconciler({ state, driftDetection: false }); - - const plan = await reconciler.plan([]); - - expect(plan.nodes).toEqual([ - { - id: "orphan", - type: "test/service/plan-orphan", - decision: "delete-orphan", - params: { name: "orphan" }, - dependsOn: [], - }, - ]); - }); - - it("populates dependsOn from resource dependencies", async () => { - const AResource = createTestResourceClass({ - type: "test/service/plan-dep-a", - }); - const BResource = createTestResourceClass({ - type: "test/service/plan-dep-b", - }); - - const resourceA = new AResource({ id: "a", config: { name: "a" } }); - const resourceB = new BResource({ - id: "b", - config: { name: "b" }, - dependencies: { a: resourceA }, - }); - - const state = createMemoryState(); - const reconciler = new Reconciler({ state, driftDetection: false }); - - const plan = await reconciler.plan([resourceA, resourceB]); - - const nodeB = plan.nodes.find((node) => node.id === "b"); - expect(nodeB?.dependsOn).toEqual(["a"]); - }); - - it("marks params derived from uncreated dependencies as unknown after apply", async () => { - const AResource = createTestResourceClass({ - type: "test/service/plan-unknown-a", - }); - const BResource = createTestResourceClass({ - type: "test/service/plan-unknown-b", - }) - .requireDependencies<{ a: BaseResource }>() - .deriveParams(({ deps }) => ({ - name: (deps.a.output as { name: string }).name, - })); - - const resourceA = new AResource({ id: "a", config: { name: "a" } }); - const resourceB = new BResource({ - id: "b", - config: { tag: "known" }, - dependencies: { a: resourceA }, - }); - - const state = createMemoryState(); - const reconciler = new Reconciler({ state, driftDetection: false }); - - const plan = await reconciler.plan([resourceA, resourceB]); - - const nodeB = plan.nodes.find((node) => node.id === "b"); - expect(nodeB).toMatchObject({ - decision: "create", - params: { - name: UNKNOWN_AFTER_APPLY, - tag: "known", - }, - }); - }); - - it("produces a JSON-round-trippable plan", async () => { - const CreateResource = createTestResourceClass({ - type: "test/service/plan-json-create", - }); - const UpdateResource = createTestResourceClass({ - type: "test/service/plan-json-update", - }); - - const state = createMemoryState({ - existing: createStateNode("existing", "test/service/plan-json-update", { - name: "old", - tag: "gone", - }), - orphan: createStateNode("orphan", "test/service/plan-json-orphan", { - name: "orphan", - }), - }); - const reconciler = new Reconciler({ state, driftDetection: false }); - - const plan = await reconciler.plan([ - new CreateResource({ id: "new", config: { name: "new" } }), - new UpdateResource({ id: "existing", config: { name: "new" } }), - ]); - - expect(JSON.parse(JSON.stringify(plan))).toStrictEqual(plan); - }); - - it("performs no state writes or resource operations", async () => { - const createSpy = vi.fn(async () => ({ name: "new" })); - const updateSpy = vi.fn(async () => undefined); - const deleteSpy = vi.fn(async () => undefined); - - const CreateResource = createTestResourceClass({ - type: "test/service/plan-pure-create", - create: createSpy, - update: updateSpy, - delete: deleteSpy, - }); - const UpdateResource = createTestResourceClass({ - type: "test/service/plan-pure-update", - create: createSpy, - update: updateSpy, - delete: deleteSpy, - read: async () => ({ name: "drifted" }), - }); - - const state = createMemoryState({ - existing: createStateNode("existing", "test/service/plan-pure-update", { - name: "same", - }), - orphan: createStateNode("orphan", "test/service/plan-pure-orphan", { - name: "orphan", - }), - }); - const reconciler = new Reconciler({ state, driftDetection: true }); - - await reconciler.plan([ - new CreateResource({ id: "new", config: { name: "new" } }), - new UpdateResource({ id: "existing", config: { name: "same" } }), - ]); - - expect(createSpy).not.toHaveBeenCalled(); - expect(updateSpy).not.toHaveBeenCalled(); - expect(deleteSpy).not.toHaveBeenCalled(); - expect(state.update).not.toHaveBeenCalled(); - expect(state.delete).not.toHaveBeenCalled(); - }); -}); diff --git a/packages/reconciler/test/yieldstar.integration.test.ts b/packages/reconciler/test/yieldstar.integration.test.ts index 5e1b570..043d700 100644 --- a/packages/reconciler/test/yieldstar.integration.test.ts +++ b/packages/reconciler/test/yieldstar.integration.test.ts @@ -14,10 +14,11 @@ import { createWorkflowRouter, workflow } from "yieldstar"; import { describe, expect, it, vi } from "vitest"; import { YieldStarStateBackend, - reconcileWithYieldStar, + deployWithYieldStar, + destroyWithYieldStar, yieldStarResourceStateStore, } from "../src/yieldstar"; -import type { ReconcilerEvent } from "../src/reconciler"; +import type { ReconcilerEvent } from "../src/events"; import { createResourceRegistry, type ResourceRegistry, @@ -87,6 +88,93 @@ describe("YieldStar reconciliation", () => { runtime.close(); }); + it("resumes durable destroy after a crash without repeating delete", async () => { + const remove = vi.fn(async () => undefined); + const TestResource = resource({ type: "test/yieldstar/destroy-resume" }) + .defineSchema({}) + .defineOperations({ create: async () => undefined, delete: remove }); + const runtime = createRuntime( + [new TestResource({ id: "destroyed" })], + "destroy-crash-resume", + { crashAfterStep: "notation:destroy:destroyed:delete" }, + ); + + await runtime.run("deploy-before-destroy"); + await expect(runtime.destroy("destroy-execution")).rejects.toThrow( + "simulated process crash", + ); + expect(remove).toHaveBeenCalledOnce(); + expect(await runtime.state.get("destroyed")).toBeDefined(); + + await runtime.destroy("destroy-execution"); + expect(remove).toHaveBeenCalledOnce(); + expect(await runtime.state.get("destroyed")).toBeUndefined(); + runtime.close(); + }); + + it("waits durably for a retryable delete before removing state", async () => { + let attempts = 0; + const PendingDelete = resource({ type: "test/yieldstar/pending-delete" }) + .defineSchema({}) + .defineOperations({ + create: async () => undefined, + delete: async () => { + attempts += 1; + if (attempts === 1) { + const error = new Error("delete is pending"); + error.name = "DeletePending"; + throw error; + } + }, + retryLaterOnError: [ + { name: "DeletePending", reason: "delete is pending" }, + ], + }); + const runtime = createRuntime( + [new PendingDelete({ id: "pending-delete" })], + "durable-destroy-wait", + { retryOptions: { maxAttempts: 3, retryInterval: 1 } }, + ); + + await runtime.run("deploy-before-wait"); + await runtime.destroy("destroy-wait"); + expect(attempts).toBe(1); + expect(await runtime.state.get("pending-delete")).toBeDefined(); + + await runtime.destroy("destroy-wait"); + expect(attempts).toBe(2); + expect(await runtime.state.get("pending-delete")).toBeUndefined(); + runtime.close(); + }); + + it("destroys dependents before their dependencies", async () => { + const order: string[] = []; + const Dependency = resource({ type: "test/yieldstar/dependency" }) + .defineSchema({}) + .defineOperations({ + create: async () => undefined, + delete: async () => void order.push("dependency"), + }); + const Dependent = resource({ type: "test/yieldstar/dependent" }) + .defineSchema({}) + .defineOperations({ + create: async () => undefined, + delete: async () => void order.push("dependent"), + }); + const dependency = new Dependency({ id: "dependency" }); + const dependent = new Dependent({ + id: "dependent", + dependencies: { dependency }, + }); + const runtime = createRuntime([dependency, dependent], "destroy-order"); + + await runtime.run("deploy-before-ordered-destroy"); + await runtime.destroy("ordered-destroy"); + + expect(order).toEqual(["dependent", "dependency"]); + runtime.close(); + }); + it("uses store identity and version for conditional update and delete", async () => { const runtime = createRuntime([], "conditional-state"); await runtime.state.update("resource", 0, statePatch("resource")); @@ -255,7 +343,7 @@ function createRuntime( }); const state = new YieldStarStateBackend(storeClient, deploymentId); const deploy = workflow(async function* (step, event) { - yield* reconcileWithYieldStar(step, { + yield* deployWithYieldStar(step, { deploymentId, executionId: event.executionId, resources, @@ -266,7 +354,18 @@ function createRuntime( retryOptions: options.retryOptions, }); }); - const router = createWorkflowRouter({ deploy }); + const destroy = workflow(async function* (step, event) { + yield* destroyWithYieldStar(step, { + deploymentId, + executionId: event.executionId, + resources, + state, + registry: options.registry, + emit: options.emit, + retryOptions: options.retryOptions, + }); + }); + const router = createWorkflowRouter({ deploy, destroy }); const runner = new WorkflowRunner({ router, heapClient: heap, @@ -291,6 +390,17 @@ function createRuntime( logger, ); }, + destroy(executionId: string) { + return runner.run( + { + workflowId: "destroy", + executionId, + params: {}, + context: new Map(), + }, + logger, + ); + }, close() { database.close(); }, diff --git a/packages/state-sqlite/src/index.ts b/packages/state-sqlite/src/index.ts index 057628f..c01775b 100644 --- a/packages/state-sqlite/src/index.ts +++ b/packages/state-sqlite/src/index.ts @@ -1,11 +1,8 @@ -import { randomUUID } from "node:crypto"; import { mkdirSync } from "node:fs"; import { dirname } from "node:path"; import { DatabaseSync } from "node:sqlite"; import { - LeaseConflict, RevConflict, - type Lease, type StateBackend, type StateNode, } from "@notation/state"; @@ -24,13 +21,6 @@ export class SqliteStateBackend implements StateBackend { value TEXT NOT NULL ) `); - this.#database.exec(` - CREATE TABLE IF NOT EXISTS resource_leases ( - scope TEXT PRIMARY KEY, - owner TEXT NOT NULL, - expires_at INTEGER NOT NULL - ) - `); } close(): void { @@ -46,9 +36,7 @@ export class SqliteStateBackend implements StateBackend { async has(id: string): Promise { return Boolean( - this.#database - .prepare("SELECT 1 FROM resources WHERE id = ?") - .get(id), + this.#database.prepare("SELECT 1 FROM resources WHERE id = ?").get(id), ); } @@ -79,9 +67,7 @@ export class SqliteStateBackend implements StateBackend { } } else { this.#database - .prepare( - "INSERT INTO resources (id, rev, value) VALUES (?, ?, ?)", - ) + .prepare("INSERT INTO resources (id, rev, value) VALUES (?, ?, ?)") .run(id, rev, JSON.stringify(node)); } this.#database.exec("COMMIT"); @@ -114,82 +100,4 @@ export class SqliteStateBackend implements StateBackend { .all() as { value: string }[]; return rows.map(({ value }) => JSON.parse(value) as StateNode); } - - async lease(scope: string, ttl: number): Promise { - if (!Number.isFinite(ttl) || ttl <= 0) { - throw new RangeError( - "Lease TTL must be a positive number of milliseconds", - ); - } - - const owner = randomUUID(); - const expiresAtMs = Date.now() + ttl; - this.#database.exec("BEGIN IMMEDIATE"); - try { - this.#database - .prepare( - "DELETE FROM resource_leases WHERE scope = ? AND expires_at <= ?", - ) - .run(scope, Date.now()); - const current = this.#database - .prepare("SELECT expires_at FROM resource_leases WHERE scope = ?") - .get(scope) as { expires_at: number } | undefined; - if (current) { - throw new LeaseConflict( - scope, - new Date(current.expires_at).toISOString(), - ); - } - this.#database - .prepare( - "INSERT INTO resource_leases (scope, owner, expires_at) VALUES (?, ?, ?)", - ) - .run(scope, owner, expiresAtMs); - this.#database.exec("COMMIT"); - } catch (error) { - this.#database.exec("ROLLBACK"); - throw error; - } - - let released = false; - let currentExpiresAtMs = expiresAtMs; - return { - scope, - get expiresAt() { - return new Date(currentExpiresAtMs).toISOString(); - }, - renew: async (nextTtl) => { - if (!Number.isFinite(nextTtl) || nextTtl <= 0) { - throw new RangeError( - "Lease TTL must be a positive number of milliseconds", - ); - } - const now = Date.now(); - const nextExpiresAtMs = now + nextTtl; - const result = this.#database - .prepare( - "UPDATE resource_leases SET expires_at = ? WHERE scope = ? AND owner = ? AND expires_at > ?", - ) - .run(nextExpiresAtMs, scope, owner, now); - if (result.changes !== 1) { - const current = this.#database - .prepare("SELECT expires_at FROM resource_leases WHERE scope = ?") - .get(scope) as { expires_at: number } | undefined; - throw new LeaseConflict( - scope, - new Date(current?.expires_at ?? 0).toISOString(), - ); - } - currentExpiresAtMs = nextExpiresAtMs; - return new Date(nextExpiresAtMs).toISOString(); - }, - release: async () => { - if (released) return; - this.#database - .prepare("DELETE FROM resource_leases WHERE scope = ? AND owner = ?") - .run(scope, owner); - released = true; - }, - }; - } } diff --git a/packages/state-sqlite/test/state-sqlite.test.ts b/packages/state-sqlite/test/state-sqlite.test.ts index 31eb35d..f50456e 100644 --- a/packages/state-sqlite/test/state-sqlite.test.ts +++ b/packages/state-sqlite/test/state-sqlite.test.ts @@ -46,33 +46,6 @@ describe("SqliteStateBackend", () => { }); }); - it("coordinates leases across backend instances and releases by owner", async () => { - const directory = await mkdtemp( - path.join(tmpdir(), "notation-sqlite-lease-"), - ); - const databasePath = path.join(directory, "state.db"); - const first = new SqliteStateBackend(databasePath); - const second = new SqliteStateBackend(databasePath); - cleanups.push(async () => { - first.close(); - second.close(); - await rm(directory, { recursive: true, force: true }); - }); - - const lease = await first.lease("orphans", 10_000); - await expect(second.lease("orphans", 10_000)).rejects.toMatchObject({ - name: "LeaseConflict", - scope: "orphans", - }); - const firstExpiry = lease.expiresAt; - await lease.renew(20_000); - expect(lease.expiresAt).not.toBe(firstExpiry); - await lease.release(); - const nextLease = await second.lease("orphans", 10_000); - expect(nextLease).toMatchObject({ scope: "orphans" }); - await nextLease.release(); - }); - it("waits for a concurrent writer instead of raising database locked", async () => { const directory = await mkdtemp( path.join(tmpdir(), "notation-sqlite-busy-"), diff --git a/packages/state/src/conflicts.ts b/packages/state/src/conflicts.ts index 81559dc..a1e666c 100644 --- a/packages/state/src/conflicts.ts +++ b/packages/state/src/conflicts.ts @@ -11,14 +11,3 @@ export class RevConflict extends Error { ); } } - -export class LeaseConflict extends Error { - readonly name = "LeaseConflict"; - - constructor( - readonly scope: string, - readonly expiresAt: string, - ) { - super(`State lease conflict for ${scope}: held until ${expiresAt}`); - } -} diff --git a/packages/state/src/state.ts b/packages/state/src/state.ts index 003faa1..3f3f3d3 100644 --- a/packages/state/src/state.ts +++ b/packages/state/src/state.ts @@ -9,7 +9,7 @@ import { } from "node:fs/promises"; import path from "node:path"; import { setTimeout as sleep } from "node:timers/promises"; -import { LeaseConflict, RevConflict } from "./conflicts"; +import { RevConflict } from "./conflicts"; export type StateNode = { rev: number; @@ -37,21 +37,12 @@ export interface StateBackend { ): Promise<{ rev: number }>; delete(id: string, expectedRev: number): Promise; values(): Promise; - lease(scope: string, ttl: number): Promise; -} - -export interface Lease { - readonly scope: string; - readonly expiresAt: string; - renew(ttl: number): Promise; - release(): Promise; } export type State = StateBackend; export class MemoryStateBackend implements StateBackend { #state: Record; - #leases = new Map(); constructor(initialState: Record = {}) { this.#state = cloneAsPersistedState(initialState); @@ -108,46 +99,6 @@ export class MemoryStateBackend implements StateBackend { .map(([, value]) => value); } - async lease(scope: string, ttl: number): Promise { - assertLeaseTtl(ttl); - const now = Date.now(); - const current = this.#leases.get(scope); - if (current && current.expiresAtMs > now) { - throw new LeaseConflict( - scope, - new Date(current.expiresAtMs).toISOString(), - ); - } - - const owner = randomUUID(); - let expiresAtMs = now + ttl; - this.#leases.set(scope, { owner, expiresAtMs }); - - return { - scope, - get expiresAt() { - return new Date(expiresAtMs).toISOString(); - }, - renew: async (nextTtl) => { - assertLeaseTtl(nextTtl); - const held = this.#leases.get(scope); - if (!held || held.owner !== owner || held.expiresAtMs <= Date.now()) { - throw new LeaseConflict( - scope, - new Date(held?.expiresAtMs ?? 0).toISOString(), - ); - } - expiresAtMs = Date.now() + nextTtl; - held.expiresAtMs = expiresAtMs; - return new Date(expiresAtMs).toISOString(); - }, - release: async () => { - if (this.#leases.get(scope)?.owner === owner) - this.#leases.delete(scope); - }, - }; - } - private async readState(): Promise> { return cloneAsPersistedState(this.#state); } @@ -207,65 +158,6 @@ export class FileStateBackend implements StateBackend { return Object.values(state); } - async lease(scope: string, ttl: number): Promise { - assertLeaseTtl(ttl); - const leaseFilePath = `${this.stateFilePath}.${encodeURIComponent(scope)}.lease`; - const owner = randomUUID(); - let expiresAtMs: number; - await mkdir(path.dirname(this.stateFilePath), { recursive: true }); - - for (;;) { - expiresAtMs = Date.now() + ttl; - try { - await writeFile(leaseFilePath, JSON.stringify({ owner, expiresAtMs }), { - flag: "wx", - }); - break; - } catch (error) { - if (!isFileExistsError(error)) throw error; - const current = await readFileLease(leaseFilePath); - if (!current || current.expiresAtMs <= Date.now()) { - await unlink(leaseFilePath).catch(() => undefined); - continue; - } - throw new LeaseConflict( - scope, - new Date(current.expiresAtMs).toISOString(), - ); - } - } - - return { - scope, - get expiresAt() { - return new Date(expiresAtMs).toISOString(); - }, - renew: async (nextTtl) => { - assertLeaseTtl(nextTtl); - const current = await readFileLease(leaseFilePath); - if ( - !current || - current.owner !== owner || - current.expiresAtMs <= Date.now() - ) { - throw new LeaseConflict( - scope, - new Date(current?.expiresAtMs ?? 0).toISOString(), - ); - } - expiresAtMs = Date.now() + nextTtl; - await writeFile(leaseFilePath, JSON.stringify({ owner, expiresAtMs })); - return new Date(expiresAtMs).toISOString(); - }, - release: async () => { - const current = await readFileLease(leaseFilePath); - if (current?.owner === owner) { - await unlink(leaseFilePath).catch(() => undefined); - } - }, - }; - } - private async readState(): Promise> { try { const file = await readFile(this.stateFilePath, "utf8"); @@ -353,26 +245,6 @@ function assertExpectedRev( } } -function assertLeaseTtl(ttl: number): void { - if (!Number.isFinite(ttl) || ttl <= 0) { - throw new RangeError("Lease TTL must be a positive number of milliseconds"); - } -} - -type FileLeaseRecord = { owner: string; expiresAtMs: number }; - -async function readFileLease( - filePath: string, -): Promise { - try { - return JSON.parse(await readFile(filePath, "utf8")) as FileLeaseRecord; - } catch (error) { - if (isFileMissingError(error) || error instanceof SyntaxError) - return undefined; - throw error; - } -} - function isFileMissingError(error: unknown): boolean { return isErrorWithCode(error, "ENOENT"); } diff --git a/packages/state/test/state-backend.test.ts b/packages/state/test/state-backend.test.ts index fde5e5b..888055a 100644 --- a/packages/state/test/state-backend.test.ts +++ b/packages/state/test/state-backend.test.ts @@ -157,27 +157,6 @@ function runStateBackendContractTests( await fixture.cleanup(); } }); - - it("holds and renews an exclusive lease", async () => { - const fixture = await createBackend(); - - try { - const lease = await fixture.backend.lease("resource:a", 1_000); - const firstExpiry = lease.expiresAt; - await expect( - fixture.backend.lease("resource:a", 1_000), - ).rejects.toMatchObject({ name: "LeaseConflict" }); - - await lease.renew(2_000); - expect(lease.expiresAt).not.toBe(firstExpiry); - await lease.release(); - - const next = await fixture.backend.lease("resource:a", 1_000); - await next.release(); - } finally { - await fixture.cleanup(); - } - }); }); } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7621ec9..762baba 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -230,12 +230,12 @@ importers: '@notation/resource': specifier: workspace:* version: link:../resource - '@notation/state': - specifier: workspace:* - version: link:../state - '@notation/state-sqlite': - specifier: workspace:* - version: link:../state-sqlite + '@yieldstar/core': + specifier: 0.5.0 + version: 0.5.0 + '@yieldstar/sqlite-runtime': + specifier: 0.5.0 + version: 0.5.0 deep-object-diff: specifier: ^1.1.9 version: 1.1.9 @@ -248,6 +248,12 @@ importers: pako: specifier: ^2.1.0 version: 2.1.0 + pino: + specifier: ^9.14.0 + version: 9.14.0 + yieldstar: + specifier: 0.5.0 + version: 0.5.0 devDependencies: '@types/common-tags': specifier: ^1.8.4 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 3c6d254..d1d8a41 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -4,8 +4,8 @@ packages: allowBuilds: esbuild: true minimumReleaseAgeExclude: - - '@yieldstar/core@0.5.0' - - '@yieldstar/sqlite-runtime@0.5.0' + - "@yieldstar/core@0.5.0" + - "@yieldstar/sqlite-runtime@0.5.0" - yieldstar@0.5.0 overrides: lodash-es@<4.18.1: ^4.18.1