From 9167eb8ed15ee37866d70faaf77e8a16742876fc Mon Sep 17 00:00:00 2001 From: djgrant <1670902+djgrant@users.noreply.github.com> Date: Wed, 22 Jul 2026 14:14:27 +0100 Subject: [PATCH 1/2] Cut core and CLI over to the durable runtime Run deploy, plan, and destroy through NodeDurableRuntime, unify CLI error handling, and remove the legacy reconciler, operation workflows, and state leasing that the old runtime required. --- .changeset/reconciler.md | 4 +- docs/cli/dashboard.md | 16 +- docs/cli/deploy.md | 38 +- docs/cli/destroy.md | 10 +- docs/internals/reconciler.md | 111 +-- docs/internals/state.md | 122 +-- docs/manual/introduction.md | 3 +- docs/manual/reconciler.md | 78 +- docs/rfcs/reconciler.md | 102 +-- examples/reconciler/README.md | 14 +- examples/reconciler/package.json | 6 +- examples/reconciler/src/index.ts | 55 +- packages/cli/src/deploy.ts | 24 +- packages/cli/src/destroy.ts | 7 +- packages/cli/src/index.ts | 27 +- packages/cli/src/plan.ts | 60 +- packages/cli/src/run-with-error-handling.ts | 22 + .../cli/test/run-with-error-handling.test.ts | 41 + packages/core/package.json | 8 +- .../core/src/provisioner/durable-runtime.ts | 142 +++ packages/core/src/provisioner/index.ts | 2 +- .../core/src/provisioner/state-backend.ts | 16 - .../core/src/provisioner/workflows/index.ts | 1 - .../provisioner/workflows/workflow.deploy.ts | 49 +- .../provisioner/workflows/workflow.destroy.ts | 46 +- .../provisioner/workflows/workflow.plan.ts | 44 +- .../provisioner/workflows/workflow.refresh.ts | 39 - .../test/provisioner/durable-runtime.test.ts | 60 ++ .../test/provisioner/operation.create.test.ts | 54 -- .../test/provisioner/state-backend.test.ts | 59 -- packages/reconciler/src/durable/operations.ts | 2 +- packages/reconciler/src/index.ts | 5 +- packages/reconciler/src/logger-subscriber.ts | 4 +- packages/reconciler/src/operations/index.ts | 6 - .../src/operations/operation.create.ts | 82 -- .../src/operations/operation.delete.ts | 52 -- .../src/operations/operation.read.ts | 68 -- .../src/operations/operation.types.ts | 92 -- .../src/operations/operation.update.ts | 94 -- ...ration.pending.ts => pending-operation.ts} | 11 +- packages/reconciler/src/planner.ts | 43 +- packages/reconciler/src/protocol.ts | 2 +- packages/reconciler/src/reconciler.ts | 602 ------------- packages/reconciler/src/resource-registry.ts | 10 +- .../reconciler/test/logger-subscriber.test.ts | 14 +- .../test/operation.workflows.test.ts | 346 -------- .../reconciler/test/reconciler.deploy.test.ts | 807 ------------------ .../reconciler/test/reconciler.plan.test.ts | 463 ---------- packages/state-sqlite/src/index.ts | 96 +-- .../state-sqlite/test/state-sqlite.test.ts | 27 - packages/state/src/conflicts.ts | 11 - packages/state/src/state.ts | 130 +-- packages/state/test/state-backend.test.ts | 21 - pnpm-lock.yaml | 76 +- 54 files changed, 682 insertions(+), 3642 deletions(-) create mode 100644 packages/cli/src/run-with-error-handling.ts create mode 100644 packages/cli/test/run-with-error-handling.test.ts create mode 100644 packages/core/src/provisioner/durable-runtime.ts delete mode 100644 packages/core/src/provisioner/state-backend.ts delete mode 100644 packages/core/src/provisioner/workflows/workflow.refresh.ts create mode 100644 packages/core/test/provisioner/durable-runtime.test.ts delete mode 100644 packages/core/test/provisioner/operation.create.test.ts delete mode 100644 packages/core/test/provisioner/state-backend.test.ts delete mode 100644 packages/reconciler/src/operations/index.ts delete mode 100644 packages/reconciler/src/operations/operation.create.ts delete mode 100644 packages/reconciler/src/operations/operation.delete.ts delete mode 100644 packages/reconciler/src/operations/operation.read.ts delete mode 100644 packages/reconciler/src/operations/operation.types.ts delete mode 100644 packages/reconciler/src/operations/operation.update.ts rename packages/reconciler/src/{operations/operation.pending.ts => pending-operation.ts} (84%) delete mode 100644 packages/reconciler/src/reconciler.ts delete mode 100644 packages/reconciler/test/operation.workflows.test.ts delete mode 100644 packages/reconciler/test/reconciler.deploy.test.ts delete mode 100644 packages/reconciler/test/reconciler.plan.test.ts diff --git a/.changeset/reconciler.md b/.changeset/reconciler.md index 409f877..f28bafd 100644 --- a/.changeset/reconciler.md +++ b/.changeset/reconciler.md @@ -9,6 +9,4 @@ "@notation/state-sqlite": minor --- -Add the reconciler API, versioned event streams, renewable mutation leases, -SQLite state, backend-neutral dashboard state, and compiled infrastructure -graphs. +Add durable Yieldstar 0.5.0 deploy and destroy workflows, a resident Node SQLite runtime for CLI execution, versioned event streams, backend-neutral dashboard state, and compiled infrastructure graphs. diff --git a/docs/cli/dashboard.md b/docs/cli/dashboard.md index f630ad7..2baffaf 100644 --- a/docs/cli/dashboard.md +++ b/docs/cli/dashboard.md @@ -1,21 +1,13 @@ # notation dashboard ```sh -notation dashboard +notation dashboard ``` -Starts a local web dashboard for observing deployment state. +Starts a local web dashboard for observing the deployment's Yieldstar resource stores. ```sh -notation dashboard +notation dashboard infra/api.ts ``` -The dashboard uses the same state backend as deploy and destroy. Set -`NOTATION_STATE_PATH` to select SQLite: - -```sh -NOTATION_STATE_PATH=.notation/state.db notation dashboard -``` - -The server reads through `StateBackend`, so file and SQLite state produce the same -dashboard payload. +The dashboard reads `.notation/workflows.db`, the same database used by deploy, destroy, and plan. Set `NOTATION_STATE_PATH` to choose another SQLite database path. diff --git a/docs/cli/deploy.md b/docs/cli/deploy.md index 940cf7e..3e81379 100644 --- a/docs/cli/deploy.md +++ b/docs/cli/deploy.md @@ -4,7 +4,7 @@ notation deploy ``` -Compiles and deploys the stack to AWS. +Compiles and durably deploys the stack through the resident Yieldstar 0.5.0 Node runtime. ```sh notation deploy infra/api.ts @@ -12,32 +12,36 @@ notation deploy infra/api.ts ## Event stream -`--json` writes versioned reconciler events to stdout as newline-delimited JSON. Build -output and diagnostics move to stderr. +`--json` writes versioned reconciler events to stdout as newline-delimited JSON. Build output, the execution ID, and diagnostics move to stderr. ```sh notation deploy infra/api.ts --json > deploy.ndjson ``` +## Durable execution + +The command prints its Yieldstar execution ID before starting provider work. If the process crashes, resume the same durable heap with that ID: + +```sh +notation deploy infra/api.ts --execution-id +``` + +Do not reuse a completed execution ID for a new deploy or for destroy. + +Retryable provider conditions and consistency reads suspend on durable SQLite timers. The CLI stays resident until the scheduler wakes the execution and the workflow completes; completed provider calls are replayed from the heap rather than repeated. + ## What happens -1. **Compile** – esbuild compiles infra and runtime modules to `dist/`. +1. **Compile** – esbuild compiles infrastructure and runtime modules to `dist/`. -2. **Build resource graph** – imports the compiled output and collects the declared resources. +2. **Build resource graph** – the worker imports the compiled output and collects declared resources. -3. **Reconcile** – the reconciler compares desired state (graph) against current state (`.notation/state.json`): - - New resources → **create** - - Changed params → **update** - - No changes → **noop** - - Orphaned resources (in state but not in graph) → **delete** +3. **Reconcile** – Notation compares desired resources with Yieldstar stores, then creates, updates, recreates, or leaves each resource unchanged. -4. **Topological deployment** – resources deploy in dependency order (levels). Resources at the same level deploy concurrently. +4. **Order dependencies** – dependency levels run in topological order. -5. **Drift detection** – enabled by default. Reads actual AWS state and compares against stored state. If drifted, Notation updates to match your definition. +5. **Detect drift** – unchanged resources are read from the provider and repaired when their remote state differs. -State is persisted to `.notation/state.json` after each operation. Set -`NOTATION_STATE_PATH` to a path ending in `.db` or `.sqlite` to use SQLite: +6. **Delete orphans** – persisted resources absent from the graph are deleted when their resource type is registered. -```sh -NOTATION_STATE_PATH=.notation/state.db notation deploy infra/api.ts -``` +State, step results, timers, task coordination, and resource stores are persisted to `.notation/workflows.db`. Set `NOTATION_STATE_PATH` to choose another SQLite database path. diff --git a/docs/cli/destroy.md b/docs/cli/destroy.md index a74ff30..c25a386 100644 --- a/docs/cli/destroy.md +++ b/docs/cli/destroy.md @@ -4,7 +4,7 @@ notation destroy ``` -Removes all resources in the stack. Tears down runs in reverse dependency order, so routes are removed before APIs and Lambdas before IAM roles etc. +Compiles the application and runs durable destroy through the resident Yieldstar 0.5.0 Node runtime. Resources are removed in reverse dependency order, then registered persisted orphans are removed. ```sh notation destroy infra/api.ts @@ -15,3 +15,11 @@ notation destroy infra/api.ts ```sh notation destroy infra/api.ts --json > destroy.ndjson ``` + +The command prints its execution ID. Resume a crashed destroy with the same ID so a provider delete that already completed is replayed instead of repeated: + +```sh +notation destroy infra/api.ts --execution-id +``` + +Retryable deletes suspend on durable SQLite timers. Resource state is removed only after the provider delete succeeds or reports that the resource is already absent. diff --git a/docs/internals/reconciler.md b/docs/internals/reconciler.md index 6b0d413..05034a8 100644 --- a/docs/internals/reconciler.md +++ b/docs/internals/reconciler.md @@ -1,115 +1,50 @@ # Reconciler -The reconciler runs deployment operations to transition infrastructure from its current state to the state defined in the project. - -Source: `@notation/reconciler` +The reconciler expresses deployment and destruction as Yieldstar async generators. Notation owns desired-state decisions and provider lifecycle; the caller's Yieldstar runtime owns durable execution, waiting, shared state, and coordination. ## Deploy flow -```ts [packages/reconciler/src/index.ts] -const reconciler = new Reconciler({ state, registry, emit }); -await reconciler.deploy(resources, { dryRun, driftDetection }); -``` - -The reconciler walks the resource graph and, for each resource, determines an action: - -| Condition | Decision | -| --------------------------------------------- | ------------------ | -| Not in state | **create** | -| In state, params changed | **update** | -| In state, params unchanged, no drift | **noop** | -| In state, but deleted from AWS | **drift-recreate** | -| In state, AWS state differs from stored state | **drift-update** | -| In state, not in graph (orphan) | **delete** | - -The `dryRun` flag runs the full diffing pipeline without executing any operations, so you can preview what a deploy would do. - -## Topological ordering +`deploy` acquires the deployment coordination store, walks dependency levels in order, decides an action for every resource, executes provider calls as durable steps, persists the result in a resource store, and deletes registered orphans. -Resources are deployed in dependency order using `buildResourceDepthLevels()`. This function partitions the resource graph into levels – each level contains resources whose dependencies have all been satisfied by previous levels. +| Condition | Decision | +| --- | --- | +| Not in state | **create** | +| In state, params changed | **update** | +| In state, params unchanged, no drift | **noop** | +| In state, but deleted from the provider | **drift-recreate** | +| In state, provider state differs from stored state | **drift-update** | +| In state, not in graph | **delete** | -``` -Level 0: IAM Role, CloudWatch LogGroup -Level 1: Lambda Function (depends on Role, LogGroup) -Level 2: API Gateway Integration (depends on Lambda) -Level 3: API Gateway Route (depends on Integration) -``` +Dry-run deploy performs decisions and emits lifecycle events without calling providers or mutating state. -Resources within a level deploy concurrently, so independent resources like the IAM Role and LogGroup above are provisioned in parallel. Dependent resources wait for their dependencies. +## Destroy flow -Destroy operates in reverse order with dependents getting removed before their dependencies. +`destroy` is a first-class durable operation. It acquires the same deployment coordination store as deploy, deletes desired resources in reverse dependency order, deletes hydratable persisted orphans, and conditionally removes each resource store only after the provider delete succeeds or reports that the resource is already absent. -### Cycle detection +Provider delete is a stable durable step. If the process crashes after the provider acknowledges deletion but before state removal, replay uses the cached delete result and continues at the conditional store delete. -Cycle detection is built in. If resources form a circular dependency, the build fails with: +## Waiting and replay -``` -Resource dependency cycle detected -``` +A resource operation throws `ResourceOperationPendingError` when it has not finished. The error gives the reconciler a delay and optional callback context. The runtime stores the context, waits without keeping the process busy, and calls the same operation again. See [Operation errors](./resource.md#operation-errors) for the complete API. -This catches configuration errors before any cloud operations are attempted. +Each attempt, delay, event, state read, state write, and coordination change has a stable step key. A resumed execution must use the same execution ID. A new deploy or destroy must use a new execution ID. -## Drift detection +## State and coordination -Drift detection is enabled by default. After confirming no local changes to a resource, the reconciler reads the resource's current state from AWS (via the resource's `read()` operation) and diffs it against stored state. +Each resource is stored under `notation/resource-state` with a deployment-scoped ID. Conditional updates and deletes compare the snapshot's UUIDv7 `instanceId` and version, so a stale execution cannot modify a deleted and recreated store. -If AWS has drifted (e.g. someone changed a Lambda timeout in the console, or an IAM policy was modified by another tool), Notation updates the resource to match the canoncial definition in the source code. - -Properties marked as `volatile` in the schema (like `LastModified` timestamps) are excluded from drift comparison. +Deploy and destroy share one `notation/deployment-coordination` store per deployment. `store.take` suspends a competing execution as a durable waiter and wakes it after the holder releases. Before suspending, the waiter emits `reconciler.coordination.waiting` naming the holding execution ID, so a wait behind a crashed execution is visible instead of silent. ## Events -The reconciler emits events at each step of an operation's lifecycle. The default `createConsoleReconcilerSubscriber()` logs these to the console with formatted output. +The durable workflows emit these events: | Event | When | | ------------------------------------ | --------------------------------------------------- | | `reconciler.deploy.decision` | After deciding what action to take for a resource | | `reconciler.drift.detected` | When drift is found between stored and actual state | | `reconciler.operation.lifecycle` | When an operation starts, finishes, skips, or fails | +| `reconciler.coordination.waiting` | When another deployment holds the coordination store | | `reconciler.orphan-deletion.skipped` | When no registered class can delete an orphan | -Lifecycle events contain the operation (`create`, `read`, `update`, or `delete`) and its -status (`start`, `success`, `error`, `skip`, or `dry-run`). Events carry the resource ID, -type, and relevant diff or error details. - -## Operations - -Each CRUD operation is implemented as an async generator with retry support: - -- **`createResourceOperation`** – creates the resource, reads back its state, persists to state backend -- **`updateResourceOperation`** – applies the update, reads back new state, persists to state backend -- **`deleteResourceOperation`** – deletes the resource, removes the entry from state backend -- **`readResourceOperation`** – reads current state from the cloud provider (used for drift detection) - -### Pending operations - -A resource operation throws `ResourceOperationPendingError` when it has not finished. The reconciler reads two fields from the error: - -| Field | Action | -| ----- | ------ | -| `retryAfterMs` | Wait this many milliseconds. | -| `callbackContext` | Pass this value to the next call of the same operation. | - -The reconciler then calls the same operation again. Any other error fails the operation. See [Operation errors](./resource.md#operation-errors) for the complete API. - -The default limit is 30 calls to one operation: - -```ts [packages/reconciler/src/index.ts] -{ - maxOperationAttempts: 30, -} -``` - -The last pending error becomes a failure when the limit is reached. - -### Operation lifecycle - -Each operation follows the following pattern: - -1. Emit `started` event -2. Execute the cloud operation (with retries) -3. Read back the resource state -4. Persist to state backend -5. Emit `completed` event (or `failed` on error) - -State is updated after the provider operation and read-back complete. +Lifecycle events cover create, read, update, and delete with `start`, `success`, `error`, `skip`, or `dry-run` status. diff --git a/docs/internals/state.md b/docs/internals/state.md index 44d5df0..77647e4 100644 --- a/docs/internals/state.md +++ b/docs/internals/state.md @@ -1,129 +1,25 @@ # State -Notation tracks deployed resources in a state backend. State is the bridge between what is defined and what actually exists in the cloud. +Notation CLI deploy, destroy, plan, and dashboard use Yieldstar 0.5.0 stores in `.notation/workflows.db`. Override the database path with `NOTATION_STATE_PATH`. -Source: `@notation/state` - -## State file - -Default location: `.notation/state.json`. Override with the `NOTATION_STATE_PATH` environment variable. - -Each resource entry records everything needed to diff, update, or delete the resource: - -```json -{ - "my-api-lambda-getTodos": { - "rev": 3, - "id": "my-api-lambda-getTodos", - "type": "aws/lambda/LambdaFunction", - "config": { - "service": "aws/lambda", - "timeout": 5, - "memory": 64 - }, - "params": { - "FunctionName": "my-api-getTodos", - "Runtime": "nodejs18.x", - "Handler": "index.getTodos", - "MemorySize": 64, - "Timeout": 5 - }, - "output": { - "FunctionArn": "arn:aws:lambda:us-east-1:123456789:function:my-api-getTodos", - "FunctionUrl": "https://xyz.lambda-url.us-east-1.on.aws/" - }, - "lastOperation": "create", - "lastOperationAt": "2027-01-15T10:30:00.000Z" - } -} -``` - -Key fields: - -- **`id`** – unique identifier derived from the resource's position in the graph -- **`rev`** – monotonically increasing revision used for compare-and-swap writes -- **`type`** – the resource type string (e.g., `aws/lambda/LambdaFunction`) -- **`config`** – user-facing configuration values -- **`params`** – the full set of parameters sent to the cloud provider -- **`output`** – computed values returned by the provider after creation -- **`lastOperation`** – what the reconciler last did (`create`, `update`, `delete`) -- **`lastOperationAt`** – ISO timestamp of the last operation - -## Backends - -Three built-in backends: - -### `FileStateBackend` (default) - -Reads and writes JSON to disk. Uses atomic writes – writes to a temporary file first, then renames – to prevent corruption if the process is interrupted mid-write. - -```ts [packages/state/src/file.ts] -const state = new FileStateBackend(".notation/state.json"); -``` - -### `MemoryStateBackend` - -In-memory backend used for testing. Deep-clones on read and write to simulate persistence semantics (mutations to returned objects don't affect stored data). - -```ts [packages/state/src/memory.ts] -const state = new MemoryStateBackend(); -``` - -### `SqliteStateBackend` - -Stores state and leases in SQLite. Select it in the CLI by setting -`NOTATION_STATE_PATH` to a path ending in `.db` or `.sqlite`. +Each live resource is a `notation/resource-state` store scoped by deployment and resource ID. A missing store means the resource is absent. No application tombstone is written. ```ts -const state = new SqliteStateBackend(".notation/state.db"); +const state = new DurableStateBackend(storeClient, "infra/api.ts"); ``` -### `StateBackend` interface +The runtime assigns a UUIDv7 `instanceId` when a store is created and increments its version on update. Conditional workflow updates and deletes compare both values, preventing a stale snapshot from modifying a deleted and recreated resource. The one-based value exposed as `StateNode.rev` is derived from the authoritative Yieldstar store version. -All backends implement the same interface: - -```ts [@notation/state/src/backend.ts] +```ts interface StateBackend { get(id: string): Promise; has(id: string): Promise; - update( - id: string, - patch: Partial, - expectedRev?: number, - ): Promise<{ rev: number }>; - delete(id: string, expectedRev?: number): Promise; + update(id: string, expectedRev: number, patch: Partial): Promise<{ rev: number }>; + delete(id: string, expectedRev: number): Promise; values(): Promise; - lease(scope: string, ttl: number): Promise; } ``` -Every backend provides compare-and-swap writes and renewable exclusive leases. The -reconciler holds a per-resource lease across the provider operation and state write, so -concurrent deploys cannot both perform the same create or update. It renews long-running -leases until the mutation finishes. Orphan deletion additionally holds a snapshot lease -while it decides which state records no longer appear in the desired graph. - -## How state is used - -### Deploy - -The reconciler reads state to diff against the desired resource graph: - -1. For each resource in the graph, check if it exists in state -2. If it exists, compare `params` to detect changes -3. Execute the appropriate operation (create, update, noop) -4. After each operation, update the state entry with new params and output - -### Destroy - -The reconciler reads state to find resources to delete: - -1. Load all state entries -2. Delete resources in reverse dependency order -3. Remove each entry from state after successful deletion - -### Orphan detection - -The reconciler checks for orphaned resources – resources that exist in state but are no longer present in the resource graph. This happens when you remove a function export or delete a `.fn.ts` file. +Coordination is not part of the state backend contract. The outer Yieldstar workflow serializes deploy and destroy through a deployment coordination store and records applied store steps for crash-safe replay. -Orphaned resources are deleted from AWS and removed from state. +`MemoryStateBackend`, `FileStateBackend`, and `SqliteStateBackend` remain data adapters for tests and embedded read/write consumers. They are not CLI execution runtimes and do not provide mutation coordination. diff --git a/docs/manual/introduction.md b/docs/manual/introduction.md index bb629e0..41cafca 100644 --- a/docs/manual/introduction.md +++ b/docs/manual/introduction.md @@ -13,8 +13,7 @@ todoRouter.get("/todos", getTodos); Notation is a compiler, reconciler, and deployment engine. -The reconciler is also available as an embedded library. A Node.js host can construct -resources, choose a state backend, and run plan, deploy, or destroy without the CLI. +The reconciler is also available as an embedded library. A Node.js host can construct resources and compose durable reconciliation inside its own Yieldstar workflow without the CLI. The compiler runs two passes over your codebase: diff --git a/docs/manual/reconciler.md b/docs/manual/reconciler.md index 87b02f7..eee8d06 100644 --- a/docs/manual/reconciler.md +++ b/docs/manual/reconciler.md @@ -1,55 +1,45 @@ # Reconciler -Use the reconciler directly when a Node.js application needs to deploy resources without -starting the Notation CLI. - -This complete program deploys two static sites and keeps their deployment state in -SQLite: +Use `deploy` and `destroy` when a Node.js application needs durable resource lifecycle operations without starting the Notation CLI. Notation owns reconciliation intent, graph ordering, provider calls, and resource state; the application owns the outer Yieldstar workflow and runtime. ```ts -import { Reconciler, createResourceRegistry } from "@notation/reconciler"; -import { SqliteStateBackend } from "@notation/state-sqlite"; -import { StaticSite } from "./static-site"; - -const state = new SqliteStateBackend("sites.db"); - -const resources = [ - new StaticSite({ - id: "documentation", - config: { - siteDirectory: "sites/docs", - html: "

Documentation

\n", - }, - }), - new StaticSite({ - id: "status", - config: { - siteDirectory: "sites/status", - html: "

All systems operational

\n", - }, - }), -]; - -const reconciler = new Reconciler({ - state, - registry: createResourceRegistry([StaticSite]), +import { SqliteSchedulerClient, SqliteStoreClient, SqliteTaskQueueClient, SqliteTimersClient, createSqliteDb } from "@yieldstar/sqlite-runtime/node"; +import { DurableStateBackend, deploy, destroy } from "@notation/reconciler"; +import { workflow } from "yieldstar"; + +const database = createSqliteDb({ path: ".notation/workflows.db" }); +const schedulerClient = new SqliteSchedulerClient({ + taskQueueClient: new SqliteTaskQueueClient(database), + timersClient: new SqliteTimersClient(database), +}); +const storeClient = new SqliteStoreClient({ db: database, schedulerClient }); +const state = new DurableStateBackend(storeClient, "my-application"); + +export const deploy = workflow(async function* (step, event) { + yield* deploy(step, { + deploymentId: "my-application", + executionId: event.executionId, + resources, + state, + }); }); -try { - await reconciler.deploy(resources); -} finally { - state.close(); -} +export const destroy = workflow(async function* (step, event) { + yield* destroy(step, { + deploymentId: "my-application", + executionId: event.executionId, + resources, + state, + }); +}); ``` -`StaticSite` contains the provider operations which create, read, update, and delete a -site. A real provider would call its infrastructure API instead of writing local files. +The outer workflow supplies durable step execution, timers, shared stores, waiting, scheduling, and coordination. Completed provider calls are replayed from the heap after a crash, retryable provider conditions suspend on a durable timer, and conditional state writes use Yieldstar store identity and version. + +Each live resource is one Yieldstar store. Absence is represented by no store, not a tombstone. Yieldstar's UUIDv7 store `instanceId` and version are authoritative for conditional update and delete; Notation exposes the version as the resource state's `rev`. -Pass the complete desired set to `deploy`. A resource which remains in deployment state -but is absent from that set is deleted. The explicit registry lets the reconciler find -its delete operation. +Operations against the same `deploymentId` are serialized through a coordination store keyed by `executionId`. Resume a crashed operation with the same execution ID; use a new globally unique execution ID for every new deploy or destroy. An execution that must wait emits a `reconciler.coordination.waiting` event naming the holder before it suspends, which also identifies a crashed holder that should be resumed instead. -Notation's state records what was deployed. It does not replace application data which -owns the desired configuration. +Pass the complete desired set on every deployment. Persisted resources absent from that set are deleted through the supplied resource registry. Destroy removes current resources in reverse dependency order and then removes any persisted orphans that the registry can hydrate. -The runnable version is in `examples/reconciler`. +The runnable Node SQLite composition is in `examples/reconciler`. diff --git a/docs/rfcs/reconciler.md b/docs/rfcs/reconciler.md index dd33e92..7ec505a 100644 --- a/docs/rfcs/reconciler.md +++ b/docs/rfcs/reconciler.md @@ -1,102 +1,28 @@ -# RFC: Reconciler +# RFC: Durable Yieldstar reconciliation **Status:** implemented -**Scope:** `@notation/state`, `@notation/reconciler` +**Scope:** `@notation/reconciler`, `@notation/core`, Yieldstar 0.5.0 -Notation evaluates an infrastructure program into resources, then reconciles those -resources against recorded state. The same engine now runs behind the CLI, the dashboard, -and direct library integrations. +Notation describes reconciliation intent and resource lifecycle operations. An outer Yieldstar workflow supplies durable execution, waiting, state, and coordination by composing `deploy` or `destroy`. -```ts -import { Reconciler } from "@notation/reconciler"; -import { SqliteStateBackend } from "@notation/state-sqlite"; +## Boundary -const state = new SqliteStateBackend(".notation/state.db"); -const reconciler = new Reconciler({ state }); +Live resource objects remain in the workflow process. They are not serialized into workflow parameters. This keeps provider clients and operation closures under Notation's lifecycle control while Yieldstar persists step results and shared state. -await reconciler.deploy(resources); -state.close(); -``` +Provider create, update, read, and delete calls are durable steps with stable resource-scoped keys. A process crash after a completed provider call replays the cached result and continues at state persistence instead of repeating the call. Retryable provider conditions become Yieldstar delays, allowing the process to wait without polling the provider continuously. -The reconciler boundary consists of live resource objects, a state backend, and an event -subscriber. Resource operations run in the host process. +## State lifecycle -## State +`DurableStateBackend` stores one live resource per `notation/resource-state` store. The store ID is scoped by deployment and resource ID. Store absence is resource absence. -Each state record carries a revision. Updates and deletes can require the revision which -the caller previously read: +The runtime-assigned UUIDv7 `instanceId` distinguishes a deleted store from a later store created under the same logical ID. Yieldstar's version is the concurrency token and is exposed as Notation's one-based `rev`. Workflow updates use `store.updateFrom` and deletes use `store.deleteFrom`, so both the instance and version must match the snapshot that informed the operation. -```ts -await state.update(resource.id, patch, resource.rev); -``` +## Coordination -A stale writer receives `RevConflict`. A missing record has revision zero, so -`expectedRev: 0` means that the record must not exist. +Each deployment has a `notation/deployment-coordination` store shared by deploy and destroy. The workflow atomically claims it with `store.take`. A concurrent execution suspends as a durable waiter and is woken when the holder releases the store. The same execution can recover an acquisition across the store-commit and heap-write crash gap through Yieldstar's applied-step ledger. -The reconciler also takes a renewable per-resource lease before it reads a resource for -mutation. The lease remains held across the provider operation and state write. Two -hosts therefore cannot create or update the same resource concurrently through the same -backend. +## Node CLI runtime -Orphan deletion takes an additional snapshot lease. The snapshot remains stable while -the reconciler decides which state records no longer appear in the desired graph. +`NodeDurableRuntime` wires `WorkflowRunner`, `SqliteHeapClient`, `SqliteStoreClient`, `SqliteSchedulerClient`, and `SqliteEventLoop` against one Node SQLite database. CLI deploy and destroy run through this resident runtime and wait for a workflow result across timer and store wake-ups. -## Backends - -`@notation/state` provides file and memory backends. `@notation/state-sqlite` provides -the reference database backend. - -Every backend implements the same contract: - -```ts -interface StateBackend { - get(id: string): Promise; - has(id: string): Promise; - update( - id: string, - patch: Partial, - expectedRev?: number, - ): Promise<{ rev: number }>; - delete(id: string, expectedRev?: number): Promise; - values(): Promise; - lease(scope: string, ttl: number): Promise; -} -``` - -The dashboard reads this interface. It does not inspect a state file directly. - -## Events - -The reconciler accepts one subscriber: - -```ts -const reconciler = new Reconciler({ - state, - emit: async (event) => auditLog.write(event), -}); -``` - -`createNdjsonEventEmitter` adapts the subscriber to a versioned newline-delimited JSON -stream. The CLI uses the same adapter for `deploy --json` and `destroy --json`. - -## Package boundary - -The CLI creates resources from compiled Notation programs, then hands those live objects -to `Reconciler`. An application can construct the same resource classes directly. - -The reconciler does not serialise resource classes or execute operations in another -process. Detached execution needs manifests, resource-reference encoding, actuator -binding, and a runtime consumer. That work has its own RFC and release. - -## Acceptance - -The reconciler example is the compatibility test for this boundary. It must: - -1. Construct a resource without the CLI. -2. Plan and deploy it through `Reconciler`. -3. Close and reopen SQLite state. -4. Plan and apply an update. -5. Receive versioned events. -6. Destroy the resource and remove its state. - -The example lives in `examples/reconciler` and runs without cloud credentials. +The CLI prints a new execution ID for each operation. Re-running with `--execution-id ` resumes that operation from its durable heap after a process crash. diff --git a/examples/reconciler/README.md b/examples/reconciler/README.md index 428eea3..19a6ae0 100644 --- a/examples/reconciler/README.md +++ b/examples/reconciler/README.md @@ -1,12 +1,8 @@ -# Reconciler +# Durable reconciler -This example deploys two static sites from an ordinary Node.js program. It does not -compile a Notation project or start the Notation CLI. +This example deploys two static sites from an ordinary Node.js program using Yieldstar 0.5.0 for durable execution, state, retries, waiting, and deployment coordination. -[`src/index.ts`](./src/index.ts) is the complete program. It defines the desired -resources inline, opens a SQLite state backend, and passes the resources directly to the -reconciler. [`src/static-site.ts`](./src/static-site.ts) defines the local provider -operations used to create, read, update, and delete each site. +[`src/index.ts`](./src/index.ts) owns the outer workflow and Node SQLite runtime. It passes Yieldstar's `step` context to `deploy`, while [`src/static-site.ts`](./src/static-site.ts) contains only the desired resources and provider lifecycle operations. Run it from the repository root: @@ -14,9 +10,7 @@ Run it from the repository root: pnpm --filter reconciler-example demo ``` -The generated sites are written to `sites/`, and deployment state is stored in -`sites.db`. Change the resource configuration and run the command again to update the -sites. Remove a resource from the array and run it again to delete that site. +The generated sites are written to `sites/`, and the workflow heap, resource stores, timers, and coordination state are stored in `sites.db`. Change the resource configuration and run the command again to update the sites. Remove a resource from the array and run it again to delete that site. Run the integration test with: diff --git a/examples/reconciler/package.json b/examples/reconciler/package.json index 55cbaa1..c5d8f2e 100644 --- a/examples/reconciler/package.json +++ b/examples/reconciler/package.json @@ -11,8 +11,10 @@ "dependencies": { "@notation/reconciler": "workspace:*", "@notation/resource": "workspace:*", - "@notation/state-sqlite": "workspace:*", - "@notation/utils": "workspace:*" + "@yieldstar/core": "0.5.0", + "@yieldstar/sqlite-runtime": "0.5.0", + "pino": "^9.9.0", + "yieldstar": "0.5.0" }, "devDependencies": { "@types/node": "^22.13.4", diff --git a/examples/reconciler/src/index.ts b/examples/reconciler/src/index.ts index 4a85756..0253115 100644 --- a/examples/reconciler/src/index.ts +++ b/examples/reconciler/src/index.ts @@ -1,8 +1,26 @@ -import { Reconciler, createResourceRegistry } from "@notation/reconciler"; -import { SqliteStateBackend } from "@notation/state-sqlite"; +import { WorkflowRunner } from "@yieldstar/core"; +import { + SqliteHeapClient, + SqliteSchedulerClient, + SqliteStoreClient, + SqliteTaskQueueClient, + SqliteTimersClient, + createSqliteDb, +} from "@yieldstar/sqlite-runtime/node"; +import * as reconciler from "@notation/reconciler"; +import pino from "pino"; +import { createWorkflowRouter, workflow } from "yieldstar"; import { StaticSite } from "./static-site"; -const state = new SqliteStateBackend("sites.db"); +const logger = pino(); +const database = createSqliteDb({ path: "sites.db" }); +const taskQueueClient = new SqliteTaskQueueClient(database); +const schedulerClient = new SqliteSchedulerClient({ + taskQueueClient, + timersClient: new SqliteTimersClient(database), +}); +const storeClient = new SqliteStoreClient({ db: database, schedulerClient }); +const state = new reconciler.DurableStateBackend(storeClient, "static-sites"); const resources = [ new StaticSite({ @@ -21,13 +39,34 @@ const resources = [ }), ]; -const reconciler = new Reconciler({ - state, - registry: createResourceRegistry([StaticSite]), +const deploy = workflow(async function* (step, event) { + yield* reconciler.deploy(step, { + deploymentId: "static-sites", + executionId: event.executionId, + resources, + state, + registry: reconciler.createResourceRegistry([StaticSite]), + }); +}); + +const runner = new WorkflowRunner({ + router: createWorkflowRouter({ deploy }), + heapClient: new SqliteHeapClient(database), + storeClient, + schedulerClient, + logger, }); try { - await reconciler.deploy(resources); + await runner.run( + { + workflowId: "deploy", + executionId: crypto.randomUUID(), + params: {}, + context: new Map(), + }, + logger, + ); } finally { - state.close(); + database.close(); } diff --git a/packages/cli/src/deploy.ts b/packages/cli/src/deploy.ts index 07fdad1..3f141c1 100644 --- a/packages/cli/src/deploy.ts +++ b/packages/cli/src/deploy.ts @@ -3,12 +3,14 @@ import { createNdjsonEventEmitter, deployApp, } from "@notation/core"; +import { randomUUID } from "node:crypto"; import { compile } from "./compile"; import { defaultLogger, type Logger } from "./logger"; import { redirectStdoutToStderr } from "./stdio"; export type DeployCommandOptions = { json?: boolean; + executionId?: string; logger?: Logger; }; @@ -17,30 +19,14 @@ export async function deploy( opts: DeployCommandOptions = {}, ) { const logger = opts.logger ?? defaultLogger; - // In --json mode console output moves to stderr so stdout carries only the - // NDJSON event stream; capture the real stdout for the emitter first. const emit = opts.json ? createNdjsonEventEmitter(redirectStdoutToStderr().write) : createLoggerReconcilerSubscriber({ logger }); await compile(entryPoint, { logger }); logger.info(`Deploying ${entryPoint}`); + const executionId = opts.executionId ?? randomUUID(); + logger.info(`Yieldstar execution ${executionId}`); - try { - await deployApp({ - entryPoint, - emit, - }); - } catch (err: any) { - if (err.name === "CredentialsProviderError") { - logger.error( - "\nAWS credentials not found.", - "\n\nEnsure you have a default profile set up in ~/.aws/credentials.", - "\n\nIf using another profile run AWS_PROFILE=otherProfile notation deploy.\n", - ); - process.exit(1); - } - logger.error(err); - process.exit(1); - } + await deployApp({ entryPoint, emit, executionId }); } diff --git a/packages/cli/src/destroy.ts b/packages/cli/src/destroy.ts index acdc5b0..9352f3c 100644 --- a/packages/cli/src/destroy.ts +++ b/packages/cli/src/destroy.ts @@ -3,12 +3,14 @@ import { createNdjsonEventEmitter, destroyApp, } from "@notation/core"; +import { randomUUID } from "node:crypto"; import { compile } from "./compile"; import { defaultLogger, type Logger } from "./logger"; import { redirectStdoutToStderr } from "./stdio"; export type DestroyCommandOptions = { json?: boolean; + executionId?: string; logger?: Logger; }; @@ -23,5 +25,8 @@ export async function destroy( await compile(entryPoint, { logger }); logger.info(`Destroying ${entryPoint}\n`); - await destroyApp({ entryPoint, emit }); + const executionId = opts.executionId ?? randomUUID(); + logger.info(`Yieldstar execution ${executionId}`); + + await destroyApp({ entryPoint, emit, executionId }); } diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index 3ef90b7..eee95f2 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -4,10 +4,12 @@ import { compile } from "./compile"; import { deploy } from "./deploy"; import { destroy } from "./destroy"; import { plan } from "./plan"; +import { defaultLogger } from "./logger"; +import { runWithCliErrorHandling } from "./run-with-error-handling"; import { visualise } from "./visualise"; import { watch } from "./watch"; import { startDashboardServer } from "@notation/dashboard"; -import { createDefaultStateBackend } from "@notation/core"; +import { NodeDurableRuntime } from "@notation/core"; program .command("compile") @@ -19,9 +21,11 @@ program program .command("dashboard") + .argument("", "entryPoint") .description("Start Notation Dashboard") - .action(async () => { - await startDashboardServer({ state: createDefaultStateBackend() }); + .action(async (entryPoint) => { + const runtime = new NodeDurableRuntime({ deploymentId: entryPoint }); + await startDashboardServer({ state: runtime.state }); }); program @@ -29,8 +33,12 @@ program .argument("", "entryPoint") .description("Deploy Notation App") .option("--json", "stream reconciler events as NDJSON") + .option("--execution-id ", "resume a durable execution") .action(async (entryPoint, options) => { - await deploy(entryPoint, { json: options.json }); + await deploy(entryPoint, { + json: options.json, + executionId: options.executionId, + }); }); program @@ -38,8 +46,12 @@ program .argument("", "entryPoint") .description("Destroy Notation App") .option("--json", "stream reconciler events as NDJSON") + .option("--execution-id ", "resume a durable execution") .action(async (entryPoint, options) => { - await destroy(entryPoint, { json: options.json }); + await destroy(entryPoint, { + json: options.json, + executionId: options.executionId, + }); }); program @@ -67,4 +79,7 @@ program await watch(entryPoint); }); -program.parse(process.argv); +process.exitCode = await runWithCliErrorHandling( + () => program.parseAsync(process.argv), + { logger: defaultLogger, command: process.argv[2] ?? program.name() }, +); diff --git a/packages/cli/src/plan.ts b/packages/cli/src/plan.ts index ef52ec8..35841cc 100644 --- a/packages/cli/src/plan.ts +++ b/packages/cli/src/plan.ts @@ -1,9 +1,4 @@ -import { - createLoggerReconcilerSubscriber, - planApp, - type Plan, - type PlanNode, -} from "@notation/core"; +import { planApp, type Plan, type PlanNode } from "@notation/core"; import { compile } from "./compile"; import { defaultLogger, type Logger } from "./logger"; import { redirectStdoutToStderr } from "./stdio"; @@ -24,42 +19,27 @@ const decisionSymbols: Record = { export async function plan(entryPoint: string, opts: PlanCommandOptions = {}) { const logger = opts.logger ?? defaultLogger; - const emit = createLoggerReconcilerSubscriber({ logger }); - try { - if (opts.json) { - let result: Plan; - const { restore } = redirectStdoutToStderr(); - try { - await compile(entryPoint, { logger }); - result = await planApp({ - entryPoint, - emit, - }); - } finally { - restore(); - } - process.stdout.write(`${JSON.stringify(result, null, 2)}\n`); - return; + if (opts.json) { + let result: Plan; + const { restore } = redirectStdoutToStderr(); + try { + await compile(entryPoint, { logger }); + result = await planApp({ + entryPoint, + }); + } finally { + restore(); } - - await compile(entryPoint, { logger }); - logger.info(`Planning ${entryPoint}\n`); - const result = await planApp({ - entryPoint, - emit, - }); - printPlanSummary(result, logger); - } catch (err: any) { - if (err.name === "CredentialsProviderError") { - logger.error( - "\nAWS credentials not found.", - "\n\nEnsure you have a default profile set up in ~/.aws/credentials.", - "\n\nIf using another profile run AWS_PROFILE=otherProfile notation plan.\n", - ); - process.exit(1); - } - throw err; + process.stdout.write(`${JSON.stringify(result, null, 2)}\n`); + return; } + + await compile(entryPoint, { logger }); + logger.info(`Planning ${entryPoint}\n`); + const result = await planApp({ + entryPoint, + }); + printPlanSummary(result, logger); } function printPlanSummary(result: Plan, logger: Logger) { diff --git a/packages/cli/src/run-with-error-handling.ts b/packages/cli/src/run-with-error-handling.ts new file mode 100644 index 0000000..85f8f78 --- /dev/null +++ b/packages/cli/src/run-with-error-handling.ts @@ -0,0 +1,22 @@ +import type { Logger } from "./logger"; + +export async function runWithCliErrorHandling( + fn: () => Promise, + opts: { logger: Logger; command: string }, +): Promise<0 | 1> { + try { + await fn(); + return 0; + } catch (error: unknown) { + if (error instanceof Error && error.name === "CredentialsProviderError") { + opts.logger.error( + "\nAWS credentials not found.", + "\n\nEnsure you have a default profile set up in ~/.aws/credentials.", + `\n\nIf using another profile run AWS_PROFILE=otherProfile notation ${opts.command}.\n`, + ); + return 1; + } + opts.logger.error(error); + return 1; + } +} diff --git a/packages/cli/test/run-with-error-handling.test.ts b/packages/cli/test/run-with-error-handling.test.ts new file mode 100644 index 0000000..2ca64e9 --- /dev/null +++ b/packages/cli/test/run-with-error-handling.test.ts @@ -0,0 +1,41 @@ +import { describe, expect, it, vi } from "vitest"; +import { runWithCliErrorHandling } from "../src/run-with-error-handling"; + +describe("CLI error handling", () => { + it("reports credential failures with command-specific guidance", async () => { + const error = new Error("Could not load credentials"); + error.name = "CredentialsProviderError"; + const logger = { info: vi.fn(), warn: vi.fn(), error: vi.fn() }; + + const exitCode = await runWithCliErrorHandling( + async () => { + throw error; + }, + { logger, command: "deploy" }, + ); + + expect(exitCode).toBe(1); + expect(logger.error).toHaveBeenCalledOnce(); + expect(logger.error).toHaveBeenCalledWith( + "\nAWS credentials not found.", + "\n\nEnsure you have a default profile set up in ~/.aws/credentials.", + "\n\nIf using another profile run AWS_PROFILE=otherProfile notation deploy.\n", + ); + }); + + it("reports non-credential failures unchanged", async () => { + const error = new Error("deploy failed"); + const logger = { info: vi.fn(), warn: vi.fn(), error: vi.fn() }; + + const exitCode = await runWithCliErrorHandling( + async () => { + throw error; + }, + { logger, command: "deploy" }, + ); + + expect(exitCode).toBe(1); + expect(logger.error).toHaveBeenCalledOnce(); + expect(logger.error).toHaveBeenCalledWith(error); + }); +}); diff --git a/packages/core/package.json b/packages/core/package.json index 542912a..cc55f70 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -15,12 +15,14 @@ "dependencies": { "@notation/reconciler": "workspace:*", "@notation/resource": "workspace:*", - "@notation/state": "workspace:*", - "@notation/state-sqlite": "workspace:*", + "@yieldstar/core": "0.5.0", + "@yieldstar/sqlite-runtime": "0.5.0", "deep-object-diff": "^1.1.9", "js-base64": "^3.7.7", "lodash-es": "^4.17.21", - "pako": "^2.1.0" + "pako": "^2.1.0", + "pino": "^9.14.0", + "yieldstar": "0.5.0" }, "devDependencies": { "@types/common-tags": "^1.8.4", diff --git a/packages/core/src/provisioner/durable-runtime.ts b/packages/core/src/provisioner/durable-runtime.ts new file mode 100644 index 0000000..73face8 --- /dev/null +++ b/packages/core/src/provisioner/durable-runtime.ts @@ -0,0 +1,142 @@ +import { randomUUID } from "node:crypto"; +import { setImmediate } from "node:timers/promises"; +import { + WorkflowRunner, + type WorkflowEvent, + type WorkflowRouter, +} from "@yieldstar/core"; +import { + SqliteEventLoop, + SqliteHeapClient, + SqliteSchedulerClient, + SqliteStoreClient, + SqliteTaskQueueClient, + SqliteTimersClient, + createSqliteDb, +} from "@yieldstar/sqlite-runtime/node"; +import { DurableStateBackend } from "@notation/reconciler"; +import pino, { type Logger } from "pino"; + +export const DEFAULT_WORKFLOW_STATE_PATH = ".notation/workflows.db"; + +export function resolveWorkflowStatePath(): string { + return process.env.NOTATION_STATE_PATH ?? DEFAULT_WORKFLOW_STATE_PATH; +} + +export type NodeDurableRuntimeOptions = { + deploymentId: string; + databasePath?: string; + logger?: Logger; +}; + +export type RunWorkflowOptions = { + workflowId: string; + executionId?: string; + params?: Record; +}; + +/** Resident Yieldstar 0.5.0 Node runtime used by Notation application commands. */ +export class NodeDurableRuntime { + readonly deploymentId: string; + readonly state: DurableStateBackend; + readonly #database: ReturnType; + readonly #eventLoop: SqliteEventLoop; + readonly #heapClient: SqliteHeapClient; + readonly #schedulerClient: SqliteSchedulerClient; + readonly #storeClient: SqliteStoreClient; + readonly #logger: Logger; + #running = false; + + constructor(opts: NodeDurableRuntimeOptions) { + this.deploymentId = opts.deploymentId; + this.#logger = opts.logger ?? pino({ level: "silent" }); + this.#database = createSqliteDb({ + path: opts.databasePath ?? resolveWorkflowStatePath(), + }); + const taskQueueClient = new SqliteTaskQueueClient(this.#database); + this.#schedulerClient = new SqliteSchedulerClient({ + taskQueueClient, + timersClient: new SqliteTimersClient(this.#database), + }); + this.#storeClient = new SqliteStoreClient({ + db: this.#database, + schedulerClient: this.#schedulerClient, + }); + this.#heapClient = new SqliteHeapClient(this.#database); + this.#eventLoop = new SqliteEventLoop(this.#database); + this.state = new DurableStateBackend(this.#storeClient, this.deploymentId); + } + + async run( + router: WorkflowRouter, + opts: RunWorkflowOptions, + ): Promise { + if (this.#running) { + throw new Error( + "The Node Yieldstar runtime already has an active workflow", + ); + } + this.#running = true; + const event: WorkflowEvent = { + workflowId: opts.workflowId, + executionId: opts.executionId ?? randomUUID(), + params: opts.params ?? {}, + context: new Map(), + }; + const runner = new WorkflowRunner({ + router, + heapClient: this.#heapClient, + storeClient: this.#storeClient, + schedulerClient: this.#schedulerClient, + logger: this.#logger, + }); + + let resolveCompletion!: (value: unknown) => void; + let rejectCompletion!: (error: unknown) => void; + const completion = new Promise((resolve, reject) => { + resolveCompletion = resolve; + rejectCompletion = reject; + }); + const processEvent = async (nextEvent: WorkflowEvent, logger: Logger) => { + try { + const result = await runner.run(nextEvent, logger); + if (result && nextEvent.executionId === event.executionId) { + this.#eventLoop.stop(); + resolveCompletion(result.result); + } + } catch (error) { + if (nextEvent.executionId === event.executionId) { + this.#eventLoop.stop(); + rejectCompletion(error); + return; + } + this.#logger.error({ err: error }, "Yieldstar replay failed"); + } + }; + + try { + await processEvent(event, this.#logger); + this.#eventLoop.start({ onNewEvent: processEvent, logger: this.#logger }); + try { + return await completion; + } finally { + // Let SqliteEventLoop remove the completed queue item before callers + // close the shared database. + await setImmediate(); + } + } finally { + this.#eventLoop.stop(); + this.#running = false; + } + } + + close(): void { + if (this.#running) { + throw new Error( + "Cannot close the Node Yieldstar runtime while a workflow is active", + ); + } + this.#eventLoop.stop(); + this.#database.close(); + } +} diff --git a/packages/core/src/provisioner/index.ts b/packages/core/src/provisioner/index.ts index 89bf6e7..0b551ad 100644 --- a/packages/core/src/provisioner/index.ts +++ b/packages/core/src/provisioner/index.ts @@ -1,3 +1,3 @@ export * from "./workflows"; export * from "./resource-registry"; -export * from "./state-backend"; +export * from "./durable-runtime"; diff --git a/packages/core/src/provisioner/state-backend.ts b/packages/core/src/provisioner/state-backend.ts deleted file mode 100644 index 437e56c..0000000 --- a/packages/core/src/provisioner/state-backend.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { FileStateBackend, type StateBackend } from "@notation/state"; -import { SqliteStateBackend } from "@notation/state-sqlite"; - -export const DEFAULT_STATE_PATH = "./.notation/state.json"; - -export function resolveStatePath(): string { - return process.env.NOTATION_STATE_PATH ?? DEFAULT_STATE_PATH; -} - -export function createDefaultStateBackend(): StateBackend { - const statePath = resolveStatePath(); - if (statePath.endsWith(".db") || statePath.endsWith(".sqlite")) { - return new SqliteStateBackend(statePath); - } - return new FileStateBackend(statePath); -} diff --git a/packages/core/src/provisioner/workflows/index.ts b/packages/core/src/provisioner/workflows/index.ts index 9dd1579..835d222 100644 --- a/packages/core/src/provisioner/workflows/index.ts +++ b/packages/core/src/provisioner/workflows/index.ts @@ -7,4 +7,3 @@ export { export * from "./workflow.deploy"; export * from "./workflow.destroy"; export * from "./workflow.plan"; -export * from "./workflow.refresh"; diff --git a/packages/core/src/provisioner/workflows/workflow.deploy.ts b/packages/core/src/provisioner/workflows/workflow.deploy.ts index 3eaa88e..dc3b765 100644 --- a/packages/core/src/provisioner/workflows/workflow.deploy.ts +++ b/packages/core/src/provisioner/workflows/workflow.deploy.ts @@ -1,19 +1,22 @@ +import * as reconciler from "@notation/reconciler"; import { - Reconciler, createLoggerReconcilerSubscriber, type ReconcilerEventEmitter, type ResourceRegistry, } from "@notation/reconciler"; -import type { StateBackend } from "@notation/state"; +import { createWorkflowRouter, workflow } from "yieldstar"; import { getResourceGraph } from "src/orchestrator/graph"; -import { createDefaultStateBackend } from "../state-backend"; +import { NodeDurableRuntime } from "../durable-runtime"; export type DeployAppOptions = { entryPoint: string; driftDetection?: boolean; dryRun?: boolean; + maxOperationAttempts?: number; registry?: ResourceRegistry; - state?: StateBackend; + runtime?: NodeDurableRuntime; + executionId?: string; + databasePath?: string; emit?: ReconcilerEventEmitter; }; @@ -21,20 +24,36 @@ export async function deployApp({ entryPoint, driftDetection = true, dryRun = false, + maxOperationAttempts, registry, - state: stateBackend, + runtime: suppliedRuntime, + executionId, + databasePath, emit = createLoggerReconcilerSubscriber(), }: DeployAppOptions): Promise { const graph = await getResourceGraph(entryPoint); - const state = stateBackend ?? createDefaultStateBackend(); - const reconciler = new Reconciler({ - state, - registry, - emit, - }); - - await reconciler.deploy(graph.resources, { - dryRun, - driftDetection, + const runtime = + suppliedRuntime ?? + new NodeDurableRuntime({ deploymentId: entryPoint, databasePath }); + const deploy = workflow(async function* (step, event) { + yield* reconciler.deploy(step, { + deploymentId: runtime.deploymentId, + executionId: event.executionId, + resources: graph.resources, + state: runtime.state, + registry, + emit, + dryRun, + driftDetection, + maxOperationAttempts, + }); }); + try { + await runtime.run(createWorkflowRouter({ deploy }), { + workflowId: "deploy", + executionId, + }); + } finally { + if (!suppliedRuntime) runtime.close(); + } } diff --git a/packages/core/src/provisioner/workflows/workflow.destroy.ts b/packages/core/src/provisioner/workflows/workflow.destroy.ts index 813239a..9a534c4 100644 --- a/packages/core/src/provisioner/workflows/workflow.destroy.ts +++ b/packages/core/src/provisioner/workflows/workflow.destroy.ts @@ -1,35 +1,53 @@ +import * as reconciler from "@notation/reconciler"; import { - Reconciler, createLoggerReconcilerSubscriber, type ReconcilerEventEmitter, type ResourceRegistry, } from "@notation/reconciler"; -import type { StateBackend } from "@notation/state"; +import { createWorkflowRouter, workflow } from "yieldstar"; import { getResourceGraph } from "src/orchestrator/graph"; -import { createDefaultStateBackend } from "../state-backend"; -import { refreshState } from "./workflow.refresh"; +import { NodeDurableRuntime } from "../durable-runtime"; export type DestroyAppOptions = { entryPoint: string; + maxOperationAttempts?: number; registry?: ResourceRegistry; - state?: StateBackend; + runtime?: NodeDurableRuntime; + executionId?: string; + databasePath?: string; emit?: ReconcilerEventEmitter; }; export async function destroyApp({ entryPoint, + maxOperationAttempts, registry, - state: stateBackend, + runtime: suppliedRuntime, + executionId, + databasePath, emit = createLoggerReconcilerSubscriber(), }: DestroyAppOptions) { - const state = stateBackend ?? createDefaultStateBackend(); - await refreshState({ entryPoint, registry, state, emit }); - const graph = await getResourceGraph(entryPoint); - const reconciler = new Reconciler({ - state, - emit, + const runtime = + suppliedRuntime ?? + new NodeDurableRuntime({ deploymentId: entryPoint, databasePath }); + const destroy = workflow(async function* (step, event) { + yield* reconciler.destroy(step, { + deploymentId: runtime.deploymentId, + executionId: event.executionId, + resources: graph.resources, + state: runtime.state, + registry, + emit, + maxOperationAttempts, + }); }); - - await reconciler.destroy(graph.resources); + try { + await runtime.run(createWorkflowRouter({ destroy }), { + workflowId: "destroy", + executionId, + }); + } finally { + if (!suppliedRuntime) runtime.close(); + } } diff --git a/packages/core/src/provisioner/workflows/workflow.plan.ts b/packages/core/src/provisioner/workflows/workflow.plan.ts index 2e51cb8..b64db9d 100644 --- a/packages/core/src/provisioner/workflows/workflow.plan.ts +++ b/packages/core/src/provisioner/workflows/workflow.plan.ts @@ -1,38 +1,36 @@ -import { - Reconciler, - createLoggerReconcilerSubscriber, - type Plan, - type ReconcilerEventEmitter, - type ResourceRegistry, -} from "@notation/reconciler"; -import type { StateBackend } from "@notation/state"; +import { createPlan, type Plan } from "@notation/reconciler"; import { getResourceGraph } from "src/orchestrator/graph"; -import { createDefaultStateBackend } from "../state-backend"; +import { NodeDurableRuntime } from "../durable-runtime"; export type { Plan, PlanNode, PlanDecision } from "@notation/reconciler"; export type PlanAppOptions = { entryPoint: string; driftDetection?: boolean; - registry?: ResourceRegistry; - state?: StateBackend; - emit?: ReconcilerEventEmitter; + maxOperationAttempts?: number; + runtime?: NodeDurableRuntime; + databasePath?: string; }; export async function planApp({ entryPoint, driftDetection = true, - registry, - state: stateBackend, - emit = createLoggerReconcilerSubscriber(), + maxOperationAttempts, + runtime: suppliedRuntime, + databasePath, }: PlanAppOptions): Promise { const graph = await getResourceGraph(entryPoint); - const state = stateBackend ?? createDefaultStateBackend(); - const reconciler = new Reconciler({ - state, - registry, - emit, - }); - - return reconciler.plan(graph.resources, { driftDetection }); + const runtime = + suppliedRuntime ?? + new NodeDurableRuntime({ deploymentId: entryPoint, databasePath }); + try { + return await createPlan({ + resources: graph.resources, + state: runtime.state, + driftDetection, + maxOperationAttempts, + }); + } finally { + if (!suppliedRuntime) runtime.close(); + } } diff --git a/packages/core/src/provisioner/workflows/workflow.refresh.ts b/packages/core/src/provisioner/workflows/workflow.refresh.ts deleted file mode 100644 index b463a87..0000000 --- a/packages/core/src/provisioner/workflows/workflow.refresh.ts +++ /dev/null @@ -1,39 +0,0 @@ -import { - Reconciler, - createLoggerReconcilerSubscriber, - type ReconcilerEventEmitter, - type ResourceRegistry, -} from "@notation/reconciler"; -import type { StateBackend } from "@notation/state"; -import { getResourceGraph } from "src/orchestrator/graph"; -import { createDefaultStateBackend } from "../state-backend"; - -/** - * @description Destroy resources that are in state but not in the orchestration graph - */ -export type RefreshStateOptions = { - entryPoint: string; - dryRun?: boolean; - registry?: ResourceRegistry; - state?: StateBackend; - emit?: ReconcilerEventEmitter; -}; - -export async function refreshState({ - entryPoint, - dryRun = false, - registry, - state: stateBackend, - emit = createLoggerReconcilerSubscriber(), -}: RefreshStateOptions): Promise { - const graph = await getResourceGraph(entryPoint); - const state = stateBackend ?? createDefaultStateBackend(); - - const reconciler = new Reconciler({ - state, - registry, - emit, - }); - - await reconciler.refresh(graph.resources, { dryRun }); -} diff --git a/packages/core/test/provisioner/durable-runtime.test.ts b/packages/core/test/provisioner/durable-runtime.test.ts new file mode 100644 index 0000000..77a699b --- /dev/null +++ b/packages/core/test/provisioner/durable-runtime.test.ts @@ -0,0 +1,60 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import * as reconciler from "@notation/reconciler"; +import { + ResourceOperationPendingError, + resource, +} from "@notation/resource"; +import { createWorkflowRouter, workflow } from "yieldstar"; +import { describe, expect, it } from "vitest"; +import { NodeDurableRuntime } from "src/provisioner/durable-runtime"; + +describe("NodeDurableRuntime", () => { + it("stays resident across a provider delay and resumes from the SQLite event loop", async () => { + const directory = await mkdtemp(path.join(tmpdir(), "notation-runtime-")); + const runtime = new NodeDurableRuntime({ + deploymentId: "resident-wait", + databasePath: path.join(directory, "workflows.db"), + }); + let attempts = 0; + const PendingResource = resource({ type: "test/runtime/pending" }) + .defineSchema({}) + .defineOperations({ + create: async () => { + attempts += 1; + if (attempts === 1) { + throw new ResourceOperationPendingError("provider is not ready", { + retryAfterMs: 10, + }); + } + }, + delete: async () => undefined, + }); + const resources = [new PendingResource({ id: "pending" })]; + const deploy = workflow(async function* (step, event) { + yield* reconciler.deploy(step, { + deploymentId: runtime.deploymentId, + executionId: event.executionId, + resources, + state: runtime.state, + driftDetection: false, + maxOperationAttempts: 3, + }); + }); + + try { + await runtime.run(createWorkflowRouter({ deploy }), { + workflowId: "deploy", + executionId: "resident-execution", + }); + expect(attempts).toBe(2); + await expect(runtime.state.get("pending")).resolves.toMatchObject({ + lastOperation: "create", + }); + } finally { + runtime.close(); + await rm(directory, { recursive: true, force: true }); + } + }, 5_000); +}); diff --git a/packages/core/test/provisioner/operation.create.test.ts b/packages/core/test/provisioner/operation.create.test.ts deleted file mode 100644 index 2e4f506..0000000 --- a/packages/core/test/provisioner/operation.create.test.ts +++ /dev/null @@ -1,54 +0,0 @@ -import { describe, expect, it, vi } from "vitest"; -import { - createResourceOperation, - createStepRunner, - runOperation, -} from "@notation/reconciler"; -import { MemoryStateBackend } from "@notation/state"; -import { - TestResourceSchema, - testResourceConfig, - testOperations, - testResourceOutput, -} from "test/orchestrator/resource.doubles"; - -describe("resource creation", () => { - it("passes computed input to resource.create", async () => { - const stateBackend = new MemoryStateBackend(); - const readResult = { ...testResourceOutput, volatileComputed: "123" }; - const createMock = vi.fn(async () => ({ primaryKey: "" })); - const readMock = vi.fn(async () => readResult); - - const TestResource = TestResourceSchema.defineOperations({ - ...testOperations, - create: createMock, - read: readMock, - }); - - const testResource = new TestResource({ - id: "test-resource", - config: testResourceConfig, - }); - const step = createStepRunner(); - - await runOperation( - createResourceOperation(step, { - resource: testResource, - state: stateBackend, - expectedRev: 0, - }), - ); - - const params = await testResource.getParams(); - const persistedOutput = testResource.toState(readResult); - - expect(createMock.mock.calls[0]).toEqual([params, undefined]); - await expect(stateBackend.get(testResource.id)).resolves.toMatchObject({ - id: testResource.id, - output: persistedOutput, - lastOperation: "create", - }); - expect(testResource.output).not.toEqual(testResourceOutput); - expect(testResource.output).toEqual(readResult); - }); -}); diff --git a/packages/core/test/provisioner/state-backend.test.ts b/packages/core/test/provisioner/state-backend.test.ts deleted file mode 100644 index 5ac04a4..0000000 --- a/packages/core/test/provisioner/state-backend.test.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { mkdtempSync, rmSync } from "node:fs"; -import { tmpdir } from "node:os"; -import path from "node:path"; -import { afterEach, beforeEach, describe, expect, it } from "vitest"; -import { FileStateBackend } from "@notation/state"; -import { SqliteStateBackend } from "@notation/state-sqlite"; -import { createDefaultStateBackend } from "src/provisioner/state-backend"; - -describe("createDefaultStateBackend", () => { - let directory: string; - const originalStatePath = process.env.NOTATION_STATE_PATH; - - beforeEach(() => { - directory = mkdtempSync(path.join(tmpdir(), "notation-state-")); - }); - - afterEach(() => { - if (originalStatePath === undefined) { - delete process.env.NOTATION_STATE_PATH; - } else { - process.env.NOTATION_STATE_PATH = originalStatePath; - } - rmSync(directory, { recursive: true, force: true }); - }); - - it("uses the file backend for the default JSON path", () => { - delete process.env.NOTATION_STATE_PATH; - - expect(createDefaultStateBackend()).toBeInstanceOf(FileStateBackend); - }); - - it("uses the sqlite backend for .db paths", () => { - process.env.NOTATION_STATE_PATH = path.join(directory, "state.db"); - - const backend = createDefaultStateBackend(); - expect(backend).toBeInstanceOf(SqliteStateBackend); - (backend as SqliteStateBackend).close(); - }); - - it("uses the sqlite backend for .sqlite paths", () => { - process.env.NOTATION_STATE_PATH = path.join(directory, "state.sqlite"); - - const backend = createDefaultStateBackend(); - expect(backend).toBeInstanceOf(SqliteStateBackend); - (backend as SqliteStateBackend).close(); - }); - - it("creates missing parent directories for sqlite paths", () => { - process.env.NOTATION_STATE_PATH = path.join( - directory, - ".notation", - "state.db", - ); - - const backend = createDefaultStateBackend(); - expect(backend).toBeInstanceOf(SqliteStateBackend); - (backend as SqliteStateBackend).close(); - }); -}); diff --git a/packages/reconciler/src/durable/operations.ts b/packages/reconciler/src/durable/operations.ts index d945403..eeaa73a 100644 --- a/packages/reconciler/src/durable/operations.ts +++ b/packages/reconciler/src/durable/operations.ts @@ -9,7 +9,7 @@ import { createResourceRegistryFromResources, resolveResourceClass, } from "../resource-registry"; -import { runPendingOperation } from "../operations/operation.pending"; +import { runPendingOperation } from "../pending-operation"; import { decideAction, type DriftRead, type ResourceAction } from "../plan"; import { emitEvent, emitLifecycle } from "./emit"; import type { DurableStateBackend } from "./state-backend"; diff --git a/packages/reconciler/src/index.ts b/packages/reconciler/src/index.ts index 44fc402..1f6c3dc 100644 --- a/packages/reconciler/src/index.ts +++ b/packages/reconciler/src/index.ts @@ -4,9 +4,10 @@ export type DeepObjectDiffApi = typeof import("deep-object-diff"); export type YieldstarApi = typeof import("yieldstar"); export * from "./resource-registry"; -export * from "./operations"; export * from "./dependency-graph"; export * from "./plan"; -export * from "./reconciler"; +export * from "./planner"; +export * from "./events"; +export * from "./durable"; export * from "./logger-subscriber"; export * from "./protocol"; diff --git a/packages/reconciler/src/logger-subscriber.ts b/packages/reconciler/src/logger-subscriber.ts index 318a4cb..e756ff8 100644 --- a/packages/reconciler/src/logger-subscriber.ts +++ b/packages/reconciler/src/logger-subscriber.ts @@ -1,4 +1,4 @@ -import type { ReconcilerEvent, ReconcilerEventEmitter } from "./reconciler"; +import type { ReconcilerEvent, ReconcilerEventEmitter } from "./events"; export type Logger = Pick; @@ -17,7 +17,7 @@ export function createLoggerReconcilerSubscriber( return; } - if (event.event === "reconciler.orphan-deletion.skipped") { + if (event.level === "warn") { logger.warn(event.event, event); return; } diff --git a/packages/reconciler/src/operations/index.ts b/packages/reconciler/src/operations/index.ts deleted file mode 100644 index b0107e7..0000000 --- a/packages/reconciler/src/operations/index.ts +++ /dev/null @@ -1,6 +0,0 @@ -export * from "./operation.types"; -export * from "./operation.pending"; -export * from "./operation.create"; -export * from "./operation.read"; -export * from "./operation.update"; -export * from "./operation.delete"; diff --git a/packages/reconciler/src/operations/operation.create.ts b/packages/reconciler/src/operations/operation.create.ts deleted file mode 100644 index e0aee4f..0000000 --- a/packages/reconciler/src/operations/operation.create.ts +++ /dev/null @@ -1,82 +0,0 @@ -import { createWorkflow } from "yieldstar"; -import { - type CreateResourceParams, - type StepRunner, - emitLifecycleEvent, - getErrorDetails, -} from "./operation.types"; -import { runPendingOperation } from "./operation.pending"; -import { readResourceOperation } from "./operation.read"; - -export async function* createResourceOperation( - step: StepRunner, - params: CreateResourceParams, -): AsyncGenerator { - await emitLifecycleEvent(params, "create", "start"); - - if (params.dryRun) { - await emitLifecycleEvent(params, "create", "dry-run"); - return; - } - - try { - const resourceParams = yield* step.run("create:get-params", () => - params.resource.getParams(), - ); - - const computedPrimaryKey = yield* runPendingOperation( - step, - "create:remote", - (context) => params.resource.create(resourceParams, context), - params.maxOperationAttempts, - ); - - params.resource.setOutput(resourceParams); - if (computedPrimaryKey) { - params.resource.setOutput({ - ...computedPrimaryKey, - ...params.resource.output, - }); - } - - const readResult = yield* readResourceOperation(step, { - resource: params.resource, - state: params.state, - emit: params.emit, - maxOperationAttempts: params.maxOperationAttempts, - }); - - params.resource.setOutput({ - ...params.resource.output, - ...readResult, - }); - - yield* step.run("create:persist-state", async () => { - await params.state.update(params.resource.id, params.expectedRev, { - id: params.resource.id, - groupId: params.resource.groupId, - groupType: params.resource.groupType, - type: params.resource.type, - lastOperation: "create", - lastOperationAt: new Date().toISOString(), - config: params.resource.config, - params: params.resource.toState(resourceParams), - output: params.resource.toState(params.resource.output), - }); - }); - - await emitLifecycleEvent(params, "create", "success"); - } catch (err) { - await emitLifecycleEvent(params, "create", "error", getErrorDetails(err)); - throw err; - } -} - -export const createResourceWorkflow: unknown = createWorkflow( - async function* (step, event) { - return yield* createResourceOperation( - step as StepRunner, - event.params as CreateResourceParams, - ); - }, -); diff --git a/packages/reconciler/src/operations/operation.delete.ts b/packages/reconciler/src/operations/operation.delete.ts deleted file mode 100644 index 51975b6..0000000 --- a/packages/reconciler/src/operations/operation.delete.ts +++ /dev/null @@ -1,52 +0,0 @@ -import { createWorkflow } from "yieldstar"; -import { - type DeleteResourceParams, - type StepRunner, - emitLifecycleEvent, - getErrorDetails, -} from "./operation.types"; -import { runPendingOperation } from "./operation.pending"; - -export async function* deleteResourceOperation( - step: StepRunner, - params: DeleteResourceParams, -): AsyncGenerator { - await emitLifecycleEvent(params, "delete", "start"); - - if (params.dryRun) { - await emitLifecycleEvent(params, "delete", "dry-run"); - return; - } - - try { - yield* runPendingOperation( - step, - "delete:remote", - (context) => - params.resource.delete( - params.resource.key, - params.resource.toState(params.resource.output), - context, - ), - params.maxOperationAttempts, - ); - - yield* step.run("delete:persist-state", () => - params.state.delete(params.resource.id, params.expectedRev), - ); - - await emitLifecycleEvent(params, "delete", "success"); - } catch (err) { - await emitLifecycleEvent(params, "delete", "error", getErrorDetails(err)); - throw err; - } -} - -export const deleteResourceWorkflow: unknown = createWorkflow( - async function* (step, event) { - return yield* deleteResourceOperation( - step as StepRunner, - event.params as DeleteResourceParams, - ); - }, -); diff --git a/packages/reconciler/src/operations/operation.read.ts b/packages/reconciler/src/operations/operation.read.ts deleted file mode 100644 index e74196a..0000000 --- a/packages/reconciler/src/operations/operation.read.ts +++ /dev/null @@ -1,68 +0,0 @@ -import { createWorkflow } from "yieldstar"; -import { - type ReadResourceParams, - type StepRunner, - emitLifecycleEvent, - getErrorDetails, -} from "./operation.types"; -import { runPendingOperation } from "./operation.pending"; - -export async function* readResourceOperation( - step: StepRunner, - params: ReadResourceParams, -): AsyncGenerator, unknown> { - await emitLifecycleEvent(params, "read", "start"); - - if (params.dryRun) { - await emitLifecycleEvent(params, "read", "dry-run"); - return {}; - } - - try { - const resourceParams = yield* step.run("read:get-params", () => - params.resource.getParams(), - ); - - if (!params.resource.read) { - const stateNode = yield* step.run("read:get-state-node", () => - params.state.get(params.resource.id), - ); - const merged = stateNode - ? { ...stateNode.output, ...resourceParams } - : resourceParams; - - await emitLifecycleEvent(params, "read", "skip", { - reason: "read-not-implemented", - }); - await emitLifecycleEvent(params, "read", "success"); - return merged as Record; - } - - const remote = yield* runPendingOperation( - step, - "read:remote", - (context) => params.resource.read!(params.resource.key, context), - params.maxOperationAttempts, - ); - - const mergedOutput = { - ...resourceParams, - ...remote, - }; - - await emitLifecycleEvent(params, "read", "success"); - return mergedOutput; - } catch (err) { - await emitLifecycleEvent(params, "read", "error", getErrorDetails(err)); - throw err; - } -} - -export const readResourceWorkflow: unknown = createWorkflow( - async function* (step, event) { - return yield* readResourceOperation( - step as StepRunner, - event.params as ReadResourceParams, - ); - }, -); diff --git a/packages/reconciler/src/operations/operation.types.ts b/packages/reconciler/src/operations/operation.types.ts deleted file mode 100644 index 6cbcd1c..0000000 --- a/packages/reconciler/src/operations/operation.types.ts +++ /dev/null @@ -1,92 +0,0 @@ -import type { BaseResource, ResourceType } from "@notation/resource"; -import type { State } from "@notation/state"; - -export type OperationName = "create" | "read" | "update" | "delete"; - -export type OperationLifecycleStatus = - "start" | "success" | "error" | "skip" | "dry-run"; - -export type OperationLifecycleEvent = { - level: "info" | "error"; - event: "reconciler.operation.lifecycle"; - operation: OperationName; - status: OperationLifecycleStatus; - resourceId: string; - resourceType: ResourceType; - reason?: string; - errorName?: string; - errorMessage?: string; -}; - -export type OperationEventEmitter = ( - event: OperationLifecycleEvent, -) => void | Promise; - -export type StepRunner = { - run(fn: () => T | Promise): AsyncGenerator; - run( - key: string, - fn: () => T | Promise, - ): AsyncGenerator; - delay(ms: number): AsyncGenerator; - delay(key: string, ms: number): AsyncGenerator; -}; - -export type ResourceOperationBaseParams = { - resource: BaseResource; - state: Pick; - dryRun?: boolean; - emit?: OperationEventEmitter; - maxOperationAttempts?: number; -}; - -export type CreateResourceParams = ResourceOperationBaseParams & { - expectedRev: number; -}; - -export type ReadResourceParams = ResourceOperationBaseParams; - -export type UpdateResourceParams = ResourceOperationBaseParams & { - patch: Record; - expectedRev: number; -}; - -export type DeleteResourceParams = ResourceOperationBaseParams & { - expectedRev: number; -}; - -export function getErrorDetails(err: unknown): { - errorName: string; - errorMessage: string; -} { - if (err instanceof Error) { - return { - errorName: err.name, - errorMessage: err.message, - }; - } - - return { - errorName: "UnknownError", - errorMessage: String(err), - }; -} - -export async function emitLifecycleEvent( - params: ResourceOperationBaseParams, - operation: OperationName, - status: OperationLifecycleStatus, - extra: Partial = {}, -) { - if (!params.emit) return; - - await params.emit({ - level: status === "error" ? "error" : "info", - event: "reconciler.operation.lifecycle", - operation, - status, - resourceId: params.resource.id, - resourceType: params.resource.type, - ...extra, - }); -} diff --git a/packages/reconciler/src/operations/operation.update.ts b/packages/reconciler/src/operations/operation.update.ts deleted file mode 100644 index fc62a8c..0000000 --- a/packages/reconciler/src/operations/operation.update.ts +++ /dev/null @@ -1,94 +0,0 @@ -import { createWorkflow } from "yieldstar"; -import { - type StepRunner, - type UpdateResourceParams, - emitLifecycleEvent, - getErrorDetails, -} from "./operation.types"; -import { runPendingOperation } from "./operation.pending"; -import { readResourceOperation } from "./operation.read"; - -export async function* updateResourceOperation( - step: StepRunner, - params: UpdateResourceParams, -): AsyncGenerator { - await emitLifecycleEvent(params, "update", "start"); - - if (params.dryRun) { - await emitLifecycleEvent(params, "update", "dry-run"); - return; - } - - if (!params.resource.update) { - await emitLifecycleEvent(params, "update", "skip", { - reason: "update-not-implemented", - }); - await emitLifecycleEvent(params, "update", "success"); - return; - } - - try { - const resourceParams = yield* step.run("update:get-params", () => - params.resource.getParams(), - ); - - yield* runPendingOperation( - step, - "update:remote", - (context) => - params.resource.update!( - params.resource.key, - params.patch, - resourceParams, - params.resource.toState(params.resource.output), - context, - ), - params.maxOperationAttempts, - ); - - params.resource.setOutput({ - ...params.resource.key, - ...resourceParams, - }); - - const readResult = yield* readResourceOperation(step, { - resource: params.resource, - state: params.state, - emit: params.emit, - maxOperationAttempts: params.maxOperationAttempts, - }); - - params.resource.setOutput({ - ...params.resource.output, - ...readResult, - }); - - yield* step.run("update:persist-state", async () => { - await params.state.update(params.resource.id, params.expectedRev, { - id: params.resource.id, - groupId: params.resource.groupId, - groupType: params.resource.groupType, - type: params.resource.type, - lastOperation: "update", - lastOperationAt: new Date().toISOString(), - config: params.resource.config, - params: params.resource.toState(resourceParams), - output: params.resource.toState(params.resource.output), - }); - }); - - await emitLifecycleEvent(params, "update", "success"); - } catch (err) { - await emitLifecycleEvent(params, "update", "error", getErrorDetails(err)); - throw err; - } -} - -export const updateResourceWorkflow: unknown = createWorkflow( - async function* (step, event) { - return yield* updateResourceOperation( - step as StepRunner, - event.params as UpdateResourceParams, - ); - }, -); diff --git a/packages/reconciler/src/operations/operation.pending.ts b/packages/reconciler/src/pending-operation.ts similarity index 84% rename from packages/reconciler/src/operations/operation.pending.ts rename to packages/reconciler/src/pending-operation.ts index 04a6b7e..d9f2648 100644 --- a/packages/reconciler/src/operations/operation.pending.ts +++ b/packages/reconciler/src/pending-operation.ts @@ -2,12 +2,19 @@ import { ResourceOperationPendingError, type ResourceOperationContext, } from "@notation/resource"; -import type { StepRunner } from "./operation.types"; + +type OperationStep = { + run( + key: string, + operation: () => T | Promise, + ): AsyncGenerator; + delay(key: string, delayMs: number): AsyncGenerator; +}; export const DEFAULT_MAX_OPERATION_ATTEMPTS = 30; export async function* runPendingOperation( - step: StepRunner, + step: OperationStep, key: string, operation: (context?: ResourceOperationContext) => T | Promise, maxAttempts = DEFAULT_MAX_OPERATION_ATTEMPTS, diff --git a/packages/reconciler/src/planner.ts b/packages/reconciler/src/planner.ts index ebfa064..5dfed90 100644 --- a/packages/reconciler/src/planner.ts +++ b/packages/reconciler/src/planner.ts @@ -1,7 +1,8 @@ import { ResourceNotFoundError, type BaseResource } from "@notation/resource"; import type { StateBackend } from "@notation/state"; +import { setTimeout as sleep } from "node:timers/promises"; import { buildResourceDepthLevels } from "./dependency-graph"; -import { readResourceOperation } from "./operations"; +import { runPendingOperation } from "./pending-operation"; import { decideAction, getDependencyIds, @@ -9,18 +10,19 @@ import { type Plan, type PlanNode, } from "./plan"; -import { createStepRunner, runOperation } from "./reconciler"; export type CreatePlanOptions = { resources: BaseResource[]; state: StateBackend; driftDetection?: boolean; + maxOperationAttempts?: number; }; export async function createPlan({ resources, state, driftDetection = true, + maxOperationAttempts, }: CreatePlanOptions): Promise { const resourceById = new Map( resources.map((resource) => [resource.id, resource]), @@ -38,10 +40,12 @@ export async function createPlan({ let driftRead; try { const output = await runOperation( - readResourceOperation(createStepRunner(), { - resource, - state, - }), + runPendingOperation( + createStepRunner(), + `plan:${resource.id}:read`, + (context) => resource.read!(resource.key, context), + maxOperationAttempts, + ), ); driftRead = { kind: "present" as const, output }; } catch (error) { @@ -81,3 +85,30 @@ export async function createPlan({ return { createdAt: new Date().toISOString(), nodes }; } + +async function runOperation( + operation: AsyncGenerator, +) { + let next = await operation.next(); + while (!next.done) { + next = await operation.next(); + } + return next.value; +} + +function createStepRunner() { + return { + async *run( + _key: string, + operation: () => T | Promise, + ): AsyncGenerator { + return await operation(); + }, + async *delay( + _key: string, + delayMs: number, + ): AsyncGenerator { + await sleep(delayMs); + }, + }; +} diff --git a/packages/reconciler/src/protocol.ts b/packages/reconciler/src/protocol.ts index bf3706e..a8ab407 100644 --- a/packages/reconciler/src/protocol.ts +++ b/packages/reconciler/src/protocol.ts @@ -1,4 +1,4 @@ -import type { ReconcilerEvent, ReconcilerEventEmitter } from "./reconciler"; +import type { ReconcilerEvent, ReconcilerEventEmitter } from "./events"; export const EVENT_STREAM_VERSION = 1 as const; diff --git a/packages/reconciler/src/reconciler.ts b/packages/reconciler/src/reconciler.ts deleted file mode 100644 index fab6c0b..0000000 --- a/packages/reconciler/src/reconciler.ts +++ /dev/null @@ -1,602 +0,0 @@ -import { ResourceNotFoundError } from "@notation/resource"; -import type { BaseResource, ResourceType } from "@notation/resource"; -import { RevConflict, type State, type StateNode } from "@notation/state"; -import { setTimeout as sleep } from "node:timers/promises"; -import { buildResourceDepthLevels } from "./dependency-graph"; -import { - decideAction, - getDependencyIds, - resolvePlanParams, - type DriftRead, - type Plan, - type PlanNode, - type ResourceAction, -} from "./plan"; -import { - createResourceOperation, - deleteResourceOperation, - readResourceOperation, - type OperationLifecycleEvent, - type StepRunner, - updateResourceOperation, -} from "./operations"; -import { - createMissingResourceRegistryMatchWarningEvent, - createResourceRegistryFromResources, - resolveResourceClass, - type MissingResourceRegistryMatchWarningEvent, - type ResourceRegistry, -} from "./resource-registry"; - -export type ReconcilerDeployEvent = { - level: "info"; - event: "reconciler.deploy.decision"; - resourceId: string; - resourceType: string; - decision: "create" | "update" | "drift-update" | "drift-recreate" | "noop"; -}; - -export type ReconcilerDriftDetectedEvent = { - level: "info"; - event: "reconciler.drift.detected"; - resourceId: string; - resourceType: string; - diff: Record; -}; - -export type ReconcilerEvent = - | OperationLifecycleEvent - | ReconcilerDeployEvent - | ReconcilerDriftDetectedEvent - | MissingResourceRegistryMatchWarningEvent; - -export type ReconcilerEventEmitter = ( - event: ReconcilerEvent, -) => void | Promise; - -export type ReconcilerState = Pick< - State, - "get" | "update" | "delete" | "values" | "lease" ->; - -export type ReconcilerOptions = { - state: ReconcilerState; - registry?: ResourceRegistry; - dryRun?: boolean; - driftDetection?: boolean; - emit?: ReconcilerEventEmitter; - maxOperationAttempts?: number; - mutationLeaseTtl?: number; -}; - -export type DeployOptions = { - dryRun?: boolean; - driftDetection?: boolean; -}; - -export type DestroyOptions = { - dryRun?: boolean; -}; - -export type RefreshOptions = { - dryRun?: boolean; -}; - -export type PlanOptions = { - driftDetection?: boolean; -}; - -export class Reconciler { - readonly #state: ReconcilerState; - readonly #registry?: ResourceRegistry; - readonly #defaultDryRun: boolean; - readonly #defaultDriftDetection: boolean; - readonly #emit?: ReconcilerEventEmitter; - readonly #maxOperationAttempts?: number; - readonly #mutationLeaseTtl: number; - readonly #stepRunner: StepRunner; - - constructor(opts: ReconcilerOptions) { - this.#state = opts.state; - this.#registry = opts.registry; - this.#defaultDryRun = opts.dryRun ?? false; - this.#defaultDriftDetection = opts.driftDetection ?? true; - this.#emit = opts.emit; - this.#maxOperationAttempts = opts.maxOperationAttempts; - this.#mutationLeaseTtl = opts.mutationLeaseTtl ?? 30_000; - this.#stepRunner = createStepRunner(); - } - - async deploy( - resources: BaseResource[], - opts: DeployOptions = {}, - ): Promise { - const dryRun = opts.dryRun ?? this.#defaultDryRun; - const driftDetection = opts.driftDetection ?? this.#defaultDriftDetection; - const resourceById = new Map( - resources.map((resource) => [resource.id, resource]), - ); - - const dependencyLevels = buildResourceDepthLevels(resources); - for (const level of dependencyLevels) { - await Promise.all( - level.map((resource) => - this.#deployResource(resource, dryRun, driftDetection), - ), - ); - } - - await this.#deleteOrphans(resources, resourceById, dryRun, "deploy"); - } - - async plan(resources: BaseResource[], opts: PlanOptions = {}): Promise { - const driftDetection = opts.driftDetection ?? this.#defaultDriftDetection; - const resourceById = new Map( - resources.map((resource) => [resource.id, resource]), - ); - const nodes: PlanNode[] = []; - - const dependencyLevels = buildResourceDepthLevels(resources); - for (const level of dependencyLevels) { - for (const resource of level) { - nodes.push(await this.#planResource(resource, driftDetection)); - } - } - - const stateNodes = await this.#state.values(); - for (const stateNode of stateNodes) { - if (resourceById.has(stateNode.id)) continue; - - nodes.push({ - id: stateNode.id, - type: stateNode.type, - decision: "delete-orphan", - params: stateNode.params, - dependsOn: [], - }); - } - - return { - createdAt: new Date().toISOString(), - nodes, - }; - } - - async destroy( - resources: BaseResource[], - opts: DestroyOptions = {}, - ): Promise { - const dryRun = opts.dryRun ?? this.#defaultDryRun; - const dependencyLevels = buildResourceDepthLevels(resources); - - for ( - let levelIndex = dependencyLevels.length - 1; - levelIndex >= 0; - levelIndex -= 1 - ) { - const level = dependencyLevels[levelIndex]!; - await Promise.all( - level.map((resource) => this.#destroyResource(resource, dryRun)), - ); - } - } - - async refresh( - resources: BaseResource[], - opts: RefreshOptions = {}, - ): Promise { - const dryRun = opts.dryRun ?? this.#defaultDryRun; - const resourceById = new Map( - resources.map((resource) => [resource.id, resource]), - ); - - await this.#deleteOrphans(resources, resourceById, dryRun, "refresh"); - } - - async #deployResource( - resource: BaseResource, - dryRun: boolean, - driftDetection: boolean, - ) { - await this.#withMutationLease(resource.id, () => - this.#retryOnRevConflict((conflict) => - this.#deployResourceOnce(resource, dryRun, driftDetection, conflict), - ), - ); - } - - async #withMutationLease(resourceId: string, fn: () => Promise) { - return this.#withLease(`reconciler:resource:${resourceId}`, fn); - } - - async #withLease(scope: string, fn: () => Promise): Promise { - const lease = await this.#state.lease(scope, this.#mutationLeaseTtl); - const controller = new AbortController(); - let renewalError: unknown; - const heartbeat = (async () => { - try { - while (!controller.signal.aborted) { - await sleep( - Math.max(1, Math.floor(this.#mutationLeaseTtl / 3)), - undefined, - { - signal: controller.signal, - }, - ); - await lease.renew(this.#mutationLeaseTtl); - } - } catch (error) { - if (!controller.signal.aborted) renewalError = error; - } - })(); - - try { - const result = await fn(); - if (renewalError) throw renewalError; - return result; - } finally { - controller.abort(); - await heartbeat; - await lease.release(); - } - } - - async #retryOnRevConflict(fn: (conflict?: RevConflict) => Promise) { - let conflict: RevConflict | undefined; - for (let attempt = 0; attempt < 3; attempt += 1) { - try { - await fn(conflict); - return; - } catch (error) { - if (!(error instanceof RevConflict) || attempt === 2) throw error; - // Re-throwing the conflict supplied for recovery means the resource - // cannot be recovered safely (for example, it has no read operation). - if (error === conflict) throw error; - conflict = error; - } - } - } - - async #deployResourceOnce( - resource: BaseResource, - dryRun: boolean, - driftDetection: boolean, - conflict?: RevConflict, - ) { - if (conflict) { - await this.#recoverDeployResource(resource, dryRun, conflict); - return; - } - - const stateNode = await this.#state.get(resource.id); - - let action: ResourceAction; - if (!stateNode) { - action = decideAction({ resource }); - } else { - resource.setOutput(stateNode.output); - const params = await resource.getParams(); - action = decideAction({ resource, stateNode, params }); - - if (action.decision === "noop" && driftDetection) { - const driftRead = await this.#readForDrift(resource); - action = decideAction({ resource, stateNode, params, driftRead }); - } - } - - if (action.decision === "drift-update") { - await this.#emit?.({ - level: "info", - event: "reconciler.drift.detected", - resourceId: resource.id, - resourceType: resource.type, - diff: action.patch, - }); - } - - await this.#emit?.({ - level: "info", - event: "reconciler.deploy.decision", - resourceId: resource.id, - resourceType: resource.type, - decision: action.decision, - }); - - switch (action.decision) { - case "create": - case "drift-recreate": - await runOperation( - createResourceOperation(this.#stepRunner, { - resource, - state: this.#state, - dryRun, - emit: this.#emit, - maxOperationAttempts: this.#maxOperationAttempts, - expectedRev: stateNode?.rev ?? 0, - }), - ); - return; - case "update": - case "drift-update": - // decideAction only returns update decisions for an existing stateNode - await runOperation( - updateResourceOperation(this.#stepRunner, { - resource, - state: this.#state, - patch: action.patch, - dryRun, - emit: this.#emit, - maxOperationAttempts: this.#maxOperationAttempts, - expectedRev: stateNode!.rev, - }), - ); - return; - case "noop": - return; - } - } - - async #recoverDeployResource( - resource: BaseResource, - dryRun: boolean, - conflict: RevConflict, - ) { - if (!resource.read) throw conflict; - - const stateNode = await this.#state.get(resource.id); - if (stateNode) resource.setOutput(stateNode.output); - - const params = await resource.getParams(); - const remote = await this.#readForDrift(resource); - const action = decideAction({ - resource, - stateNode, - params, - driftRead: remote, - }); - if (remote.kind === "present") resource.setOutput(remote.output); - - await this.#emit?.({ - level: "info", - event: "reconciler.deploy.decision", - resourceId: resource.id, - resourceType: resource.type, - decision: action.decision, - }); - - switch (action.decision) { - case "create": - case "drift-recreate": - await runOperation( - createResourceOperation(this.#stepRunner, { - resource, - state: this.#state, - dryRun, - emit: this.#emit, - maxOperationAttempts: this.#maxOperationAttempts, - expectedRev: stateNode?.rev ?? 0, - }), - ); - return; - case "update": - case "drift-update": - await runOperation( - updateResourceOperation(this.#stepRunner, { - resource, - state: this.#state, - patch: action.patch, - dryRun, - emit: this.#emit, - maxOperationAttempts: this.#maxOperationAttempts, - expectedRev: stateNode?.rev ?? 0, - }), - ); - return; - case "noop": - if (dryRun) return; - await this.#state.update(resource.id, stateNode?.rev ?? 0, { - id: resource.id, - groupId: resource.groupId, - groupType: resource.groupType, - type: resource.type, - lastOperation: "drift", - lastOperationAt: new Date().toISOString(), - config: resource.config, - params: resource.toState(params), - output: resource.toState(resource.output), - }); - return; - } - } - - async #planResource( - resource: BaseResource, - driftDetection: boolean, - ): Promise { - const stateNode = await this.#state.get(resource.id); - if (stateNode) { - resource.setOutput(stateNode.output); - } - - const params = await resolvePlanParams(resource); - let action = decideAction({ resource, stateNode, params }); - - if (action.decision === "noop" && driftDetection) { - const driftRead = await this.#readForDrift(resource); - action = decideAction({ resource, stateNode, params, driftRead }); - } - - return { - id: resource.id, - type: resource.type, - decision: action.decision, - ...("diff" in action ? { diff: action.diff } : {}), - params, - dependsOn: getDependencyIds(resource), - }; - } - - async #readForDrift(resource: BaseResource): Promise { - try { - const output = await runOperation( - readResourceOperation(this.#stepRunner, { - resource, - state: this.#state, - emit: this.#emit, - maxOperationAttempts: this.#maxOperationAttempts, - }), - ); - return { kind: "present", output }; - } catch (error) { - if (ResourceNotFoundError.is(error)) return { kind: "absent" }; - throw error; - } - } - - async #deleteOrphans( - resources: BaseResource[], - resourceById: Map, - dryRun: boolean, - workflow: "deploy" | "refresh", - ) { - await this.#withLease("reconciler:orphan-deletion", async () => { - const stateNodes = await this.#state.values(); - const registry = - this.#registry ?? createResourceRegistryFromResources(resources); - - for (const stateNode of stateNodes) { - if (resourceById.has(stateNode.id)) continue; - - const stateNodeResourceType = stateNode.type as ResourceType; - - const Resource = resolveResourceClass(registry, stateNodeResourceType); - if (!Resource) { - await this.#emit?.( - createMissingResourceRegistryMatchWarningEvent({ - workflow, - resourceId: stateNode.id, - resourceType: stateNodeResourceType, - }), - ); - continue; - } - - await this.#withMutationLease(stateNode.id, () => - this.#retryOnRevConflict(async (conflict) => { - const currentNode = await this.#state.get(stateNode.id); - if (!currentNode) return; - - const orphanResource = hydrateResourceFromState( - Resource, - currentNode, - ); - - await this.#deleteResourceOnce( - orphanResource, - currentNode, - dryRun, - conflict, - ); - }), - ); - } - }); - } - - async #destroyResource(resource: BaseResource, dryRun: boolean) { - await this.#withMutationLease(resource.id, () => - this.#retryOnRevConflict(async (conflict) => { - const stateNode = await this.#state.get(resource.id); - if (!stateNode) { - return; - } - - resource.setOutput(stateNode.output); - await this.#deleteResourceOnce(resource, stateNode, dryRun, conflict); - }), - ); - } - - async #deleteResourceOnce( - resource: BaseResource, - stateNode: StateNode, - dryRun: boolean, - conflict?: RevConflict, - ) { - if (conflict) { - if (!resource.read) throw conflict; - - const remote = await this.#readForDrift(resource); - if (remote.kind !== "present") { - if (!dryRun) await this.#state.delete(resource.id, stateNode.rev); - return; - } - resource.setOutput(remote.output); - } - - await runOperation( - deleteResourceOperation(this.#stepRunner, { - resource, - state: this.#state, - dryRun, - emit: this.#emit, - maxOperationAttempts: this.#maxOperationAttempts, - expectedRev: stateNode.rev, - }), - ); - } -} - -export async function runOperation( - operation: AsyncGenerator, -) { - let next = await operation.next(); - while (!next.done) { - next = await operation.next(); - } - return next.value; -} - -function hydrateResourceFromState( - Resource: new (opts: { - id: string; - config: Record; - }) => BaseResource, - stateNode: StateNode, -): BaseResource { - const resource = new Resource({ - id: stateNode.id, - config: stateNode.config, - }); - resource.setOutput(stateNode.output); - return resource; -} - -export function createStepRunner(): StepRunner { - return { - async *run( - arg1: string | (() => T | Promise), - arg2?: () => T | Promise, - ): AsyncGenerator { - const fn = (typeof arg1 === "string" ? arg2 : arg1) as - (() => T | Promise) | undefined; - - if (!fn) { - throw new Error("Missing run function"); - } - - return await fn(); - }, - async *delay( - arg1: string | number, - arg2?: number, - ): AsyncGenerator { - const ms = typeof arg1 === "number" ? arg1 : arg2; - if (ms === undefined) { - throw new Error("Missing delay duration"); - } - - await new Promise((resolve) => setTimeout(resolve, ms)); - }, - }; -} diff --git a/packages/reconciler/src/resource-registry.ts b/packages/reconciler/src/resource-registry.ts index 6916caa..34dabfc 100644 --- a/packages/reconciler/src/resource-registry.ts +++ b/packages/reconciler/src/resource-registry.ts @@ -1,4 +1,8 @@ -import type { BaseResource, ResourceClass, ResourceType } from "@notation/resource"; +import type { + BaseResource, + ResourceClass, + ResourceType, +} from "@notation/resource"; export type ResourceRegistry = Map>; @@ -6,7 +10,7 @@ export type MissingResourceRegistryMatchWarningEvent = { level: "warn"; event: "reconciler.orphan-deletion.skipped"; reason: "resource-type-not-registered"; - workflow: "deploy" | "refresh" | "destroy"; + workflow: "deploy" | "destroy"; resourceId: string; resourceType: ResourceType; }; @@ -46,7 +50,7 @@ export function resolveResourceClass( } export function createMissingResourceRegistryMatchWarningEvent(opts: { - workflow: "deploy" | "refresh" | "destroy"; + workflow: "deploy" | "destroy"; resourceId: string; resourceType: ResourceType; }): MissingResourceRegistryMatchWarningEvent { diff --git a/packages/reconciler/test/logger-subscriber.test.ts b/packages/reconciler/test/logger-subscriber.test.ts index 49569dd..721e884 100644 --- a/packages/reconciler/test/logger-subscriber.test.ts +++ b/packages/reconciler/test/logger-subscriber.test.ts @@ -27,6 +27,13 @@ describe("logger reconciler subscriber", () => { resourceId: "resource-2", resourceType: "test/service/subscriber", }); + await emit({ + level: "warn", + event: "reconciler.coordination.waiting", + deploymentId: "deployment-1", + executionId: "execution-2", + holderExecutionId: "execution-1", + }); await emit({ level: "error", event: "reconciler.operation.lifecycle", @@ -39,7 +46,12 @@ describe("logger reconciler subscriber", () => { }); expect(info).toHaveBeenCalledOnce(); - expect(warn).toHaveBeenCalledOnce(); + expect(warn).toHaveBeenCalledTimes(2); + expect(warn).toHaveBeenNthCalledWith( + 2, + "reconciler.coordination.waiting", + expect.objectContaining({ level: "warn" }), + ); expect(error).toHaveBeenCalledOnce(); }); }); diff --git a/packages/reconciler/test/operation.workflows.test.ts b/packages/reconciler/test/operation.workflows.test.ts deleted file mode 100644 index 2afc6df..0000000 --- a/packages/reconciler/test/operation.workflows.test.ts +++ /dev/null @@ -1,346 +0,0 @@ -import { describe, expect, it, vi } from "vitest"; -import { - resource, - ResourceNotFoundError, - ResourceOperationPendingError, -} from "@notation/resource"; -import { - createResourceOperation, - deleteResourceOperation, - readResourceOperation, - type OperationLifecycleEvent, - type StepRunner, -} from "../src/operations"; - -function createStepRunnerDouble(): StepRunner { - const run = vi.fn(async function* ( - arg1: string | (() => T | Promise), - arg2?: () => T | Promise, - ): AsyncGenerator { - const fn = (typeof arg1 === "string" ? arg2 : arg1) as () => T | Promise; - if (!fn) { - throw new Error("Missing run function"); - } - - return await fn(); - }); - - const delay = vi.fn(async function* (): AsyncGenerator< - unknown, - void, - unknown - > { - return; - }); - - return { - run, - delay, - }; -} - -async function runOperation(operation: AsyncGenerator) { - let next = await operation.next(); - while (!next.done) { - next = await operation.next(); - } - return next.value; -} - -describe("operation workflows", () => { - it("create performs create + read-after-create + state persistence", async () => { - const step = createStepRunnerDouble(); - const events: OperationLifecycleEvent[] = []; - const state = { - get: vi.fn(async () => undefined), - update: vi.fn(async () => undefined), - delete: vi.fn(async () => undefined), - }; - - let createAttempts = 0; - const createMock = vi.fn(async (_params, context) => { - createAttempts += 1; - if (createAttempts === 1) { - expect(context).toBeUndefined(); - throw Object.assign(new Error("retry create"), { - _tag: "ResourceOperationPendingError", - retryAfterMs: 25, - callbackContext: { operationId: "create-123" }, - }); - } - expect(context).toEqual({ operationId: "create-123" }); - return { remoteId: "abc" }; - }); - - const TestResource = resource({ type: "test/service/create" }) - .defineSchema({}) - .defineOperations({ - create: createMock, - read: async () => ({ remoteId: "abc", status: "ready" }), - delete: async () => undefined, - }); - - const testResource = new TestResource({ id: "test-create" }); - - await runOperation( - createResourceOperation(step, { - resource: testResource, - state, - expectedRev: 0, - emit: async (event) => { - events.push(event); - }, - }), - ); - - expect(createAttempts).toBe(2); - expect(state.update).toHaveBeenCalledOnce(); - expect(createMock).toHaveBeenNthCalledWith( - 1, - await testResource.getParams(), - undefined, - ); - expect(createMock).toHaveBeenNthCalledWith( - 2, - await testResource.getParams(), - { operationId: "create-123" }, - ); - expect(testResource.output).toEqual({ remoteId: "abc", status: "ready" }); - expect(events.map((event) => `${event.operation}:${event.status}`)).toEqual( - ["create:start", "read:start", "read:success", "create:success"], - ); - expect(events[0]).toMatchObject({ - resourceId: "test-create", - resourceType: TestResource.type, - event: "reconciler.operation.lifecycle", - }); - }); - - it("read follows pending retry instructions", async () => { - const step = createStepRunnerDouble(); - const state = { - get: vi.fn(async () => undefined), - update: vi.fn(async () => undefined), - delete: vi.fn(async () => undefined), - }; - - let readAttempts = 0; - const TestResource = resource({ type: "test/service/read" }) - .defineSchema({}) - .defineOperations({ - create: async () => ({}), - read: async (_key, context) => { - readAttempts += 1; - if (readAttempts < 3) { - throw new ResourceOperationPendingError("resource is not ready", { - retryAfterMs: readAttempts * 10, - callbackContext: { readAttempts }, - }); - } - expect(context).toEqual({ readAttempts: 2 }); - return { status: "ready" } as const; - }, - delete: async () => undefined, - }); - - const testResource = new TestResource({ id: "test-read" }); - - const result = await runOperation( - readResourceOperation(step, { - resource: testResource, - state, - }), - ); - - expect(readAttempts).toBe(3); - expect(result).toEqual({ status: "ready" }); - expect(step.delay).toHaveBeenNthCalledWith( - 1, - "read:remote:retry-delay:0", - 10, - ); - expect(step.delay).toHaveBeenNthCalledWith( - 2, - "read:remote:retry-delay:1", - 20, - ); - }); - - it("fails when an operation remains pending past the safety limit", async () => { - const step = createStepRunnerDouble(); - const state = { - get: vi.fn(async () => undefined), - update: vi.fn(async () => undefined), - delete: vi.fn(async () => undefined), - }; - const read = vi.fn(async () => { - throw new ResourceOperationPendingError("still pending", { - retryAfterMs: 10, - }); - }); - const TestResource = resource({ type: "test/service/pending-limit" }) - .defineSchema({}) - .defineOperations({ - create: async () => ({}), - read, - delete: async () => undefined, - }); - - await expect( - runOperation( - readResourceOperation(step, { - resource: new TestResource({ id: "pending-limit" }), - state, - maxOperationAttempts: 2, - }), - ), - ).rejects.toThrowError("still pending after 2 attempts"); - expect(read).toHaveBeenCalledTimes(2); - expect(step.delay).toHaveBeenCalledOnce(); - }); - - it("does not infer that not-found after creation is retryable", async () => { - const step = createStepRunnerDouble(); - const state = { - get: vi.fn(async () => undefined), - update: vi.fn(async () => undefined), - delete: vi.fn(async () => undefined), - }; - const TestResource = resource({ type: "test/service/eventually-visible" }) - .defineSchema({}) - .defineOperations({ - create: async () => ({}), - read: async () => { - throw new ResourceNotFoundError("resource is absent"); - }, - delete: async () => undefined, - }); - - await expect( - runOperation( - createResourceOperation(step, { - resource: new TestResource({ id: "eventually-visible" }), - state, - expectedRev: 0, - }), - ), - ).rejects.toThrowError("resource is absent"); - expect(state.update).not.toHaveBeenCalled(); - }); - - it("delete treats an already-absent remote as success through its idempotent resource contract", async () => { - const step = createStepRunnerDouble(); - const events: OperationLifecycleEvent[] = []; - const state = { - get: vi.fn(async () => undefined), - update: vi.fn(async () => undefined), - delete: vi.fn(async () => undefined), - }; - - const TestResource = resource({ type: "test/service/delete" }) - .defineSchema({}) - .defineOperations({ - create: async () => ({}), - delete: async () => undefined, - }); - - const testResource = new TestResource({ id: "test-delete" }); - - await runOperation( - deleteResourceOperation(step, { - resource: testResource, - state, - expectedRev: 1, - emit: async (event) => { - events.push(event); - }, - }), - ); - - expect(state.delete).toHaveBeenCalledWith("test-delete", 1); - expect(events.map((event) => event.status)).toEqual(["start", "success"]); - }); - - it("delete rethrows an unclassified resource error", async () => { - const step = createStepRunnerDouble(); - const state = { - get: vi.fn(async () => undefined), - update: vi.fn(async () => undefined), - delete: vi.fn(async () => undefined), - }; - - const TestResource = resource({ type: "test/service/delete-miss" }) - .defineSchema({}) - .defineOperations({ - create: async () => ({}), - delete: async () => { - const err = new Error("still exists"); - err.name = "DifferentError"; - throw err; - }, - }); - - const testResource = new TestResource({ id: "test-delete-miss" }); - - await expect( - runOperation( - deleteResourceOperation(step, { - resource: testResource, - state, - expectedRev: 1, - }), - ), - ).rejects.toMatchObject({ - name: "DifferentError", - message: "still exists", - }); - - expect(state.delete).not.toHaveBeenCalled(); - }); - - it("emits structured error details on operation failure", async () => { - const step = createStepRunnerDouble(); - const events: OperationLifecycleEvent[] = []; - const state = { - get: vi.fn(async () => undefined), - update: vi.fn(async () => undefined), - delete: vi.fn(async () => undefined), - }; - - const TestResource = resource({ type: "test/service/create-error" }) - .defineSchema({}) - .defineOperations({ - create: async () => { - const err = new Error("boom"); - err.name = "CreateFailed"; - throw err; - }, - delete: async () => undefined, - }); - - const testResource = new TestResource({ id: "test-create-error" }); - - await expect( - runOperation( - createResourceOperation(step, { - resource: testResource, - state, - expectedRev: 0, - emit: async (event) => { - events.push(event); - }, - }), - ), - ).rejects.toMatchObject({ name: "CreateFailed", message: "boom" }); - - expect(events.map((event) => event.status)).toEqual(["start", "error"]); - expect(events[1]).toMatchObject({ - operation: "create", - status: "error", - resourceId: "test-create-error", - resourceType: TestResource.type, - errorName: "CreateFailed", - errorMessage: "boom", - }); - }); -}); diff --git a/packages/reconciler/test/reconciler.deploy.test.ts b/packages/reconciler/test/reconciler.deploy.test.ts deleted file mode 100644 index eaa5ce5..0000000 --- a/packages/reconciler/test/reconciler.deploy.test.ts +++ /dev/null @@ -1,807 +0,0 @@ -import { describe, expect, it, vi } from "vitest"; -import { resource, ResourceNotFoundError } from "@notation/resource"; -import { - LeaseConflict, - MemoryStateBackend, - RevConflict, - type StateNode, -} from "@notation/state"; -import { Reconciler, createResourceRegistry } from "../src"; - -function createMemoryState(initial: Record = {}) { - const store: Record = { ...initial }; - - return { - store, - get: vi.fn(async (id: string) => store[id]), - update: vi.fn( - async (id: string, expectedRev: number, patch: Partial) => { - const actualRev = store[id]?.rev ?? 0; - if (actualRev !== expectedRev) { - throw new RevConflict(id, expectedRev, store[id]?.rev); - } - const rev = actualRev + 1; - store[id] = { - ...(store[id] ?? {}), - ...patch, - rev, - } as StateNode; - return { rev }; - }, - ), - delete: vi.fn(async (id: string, expectedRev: number) => { - const actualRev = store[id]?.rev ?? 0; - if (actualRev !== expectedRev) { - throw new RevConflict(id, expectedRev, store[id]?.rev); - } - delete store[id]; - }), - values: vi.fn(async () => Object.values(store)), - lease: vi.fn(async (scope: string, ttl: number) => { - let expiresAt = new Date(Date.now() + ttl).toISOString(); - return { - scope, - get expiresAt() { - return expiresAt; - }, - renew: vi.fn(async (nextTtl: number) => { - expiresAt = new Date(Date.now() + nextTtl).toISOString(); - return expiresAt; - }), - release: vi.fn(async () => undefined), - }; - }), - }; -} - -function createTestResourceClass(opts: { - type: `${string}/${string}/${string}`; - create?: ( - params: Record, - ) => Promise | void>; - read?: (key: Record) => Promise>; - update?: ( - key: Record, - patch: Record, - params: Record, - state: Record, - ) => Promise; - delete?: ( - key: Record, - state: Record, - ) => Promise; -}) { - return resource({ type: opts.type }) - .defineSchema({ - name: { - presence: "required", - propertyType: "param", - valueType: "string" as any, - }, - }) - .defineOperations({ - create: opts.create ?? (async () => ({})), - read: opts.read, - update: opts.update, - delete: opts.delete ?? (async () => undefined), - }); -} - -const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); -const found = (output: Record) => output; - -describe("reconciler deploy", () => { - it("chooses create vs update from desired params vs state", async () => { - const createSpy = vi.fn(async () => ({ name: "new" })); - const updateSpy = vi.fn(async () => undefined); - - const CreateResource = createTestResourceClass({ - type: "test/service/create-choice", - create: createSpy, - read: async () => found({ name: "new" }), - }); - const UpdateResource = createTestResourceClass({ - type: "test/service/update-choice", - update: updateSpy, - read: async () => found({ name: "new" }), - }); - - const state = createMemoryState({ - existing: { - rev: 1, - id: "existing", - groupId: -1, - groupType: "", - type: UpdateResource.type, - config: { name: "old" }, - params: { name: "old" }, - output: { name: "old" }, - lastOperation: "create", - lastOperationAt: new Date().toISOString(), - }, - }); - - const events: string[] = []; - const reconciler = new Reconciler({ - state, - driftDetection: false, - emit: async (event) => { - if ("operation" in event) { - events.push(`${event.operation}:${event.status}:${event.resourceId}`); - } - }, - }); - - await reconciler.deploy([ - new CreateResource({ id: "new", config: { name: "new" } }), - new UpdateResource({ id: "existing", config: { name: "new" } }), - ]); - - expect(createSpy).toHaveBeenCalledOnce(); - expect(updateSpy).toHaveBeenCalledOnce(); - expect(updateSpy.mock.calls[0]?.[1]).toEqual({ name: "new" }); - expect(events).toContain("create:success:new"); - expect(events).toContain("update:success:existing"); - }); - - it("persists first-time creates with an expect-absent revision", async () => { - const CreateResource = createTestResourceClass({ - type: "test/service/first-create", - create: async () => ({ name: "new" }), - read: async () => found({ name: "new" }), - }); - const state = createMemoryState(); - const reconciler = new Reconciler({ state, driftDetection: false }); - - await reconciler.deploy([ - new CreateResource({ id: "new", config: { name: "new" } }), - ]); - - expect(state.update).toHaveBeenCalledWith("new", 0, expect.any(Object)); - }); - - it("leases a resource before remote create so concurrent deploys cannot duplicate it", async () => { - let signalCreateStarted!: () => void; - const createStarted = new Promise((resolve) => { - signalCreateStarted = resolve; - }); - let allowCreateToFinish!: () => void; - const createCanFinish = new Promise((resolve) => { - allowCreateToFinish = resolve; - }); - const createSpy = vi.fn(async () => { - signalCreateStarted(); - await createCanFinish; - return { name: "new" }; - }); - const CreateResource = createTestResourceClass({ - type: "test/service/concurrent-create", - create: createSpy, - read: async () => found({ name: "new" }), - }); - const state = new MemoryStateBackend(); - const first = new Reconciler({ state, driftDetection: false }); - const second = new Reconciler({ state, driftDetection: false }); - - const firstDeploy = first.deploy([ - new CreateResource({ id: "new", config: { name: "new" } }), - ]); - await createStarted; - - await expect( - second.deploy([ - new CreateResource({ id: "new", config: { name: "new" } }), - ]), - ).rejects.toBeInstanceOf(LeaseConflict); - - allowCreateToFinish(); - await firstDeploy; - expect(createSpy).toHaveBeenCalledOnce(); - }); - - it("reads remote state after an update conflict instead of repeating the update", async () => { - let remoteName = "old"; - const readSpy = vi.fn(async () => found({ name: remoteName })); - const updateSpy = vi.fn(async (_key, _patch, params) => { - remoteName = params.name as string; - }); - const UpdateResource = createTestResourceClass({ - type: "test/service/update-conflict", - read: readSpy, - update: updateSpy, - }); - const state = createMemoryState({ - existing: { - rev: 1, - id: "existing", - groupId: -1, - groupType: "", - type: UpdateResource.type, - config: { name: "old" }, - params: { name: "old" }, - output: { name: "old" }, - lastOperation: "create", - lastOperationAt: new Date().toISOString(), - }, - }); - const updateState = state.update.getMockImplementation()!; - state.update - .mockImplementationOnce(async () => { - state.store.existing = { - ...state.store.existing!, - rev: 2, - config: { name: "concurrent" }, - params: { name: "concurrent" }, - output: { name: "concurrent" }, - }; - throw new RevConflict("existing", 1, 2); - }) - .mockImplementation(updateState); - - const reconciler = new Reconciler({ state, driftDetection: false }); - await reconciler.deploy([ - new UpdateResource({ id: "existing", config: { name: "new" } }), - ]); - - expect(updateSpy).toHaveBeenCalledOnce(); - expect(readSpy).toHaveBeenCalledTimes(2); - expect(state.update).toHaveBeenLastCalledWith( - "existing", - 2, - expect.objectContaining({ - params: { name: "new" }, - output: { name: "new" }, - lastOperation: "drift", - }), - ); - expect(state.store.existing).toMatchObject({ - rev: 3, - params: { name: "new" }, - output: { name: "new" }, - }); - }); - - it("reads remote state after a create conflict instead of creating twice", async () => { - let remoteName: string | undefined; - const createSpy = vi.fn(async (params) => { - remoteName = params.name as string; - return { name: remoteName }; - }); - const readSpy = vi.fn(async () => found({ name: remoteName! })); - const CreateResource = createTestResourceClass({ - type: "test/service/create-conflict", - create: createSpy, - read: readSpy, - }); - const state = createMemoryState(); - const updateState = state.update.getMockImplementation()!; - state.update - .mockImplementationOnce(async () => { - state.store.new = { - rev: 1, - id: "new", - groupId: -1, - groupType: "", - type: CreateResource.type, - config: { name: "concurrent" }, - params: { name: "concurrent" }, - output: { name: "concurrent" }, - lastOperation: "create", - lastOperationAt: new Date().toISOString(), - }; - throw new RevConflict("new", 0, 1); - }) - .mockImplementation(updateState); - - const reconciler = new Reconciler({ state, driftDetection: false }); - await reconciler.deploy([ - new CreateResource({ id: "new", config: { name: "new" } }), - ]); - - expect(createSpy).toHaveBeenCalledOnce(); - expect(readSpy).toHaveBeenCalledTimes(2); - expect(state.store.new).toMatchObject({ - rev: 2, - params: { name: "new" }, - output: { name: "new" }, - lastOperation: "drift", - }); - }); - - it("does not blindly retry a conflicted mutation without a read operation", async () => { - const updateSpy = vi.fn(async () => undefined); - const UpdateResource = createTestResourceClass({ - type: "test/service/unreadable-conflict", - update: updateSpy, - }); - const state = createMemoryState({ - existing: { - rev: 1, - id: "existing", - groupId: -1, - groupType: "", - type: UpdateResource.type, - config: { name: "old" }, - params: { name: "old" }, - output: { name: "old" }, - lastOperation: "create", - lastOperationAt: new Date().toISOString(), - }, - }); - state.update.mockImplementationOnce(async () => { - state.store.existing = { ...state.store.existing!, rev: 2 }; - throw new RevConflict("existing", 1, 2); - }); - - const reconciler = new Reconciler({ state, driftDetection: false }); - await expect( - reconciler.deploy([ - new UpdateResource({ id: "existing", config: { name: "new" } }), - ]), - ).rejects.toMatchObject({ - id: "existing", - expectedRev: 1, - actualRev: 2, - }); - - expect(updateSpy).toHaveBeenCalledOnce(); - expect(state.update).toHaveBeenCalledOnce(); - }); - - it("runs independent resources concurrently per dependency depth", async () => { - const marks: Record = {}; - - const AResource = createTestResourceClass({ - type: "test/service/a", - create: async () => { - marks.aStart = Date.now(); - await sleep(60); - marks.aEnd = Date.now(); - return { name: "a" }; - }, - read: async () => found({ name: "a" }), - }); - const CResource = createTestResourceClass({ - type: "test/service/c", - create: async () => { - marks.cStart = Date.now(); - await sleep(60); - marks.cEnd = Date.now(); - return { name: "c" }; - }, - read: async () => found({ name: "c" }), - }); - const BResource = createTestResourceClass({ - type: "test/service/b", - create: async () => { - marks.bStart = Date.now(); - return { name: "b" }; - }, - read: async () => found({ name: "b" }), - }); - - const state = createMemoryState(); - const resourceA = new AResource({ id: "a", config: { name: "a" } }); - const resourceB = new BResource({ - id: "b", - config: { name: "b" }, - dependencies: { a: resourceA }, - }); - const resourceC = new CResource({ id: "c", config: { name: "c" } }); - - const reconciler = new Reconciler({ state, driftDetection: false }); - await reconciler.deploy([resourceA, resourceB, resourceC]); - - expect(Math.abs(marks.aStart - marks.cStart)).toBeLessThan(40); - expect(marks.bStart).toBeGreaterThanOrEqual(marks.aEnd); - }); - - it("detects drift using live read output and converges with update", async () => { - const updateSpy = vi.fn(async () => undefined); - const events: Array> = []; - const TestResource = createTestResourceClass({ - type: "test/service/drift", - read: async () => found({ name: "drifted" }), - update: updateSpy, - }); - - const state = createMemoryState({ - resource: { - rev: 1, - id: "resource", - groupId: -1, - groupType: "", - type: TestResource.type, - config: { name: "desired" }, - params: { name: "desired" }, - output: { name: "desired" }, - lastOperation: "create", - lastOperationAt: new Date().toISOString(), - }, - }); - - const reconciler = new Reconciler({ - state, - driftDetection: true, - emit: async (event) => { - events.push(event as unknown as Record); - }, - }); - await reconciler.deploy([ - new TestResource({ id: "resource", config: { name: "desired" } }), - ]); - - expect(updateSpy).toHaveBeenCalledOnce(); - expect(updateSpy.mock.calls[0]?.[1]).toEqual({ name: "desired" }); - expect(events).toContainEqual({ - level: "info", - event: "reconciler.drift.detected", - resourceId: "resource", - resourceType: TestResource.type, - diff: { name: "desired" }, - }); - }); - - it("deletes orphaned state entries by reconstructing from registry", async () => { - const deleteSpy = vi.fn(async () => undefined); - const OrphanResource = createTestResourceClass({ - type: "test/service/orphan", - delete: deleteSpy, - }); - - const state = createMemoryState({ - orphan: { - rev: 1, - id: "orphan", - groupId: -1, - groupType: "", - type: OrphanResource.type, - config: { name: "from-state" }, - params: { name: "from-state" }, - output: { name: "from-state" }, - lastOperation: "create", - lastOperationAt: new Date().toISOString(), - }, - }); - - const reconciler = new Reconciler({ - state, - registry: createResourceRegistry([OrphanResource]), - driftDetection: false, - }); - - await reconciler.deploy([]); - - expect(deleteSpy).toHaveBeenCalledOnce(); - expect(state.delete).toHaveBeenCalledWith("orphan", 1); - }); - - it("dryRun emits operation intent without applying side effects", async () => { - const createSpy = vi.fn(async () => ({ name: "new" })); - const deleteSpy = vi.fn(async () => undefined); - - const CreateResource = createTestResourceClass({ - type: "test/service/dry-run-create", - create: createSpy, - read: async () => found({ name: "new" }), - }); - const OrphanResource = createTestResourceClass({ - type: "test/service/dry-run-orphan", - delete: deleteSpy, - }); - - const state = createMemoryState({ - orphan: { - rev: 1, - id: "orphan", - groupId: -1, - groupType: "", - type: OrphanResource.type, - config: { name: "orphan" }, - params: { name: "orphan" }, - output: { name: "orphan" }, - lastOperation: "create", - lastOperationAt: new Date().toISOString(), - }, - }); - - const operationEvents: string[] = []; - const reconciler = new Reconciler({ - state, - registry: createResourceRegistry([OrphanResource]), - dryRun: true, - driftDetection: false, - emit: async (event) => { - if ("operation" in event) { - operationEvents.push( - `${event.operation}:${event.status}:${event.resourceId}`, - ); - } - }, - }); - - await reconciler.deploy([ - new CreateResource({ id: "new", config: { name: "new" } }), - ]); - - expect(createSpy).not.toHaveBeenCalled(); - expect(deleteSpy).not.toHaveBeenCalled(); - expect(state.update).not.toHaveBeenCalled(); - expect(state.delete).not.toHaveBeenCalled(); - expect(operationEvents).toContain("create:dry-run:new"); - expect(operationEvents).toContain("delete:dry-run:orphan"); - }); -}); - -describe("reconciler destroy + refresh", () => { - it("reads remote state after a delete conflict instead of deleting twice", async () => { - let remoteExists = true; - const deleteSpy = vi.fn(async () => { - remoteExists = false; - }); - const readSpy = vi.fn(async () => { - if (!remoteExists) { - throw new ResourceNotFoundError("resource is absent"); - } - return found({ name: "doomed" }); - }); - const DestroyResource = createTestResourceClass({ - type: "test/service/destroy-retry", - read: readSpy, - delete: deleteSpy, - }); - const state = createMemoryState({ - doomed: { - rev: 1, - id: "doomed", - groupId: -1, - groupType: "", - type: DestroyResource.type, - config: { name: "doomed" }, - params: { name: "doomed" }, - output: { name: "doomed" }, - lastOperation: "create", - lastOperationAt: new Date().toISOString(), - }, - }); - const deleteState = state.delete.getMockImplementation()!; - state.delete - .mockImplementationOnce(async () => { - state.store.doomed = { ...state.store.doomed!, rev: 2 }; - throw new RevConflict("doomed", 1, 2); - }) - .mockImplementation(deleteState); - - const reconciler = new Reconciler({ state }); - await reconciler.destroy([ - new DestroyResource({ id: "doomed", config: { name: "doomed" } }), - ]); - - expect(deleteSpy).toHaveBeenCalledOnce(); - expect(readSpy).toHaveBeenCalledOnce(); - expect(state.delete).toHaveBeenCalledTimes(2); - expect(state.store.doomed).toBeUndefined(); - }); - - it("holds a backend lease for the orphan snapshot", async () => { - const state = createMemoryState(); - const release = vi.fn(async () => undefined); - const lease = vi.fn(async () => ({ - scope: "reconciler:orphan-deletion", - expiresAt: new Date(Date.now() + 10_000).toISOString(), - renew: vi.fn(async () => new Date(Date.now() + 10_000).toISOString()), - release, - })); - const reconciler = new Reconciler({ - state: { ...state, lease }, - mutationLeaseTtl: 10_000, - }); - - await reconciler.refresh([]); - - expect(lease).toHaveBeenCalledWith("reconciler:orphan-deletion", 10_000); - expect(state.values).toHaveBeenCalledOnce(); - expect(release).toHaveBeenCalledOnce(); - }); - - it("destroys resources in reverse dependency order", async () => { - const destroyOrder: string[] = []; - const deleteA = vi.fn(async () => { - destroyOrder.push("a"); - }); - const deleteB = vi.fn(async () => { - destroyOrder.push("b"); - }); - const deleteC = vi.fn(async () => { - destroyOrder.push("c"); - }); - - const AResource = createTestResourceClass({ - type: "test/service/destroy-a", - delete: deleteA, - }); - const BResource = createTestResourceClass({ - type: "test/service/destroy-b", - delete: deleteB, - }); - const CResource = createTestResourceClass({ - type: "test/service/destroy-c", - delete: deleteC, - }); - - const resourceA = new AResource({ id: "a", config: { name: "a" } }); - const resourceB = new BResource({ - id: "b", - config: { name: "b" }, - dependencies: { a: resourceA }, - }); - const resourceC = new CResource({ - id: "c", - config: { name: "c" }, - dependencies: { b: resourceB }, - }); - - const state = createMemoryState({ - a: { - rev: 1, - id: "a", - groupId: -1, - groupType: "", - type: AResource.type, - config: { name: "a" }, - params: { name: "a" }, - output: { name: "a" }, - lastOperation: "create", - lastOperationAt: new Date().toISOString(), - }, - b: { - rev: 1, - id: "b", - groupId: -1, - groupType: "", - type: BResource.type, - config: { name: "b" }, - params: { name: "b" }, - output: { name: "b" }, - lastOperation: "create", - lastOperationAt: new Date().toISOString(), - }, - c: { - rev: 1, - id: "c", - groupId: -1, - groupType: "", - type: CResource.type, - config: { name: "c" }, - params: { name: "c" }, - output: { name: "c" }, - lastOperation: "create", - lastOperationAt: new Date().toISOString(), - }, - }); - - const reconciler = new Reconciler({ state }); - await reconciler.destroy([resourceA, resourceB, resourceC]); - - expect(destroyOrder).toEqual(["c", "b", "a"]); - expect(state.delete).toHaveBeenCalledWith("a", 1); - expect(state.delete).toHaveBeenCalledWith("b", 1); - expect(state.delete).toHaveBeenCalledWith("c", 1); - }); - - it("refresh removes orphan state entries", async () => { - const deleteSpy = vi.fn(async () => undefined); - const OrphanResource = createTestResourceClass({ - type: "test/service/refresh-orphan", - delete: deleteSpy, - }); - const KeepResource = createTestResourceClass({ - type: "test/service/refresh-keep", - }); - - const keep = new KeepResource({ id: "keep", config: { name: "keep" } }); - const state = createMemoryState({ - keep: { - rev: 1, - id: "keep", - groupId: -1, - groupType: "", - type: KeepResource.type, - config: { name: "keep" }, - params: { name: "keep" }, - output: { name: "keep" }, - lastOperation: "create", - lastOperationAt: new Date().toISOString(), - }, - orphan: { - rev: 1, - id: "orphan", - groupId: -1, - groupType: "", - type: OrphanResource.type, - config: { name: "orphan" }, - params: { name: "orphan" }, - output: { name: "orphan" }, - lastOperation: "create", - lastOperationAt: new Date().toISOString(), - }, - }); - - const reconciler = new Reconciler({ - state, - registry: createResourceRegistry([OrphanResource]), - }); - - await reconciler.refresh([keep]); - - expect(deleteSpy).toHaveBeenCalledOnce(); - expect(state.delete).toHaveBeenCalledWith("orphan", 1); - expect(state.delete).not.toHaveBeenCalledWith("keep", expect.anything()); - }); - - it("destroy and refresh dryRun emit operation events without side effects", async () => { - const deleteSpy = vi.fn(async () => undefined); - const DestroyResource = createTestResourceClass({ - type: "test/service/dry-run-destroy", - delete: deleteSpy, - }); - const OrphanResource = createTestResourceClass({ - type: "test/service/dry-run-refresh", - delete: deleteSpy, - }); - - const destroyResource = new DestroyResource({ - id: "destroy-me", - config: { name: "destroy-me" }, - }); - - const state = createMemoryState({ - "destroy-me": { - rev: 1, - id: "destroy-me", - groupId: -1, - groupType: "", - type: DestroyResource.type, - config: { name: "destroy-me" }, - params: { name: "destroy-me" }, - output: { name: "destroy-me" }, - lastOperation: "create", - lastOperationAt: new Date().toISOString(), - }, - orphan: { - rev: 1, - id: "orphan", - groupId: -1, - groupType: "", - type: OrphanResource.type, - config: { name: "orphan" }, - params: { name: "orphan" }, - output: { name: "orphan" }, - lastOperation: "create", - lastOperationAt: new Date().toISOString(), - }, - }); - - const operationEvents: string[] = []; - const reconciler = new Reconciler({ - state, - dryRun: true, - registry: createResourceRegistry([OrphanResource]), - emit: async (event) => { - if ("operation" in event) { - operationEvents.push( - `${event.operation}:${event.status}:${event.resourceId}`, - ); - } - }, - }); - - await reconciler.destroy([destroyResource]); - await reconciler.refresh([destroyResource]); - - expect(deleteSpy).not.toHaveBeenCalled(); - expect(state.delete).not.toHaveBeenCalled(); - expect(operationEvents).toContain("delete:dry-run:destroy-me"); - expect(operationEvents).toContain("delete:dry-run:orphan"); - }); -}); diff --git a/packages/reconciler/test/reconciler.plan.test.ts b/packages/reconciler/test/reconciler.plan.test.ts deleted file mode 100644 index d98e55a..0000000 --- a/packages/reconciler/test/reconciler.plan.test.ts +++ /dev/null @@ -1,463 +0,0 @@ -import { describe, expect, it, vi } from "vitest"; -import { - ResourceNotFoundError, - ResourceOperationPendingError, - resource, - type BaseResource, -} from "@notation/resource"; -import type { StateNode } from "@notation/state"; -import { Reconciler, UNKNOWN_AFTER_APPLY } from "../src"; - -function createMemoryState(initial: Record = {}) { - const store: Record = { ...initial }; - - return { - store, - get: vi.fn(async (id: string) => store[id]), - update: vi.fn( - async (id: string, expectedRev: number, patch: Partial) => { - store[id] = { - ...(store[id] ?? {}), - ...patch, - } as StateNode; - }, - ), - delete: vi.fn(async (id: string) => { - delete store[id]; - }), - values: vi.fn(async () => Object.values(store)), - lease: vi.fn(async (scope: string, ttl: number) => ({ - scope, - expiresAt: new Date(Date.now() + ttl).toISOString(), - renew: vi.fn(async (nextTtl: number) => - new Date(Date.now() + nextTtl).toISOString(), - ), - release: vi.fn(async () => undefined), - })), - }; -} - -function createTestResourceClass(opts: { - type: `${string}/${string}/${string}`; - create?: ( - params: Record, - ) => Promise | void>; - read?: (key: Record) => Promise>; - update?: ( - key: Record, - patch: Record, - params: Record, - state: Record, - ) => Promise; - delete?: ( - key: Record, - state: Record, - ) => Promise; -}) { - return resource({ type: opts.type }) - .defineSchema({ - name: { - presence: "required", - propertyType: "param", - valueType: "string" as any, - }, - tag: { - presence: "optional", - propertyType: "param", - valueType: "string" as any, - }, - }) - .defineOperations({ - create: opts.create ?? (async () => ({})), - read: opts.read, - update: opts.update, - delete: opts.delete ?? (async () => undefined), - }); -} - -function createStateNode( - id: string, - type: string, - params: Record, - output: Record = params, -): StateNode { - return { - id, - groupId: -1, - groupType: "", - type, - config: params, - params, - output, - lastOperation: "create", - lastOperationAt: new Date().toISOString(), - }; -} - -describe("reconciler plan", () => { - it("plans create for resources without state", async () => { - const TestResource = createTestResourceClass({ - type: "test/service/plan-create", - }); - - const state = createMemoryState(); - const reconciler = new Reconciler({ state, driftDetection: false }); - - const plan = await reconciler.plan([ - new TestResource({ id: "new", config: { name: "new" } }), - ]); - - expect(plan.nodes).toEqual([ - { - id: "new", - type: TestResource.type, - decision: "create", - params: { name: "new" }, - dependsOn: [], - }, - ]); - }); - - it("plans update with the detailed diff that justified it", async () => { - const TestResource = createTestResourceClass({ - type: "test/service/plan-update", - }); - - const state = createMemoryState({ - existing: createStateNode("existing", "test/service/plan-update", { - name: "old", - tag: "keep", - }), - }); - const reconciler = new Reconciler({ state, driftDetection: false }); - - const plan = await reconciler.plan([ - new TestResource({ id: "existing", config: { name: "new" } }), - ]); - - expect(plan.nodes).toEqual([ - { - id: "existing", - type: TestResource.type, - decision: "update", - diff: { - added: {}, - deleted: { tag: null }, - updated: { name: "new" }, - }, - params: { name: "new" }, - dependsOn: [], - }, - ]); - }); - - it("plans noop when params match state", async () => { - const TestResource = createTestResourceClass({ - type: "test/service/plan-noop", - }); - - const state = createMemoryState({ - unchanged: createStateNode("unchanged", "test/service/plan-noop", { - name: "same", - }), - }); - const reconciler = new Reconciler({ state, driftDetection: false }); - - const plan = await reconciler.plan([ - new TestResource({ id: "unchanged", config: { name: "same" } }), - ]); - - expect(plan.nodes[0]).toMatchObject({ id: "unchanged", decision: "noop" }); - }); - - it("plans drift-update from live read output when drift detection is on", async () => { - const readSpy = vi.fn(async () => ({ name: "drifted" })); - const TestResource = createTestResourceClass({ - type: "test/service/plan-drift-update", - read: readSpy, - }); - - const state = createMemoryState({ - resource: createStateNode("resource", "test/service/plan-drift-update", { - name: "desired", - }), - }); - const reconciler = new Reconciler({ state, driftDetection: true }); - - const plan = await reconciler.plan([ - new TestResource({ id: "resource", config: { name: "desired" } }), - ]); - - expect(readSpy).toHaveBeenCalledOnce(); - expect(plan.nodes[0]).toEqual({ - id: "resource", - type: TestResource.type, - decision: "drift-update", - diff: { - added: {}, - deleted: {}, - updated: { name: "desired" }, - }, - params: { name: "desired" }, - dependsOn: [], - }); - }); - - it("plans drift-recreate when the remote resource is gone", async () => { - const TestResource = createTestResourceClass({ - type: "test/service/plan-drift-recreate", - read: async () => { - throw new ResourceNotFoundError("resource is absent"); - }, - }); - - const state = createMemoryState({ - resource: createStateNode( - "resource", - "test/service/plan-drift-recreate", - { name: "desired" }, - ), - }); - const reconciler = new Reconciler({ state, driftDetection: true }); - - const plan = await reconciler.plan([ - new TestResource({ id: "resource", config: { name: "desired" } }), - ]); - - expect(plan.nodes[0]).toMatchObject({ - id: "resource", - decision: "drift-recreate", - }); - }); - - it("waits for a pending read before planning", async () => { - let attempts = 0; - const TestResource = createTestResourceClass({ - type: "test/service/plan-not-ready", - read: async () => { - attempts += 1; - if (attempts === 1) { - throw new ResourceOperationPendingError( - "Waiting for Lambda to become active", - { retryAfterMs: 0 }, - ); - } - return { name: "desired" }; - }, - }); - - const state = createMemoryState({ - resource: createStateNode("resource", "test/service/plan-not-ready", { - name: "desired", - }), - }); - const reconciler = new Reconciler({ state, driftDetection: true }); - - const plan = await reconciler.plan([ - new TestResource({ id: "resource", config: { name: "desired" } }), - ]); - - expect(plan.nodes[0]).toMatchObject({ - id: "resource", - decision: "noop", - }); - expect(attempts).toBe(2); - }); - - it("skips remote reads when drift detection is off", async () => { - const readSpy = vi.fn(async () => ({ name: "drifted" })); - const TestResource = createTestResourceClass({ - type: "test/service/plan-no-read", - read: readSpy, - }); - - const state = createMemoryState({ - resource: createStateNode("resource", "test/service/plan-no-read", { - name: "desired", - }), - }); - const reconciler = new Reconciler({ state, driftDetection: false }); - - await reconciler.plan([ - new TestResource({ id: "resource", config: { name: "desired" } }), - ]); - - expect(readSpy).not.toHaveBeenCalled(); - }); - - it("plans delete-orphan for state nodes without a matching resource", async () => { - const state = createMemoryState({ - orphan: createStateNode("orphan", "test/service/plan-orphan", { - name: "orphan", - }), - }); - const reconciler = new Reconciler({ state, driftDetection: false }); - - const plan = await reconciler.plan([]); - - expect(plan.nodes).toEqual([ - { - id: "orphan", - type: "test/service/plan-orphan", - decision: "delete-orphan", - params: { name: "orphan" }, - dependsOn: [], - }, - ]); - }); - - it("populates dependsOn from resource dependencies", async () => { - const AResource = createTestResourceClass({ - type: "test/service/plan-dep-a", - }); - const BResource = createTestResourceClass({ - type: "test/service/plan-dep-b", - }); - - const resourceA = new AResource({ id: "a", config: { name: "a" } }); - const resourceB = new BResource({ - id: "b", - config: { name: "b" }, - dependencies: { a: resourceA }, - }); - - const state = createMemoryState(); - const reconciler = new Reconciler({ state, driftDetection: false }); - - const plan = await reconciler.plan([resourceA, resourceB]); - - const nodeB = plan.nodes.find((node) => node.id === "b"); - expect(nodeB?.dependsOn).toEqual(["a"]); - }); - - it("marks params derived from uncreated dependencies as unknown after apply", async () => { - const AResource = createTestResourceClass({ - type: "test/service/plan-unknown-a", - }); - const BResource = createTestResourceClass({ - type: "test/service/plan-unknown-b", - }) - .requireDependencies<{ a: BaseResource }>() - .deriveParams(({ deps }) => ({ - name: (deps.a.output as { name: string }).name, - })); - - const resourceA = new AResource({ id: "a", config: { name: "a" } }); - const resourceB = new BResource({ - id: "b", - config: { tag: "known" }, - dependencies: { a: resourceA }, - }); - - const state = createMemoryState(); - const reconciler = new Reconciler({ state, driftDetection: false }); - - const plan = await reconciler.plan([resourceA, resourceB]); - - const nodeB = plan.nodes.find((node) => node.id === "b"); - expect(nodeB).toMatchObject({ - decision: "create", - params: { - name: UNKNOWN_AFTER_APPLY, - tag: "known", - }, - }); - }); - - it("does not disguise parameter derivation failures as unknown values", async () => { - const TestResource = resource({ - type: "test/service/plan-derive-failure", - }) - .defineSchema({ - name: { - presence: "required", - propertyType: "param", - valueType: "string" as any, - }, - }) - .defineOperations({ - create: async () => ({}), - delete: async () => undefined, - deriveParams: () => { - throw new Error("invalid derived configuration"); - }, - }); - - const reconciler = new Reconciler({ - state: createMemoryState(), - driftDetection: false, - }); - - await expect( - reconciler.plan([new TestResource({ id: "broken" })]), - ).rejects.toThrow("invalid derived configuration"); - }); - - it("produces a JSON-round-trippable plan", async () => { - const CreateResource = createTestResourceClass({ - type: "test/service/plan-json-create", - }); - const UpdateResource = createTestResourceClass({ - type: "test/service/plan-json-update", - }); - - const state = createMemoryState({ - existing: createStateNode("existing", "test/service/plan-json-update", { - name: "old", - tag: "gone", - }), - orphan: createStateNode("orphan", "test/service/plan-json-orphan", { - name: "orphan", - }), - }); - const reconciler = new Reconciler({ state, driftDetection: false }); - - const plan = await reconciler.plan([ - new CreateResource({ id: "new", config: { name: "new" } }), - new UpdateResource({ id: "existing", config: { name: "new" } }), - ]); - - expect(JSON.parse(JSON.stringify(plan))).toStrictEqual(plan); - }); - - it("performs no state writes or resource operations", async () => { - const createSpy = vi.fn(async () => ({ name: "new" })); - const updateSpy = vi.fn(async () => undefined); - const deleteSpy = vi.fn(async () => undefined); - - const CreateResource = createTestResourceClass({ - type: "test/service/plan-pure-create", - create: createSpy, - update: updateSpy, - delete: deleteSpy, - }); - const UpdateResource = createTestResourceClass({ - type: "test/service/plan-pure-update", - create: createSpy, - update: updateSpy, - delete: deleteSpy, - read: async () => ({ name: "drifted" }), - }); - - const state = createMemoryState({ - existing: createStateNode("existing", "test/service/plan-pure-update", { - name: "same", - }), - orphan: createStateNode("orphan", "test/service/plan-pure-orphan", { - name: "orphan", - }), - }); - const reconciler = new Reconciler({ state, driftDetection: true }); - - await reconciler.plan([ - new CreateResource({ id: "new", config: { name: "new" } }), - new UpdateResource({ id: "existing", config: { name: "same" } }), - ]); - - expect(createSpy).not.toHaveBeenCalled(); - expect(updateSpy).not.toHaveBeenCalled(); - expect(deleteSpy).not.toHaveBeenCalled(); - expect(state.update).not.toHaveBeenCalled(); - expect(state.delete).not.toHaveBeenCalled(); - }); -}); diff --git a/packages/state-sqlite/src/index.ts b/packages/state-sqlite/src/index.ts index 057628f..c01775b 100644 --- a/packages/state-sqlite/src/index.ts +++ b/packages/state-sqlite/src/index.ts @@ -1,11 +1,8 @@ -import { randomUUID } from "node:crypto"; import { mkdirSync } from "node:fs"; import { dirname } from "node:path"; import { DatabaseSync } from "node:sqlite"; import { - LeaseConflict, RevConflict, - type Lease, type StateBackend, type StateNode, } from "@notation/state"; @@ -24,13 +21,6 @@ export class SqliteStateBackend implements StateBackend { value TEXT NOT NULL ) `); - this.#database.exec(` - CREATE TABLE IF NOT EXISTS resource_leases ( - scope TEXT PRIMARY KEY, - owner TEXT NOT NULL, - expires_at INTEGER NOT NULL - ) - `); } close(): void { @@ -46,9 +36,7 @@ export class SqliteStateBackend implements StateBackend { async has(id: string): Promise { return Boolean( - this.#database - .prepare("SELECT 1 FROM resources WHERE id = ?") - .get(id), + this.#database.prepare("SELECT 1 FROM resources WHERE id = ?").get(id), ); } @@ -79,9 +67,7 @@ export class SqliteStateBackend implements StateBackend { } } else { this.#database - .prepare( - "INSERT INTO resources (id, rev, value) VALUES (?, ?, ?)", - ) + .prepare("INSERT INTO resources (id, rev, value) VALUES (?, ?, ?)") .run(id, rev, JSON.stringify(node)); } this.#database.exec("COMMIT"); @@ -114,82 +100,4 @@ export class SqliteStateBackend implements StateBackend { .all() as { value: string }[]; return rows.map(({ value }) => JSON.parse(value) as StateNode); } - - async lease(scope: string, ttl: number): Promise { - if (!Number.isFinite(ttl) || ttl <= 0) { - throw new RangeError( - "Lease TTL must be a positive number of milliseconds", - ); - } - - const owner = randomUUID(); - const expiresAtMs = Date.now() + ttl; - this.#database.exec("BEGIN IMMEDIATE"); - try { - this.#database - .prepare( - "DELETE FROM resource_leases WHERE scope = ? AND expires_at <= ?", - ) - .run(scope, Date.now()); - const current = this.#database - .prepare("SELECT expires_at FROM resource_leases WHERE scope = ?") - .get(scope) as { expires_at: number } | undefined; - if (current) { - throw new LeaseConflict( - scope, - new Date(current.expires_at).toISOString(), - ); - } - this.#database - .prepare( - "INSERT INTO resource_leases (scope, owner, expires_at) VALUES (?, ?, ?)", - ) - .run(scope, owner, expiresAtMs); - this.#database.exec("COMMIT"); - } catch (error) { - this.#database.exec("ROLLBACK"); - throw error; - } - - let released = false; - let currentExpiresAtMs = expiresAtMs; - return { - scope, - get expiresAt() { - return new Date(currentExpiresAtMs).toISOString(); - }, - renew: async (nextTtl) => { - if (!Number.isFinite(nextTtl) || nextTtl <= 0) { - throw new RangeError( - "Lease TTL must be a positive number of milliseconds", - ); - } - const now = Date.now(); - const nextExpiresAtMs = now + nextTtl; - const result = this.#database - .prepare( - "UPDATE resource_leases SET expires_at = ? WHERE scope = ? AND owner = ? AND expires_at > ?", - ) - .run(nextExpiresAtMs, scope, owner, now); - if (result.changes !== 1) { - const current = this.#database - .prepare("SELECT expires_at FROM resource_leases WHERE scope = ?") - .get(scope) as { expires_at: number } | undefined; - throw new LeaseConflict( - scope, - new Date(current?.expires_at ?? 0).toISOString(), - ); - } - currentExpiresAtMs = nextExpiresAtMs; - return new Date(nextExpiresAtMs).toISOString(); - }, - release: async () => { - if (released) return; - this.#database - .prepare("DELETE FROM resource_leases WHERE scope = ? AND owner = ?") - .run(scope, owner); - released = true; - }, - }; - } } diff --git a/packages/state-sqlite/test/state-sqlite.test.ts b/packages/state-sqlite/test/state-sqlite.test.ts index 31eb35d..f50456e 100644 --- a/packages/state-sqlite/test/state-sqlite.test.ts +++ b/packages/state-sqlite/test/state-sqlite.test.ts @@ -46,33 +46,6 @@ describe("SqliteStateBackend", () => { }); }); - it("coordinates leases across backend instances and releases by owner", async () => { - const directory = await mkdtemp( - path.join(tmpdir(), "notation-sqlite-lease-"), - ); - const databasePath = path.join(directory, "state.db"); - const first = new SqliteStateBackend(databasePath); - const second = new SqliteStateBackend(databasePath); - cleanups.push(async () => { - first.close(); - second.close(); - await rm(directory, { recursive: true, force: true }); - }); - - const lease = await first.lease("orphans", 10_000); - await expect(second.lease("orphans", 10_000)).rejects.toMatchObject({ - name: "LeaseConflict", - scope: "orphans", - }); - const firstExpiry = lease.expiresAt; - await lease.renew(20_000); - expect(lease.expiresAt).not.toBe(firstExpiry); - await lease.release(); - const nextLease = await second.lease("orphans", 10_000); - expect(nextLease).toMatchObject({ scope: "orphans" }); - await nextLease.release(); - }); - it("waits for a concurrent writer instead of raising database locked", async () => { const directory = await mkdtemp( path.join(tmpdir(), "notation-sqlite-busy-"), diff --git a/packages/state/src/conflicts.ts b/packages/state/src/conflicts.ts index 81559dc..a1e666c 100644 --- a/packages/state/src/conflicts.ts +++ b/packages/state/src/conflicts.ts @@ -11,14 +11,3 @@ export class RevConflict extends Error { ); } } - -export class LeaseConflict extends Error { - readonly name = "LeaseConflict"; - - constructor( - readonly scope: string, - readonly expiresAt: string, - ) { - super(`State lease conflict for ${scope}: held until ${expiresAt}`); - } -} diff --git a/packages/state/src/state.ts b/packages/state/src/state.ts index d0c10c1..971f802 100644 --- a/packages/state/src/state.ts +++ b/packages/state/src/state.ts @@ -10,7 +10,7 @@ import { import path from "node:path"; import { setTimeout as sleep } from "node:timers/promises"; import { isErrorWithCode } from "@notation/utils"; -import { LeaseConflict, RevConflict } from "./conflicts"; +import { RevConflict } from "./conflicts"; export type StateNode = { rev: number; @@ -38,21 +38,12 @@ export interface StateBackend { ): Promise<{ rev: number }>; delete(id: string, expectedRev: number): Promise; values(): Promise; - lease(scope: string, ttl: number): Promise; -} - -export interface Lease { - readonly scope: string; - readonly expiresAt: string; - renew(ttl: number): Promise; - release(): Promise; } export type State = StateBackend; export class MemoryStateBackend implements StateBackend { #state: Record; - #leases = new Map(); constructor(initialState: Record = {}) { this.#state = cloneAsPersistedState(initialState); @@ -109,46 +100,6 @@ export class MemoryStateBackend implements StateBackend { .map(([, value]) => value); } - async lease(scope: string, ttl: number): Promise { - assertLeaseTtl(ttl); - const now = Date.now(); - const current = this.#leases.get(scope); - if (current && current.expiresAtMs > now) { - throw new LeaseConflict( - scope, - new Date(current.expiresAtMs).toISOString(), - ); - } - - const owner = randomUUID(); - let expiresAtMs = now + ttl; - this.#leases.set(scope, { owner, expiresAtMs }); - - return { - scope, - get expiresAt() { - return new Date(expiresAtMs).toISOString(); - }, - renew: async (nextTtl) => { - assertLeaseTtl(nextTtl); - const held = this.#leases.get(scope); - if (!held || held.owner !== owner || held.expiresAtMs <= Date.now()) { - throw new LeaseConflict( - scope, - new Date(held?.expiresAtMs ?? 0).toISOString(), - ); - } - expiresAtMs = Date.now() + nextTtl; - held.expiresAtMs = expiresAtMs; - return new Date(expiresAtMs).toISOString(); - }, - release: async () => { - if (this.#leases.get(scope)?.owner === owner) - this.#leases.delete(scope); - }, - }; - } - private async readState(): Promise> { return cloneAsPersistedState(this.#state); } @@ -208,65 +159,6 @@ export class FileStateBackend implements StateBackend { return Object.values(state); } - async lease(scope: string, ttl: number): Promise { - assertLeaseTtl(ttl); - const leaseFilePath = `${this.stateFilePath}.${encodeURIComponent(scope)}.lease`; - const owner = randomUUID(); - let expiresAtMs: number; - await mkdir(path.dirname(this.stateFilePath), { recursive: true }); - - for (;;) { - expiresAtMs = Date.now() + ttl; - try { - await writeFile(leaseFilePath, JSON.stringify({ owner, expiresAtMs }), { - flag: "wx", - }); - break; - } catch (error) { - if (!isErrorWithCode(error, "EEXIST")) throw error; - const current = await readFileLease(leaseFilePath); - if (!current || current.expiresAtMs <= Date.now()) { - await unlink(leaseFilePath).catch(() => undefined); - continue; - } - throw new LeaseConflict( - scope, - new Date(current.expiresAtMs).toISOString(), - ); - } - } - - return { - scope, - get expiresAt() { - return new Date(expiresAtMs).toISOString(); - }, - renew: async (nextTtl) => { - assertLeaseTtl(nextTtl); - const current = await readFileLease(leaseFilePath); - if ( - !current || - current.owner !== owner || - current.expiresAtMs <= Date.now() - ) { - throw new LeaseConflict( - scope, - new Date(current?.expiresAtMs ?? 0).toISOString(), - ); - } - expiresAtMs = Date.now() + nextTtl; - await writeFile(leaseFilePath, JSON.stringify({ owner, expiresAtMs })); - return new Date(expiresAtMs).toISOString(); - }, - release: async () => { - const current = await readFileLease(leaseFilePath); - if (current?.owner === owner) { - await unlink(leaseFilePath).catch(() => undefined); - } - }, - }; - } - private async readState(): Promise> { try { const file = await readFile(this.stateFilePath, "utf8"); @@ -354,26 +246,6 @@ function assertExpectedRev( } } -function assertLeaseTtl(ttl: number): void { - if (!Number.isFinite(ttl) || ttl <= 0) { - throw new RangeError("Lease TTL must be a positive number of milliseconds"); - } -} - -type FileLeaseRecord = { owner: string; expiresAtMs: number }; - -async function readFileLease( - filePath: string, -): Promise { - try { - return JSON.parse(await readFile(filePath, "utf8")) as FileLeaseRecord; - } catch (error) { - if (isErrorWithCode(error, "ENOENT") || error instanceof SyntaxError) - return undefined; - throw error; - } -} - function cloneAsPersistedState( state: Record, ): Record { diff --git a/packages/state/test/state-backend.test.ts b/packages/state/test/state-backend.test.ts index fde5e5b..888055a 100644 --- a/packages/state/test/state-backend.test.ts +++ b/packages/state/test/state-backend.test.ts @@ -157,27 +157,6 @@ function runStateBackendContractTests( await fixture.cleanup(); } }); - - it("holds and renews an exclusive lease", async () => { - const fixture = await createBackend(); - - try { - const lease = await fixture.backend.lease("resource:a", 1_000); - const firstExpiry = lease.expiresAt; - await expect( - fixture.backend.lease("resource:a", 1_000), - ).rejects.toMatchObject({ name: "LeaseConflict" }); - - await lease.renew(2_000); - expect(lease.expiresAt).not.toBe(firstExpiry); - await lease.release(); - - const next = await fixture.backend.lease("resource:a", 1_000); - await next.release(); - } finally { - await fixture.cleanup(); - } - }); }); } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index fd80fe6..02ef3b9 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -123,12 +123,18 @@ importers: '@notation/resource': specifier: workspace:* version: link:../../packages/resource - '@notation/state-sqlite': - specifier: workspace:* - version: link:../../packages/state-sqlite - '@notation/utils': - specifier: workspace:* - version: link:../../packages/utils + '@yieldstar/core': + specifier: 0.5.0 + version: 0.5.0 + '@yieldstar/sqlite-runtime': + specifier: 0.5.0 + version: 0.5.0 + pino: + specifier: ^9.9.0 + version: 9.14.0 + yieldstar: + specifier: 0.5.0 + version: 0.5.0 devDependencies: '@types/node': specifier: ^22.13.4 @@ -224,12 +230,12 @@ importers: '@notation/resource': specifier: workspace:* version: link:../resource - '@notation/state': - specifier: workspace:* - version: link:../state - '@notation/state-sqlite': - specifier: workspace:* - version: link:../state-sqlite + '@yieldstar/core': + specifier: 0.5.0 + version: 0.5.0 + '@yieldstar/sqlite-runtime': + specifier: 0.5.0 + version: 0.5.0 deep-object-diff: specifier: ^1.1.9 version: 1.1.9 @@ -242,6 +248,12 @@ importers: pako: specifier: ^2.1.0 version: 2.1.0 + pino: + specifier: ^9.14.0 + version: 9.14.0 + yieldstar: + specifier: 0.5.0 + version: 0.5.0 devDependencies: '@types/common-tags': specifier: ^1.8.4 @@ -2242,16 +2254,9 @@ packages: pino-abstract-transport@2.0.0: resolution: {integrity: sha512-F63x5tizV6WCh4R6RHyi2Ml+M70DNRXt/+HANowMflpgGFMAym/VKm6G7ZOQRjqN7XbGxK1Lg9t6ZrtzOaivMw==} - pino-abstract-transport@3.0.0: - resolution: {integrity: sha512-wlfUczU+n7Hy/Ha5j9a/gZNy7We5+cXp8YL+X+PG8S0KXxw7n/JXA3c46Y0zQznIJ83URJiwy7Lh56WLokNuxg==} - pino-std-serializers@7.1.0: resolution: {integrity: sha512-BndPH67/JxGExRgiX1dX0w1FvZck5Wa4aal9198SrRhZjH3GxKQUKIBnYJTdj2HDN3UQAS06HlfcSbQj2OHmaw==} - pino@10.3.1: - resolution: {integrity: sha512-r34yH/GlQpKZbU1BvFFqOjhISRo1MNx1tWYsYvmj6KIRHSPMT2+yHOEb1SG6NMvRoHRF0a07kCOox/9yakl1vg==} - hasBin: true - pino@9.14.0: resolution: {integrity: sha512-8OEwKp5juEvb/MjpIc4hjqfgCNysrS94RIOMXYvpYCdm/jglrKEiAYmiumbmGhCvs+IcInsphYDFwqrjr7398w==} hasBin: true @@ -2340,9 +2345,6 @@ packages: resolution: {integrity: sha512-57frrGM/OCTLqLOAh0mhVA9VBMHd+9U7Zb2THMGdBUoZVOtGbJzjxsYGDJ3A9AYYCP4hn6y1TVbaOfzWtm5GFg==} engines: {node: '>= 12.13.0'} - real-require@1.0.0: - resolution: {integrity: sha512-P4nbQYQfePJxRSmY+v/KINxVucm4NF3p3s7pJveMTtom52FR4YGltUQLB8idDXwDDWW+eYrWDFbuzUnjoWHF7g==} - require-from-string@2.0.2: resolution: {integrity: sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==} engines: {node: '>=0.10.0'} @@ -2504,10 +2506,6 @@ packages: thread-stream@3.1.0: resolution: {integrity: sha512-OqyPZ9u96VohAyMfJykzmivOrY2wfMSf3C5TtFJVgN+Hm6aj+voFhlK+kZEIv2FBh1X6Xp3DlnCOfEQ3B2J86A==} - thread-stream@4.2.0: - resolution: {integrity: sha512-e2zZ96wSChazBsbENf/Pcm/4swHt2cEKQ92rhUjkL9GCKiTDJIaTBenjE/m9DXi0QBmTMDkFDdOomUy20A1tDQ==} - engines: {node: '>=20'} - tinybench@2.9.0: resolution: {integrity: sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==} @@ -4139,7 +4137,7 @@ snapshots: fast-json-stringify: 7.0.0 find-my-way: 9.6.0 light-my-request: 6.6.0 - pino: 10.3.1 + pino: 9.14.0 process-warning: 5.0.0 rfdc: 1.4.1 secure-json-parse: 4.1.0 @@ -4488,26 +4486,8 @@ snapshots: dependencies: split2: 4.2.0 - pino-abstract-transport@3.0.0: - dependencies: - split2: 4.2.0 - pino-std-serializers@7.1.0: {} - pino@10.3.1: - dependencies: - '@pinojs/redact': 0.4.0 - atomic-sleep: 1.0.0 - on-exit-leak-free: 2.1.2 - pino-abstract-transport: 3.0.0 - pino-std-serializers: 7.1.0 - process-warning: 5.0.0 - quick-format-unescaped: 4.0.4 - real-require: 0.2.0 - safe-stable-stringify: 2.5.0 - sonic-boom: 4.2.1 - thread-stream: 4.2.0 - pino@9.14.0: dependencies: '@pinojs/redact': 0.4.0 @@ -4584,8 +4564,6 @@ snapshots: real-require@0.2.0: {} - real-require@1.0.0: {} - require-from-string@2.0.2: {} resolve-from@5.0.0: {} @@ -4749,10 +4727,6 @@ snapshots: dependencies: real-require: 0.2.0 - thread-stream@4.2.0: - dependencies: - real-require: 1.0.0 - tinybench@2.9.0: {} tinyexec@0.3.2: {} From a53ce8df5c3e9e4e7b0aece2033da203d948fa3d Mon Sep 17 00:00:00 2001 From: djgrant <1670902+djgrant@users.noreply.github.com> Date: Wed, 22 Jul 2026 16:05:02 +0100 Subject: [PATCH 2/2] Harden durable runtime cutover --- docs/cli/deploy.md | 4 +- docs/cli/destroy.md | 4 +- docs/internals/reconciler.md | 4 +- docs/internals/resource.md | 2 +- docs/manual/reconciler.md | 8 +- docs/rfcs/reconciler.md | 2 +- examples/reconciler/package.json | 2 + examples/reconciler/src/index.ts | 46 +---- packages/cli/src/index.ts | 7 +- packages/core/package.json | 1 + .../core/src/provisioner/durable-runtime.ts | 172 +++++++++++++++++- .../provisioner/workflows/workflow.deploy.ts | 7 +- .../provisioner/workflows/workflow.destroy.ts | 7 +- .../provisioner/workflows/workflow.plan.ts | 8 +- .../test/provisioner/durable-runtime.test.ts | 155 +++++++++++++++- .../test/durable-reconciliation.test.ts | 12 +- pnpm-lock.yaml | 9 + 17 files changed, 374 insertions(+), 76 deletions(-) diff --git a/docs/cli/deploy.md b/docs/cli/deploy.md index 3e81379..4df0e43 100644 --- a/docs/cli/deploy.md +++ b/docs/cli/deploy.md @@ -28,7 +28,7 @@ notation deploy infra/api.ts --execution-id Do not reuse a completed execution ID for a new deploy or for destroy. -Retryable provider conditions and consistency reads suspend on durable SQLite timers. The CLI stays resident until the scheduler wakes the execution and the workflow completes; completed provider calls are replayed from the heap rather than repeated. +Retryable provider conditions and consistency reads suspend on durable SQLite timers. The CLI stays resident until the scheduler wakes the execution and the workflow completes. Provider results are replayed after their heap checkpoint, but a crash after the provider accepts a create or update and before that checkpoint repeats the call, so provider mutations must be idempotent. Reconciler event consumers must tolerate the equivalent duplicate-delivery window. ## What happens @@ -45,3 +45,5 @@ Retryable provider conditions and consistency reads suspend on durable SQLite ti 6. **Delete orphans** – persisted resources absent from the graph are deleted when their resource type is registered. State, step results, timers, task coordination, and resource stores are persisted to `.notation/workflows.db`. Set `NOTATION_STATE_PATH` to choose another SQLite database path. + +On first use, Notation imports resource state from the legacy `.notation/state.json` file and archives it as `.notation/state.json.migrated`. If the durable database already contains conflicting resource state, Notation stops with recovery instructions instead of attempting to create resources from an empty namespace. diff --git a/docs/cli/destroy.md b/docs/cli/destroy.md index c25a386..aa1be47 100644 --- a/docs/cli/destroy.md +++ b/docs/cli/destroy.md @@ -16,10 +16,12 @@ notation destroy infra/api.ts notation destroy infra/api.ts --json > destroy.ndjson ``` -The command prints its execution ID. Resume a crashed destroy with the same ID so a provider delete that already completed is replayed instead of repeated: +The command prints its execution ID. Resume a crashed destroy with the same ID so checkpointed work can be replayed: ```sh notation destroy infra/api.ts --execution-id ``` Retryable deletes suspend on durable SQLite timers. Resource state is removed only after the provider delete succeeds or reports that the resource is already absent. + +The provider acknowledgement and heap checkpoint are not atomic. A crash between them repeats the delete, so provider delete operations must be idempotent and event consumers must tolerate duplicate delivery. diff --git a/docs/internals/reconciler.md b/docs/internals/reconciler.md index 05034a8..41de5e3 100644 --- a/docs/internals/reconciler.md +++ b/docs/internals/reconciler.md @@ -15,13 +15,13 @@ The reconciler expresses deployment and destruction as Yieldstar async generator | In state, provider state differs from stored state | **drift-update** | | In state, not in graph | **delete** | -Dry-run deploy performs decisions and emits lifecycle events without calling providers or mutating state. +Dry-run deploy performs decisions and emits lifecycle events without provider mutations or state mutations. When drift detection is enabled, it can still call provider read operations to decide whether a nominal noop has drifted. ## Destroy flow `destroy` is a first-class durable operation. It acquires the same deployment coordination store as deploy, deletes desired resources in reverse dependency order, deletes hydratable persisted orphans, and conditionally removes each resource store only after the provider delete succeeds or reports that the resource is already absent. -Provider delete is a stable durable step. If the process crashes after the provider acknowledges deletion but before state removal, replay uses the cached delete result and continues at the conditional store delete. +Provider delete is a stable durable step, but the provider acknowledgement and Yieldstar heap checkpoint are not atomic. If the process crashes between them, replay repeats the delete, so provider create, update, and delete operations must be idempotent. Event subscribers must likewise tolerate duplicate delivery when a crash occurs before the event checkpoint. ## Waiting and replay diff --git a/docs/internals/resource.md b/docs/internals/resource.md index baae261..851be55 100644 --- a/docs/internals/resource.md +++ b/docs/internals/resource.md @@ -187,7 +187,7 @@ new ResourceOperationPendingError(message: string, { | `callbackContext` | `Readonly>` | no | Plain serializable data for the next attempt. | | `cause` | `unknown` | no | The provider error that caused this result. | -The default limit is 30 attempts. Set `maxOperationAttempts` on the reconciler to change it. Reaching the limit fails the operation. +The default limit is 30 attempts. Set `maxOperationAttempts` in the deploy, plan, or destroy options to change it. Reaching the limit fails the operation. ```ts read: async (key, context) => { diff --git a/docs/manual/reconciler.md b/docs/manual/reconciler.md index eee8d06..9d4435f 100644 --- a/docs/manual/reconciler.md +++ b/docs/manual/reconciler.md @@ -4,7 +4,7 @@ Use `deploy` and `destroy` when a Node.js application needs durable resource lif ```ts import { SqliteSchedulerClient, SqliteStoreClient, SqliteTaskQueueClient, SqliteTimersClient, createSqliteDb } from "@yieldstar/sqlite-runtime/node"; -import { DurableStateBackend, deploy, destroy } from "@notation/reconciler"; +import { DurableStateBackend, deploy as deployResources, destroy as destroyResources } from "@notation/reconciler"; import { workflow } from "yieldstar"; const database = createSqliteDb({ path: ".notation/workflows.db" }); @@ -16,7 +16,7 @@ const storeClient = new SqliteStoreClient({ db: database, schedulerClient }); const state = new DurableStateBackend(storeClient, "my-application"); export const deploy = workflow(async function* (step, event) { - yield* deploy(step, { + yield* deployResources(step, { deploymentId: "my-application", executionId: event.executionId, resources, @@ -25,7 +25,7 @@ export const deploy = workflow(async function* (step, event) { }); export const destroy = workflow(async function* (step, event) { - yield* destroy(step, { + yield* destroyResources(step, { deploymentId: "my-application", executionId: event.executionId, resources, @@ -34,7 +34,7 @@ export const destroy = workflow(async function* (step, event) { }); ``` -The outer workflow supplies durable step execution, timers, shared stores, waiting, scheduling, and coordination. Completed provider calls are replayed from the heap after a crash, retryable provider conditions suspend on a durable timer, and conditional state writes use Yieldstar store identity and version. +The outer workflow supplies durable step execution, timers, shared stores, waiting, scheduling, and coordination. Checkpointed provider results are replayed from the heap after a crash, retryable provider conditions suspend on a durable timer, and conditional state writes use Yieldstar store identity and version. Provider mutations must be idempotent because a crash after provider acknowledgement but before the heap checkpoint repeats the call; event consumers must tolerate the same duplicate-delivery window. Each live resource is one Yieldstar store. Absence is represented by no store, not a tombstone. Yieldstar's UUIDv7 store `instanceId` and version are authoritative for conditional update and delete; Notation exposes the version as the resource state's `rev`. diff --git a/docs/rfcs/reconciler.md b/docs/rfcs/reconciler.md index 7ec505a..6cc9d1c 100644 --- a/docs/rfcs/reconciler.md +++ b/docs/rfcs/reconciler.md @@ -9,7 +9,7 @@ Notation describes reconciliation intent and resource lifecycle operations. An o Live resource objects remain in the workflow process. They are not serialized into workflow parameters. This keeps provider clients and operation closures under Notation's lifecycle control while Yieldstar persists step results and shared state. -Provider create, update, read, and delete calls are durable steps with stable resource-scoped keys. A process crash after a completed provider call replays the cached result and continues at state persistence instead of repeating the call. Retryable provider conditions become Yieldstar delays, allowing the process to wait without polling the provider continuously. +Provider create, update, read, and delete calls are durable steps with stable resource-scoped keys. Once a result reaches the heap checkpoint, replay uses the cached result and continues at state persistence. Provider mutations must be idempotent because a crash after provider acknowledgement but before that checkpoint repeats the call. Retryable provider conditions become Yieldstar delays, allowing the process to wait without polling the provider continuously. ## State lifecycle diff --git a/examples/reconciler/package.json b/examples/reconciler/package.json index c5d8f2e..8a26c85 100644 --- a/examples/reconciler/package.json +++ b/examples/reconciler/package.json @@ -9,8 +9,10 @@ "typecheck": "tsc --noEmit" }, "dependencies": { + "@notation/core": "workspace:*", "@notation/reconciler": "workspace:*", "@notation/resource": "workspace:*", + "@notation/utils": "workspace:*", "@yieldstar/core": "0.5.0", "@yieldstar/sqlite-runtime": "0.5.0", "pino": "^9.9.0", diff --git a/examples/reconciler/src/index.ts b/examples/reconciler/src/index.ts index 0253115..ee4360a 100644 --- a/examples/reconciler/src/index.ts +++ b/examples/reconciler/src/index.ts @@ -1,26 +1,12 @@ -import { WorkflowRunner } from "@yieldstar/core"; -import { - SqliteHeapClient, - SqliteSchedulerClient, - SqliteStoreClient, - SqliteTaskQueueClient, - SqliteTimersClient, - createSqliteDb, -} from "@yieldstar/sqlite-runtime/node"; +import { NodeDurableRuntime } from "@notation/core"; import * as reconciler from "@notation/reconciler"; -import pino from "pino"; import { createWorkflowRouter, workflow } from "yieldstar"; import { StaticSite } from "./static-site"; -const logger = pino(); -const database = createSqliteDb({ path: "sites.db" }); -const taskQueueClient = new SqliteTaskQueueClient(database); -const schedulerClient = new SqliteSchedulerClient({ - taskQueueClient, - timersClient: new SqliteTimersClient(database), +const runtime = new NodeDurableRuntime({ + deploymentId: "static-sites", + databasePath: "sites.db", }); -const storeClient = new SqliteStoreClient({ db: database, schedulerClient }); -const state = new reconciler.DurableStateBackend(storeClient, "static-sites"); const resources = [ new StaticSite({ @@ -44,29 +30,15 @@ const deploy = workflow(async function* (step, event) { deploymentId: "static-sites", executionId: event.executionId, resources, - state, + state: runtime.state, registry: reconciler.createResourceRegistry([StaticSite]), }); }); -const runner = new WorkflowRunner({ - router: createWorkflowRouter({ deploy }), - heapClient: new SqliteHeapClient(database), - storeClient, - schedulerClient, - logger, -}); - try { - await runner.run( - { - workflowId: "deploy", - executionId: crypto.randomUUID(), - params: {}, - context: new Map(), - }, - logger, - ); + await runtime.run(createWorkflowRouter({ deploy }), { + workflowId: "deploy", + }); } finally { - database.close(); + runtime.close(); } diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index eee95f2..1bfade1 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -9,7 +9,7 @@ import { runWithCliErrorHandling } from "./run-with-error-handling"; import { visualise } from "./visualise"; import { watch } from "./watch"; import { startDashboardServer } from "@notation/dashboard"; -import { NodeDurableRuntime } from "@notation/core"; +import { NodeDurableRuntime, resolveDeploymentId } from "@notation/core"; program .command("compile") @@ -24,7 +24,10 @@ program .argument("", "entryPoint") .description("Start Notation Dashboard") .action(async (entryPoint) => { - const runtime = new NodeDurableRuntime({ deploymentId: entryPoint }); + const runtime = new NodeDurableRuntime({ + deploymentId: resolveDeploymentId(entryPoint), + }); + await runtime.initialize(); await startDashboardServer({ state: runtime.state }); }); diff --git a/packages/core/package.json b/packages/core/package.json index cc55f70..c3e0fae 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -15,6 +15,7 @@ "dependencies": { "@notation/reconciler": "workspace:*", "@notation/resource": "workspace:*", + "@notation/state": "workspace:*", "@yieldstar/core": "0.5.0", "@yieldstar/sqlite-runtime": "0.5.0", "deep-object-diff": "^1.1.9", diff --git a/packages/core/src/provisioner/durable-runtime.ts b/packages/core/src/provisioner/durable-runtime.ts index 73face8..10d902c 100644 --- a/packages/core/src/provisioner/durable-runtime.ts +++ b/packages/core/src/provisioner/durable-runtime.ts @@ -1,5 +1,8 @@ import { randomUUID } from "node:crypto"; +import { access, rename } from "node:fs/promises"; +import path from "node:path"; import { setImmediate } from "node:timers/promises"; +import { isDeepStrictEqual } from "node:util"; import { WorkflowRunner, type WorkflowEvent, @@ -15,17 +18,25 @@ import { createSqliteDb, } from "@yieldstar/sqlite-runtime/node"; import { DurableStateBackend } from "@notation/reconciler"; +import { FileStateBackend, type StateNode } from "@notation/state"; import pino, { type Logger } from "pino"; +import { defineStore } from "yieldstar"; export const DEFAULT_WORKFLOW_STATE_PATH = ".notation/workflows.db"; +export const DEFAULT_LEGACY_STATE_PATH = ".notation/state.json"; export function resolveWorkflowStatePath(): string { return process.env.NOTATION_STATE_PATH ?? DEFAULT_WORKFLOW_STATE_PATH; } +export function resolveDeploymentId(entryPoint: string): string { + return path.resolve(entryPoint); +} + export type NodeDurableRuntimeOptions = { deploymentId: string; databasePath?: string; + legacyStatePath?: string | false; logger?: Logger; }; @@ -35,6 +46,31 @@ export type RunWorkflowOptions = { params?: Record; }; +type ExecutionBinding = { + deploymentId: string; + workflowId: string; +}; + +const executionBindingStore = defineStore("notation/execution-binding", { + "~standard": { + version: 1 as const, + vendor: "notation", + validate(value: unknown) { + if ( + typeof value === "object" && + value !== null && + "deploymentId" in value && + typeof value.deploymentId === "string" && + "workflowId" in value && + typeof value.workflowId === "string" + ) { + return { value: value as ExecutionBinding }; + } + return { issues: [{ message: "Execution binding is invalid" }] }; + }, + }, +}); + /** Resident Yieldstar 0.5.0 Node runtime used by Notation application commands. */ export class NodeDurableRuntime { readonly deploymentId: string; @@ -45,13 +81,22 @@ export class NodeDurableRuntime { readonly #schedulerClient: SqliteSchedulerClient; readonly #storeClient: SqliteStoreClient; readonly #logger: Logger; + readonly #legacyStatePath: string | undefined; #running = false; constructor(opts: NodeDurableRuntimeOptions) { this.deploymentId = opts.deploymentId; this.#logger = opts.logger ?? pino({ level: "silent" }); + const databasePath = opts.databasePath ?? resolveWorkflowStatePath(); + this.#legacyStatePath = + opts.legacyStatePath === false + ? undefined + : (opts.legacyStatePath ?? + (databasePath === DEFAULT_WORKFLOW_STATE_PATH + ? DEFAULT_LEGACY_STATE_PATH + : undefined)); this.#database = createSqliteDb({ - path: opts.databasePath ?? resolveWorkflowStatePath(), + path: databasePath, }); const taskQueueClient = new SqliteTaskQueueClient(this.#database); this.#schedulerClient = new SqliteSchedulerClient({ @@ -77,9 +122,10 @@ export class NodeDurableRuntime { ); } this.#running = true; + const executionId = opts.executionId ?? randomUUID(); const event: WorkflowEvent = { workflowId: opts.workflowId, - executionId: opts.executionId ?? randomUUID(), + executionId, params: opts.params ?? {}, context: new Map(), }; @@ -93,6 +139,7 @@ export class NodeDurableRuntime { let resolveCompletion!: (value: unknown) => void; let rejectCompletion!: (error: unknown) => void; + let completed = false; const completion = new Promise((resolve, reject) => { resolveCompletion = resolve; rejectCompletion = reject; @@ -101,12 +148,12 @@ export class NodeDurableRuntime { try { const result = await runner.run(nextEvent, logger); if (result && nextEvent.executionId === event.executionId) { - this.#eventLoop.stop(); + completed = true; resolveCompletion(result.result); } } catch (error) { if (nextEvent.executionId === event.executionId) { - this.#eventLoop.stop(); + completed = true; rejectCompletion(error); return; } @@ -115,28 +162,135 @@ export class NodeDurableRuntime { }; try { + await this.initialize(); + await this.#bindExecution(executionId, opts.workflowId); await processEvent(event, this.#logger); - this.#eventLoop.start({ onNewEvent: processEvent, logger: this.#logger }); + const eventPump = this.#processQueuedEvents( + executionId, + processEvent, + rejectCompletion, + () => completed, + ); try { return await completion; } finally { - // Let SqliteEventLoop remove the completed queue item before callers - // close the shared database. + await eventPump; + // Let the queue transaction finish before callers close the shared database. await setImmediate(); } } finally { - this.#eventLoop.stop(); this.#running = false; } } + async initialize(): Promise { + await this.#migrateLegacyState(); + } + + async #bindExecution(executionId: string, workflowId: string): Promise { + const expected: ExecutionBinding = { + deploymentId: this.deploymentId, + workflowId, + }; + const binding = await this.#storeClient.getOrCreateStore({ + definition: executionBindingStore, + id: executionId, + initial: expected, + }); + const existing = binding.state as ExecutionBinding; + if (!isDeepStrictEqual(existing, expected)) { + throw new Error( + `Yieldstar execution ${executionId} is bound to deployment ${existing.deploymentId} workflow ${existing.workflowId}, not deployment ${this.deploymentId} workflow ${workflowId}`, + ); + } + } + + async #processQueuedEvents( + executionId: string, + processEvent: (event: WorkflowEvent, logger: Logger) => Promise, + rejectCompletion: (error: unknown) => void, + isCompleted: () => boolean, + ): Promise { + const deferredTaskIds: number[] = []; + try { + while (this.#running && !isCompleted()) { + let task = this.#eventLoop.taskQueue.process(); + while (task) { + if (task.event.executionId !== executionId) { + deferredTaskIds.push(task.taskId); + } else { + try { + await this.#bindExecution( + task.event.executionId, + task.event.workflowId, + ); + await processEvent(task.event, this.#logger); + } finally { + this.#eventLoop.taskQueue.remove(task.taskId); + } + } + if (!this.#running || isCompleted()) return; + task = this.#eventLoop.taskQueue.process(); + } + this.#eventLoop.timers.processTimers(); + await new Promise((resolve) => setTimeout(resolve, 10)); + } + } catch (error) { + rejectCompletion(error); + } finally { + for (const taskId of deferredTaskIds) { + this.#eventLoop.taskQueue.makeVisible(taskId); + } + } + } + + async #migrateLegacyState(): Promise { + const legacyStatePath = this.#legacyStatePath; + if (!legacyStatePath) return; + try { + await access(legacyStatePath); + } catch { + return; + } + + const legacyState = await new FileStateBackend(legacyStatePath).values(); + const durableState = await this.state.values(); + const legacyById = new Map(legacyState.map((node) => [node.id, node])); + for (const current of durableState) { + const legacy = legacyById.get(current.id); + if (!legacy || !statesMatchIgnoringRevision(current, legacy)) { + throw legacyMigrationConflict(legacyStatePath); + } + } + for (const node of legacyState) { + const current = await this.state.get(node.id); + if (!current) { + await this.state.update(node.id, 0, node); + } else if (!statesMatchIgnoringRevision(current, node)) { + throw legacyMigrationConflict(legacyStatePath); + } + } + await rename(legacyStatePath, `${legacyStatePath}.migrated`); + } + close(): void { if (this.#running) { throw new Error( "Cannot close the Node Yieldstar runtime while a workflow is active", ); } - this.#eventLoop.stop(); this.#database.close(); } } + +function statesMatchIgnoringRevision(left: StateNode, right: StateNode) { + const { rev: _leftRev, ...leftState } = left; + const { rev: _rightRev, ...rightState } = right; + return isDeepStrictEqual(leftState, rightState); +} + +function legacyMigrationConflict(legacyStatePath: string) { + return new Error( + `Cannot migrate legacy state from ${legacyStatePath} because the durable database already contains different resource state. Back up both files, then remove the new durable database and retry the command to import the legacy state.`, + ); +} diff --git a/packages/core/src/provisioner/workflows/workflow.deploy.ts b/packages/core/src/provisioner/workflows/workflow.deploy.ts index dc3b765..ab11ecf 100644 --- a/packages/core/src/provisioner/workflows/workflow.deploy.ts +++ b/packages/core/src/provisioner/workflows/workflow.deploy.ts @@ -6,7 +6,7 @@ import { } from "@notation/reconciler"; import { createWorkflowRouter, workflow } from "yieldstar"; import { getResourceGraph } from "src/orchestrator/graph"; -import { NodeDurableRuntime } from "../durable-runtime"; +import { NodeDurableRuntime, resolveDeploymentId } from "../durable-runtime"; export type DeployAppOptions = { entryPoint: string; @@ -32,9 +32,10 @@ export async function deployApp({ emit = createLoggerReconcilerSubscriber(), }: DeployAppOptions): Promise { const graph = await getResourceGraph(entryPoint); + const deploymentId = + suppliedRuntime?.deploymentId ?? resolveDeploymentId(entryPoint); const runtime = - suppliedRuntime ?? - new NodeDurableRuntime({ deploymentId: entryPoint, databasePath }); + suppliedRuntime ?? new NodeDurableRuntime({ deploymentId, databasePath }); const deploy = workflow(async function* (step, event) { yield* reconciler.deploy(step, { deploymentId: runtime.deploymentId, diff --git a/packages/core/src/provisioner/workflows/workflow.destroy.ts b/packages/core/src/provisioner/workflows/workflow.destroy.ts index 9a534c4..c5d9303 100644 --- a/packages/core/src/provisioner/workflows/workflow.destroy.ts +++ b/packages/core/src/provisioner/workflows/workflow.destroy.ts @@ -6,7 +6,7 @@ import { } from "@notation/reconciler"; import { createWorkflowRouter, workflow } from "yieldstar"; import { getResourceGraph } from "src/orchestrator/graph"; -import { NodeDurableRuntime } from "../durable-runtime"; +import { NodeDurableRuntime, resolveDeploymentId } from "../durable-runtime"; export type DestroyAppOptions = { entryPoint: string; @@ -28,9 +28,10 @@ export async function destroyApp({ emit = createLoggerReconcilerSubscriber(), }: DestroyAppOptions) { const graph = await getResourceGraph(entryPoint); + const deploymentId = + suppliedRuntime?.deploymentId ?? resolveDeploymentId(entryPoint); const runtime = - suppliedRuntime ?? - new NodeDurableRuntime({ deploymentId: entryPoint, databasePath }); + suppliedRuntime ?? new NodeDurableRuntime({ deploymentId, databasePath }); const destroy = workflow(async function* (step, event) { yield* reconciler.destroy(step, { deploymentId: runtime.deploymentId, diff --git a/packages/core/src/provisioner/workflows/workflow.plan.ts b/packages/core/src/provisioner/workflows/workflow.plan.ts index b64db9d..5b06aba 100644 --- a/packages/core/src/provisioner/workflows/workflow.plan.ts +++ b/packages/core/src/provisioner/workflows/workflow.plan.ts @@ -1,6 +1,6 @@ import { createPlan, type Plan } from "@notation/reconciler"; import { getResourceGraph } from "src/orchestrator/graph"; -import { NodeDurableRuntime } from "../durable-runtime"; +import { NodeDurableRuntime, resolveDeploymentId } from "../durable-runtime"; export type { Plan, PlanNode, PlanDecision } from "@notation/reconciler"; @@ -20,10 +20,12 @@ export async function planApp({ databasePath, }: PlanAppOptions): Promise { const graph = await getResourceGraph(entryPoint); + const deploymentId = + suppliedRuntime?.deploymentId ?? resolveDeploymentId(entryPoint); const runtime = - suppliedRuntime ?? - new NodeDurableRuntime({ deploymentId: entryPoint, databasePath }); + suppliedRuntime ?? new NodeDurableRuntime({ deploymentId, databasePath }); try { + await runtime.initialize(); return await createPlan({ resources: graph.resources, state: runtime.state, diff --git a/packages/core/test/provisioner/durable-runtime.test.ts b/packages/core/test/provisioner/durable-runtime.test.ts index 77a699b..09aa6d8 100644 --- a/packages/core/test/provisioner/durable-runtime.test.ts +++ b/packages/core/test/provisioner/durable-runtime.test.ts @@ -1,4 +1,4 @@ -import { mkdtemp, rm } from "node:fs/promises"; +import { access, mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; import * as reconciler from "@notation/reconciler"; @@ -6,9 +6,17 @@ import { ResourceOperationPendingError, resource, } from "@notation/resource"; -import { createWorkflowRouter, workflow } from "yieldstar"; +import { + SqliteEventLoop, + SqliteTaskQueueClient, + createSqliteDb, +} from "@yieldstar/sqlite-runtime/node"; +import { RetryableError, createWorkflowRouter, workflow } from "yieldstar"; import { describe, expect, it } from "vitest"; -import { NodeDurableRuntime } from "src/provisioner/durable-runtime"; +import { + NodeDurableRuntime, + resolveDeploymentId, +} from "src/provisioner/durable-runtime"; describe("NodeDurableRuntime", () => { it("stays resident across a provider delay and resumes from the SQLite event loop", async () => { @@ -57,4 +65,145 @@ describe("NodeDurableRuntime", () => { await rm(directory, { recursive: true, force: true }); } }, 5_000); + + it("binds an execution ID to its deployment and workflow", async () => { + const directory = await mkdtemp(path.join(tmpdir(), "notation-binding-")); + const databasePath = path.join(directory, "workflows.db"); + const completed = workflow(async function* () {}); + const router = createWorkflowRouter({ + deploy: completed, + destroy: completed, + }); + const first = new NodeDurableRuntime({ + deploymentId: "first-deployment", + databasePath, + }); + + try { + await first.run(router, { + workflowId: "deploy", + executionId: "bound-execution", + }); + await expect( + first.run(router, { + workflowId: "destroy", + executionId: "bound-execution", + }), + ).rejects.toThrow("bound to deployment first-deployment workflow deploy"); + } finally { + first.close(); + } + + const second = new NodeDurableRuntime({ + deploymentId: "second-deployment", + databasePath, + }); + try { + await expect( + second.run(router, { + workflowId: "deploy", + executionId: "bound-execution", + }), + ).rejects.toThrow("bound to deployment first-deployment workflow deploy"); + } finally { + second.close(); + await rm(directory, { recursive: true, force: true }); + } + }); + + it("does not acknowledge queued events from another execution", async () => { + const directory = await mkdtemp(path.join(tmpdir(), "notation-queue-")); + const databasePath = path.join(directory, "workflows.db"); + const database = createSqliteDb({ path: databasePath }); + new SqliteEventLoop(database); + new SqliteTaskQueueClient(database).add({ + workflowId: "deploy", + executionId: "unrelated-execution", + params: {}, + context: new Map(), + }); + database.close(); + + let attempts = 0; + const delayed = workflow(async function* (step) { + yield* step.run("delay", async () => { + attempts += 1; + if (attempts === 1) { + throw new RetryableError("not ready", { + maxAttempts: 2, + retryInterval: 10, + }); + } + }); + }); + const runtime = new NodeDurableRuntime({ + deploymentId: "queue-test", + databasePath, + }); + try { + await runtime.run(createWorkflowRouter({ deploy: delayed }), { + workflowId: "deploy", + executionId: "current-execution", + }); + } finally { + runtime.close(); + } + + const reopened = createSqliteDb({ path: databasePath }); + const queued = new SqliteEventLoop(reopened).taskQueue.process(); + expect(queued?.event.executionId).toBe("unrelated-execution"); + reopened.close(); + await rm(directory, { recursive: true, force: true }); + }, 5_000); + + it("imports and archives legacy JSON state before running", async () => { + const directory = await mkdtemp(path.join(tmpdir(), "notation-migrate-")); + const databasePath = path.join(directory, "workflows.db"); + const legacyStatePath = path.join(directory, "state.json"); + await writeFile( + legacyStatePath, + JSON.stringify({ + existing: { + rev: 7, + id: "existing", + type: "test/legacy", + config: {}, + params: {}, + output: { remoteId: "provider-123" }, + lastOperation: "create", + lastOperationAt: "2026-07-22T00:00:00.000Z", + }, + }), + ); + const runtime = new NodeDurableRuntime({ + deploymentId: "legacy-deployment", + databasePath, + legacyStatePath, + }); + const completed = workflow(async function* () {}); + + try { + await runtime.run(createWorkflowRouter({ deploy: completed }), { + workflowId: "deploy", + executionId: "migration-execution", + }); + await expect(runtime.state.get("existing")).resolves.toMatchObject({ + output: { remoteId: "provider-123" }, + }); + await expect(access(legacyStatePath)).rejects.toThrow(); + await expect( + access(`${legacyStatePath}.migrated`), + ).resolves.toBeUndefined(); + } finally { + runtime.close(); + await rm(directory, { recursive: true, force: true }); + } + }); + + it("canonicalises equivalent entry-point spellings", () => { + const absolute = path.resolve("infra/api.ts"); + expect(resolveDeploymentId("infra/api.ts")).toBe(absolute); + expect(resolveDeploymentId("./infra/api.ts")).toBe(absolute); + expect(resolveDeploymentId(absolute)).toBe(absolute); + }); }); diff --git a/packages/reconciler/test/durable-reconciliation.test.ts b/packages/reconciler/test/durable-reconciliation.test.ts index 1a51cd7..47f471e 100644 --- a/packages/reconciler/test/durable-reconciliation.test.ts +++ b/packages/reconciler/test/durable-reconciliation.test.ts @@ -38,7 +38,7 @@ describe("durable execution and replay", () => { if (attempts === 1) { expect(context).toBeUndefined(); throw new ResourceOperationPendingError("provider is not ready", { - retryAfterMs: 1, + retryAfterMs: 250, callbackContext: { requestId: "request-123" }, }); } @@ -56,7 +56,7 @@ describe("durable execution and replay", () => { expect(attempts).toBe(1); expect(runtime.scheduler.events).toHaveLength(1); - await sleep(5); + await sleep(275); await runtime.run("wait-execution"); expect(attempts).toBe(2); expect(await runtime.state.get("pending")).toMatchObject({ @@ -124,7 +124,7 @@ describe("durable execution and replay", () => { attempts += 1; if (attempts === 1) { throw new ResourceOperationPendingError("delete is not ready", { - retryAfterMs: 1, + retryAfterMs: 250, }); } }, @@ -140,7 +140,7 @@ describe("durable execution and replay", () => { expect(attempts).toBe(1); expect(await runtime.state.get("pending-delete")).toBeDefined(); - await sleep(5); + await sleep(275); await runtime.destroy("destroy-wait"); expect(attempts).toBe(2); expect(await runtime.state.get("pending-delete")).toBeUndefined(); @@ -160,7 +160,7 @@ describe("durable execution and replay", () => { if (reads === 1) { throw new ResourceOperationPendingError( "resource is not visible yet", - { retryAfterMs: 1 }, + { retryAfterMs: 250 }, ); } return {} as const; @@ -177,7 +177,7 @@ describe("durable execution and replay", () => { expect(reads).toBe(1); expect(await runtime.state.get("eventually-readable")).toBeUndefined(); - await sleep(5); + await sleep(275); await runtime.run("post-write-read-execution"); expect(reads).toBe(2); expect(await runtime.state.get("eventually-readable")).toMatchObject({ diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 02ef3b9..5f1ee6d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -117,12 +117,18 @@ importers: examples/reconciler: dependencies: + '@notation/core': + specifier: workspace:* + version: link:../../packages/core '@notation/reconciler': specifier: workspace:* version: link:../../packages/reconciler '@notation/resource': specifier: workspace:* version: link:../../packages/resource + '@notation/utils': + specifier: workspace:* + version: link:../../packages/utils '@yieldstar/core': specifier: 0.5.0 version: 0.5.0 @@ -230,6 +236,9 @@ importers: '@notation/resource': specifier: workspace:* version: link:../resource + '@notation/state': + specifier: workspace:* + version: link:../state '@yieldstar/core': specifier: 0.5.0 version: 0.5.0