From b01ef5d54184a57a525ea48a8d67015aaf453b5e Mon Sep 17 00:00:00 2001 From: djgrant <1670902+djgrant@users.noreply.github.com> Date: Sun, 19 Jul 2026 17:53:54 +0100 Subject: [PATCH] Document reconciler delivery --- .changeset/reconciler.md | 14 +++++ docs/cli/dashboard.md | 10 ++++ docs/cli/deploy.md | 18 ++++++- docs/cli/destroy.md | 6 +++ docs/cli/plan.md | 22 ++++++++ docs/index.ts | 2 + docs/internals/compiler.md | 10 +--- docs/internals/reconciler.md | 25 ++++----- docs/internals/resource.md | 20 ++++--- docs/internals/state.md | 30 +++++++++-- docs/manual/introduction.md | 11 ++-- docs/manual/reconciler.md | 55 +++++++++++++++++++ docs/resources/lambda.md | 5 +- docs/rfcs/reconciler.md | 102 +++++++++++++++++++++++++++++++++++ 14 files changed, 290 insertions(+), 40 deletions(-) create mode 100644 .changeset/reconciler.md create mode 100644 docs/cli/plan.md create mode 100644 docs/manual/reconciler.md create mode 100644 docs/rfcs/reconciler.md diff --git a/.changeset/reconciler.md b/.changeset/reconciler.md new file mode 100644 index 0000000..409f877 --- /dev/null +++ b/.changeset/reconciler.md @@ -0,0 +1,14 @@ +--- +"@notation/aws.iac": minor +"@notation/aws": minor +"@notation/cli": minor +"@notation/core": minor +"@notation/reconciler": minor +"@notation/resource": minor +"@notation/state": minor +"@notation/state-sqlite": minor +--- + +Add the reconciler API, versioned event streams, renewable mutation leases, +SQLite state, backend-neutral dashboard state, and compiled infrastructure +graphs. diff --git a/docs/cli/dashboard.md b/docs/cli/dashboard.md index 9c3fcac..f630ad7 100644 --- a/docs/cli/dashboard.md +++ b/docs/cli/dashboard.md @@ -9,3 +9,13 @@ Starts a local web dashboard for observing deployment state. ```sh notation dashboard ``` + +The dashboard uses the same state backend as deploy and destroy. Set +`NOTATION_STATE_PATH` to select SQLite: + +```sh +NOTATION_STATE_PATH=.notation/state.db notation dashboard +``` + +The server reads through `StateBackend`, so file and SQLite state produce the same +dashboard payload. diff --git a/docs/cli/deploy.md b/docs/cli/deploy.md index c513b4d..940cf7e 100644 --- a/docs/cli/deploy.md +++ b/docs/cli/deploy.md @@ -10,11 +10,20 @@ Compiles and deploys the stack to AWS. notation deploy infra/api.ts ``` +## Event stream + +`--json` writes versioned reconciler events to stdout as newline-delimited JSON. Build +output and diagnostics move to stderr. + +```sh +notation deploy infra/api.ts --json > deploy.ndjson +``` + ## What happens 1. **Compile** – esbuild compiles infra and runtime modules to `dist/`. -2. **Build resource graph** – imports the compiled output, calls `register(collector)`, returns the resource graph. +2. **Build resource graph** – imports the compiled output and collects the declared resources. 3. **Reconcile** – the reconciler compares desired state (graph) against current state (`.notation/state.json`): - New resources → **create** @@ -26,4 +35,9 @@ notation deploy infra/api.ts 5. **Drift detection** – enabled by default. Reads actual AWS state and compares against stored state. If drifted, Notation updates to match your definition. -State is persisted to `.notation/state.json` after each operation. Configurable via `NOTATION_STATE_PATH`. +State is persisted to `.notation/state.json` after each operation. Set +`NOTATION_STATE_PATH` to a path ending in `.db` or `.sqlite` to use SQLite: + +```sh +NOTATION_STATE_PATH=.notation/state.db notation deploy infra/api.ts +``` diff --git a/docs/cli/destroy.md b/docs/cli/destroy.md index 8893da7..a74ff30 100644 --- a/docs/cli/destroy.md +++ b/docs/cli/destroy.md @@ -9,3 +9,9 @@ Removes all resources in the stack. Tears down runs in reverse dependency order, ```sh notation destroy infra/api.ts ``` + +`--json` writes versioned reconciler events to stdout as newline-delimited JSON: + +```sh +notation destroy infra/api.ts --json > destroy.ndjson +``` diff --git a/docs/cli/plan.md b/docs/cli/plan.md new file mode 100644 index 0000000..e6638e4 --- /dev/null +++ b/docs/cli/plan.md @@ -0,0 +1,22 @@ +# notation plan + +```sh +notation plan +``` + +Compiles the application and reports the changes a deployment would make: + +```sh +notation plan infra/api.ts +``` + +The plan reads remote resources when drift detection applies. It does not perform create, +update, or delete operations. + +## JSON output + +`--json` writes the complete plan to stdout. Build output and diagnostics move to stderr. + +```sh +notation plan infra/api.ts --json > plan.json +``` diff --git a/docs/index.ts b/docs/index.ts index 163f164..7750a49 100644 --- a/docs/index.ts +++ b/docs/index.ts @@ -12,6 +12,7 @@ export const categories: DocCategory[] = [ { label: "Introduction", slug: "manual/introduction" }, { label: "Installation", slug: "manual/installation" }, { label: "Quick Start", slug: "manual/quickstart" }, + { label: "Reconciler", slug: "manual/reconciler" }, ], }, { @@ -42,6 +43,7 @@ export const categories: DocCategory[] = [ links: [ { label: "notation compile", slug: "cli/compile" }, { label: "notation watch", slug: "cli/watch" }, + { label: "notation plan", slug: "cli/plan" }, { label: "notation deploy", slug: "cli/deploy" }, { label: "notation destroy", slug: "cli/destroy" }, { label: "notation dashboard", slug: "cli/dashboard" }, diff --git a/docs/internals/compiler.md b/docs/internals/compiler.md index 17ca135..216afd6 100644 --- a/docs/internals/compiler.md +++ b/docs/internals/compiler.md @@ -73,16 +73,10 @@ Each named export in a `.fn.ts` file becomes a Lambda function (or other serverl After compilation, the resource graph is built by dynamically importing the compiled infrastructure module: ```ts [packages/core/src/orchestrator/graph.ts] -const mod = await import(outFilePath); -const register = mod.register ?? mod.default; -await register(collector); -return { - resourceGroups: collector.getResourceGroups(), - resources: collector.getResources(), -}; +return collectResourceGraph(() => import(outFilePath)); ``` -The `collector` tracks resources as they're created during module execution. When `lambda({ ... })` is called, it registers the Lambda function and its associated resources (IAM role, log group, zip package) with the collector. +While the compiled entry point is imported, calls such as `lambda({ ... })` add the Lambda function and its associated resources (IAM role, log group, zip package) to the graph. The reconciler uses the resulting `{ resources, resourceGroups }` object to plan deployments. diff --git a/docs/internals/reconciler.md b/docs/internals/reconciler.md index 1cb0f4e..906273d 100644 --- a/docs/internals/reconciler.md +++ b/docs/internals/reconciler.md @@ -61,19 +61,16 @@ Properties marked as `volatile` in the schema (like `LastModified` timestamps) a The reconciler emits events at each step of an operation's lifecycle. The default `createConsoleReconcilerSubscriber()` logs these to the console with formatted output. -| Event | When | -| ----------------------------- | --------------------------------------------------- | -| `reconciler.deploy.decision` | After deciding what action to take for a resource | -| `reconciler.drift.detected` | When drift is found between stored and actual state | -| `reconciler.create.started` | Before creating a resource | -| `reconciler.create.completed` | After successful creation | -| `reconciler.update.started` | Before updating a resource | -| `reconciler.update.completed` | After successful update | -| `reconciler.delete.started` | Before deleting a resource | -| `reconciler.delete.completed` | After successful deletion | -| `reconciler.operation.failed` | When any operation fails | - -Events carry the resource ID, type, and relevant data (params, diff, error) so subscribers can build custom UIs or logging. +| Event | When | +| ------------------------------------ | --------------------------------------------------- | +| `reconciler.deploy.decision` | After deciding what action to take for a resource | +| `reconciler.drift.detected` | When drift is found between stored and actual state | +| `reconciler.operation.lifecycle` | When an operation starts, finishes, skips, or fails | +| `reconciler.orphan-deletion.skipped` | When no registered class can delete an orphan | + +Lifecycle events contain the operation (`create`, `read`, `update`, or `delete`) and its +status (`start`, `success`, `error`, `skip`, or `dry-run`). Events carry the resource ID, +type, and relevant diff or error details. ## Operations @@ -107,4 +104,4 @@ Each operation follows the following pattern: 4. Persist to state backend 5. Emit `completed` event (or `failed` on error) -State is updated after each step, so the workflow can be resumed if it stops prematurely. +State is updated after the provider operation and read-back complete. diff --git a/docs/internals/resource.md b/docs/internals/resource.md index a26ce45..4cf0921 100644 --- a/docs/internals/resource.md +++ b/docs/internals/resource.md @@ -14,7 +14,12 @@ import { z } from "zod"; const LambdaFunction = defineResource<{ Key: { FunctionName: string }; - CreateParams: { FunctionName: string; Runtime: string; Handler: string; Code: Buffer }; + CreateParams: { + FunctionName: string; + Runtime: string; + Handler: string; + Code: Buffer; + }; UpdateParams: { FunctionName: string }; ReadResult: { FunctionArn: string }; }>({ type: "aws/lambda/LambdaFunction" }) @@ -26,7 +31,11 @@ const LambdaFunction = defineResource<{ presence: "required", immutable: true, }, - Runtime: { propertyType: "param", valueType: z.string(), presence: "required" }, + Runtime: { + propertyType: "param", + valueType: z.string(), + presence: "required", + }, Handler: { propertyType: "param", valueType: z.string(), @@ -178,12 +187,11 @@ abstract class ResourceGroup { } ``` -Construction accepts a `ResourceGroupOptions`: +Construction accepts `ResourceGroupOptions` with: -- `collector` – a `ResourceCollector` used during graph construction to allocate group IDs and register resources. -- `id` – pre-assigned ID, used when no collector is provided. +- `id` – an optional pre-assigned ID used outside graph collection. - `dependencies` – map of dependency names to group IDs. -When a collector is provided, the group registers itself and each resource added via `add()` is registered into the collector. This is how a construct like `export const getTodos = lambda({ ... })` maps to the 4–6 actual AWS resources required to run it. +During graph construction, the group and each resource added via `add()` become part of the active graph automatically. This is how a construct like `export const getTodos = lambda({ ... })` maps to the 4–6 actual AWS resources required to run it. Resource groups are collected during graph construction and used by the reconciler to determine the full set of resources to deploy or destroy. diff --git a/docs/internals/state.md b/docs/internals/state.md index 1cf000a..44d5df0 100644 --- a/docs/internals/state.md +++ b/docs/internals/state.md @@ -13,6 +13,7 @@ Each resource entry records everything needed to diff, update, or delete the res ```json { "my-api-lambda-getTodos": { + "rev": 3, "id": "my-api-lambda-getTodos", "type": "aws/lambda/LambdaFunction", "config": { @@ -40,6 +41,7 @@ Each resource entry records everything needed to diff, update, or delete the res Key fields: - **`id`** – unique identifier derived from the resource's position in the graph +- **`rev`** – monotonically increasing revision used for compare-and-swap writes - **`type`** – the resource type string (e.g., `aws/lambda/LambdaFunction`) - **`config`** – user-facing configuration values - **`params`** – the full set of parameters sent to the cloud provider @@ -49,7 +51,7 @@ Key fields: ## Backends -Two built-in backends: +Three built-in backends: ### `FileStateBackend` (default) @@ -67,21 +69,39 @@ In-memory backend used for testing. Deep-clones on read and write to simulate pe const state = new MemoryStateBackend(); ``` +### `SqliteStateBackend` + +Stores state and leases in SQLite. Select it in the CLI by setting +`NOTATION_STATE_PATH` to a path ending in `.db` or `.sqlite`. + +```ts +const state = new SqliteStateBackend(".notation/state.db"); +``` + ### `StateBackend` interface -Both backends implement the same interface: +All backends implement the same interface: ```ts [@notation/state/src/backend.ts] interface StateBackend { get(id: string): Promise; has(id: string): Promise; - update(id: string, patch: Partial): Promise; - delete(id: string): Promise; + update( + id: string, + patch: Partial, + expectedRev?: number, + ): Promise<{ rev: number }>; + delete(id: string, expectedRev?: number): Promise; values(): Promise; + lease(scope: string, ttl: number): Promise; } ``` -The interface is deliberately minimal. There are no queries, transactions, or locking. Each operation targets a single resource by ID, which will make it straightforward to add new backends (e.g., Sqlite, S3). +Every backend provides compare-and-swap writes and renewable exclusive leases. The +reconciler holds a per-resource lease across the provider operation and state write, so +concurrent deploys cannot both perform the same create or update. It renews long-running +leases until the mutation finishes. Orphan deletion additionally holds a snapshot lease +while it decides which state records no longer appear in the desired graph. ## How state is used diff --git a/docs/manual/introduction.md b/docs/manual/introduction.md index 771ae6a..bb629e0 100644 --- a/docs/manual/introduction.md +++ b/docs/manual/introduction.md @@ -11,12 +11,15 @@ const todoRouter = router(todoApi); todoRouter.get("/todos", getTodos); ``` -Notation is a compiler, reconciler, and deployment engine. +Notation is a compiler, reconciler, and deployment engine. -The compiler runs two passes over your codebase: +The reconciler is also available as an embedded library. A Node.js host can construct +resources, choose a state backend, and run plan, deploy, or destroy without the CLI. -1. An infrastructure pass that resolves resource declarations into a graph, -2. A function pass that bundles each `.fn.ts` module into a deployable Lambda artifact. +The compiler runs two passes over your codebase: + +1. An infrastructure pass that resolves resource declarations into a graph, +2. A function pass that bundles each `.fn.ts` module into a deployable Lambda artifact. The reconciler diffs the compiled graph against persisted state and produces a plan. The deployment engine executes that plan, provisioning, updating, or destroying resources in the correct order. diff --git a/docs/manual/reconciler.md b/docs/manual/reconciler.md new file mode 100644 index 0000000..87b02f7 --- /dev/null +++ b/docs/manual/reconciler.md @@ -0,0 +1,55 @@ +# Reconciler + +Use the reconciler directly when a Node.js application needs to deploy resources without +starting the Notation CLI. + +This complete program deploys two static sites and keeps their deployment state in +SQLite: + +```ts +import { Reconciler, createResourceRegistry } from "@notation/reconciler"; +import { SqliteStateBackend } from "@notation/state-sqlite"; +import { StaticSite } from "./static-site"; + +const state = new SqliteStateBackend("sites.db"); + +const resources = [ + new StaticSite({ + id: "documentation", + config: { + siteDirectory: "sites/docs", + html: "

Documentation

\n", + }, + }), + new StaticSite({ + id: "status", + config: { + siteDirectory: "sites/status", + html: "

All systems operational

\n", + }, + }), +]; + +const reconciler = new Reconciler({ + state, + registry: createResourceRegistry([StaticSite]), +}); + +try { + await reconciler.deploy(resources); +} finally { + state.close(); +} +``` + +`StaticSite` contains the provider operations which create, read, update, and delete a +site. A real provider would call its infrastructure API instead of writing local files. + +Pass the complete desired set to `deploy`. A resource which remains in deployment state +but is absent from that set is deleted. The explicit registry lets the reconciler find +its delete operation. + +Notation's state records what was deployed. It does not replace application data which +owns the desired configuration. + +The runnable version is in `examples/reconciler`. diff --git a/docs/resources/lambda.md b/docs/resources/lambda.md index 34cf431..397591d 100644 --- a/docs/resources/lambda.md +++ b/docs/resources/lambda.md @@ -44,7 +44,10 @@ export const config: LambdaConfig = { ```ts [dist/todos.fn.ts] export const getTodos = async () => { - return { body: JSON.stringify({ id: 1, text: "Build with Notation" }), statusCode: 200 }; + return { + body: JSON.stringify({ id: 1, text: "Build with Notation" }), + statusCode: 200, + }; }; ``` diff --git a/docs/rfcs/reconciler.md b/docs/rfcs/reconciler.md new file mode 100644 index 0000000..dd33e92 --- /dev/null +++ b/docs/rfcs/reconciler.md @@ -0,0 +1,102 @@ +# RFC: Reconciler + +**Status:** implemented +**Scope:** `@notation/state`, `@notation/reconciler` + +Notation evaluates an infrastructure program into resources, then reconciles those +resources against recorded state. The same engine now runs behind the CLI, the dashboard, +and direct library integrations. + +```ts +import { Reconciler } from "@notation/reconciler"; +import { SqliteStateBackend } from "@notation/state-sqlite"; + +const state = new SqliteStateBackend(".notation/state.db"); +const reconciler = new Reconciler({ state }); + +await reconciler.deploy(resources); +state.close(); +``` + +The reconciler boundary consists of live resource objects, a state backend, and an event +subscriber. Resource operations run in the host process. + +## State + +Each state record carries a revision. Updates and deletes can require the revision which +the caller previously read: + +```ts +await state.update(resource.id, patch, resource.rev); +``` + +A stale writer receives `RevConflict`. A missing record has revision zero, so +`expectedRev: 0` means that the record must not exist. + +The reconciler also takes a renewable per-resource lease before it reads a resource for +mutation. The lease remains held across the provider operation and state write. Two +hosts therefore cannot create or update the same resource concurrently through the same +backend. + +Orphan deletion takes an additional snapshot lease. The snapshot remains stable while +the reconciler decides which state records no longer appear in the desired graph. + +## Backends + +`@notation/state` provides file and memory backends. `@notation/state-sqlite` provides +the reference database backend. + +Every backend implements the same contract: + +```ts +interface StateBackend { + get(id: string): Promise; + has(id: string): Promise; + update( + id: string, + patch: Partial, + expectedRev?: number, + ): Promise<{ rev: number }>; + delete(id: string, expectedRev?: number): Promise; + values(): Promise; + lease(scope: string, ttl: number): Promise; +} +``` + +The dashboard reads this interface. It does not inspect a state file directly. + +## Events + +The reconciler accepts one subscriber: + +```ts +const reconciler = new Reconciler({ + state, + emit: async (event) => auditLog.write(event), +}); +``` + +`createNdjsonEventEmitter` adapts the subscriber to a versioned newline-delimited JSON +stream. The CLI uses the same adapter for `deploy --json` and `destroy --json`. + +## Package boundary + +The CLI creates resources from compiled Notation programs, then hands those live objects +to `Reconciler`. An application can construct the same resource classes directly. + +The reconciler does not serialise resource classes or execute operations in another +process. Detached execution needs manifests, resource-reference encoding, actuator +binding, and a runtime consumer. That work has its own RFC and release. + +## Acceptance + +The reconciler example is the compatibility test for this boundary. It must: + +1. Construct a resource without the CLI. +2. Plan and deploy it through `Reconciler`. +3. Close and reopen SQLite state. +4. Plan and apply an update. +5. Receive versioned events. +6. Destroy the resource and remove its state. + +The example lives in `examples/reconciler` and runs without cloud credentials.