diff --git a/.changeset/webhook-cache-crypto-registration-ordering.md b/.changeset/webhook-cache-crypto-registration-ordering.md new file mode 100644 index 0000000000..baf7b3fb00 --- /dev/null +++ b/.changeset/webhook-cache-crypto-registration-ordering.md @@ -0,0 +1,55 @@ +--- +"@objectstack/plugin-webhooks": patch +"@objectstack/objectql": patch +--- + +fix(plugin-webhooks): a webhook holding an encrypted signing secret re-arms the moment the CryptoProvider registers, instead of ~60s later (#8022) + +For roughly **60 seconds after every server start**, a webhook whose +`signing_secret` is encrypted (the population #7799 created) was **not +subscribed**. A record change in that window produced no delivery **and no +`sys_http_delivery` row at all** — no dead letter, no retry, no durable trace +that anything was missed — while `GET /api/v1/data/sys_webhook/` kept reading +`active: true`, so the webhook looked armed in Setup the whole time. It +self-healed at the next periodic cache refresh, which is why it was invisible to +anyone not watching that window. + +**The fail-closed behaviour is unchanged and is not the bug.** Dropping a +subscription whose stored key cannot be recovered — rather than delivering it +unsigned — is #7799's whole point and still holds: the signature is the +receiver's only proof of origin, and a webhook that stops arriving gets +investigated while one that keeps arriving unsigned teaches the receiver to +accept unauthenticated traffic. What was wrong is that a fail-closed drop +outlived its own cause. + +**The ordering.** It was never a race that sometimes went the other way. Plugins +run inside `kernel:ready`, which `runtime.start()` completes; the host's +composition root calls `engine.setCryptoProvider(...)` only *after* +`runtime.start()` returns (`packages/cli/src/commands/serve.ts`, +`packages/verify/src/harness.ts`). So `AutoEnqueuer`'s first subscription-cache +build reliably preceded the capability it needs, dropped every secret-bearing +row on what it could see, and nothing re-read until the periodic refresh. + +`ObjectQL` now reports the registration (`onCryptoProviderChange(listener)`, +fired after the provider is in place), and the auto-enqueuer subscribes +**before** its first build and rebuilds the cache when it fires. Re-arming is +immediate and event-driven — no polling, and no shorter-but-still-present +window. The re-arm deliberately does not join an in-flight refresh: the build +most likely running at that moment is the pre-registration one, and joining it +would report success having re-armed nothing. + +The channel is feature-detected, as `resolveSecretField` already was — this +plugin takes no dependency on `@objectstack/objectql`. An engine without it keeps +the previous behaviour, with the periodic refresh as the backstop. + +**The drop is also no longer quiet.** A subscription dropped for an unresolvable +key now reports at `error` with the consequence and the fix stated in the +message, and carries an ADR-0112 `code`/`status` pair (`INTERNAL_ERROR`/500) in +its metadata — the same pair the seeder's refusal for the same cause already +carried. Per AGENTS.md it is said **once** per outage per webhook rather than +every refresh cycle, and a webhook that recovers and breaks again is loud again. + +Unaffected, and verified still true: the secret's bytes appear nowhere in +`sys_webhook` or in a delivery row, deliveries carry `signature` and never the +key (#7722), and a delivery whose key exists only as ciphertext after a restart +still produces the byte-identical HMAC receivers already verify. diff --git a/packages/objectql/src/engine.ts b/packages/objectql/src/engine.ts index c08be93f7d..7f3da68bff 100644 --- a/packages/objectql/src/engine.ts +++ b/packages/objectql/src/engine.ts @@ -1661,6 +1661,16 @@ export class ObjectQL implements IObjectQLEngine { // persists cleartext). Injected by the host via setCryptoProvider(). private cryptoProvider?: ICryptoProvider; + // [#8022] Listeners notified when a crypto provider is (re)registered. + // Server-side consumers that dereference a secret at BOOT — the webhook + // auto-enqueuer's subscription cache is the one this was built for — run + // inside `kernel:ready`, which every host completes BEFORE its composition + // root injects a provider. Their first read therefore fails closed against a + // capability that is about to exist, and without a notification the only way + // back is to poll. The engine is the sole party that knows the moment it + // arrives, so the notification belongs here. + private readonly cryptoProviderListeners = new Set<() => void>(); + // [ADR-0105 D2 / #3623] Posture accessor for driver-scope widening under the // `group` posture. Injected by SecurityPlugin via setTenancyPostureProvider(); // absent = equality scoping (fail toward isolation). @@ -4613,10 +4623,51 @@ export class ObjectQL implements IObjectQLEngine { * Mirrors the Settings subsystem's ICryptoProvider wiring; the host (e.g. * `serve`) injects `LocalCryptoProvider` in dev and a KMS/Vault-backed * provider in production. + * + * Notifies {@link onCryptoProviderChange} listeners AFTER the provider is in + * place, so a listener that immediately re-reads a secret sees the new + * capability rather than the state that made it fail (#8022). */ setCryptoProvider(provider: ICryptoProvider): void { this.cryptoProvider = provider; this.logger.info('CryptoProvider configured for secret fields'); + // A listener is a re-arm, never part of this call's contract: one that + // throws must not fail the host's composition root, and must not stop the + // listeners behind it from re-arming. + for (const listener of [...this.cryptoProviderListeners]) { + try { + listener(); + } catch (err) { + this.logger.warn('CryptoProvider registration listener failed', { + error: (err as Error)?.message ?? String(err), + }); + } + } + } + + /** + * [#8022] Observe crypto-provider registration. + * + * Exists for consumers that must dereference a `secret` field on a schedule + * they do not control — the boot path. `secret` reads are fail-closed by + * design (#7799), which is correct, but "no provider" at boot is a + * *transient* state on every host: `kernel:ready` runs plugins, and only + * after `runtime.start()` returns does the composition root call + * {@link setCryptoProvider}. A consumer whose cache was built in that gap is + * wrong until it rebuilds, and polling is the only alternative to being told. + * + * Fires on every registration, including a later replacement (a KMS provider + * swapped in over the dev one) — a listener that re-reads is correct in both + * cases, and a re-read is cheap next to signing with a key from the wrong + * provider. + * + * @returns an unsubscribe function; call it when the listener's owner stops. + */ + onCryptoProviderChange(listener: () => void): () => void { + this.cryptoProviderListeners.add(listener); + return () => { + this.cryptoProviderListeners.delete(listener); + }; } /** diff --git a/packages/plugins/plugin-webhooks/src/auto-enqueuer.ts b/packages/plugins/plugin-webhooks/src/auto-enqueuer.ts index 47afd95c2a..355c951610 100644 --- a/packages/plugins/plugin-webhooks/src/auto-enqueuer.ts +++ b/packages/plugins/plugin-webhooks/src/auto-enqueuer.ts @@ -3,7 +3,14 @@ import type { IDataEngine, IRealtimeService, RealtimeEventPayload } from '@objectstack/spec/contracts'; import type { WebhookTriggerType } from '@objectstack/spec/automation'; import type { EnqueueHttpInput } from '@objectstack/service-messaging'; -import { WEBHOOK_SECRET_FIELD, readLegacySecret, resolveWebhookSecret } from './webhook-secret.js'; +import { + WEBHOOK_SECRET_FIELD, + WEBHOOK_SECRET_REFUSAL_CODE, + WEBHOOK_SECRET_REFUSAL_STATUS, + onCryptoProviderChange, + readLegacySecret, + resolveWebhookSecret, +} from './webhook-secret.js'; /** * The authored trigger vocabulary, taken from the spec rather than restated @@ -116,6 +123,16 @@ export class AutoEnqueuer { private refreshTimer: ReturnType | undefined; private running = false; private refreshing: Promise | undefined; + /** [#8022] Detach for the engine's crypto-registration listener. */ + private unbindCryptoListener: (() => void) | undefined; + /** + * [#8022] Webhook ids currently dropped for an unresolvable signing key. + * Held so the loud first report is said ONCE per outage (AGENTS.md + * "Degradation log levels": *say it once, at the first degradation*) and + * again if the same webhook breaks after recovering — not once per row per + * refresh, forever. + */ + private readonly droppedForSecret = new Set(); constructor( private readonly engine: IDataEngine, @@ -135,6 +152,17 @@ export class AutoEnqueuer { if (this.running) return; this.running = true; + // [#8022] Bound BEFORE the first build, not after: on every host the + // composition root wires the CryptoProvider after `runtime.start()` + // returns, i.e. after the `kernel:ready` handler that runs this method + // — so the registration we need to hear about can land at any point + // from here on, including while the await below is still in flight. + // Subscribing first makes that unmissable; subscribing after the + // refresh would reintroduce the same race in miniature. + this.unbindCryptoListener = onCryptoProviderChange(this.engine, () => + this.rearmAfterCryptoRegistered(), + ); + await this.refresh(); // Main subscription: every data event → match → enqueue. @@ -167,9 +195,38 @@ export class AutoEnqueuer { if (this.subId) await this.realtime.unsubscribe(this.subId); if (this.subIdSelfHeal) await this.realtime.unsubscribe(this.subIdSelfHeal); if (this.refreshTimer) clearInterval(this.refreshTimer); + this.unbindCryptoListener?.(); this.subId = undefined; this.subIdSelfHeal = undefined; this.refreshTimer = undefined; + this.unbindCryptoListener = undefined; + } + + /** + * [#8022] The engine just gained a CryptoProvider — rebuild the cache so + * subscriptions dropped for an unresolvable signing key re-arm now, instead + * of at the next periodic refresh up to {@link refreshIntervalMs} away. + * + * It deliberately does NOT call {@link refresh} directly. `refresh()` + * coalesces onto an in-flight build, and the build most likely to be in + * flight right now is the one from `start()` — the very build whose rows + * were read while there was no provider. Joining it would return "refreshed" + * having re-armed nothing, which is this issue with an extra step. So: let + * whatever is running finish, then read again. + */ + private rearmAfterCryptoRegistered(): void { + const inFlight = this.refreshing ?? Promise.resolve(); + void inFlight + // A failed in-flight refresh already logged; it must not stop the + // re-arm, which is the whole point of this callback. + .catch(() => undefined) + .then(() => (this.running ? this.refresh() : undefined)) + .catch((err) => + this.logger.warn?.( + '[webhook-auto-enqueuer] re-arm after CryptoProvider registration failed', + err, + ), + ); } /** @@ -220,6 +277,17 @@ export class AutoEnqueuer { this.subscriptions.clear(); for (const [k, v] of next) this.subscriptions.set(k, v); + // [#8022] Forget rows this refresh no longer sees — deleted, or + // deactivated. Otherwise the set grows for the life of the process, and + // a webhook turned off while broken and later turned back on still + // broken would have its first report suppressed as a repeat. + if (this.droppedForSecret.size > 0) { + const live = new Set(rows.map((r) => String(r?.id))); + for (const id of this.droppedForSecret) { + if (!live.has(id)) this.droppedForSecret.delete(id); + } + } + this.logger.debug?.('[webhook-auto-enqueuer] cache refreshed', { objects: this.subscriptions.size, rows: rows.length, @@ -258,15 +326,13 @@ export class AutoEnqueuer { const stored = await resolveWebhookSecret(this.engine, row, this.subscriptionsObject); if (stored) { sub.secret = stored; + // Recovered — a later break is a new outage and gets said loudly + // again rather than being swallowed as a repeat. + this.droppedForSecret.delete(sub.id); return true; } } catch (err) { - this.logger.warn?.( - `[webhook-auto-enqueuer] webhook '${sub.name}' holds an encrypted signing secret that ` + - `could not be decrypted — the subscription is DROPPED rather than delivered unsigned ` + - `(#7799). Deliveries resume once the sys_secret row and CryptoProvider are reachable.`, - { id: sub.id, field: WEBHOOK_SECRET_FIELD, err: (err as Error)?.message ?? err }, - ); + this.reportDrop(sub, err); return false; } @@ -281,9 +347,72 @@ export class AutoEnqueuer { ); sub.secret = legacy; } + this.droppedForSecret.delete(sub.id); return true; } + /** + * [#8022] Report a subscription dropped for an unresolvable signing key. + * + * ## Why `error`, and why only the first time + * AGENTS.md decides the level with one question: *after the degradation, + * does the system still look normal from the outside while something the + * system claims is happening is not?* Here the answer is yes, and it is the + * whole defect — `GET /api/v1/data/sys_webhook` keeps reading + * `active: true`, Setup keeps showing the webhook armed, and every matching + * record change is discarded with no delivery and no `sys_http_delivery` + * row to find afterwards. That is a durability degradation wearing a + * functional degradation's clothes, so it owes the two things an `error` + * owes: the consequence, concretely, and the fix. + * + * Said ONCE per outage per webhook, per the same section. The cache is + * rebuilt every {@link refreshIntervalMs}; an unfixed misconfiguration would + * otherwise print this line every 60s forever, which is how an `error` + * channel becomes unreadable — the failure mode that made the founding + * incident's `warn` invisible. Repeats drop to `debug`; a recovery clears + * the id, so a re-break is loud again. + * + * ADR-0112: `code` + `status` travel in the meta so a consumer branches on + * the pair, not on message text. Same pair the seeder's refusal carries for + * the same underlying cause. + */ + private reportDrop(sub: CachedSubscription, err: unknown): void { + const meta = { + id: sub.id, + webhook: sub.name, + field: WEBHOOK_SECRET_FIELD, + code: WEBHOOK_SECRET_REFUSAL_CODE, + status: WEBHOOK_SECRET_REFUSAL_STATUS, + err: (err as Error)?.message ?? err, + }; + if (this.droppedForSecret.has(sub.id)) { + this.logger.debug?.( + `[webhook-auto-enqueuer] webhook '${sub.name}' is still dropped for an unresolvable ` + + 'signing secret (#7799/#8022)', + meta, + ); + return; + } + this.droppedForSecret.add(sub.id); + const message = + `[webhook-auto-enqueuer] webhook '${sub.name}' holds an encrypted signing secret that ` + + 'could not be decrypted — the subscription is DROPPED rather than delivered unsigned ' + + '(#7799), so every matching record change is discarded with NO delivery and NO ' + + 'sys_http_delivery row, while the row keeps reading active:true in Setup. Fix: register a ' + + 'CryptoProvider (engine.setCryptoProvider — LocalCryptoProvider in dev, KMS/Vault in ' + + 'production) with the same key the secret was written under, and make sure the sys_secret ' + + 'row is reachable; the subscription re-arms on registration (#8022) and at the next ' + + 'periodic refresh.'; + // The logger surface is a subset of console/kernel logger — `error` is + // optional on it, so fall back rather than silently losing the report + // on a logger that only implements `warn`. + if (typeof this.logger.error === 'function') { + this.logger.error(message, err, meta); + } else { + this.logger.warn?.(message, meta); + } + } + private parseRow(row: any): CachedSubscription | null { if (!row?.id || !row?.url) return null; // `triggers` is now authored as a multi-select (stored as an array), but diff --git a/packages/plugins/plugin-webhooks/src/webhook-secret-at-rest.test.ts b/packages/plugins/plugin-webhooks/src/webhook-secret-at-rest.test.ts index e8ce1f9a3d..2a0166a19b 100644 --- a/packages/plugins/plugin-webhooks/src/webhook-secret-at-rest.test.ts +++ b/packages/plugins/plugin-webhooks/src/webhook-secret-at-rest.test.ts @@ -176,9 +176,19 @@ const sysSecretObject = { }, }; -async function buildEngine(opts: { withCrypto?: boolean } = {}) { +/** + * A booted engine over a store. + * + * `reuse` hands back the SAME driver (and therefore the same rows) under a + * fresh engine — a process restart against an existing database, which is the + * only way to reach a state where the ciphertext predates the engine reading + * it (#8022). + */ +async function buildEngine( + opts: { withCrypto?: boolean; reuse?: { driver: any; stores: Map>> } } = {}, +) { const engine = new ObjectQL(); - const { driver, stores } = makeStubDriver(); + const { driver, stores } = opts.reuse ?? makeStubDriver(); engine.registerDriver(driver, true); await engine.init(); engine.registry.registerObject(sysSecretObject as any, 'test'); @@ -455,3 +465,139 @@ describe('fail-closed: no CryptoProvider (#7799)', () => { expect(row[WEBHOOK_SECRET_FIELD] ?? null).toBeNull(); }); }); + +/** + * #8022 — the boot window #7799 opened. + * + * Every host wires its CryptoProvider from the composition root AFTER + * `runtime.start()` returns, and `runtime.start()` is what runs `kernel:ready` + * — the hook under which `AutoEnqueuer.start()` builds its first subscription + * cache. So the first build does not merely *race* crypto registration, it + * reliably precedes it: on `packages/cli/src/commands/serve.ts` the + * `setCryptoProvider` call sits below `await runtime.start()` unconditionally. + * A secret-bearing webhook was therefore dropped on every boot — correctly, on + * what the enqueuer could see — and nothing re-armed it until the periodic + * refresh up to 60s later. In that window a record change produced no delivery + * AND no `sys_http_delivery` row: not a dead letter, not a retry, nothing. + * + * The fail-closed drop is NOT what these tests relax. #7799's refusal to + * deliver unsigned is asserted below to still hold, before and after. What is + * fixed is the ORDERING: the drop must not outlive the reason for it. + */ +describe('boot ordering: the cache is built before the CryptoProvider (#8022)', () => { + /** Reboot onto the same rows in the host's real order: kernel first, crypto after. */ + async function bootWithoutCrypto() { + const first = await buildEngine(); + await bootstrapDeclaredWebhooks(first.engine, metadataWith([declaredWebhook()])); + // Precondition: the key is at rest as ciphertext only — the exact + // population #7799 created, and the only one this defect can reach. + expect(first.stores.get('sys_secret')!.size).toBe(1); + + return buildEngine({ + withCrypto: false, + reuse: { driver: first.driver, stores: first.stores }, + }); + } + + /** Let the engine-driven re-arm settle. Two macrotasks — no polling, no 60s. */ + const settle = async () => { + await new Promise((r) => setTimeout(r, 0)); + await new Promise((r) => setTimeout(r, 0)); + }; + + it('re-arms the dropped subscription when the CryptoProvider registers, without the 60s refresh', async () => { + const { engine } = await bootWithoutCrypto(); + const realtime = new FakeRealtime(); + const outbox = new MemoryHttpOutbox(); + const enqueuer = new AutoEnqueuer(engine, realtime, (i) => outbox.enqueue(i), { + // The escape hatch this defect self-heals through, held shut. With a + // periodic refresh armed, a passing test proves only that waiting + // works — which it already did, 60s late. Zero means the ONLY thing + // that can re-arm the cache is the registration itself. + refreshIntervalMs: 0, + logger: { error: () => {}, warn: () => {} }, + }); + await enqueuer.start(); + + // ── The window, as the filer measured it ────────────────────────── + // Asserted on the durable record, not on a cache internal: a mutation + // here reaches neither the receiver nor `sys_http_delivery`. This is + // the state BOTH revisions are in at this point — it is the next half + // that separates them. + await realtime.publish(recordEvent('contact', { id: 'c_window', name: 'Ada' })); + await settle(); + expect(await outbox.list()).toHaveLength(0); + + // ── The composition root wires crypto, exactly as `serve` does ───── + engine.setCryptoProvider(makeFakeCrypto()); + await settle(); + + // ── A mutation in what used to be the hole ──────────────────────── + await realtime.publish(recordEvent('contact', { id: 'c_rearmed', name: 'Grace' })); + await settle(); + const { impl, calls } = makeFetch(); + await new HttpDispatcher({ nodeId: 'n1', outbox, fetchImpl: impl, partitionCount: 1 }).tick(); + await enqueuer.stop(); + + // The durable record exists… + const rows = await outbox.list(); + expect(rows).toHaveLength(1); + expect(rows[0].refId).toBe(Array.from((await engine.find('sys_webhook', {})).map((r: any) => r.id))[0]); + // …the receiver was actually hit, and #7799's whole point still holds: + // the delivery is SIGNED, with the key recovered from ciphertext alone. + expect(calls).toHaveLength(1); + const expected = createHmac('sha256', SECRET).update(calls[0].body).digest('hex'); + expect(calls[0].headers['X-Objectstack-Signature']).toBe(`sha256=${expected}`); + expect(JSON.parse(calls[0].body)).toMatchObject({ object: 'contact', recordId: 'c_rearmed' }); + // …and #7722's: the row carries the signature, never the key. + expect(rows[0].signature).toBe(`sha256=${expected}`); + expect(JSON.stringify(rows)).not.toContain(SECRET); + }); + + // NOT pinned here, deliberately: that the re-arm does not COALESCE onto the + // in-flight pre-registration build (see `rearmAfterCryptoRegistered`). Every + // harness that can inject the registration at that instant also moves the + // secret resolution to after it, so the naive `() => this.refresh()` passes + // too — a test that cannot separate the two revisions is a false green, and + // this repo has paid for those. The guard stays because the reasoning is + // sound, not because a test proves it. + + it('still refuses to deliver unsigned while the key stays unresolvable, and says so once, loudly', async () => { + const { engine } = await bootWithoutCrypto(); + const realtime = new FakeRealtime(); + const outbox = new MemoryHttpOutbox(); + const errors: Array<{ msg: string; meta: any }> = []; + const debugs: string[] = []; + const enqueuer = new AutoEnqueuer(engine, realtime, (i) => outbox.enqueue(i), { + refreshIntervalMs: 0, + logger: { + error: (msg: string, _err?: unknown, meta?: unknown) => { errors.push({ msg, meta: meta as any }); }, + debug: (msg: string) => { debugs.push(msg); }, + warn: () => {}, + }, + }); + await enqueuer.start(); + // No provider ever arrives. Rebuild anyway — the periodic refresh would. + await enqueuer.refresh(); + await realtime.publish(recordEvent('contact', { id: 'c1', name: 'Ada' })); + await settle(); + const { impl, calls } = makeFetch(); + await new HttpDispatcher({ nodeId: 'n1', outbox, fetchImpl: impl, partitionCount: 1 }).tick(); + await enqueuer.stop(); + + // The #7799 boundary: nothing is delivered, and nothing is delivered + // UNSIGNED, which is the outcome this whole card must not buy. + expect(calls).toHaveLength(0); + expect(await outbox.list()).toHaveLength(0); + + // ADR-0112 — a consumer branches on the pair, not on message text. + expect(errors).toHaveLength(1); + expect(errors[0].meta).toMatchObject({ code: 'INTERNAL_ERROR', status: 500 }); + // An `error` owes the consequence and the fix (AGENTS.md), and owes + // them ONCE: the second refresh repeats at debug, not at error, or an + // unfixed deployment prints this every 60s until nobody reads `error`. + expect(errors[0].msg).toMatch(/NO delivery and NO sys_http_delivery row/); + expect(errors[0].msg).toMatch(/setCryptoProvider/); + expect(debugs.join('\n')).toMatch(/still dropped for an unresolvable signing secret/); + }); +}); diff --git a/packages/plugins/plugin-webhooks/src/webhook-secret.ts b/packages/plugins/plugin-webhooks/src/webhook-secret.ts index 1d1b07f5f6..ec3632b644 100644 --- a/packages/plugins/plugin-webhooks/src/webhook-secret.ts +++ b/packages/plugins/plugin-webhooks/src/webhook-secret.ts @@ -149,6 +149,7 @@ export const __objectqlSecretWireForms = { /** Engines that expose the privileged dereference (ObjectQL ≥ #7799). */ type SecretResolvingEngine = IDataEngine & { resolveSecretField?(object: string, recordId: string, field: string): Promise; + onCryptoProviderChange?(listener: () => void): () => void; }; /** True when this engine can dereference an encrypted field. */ @@ -156,6 +157,36 @@ export function canResolveSecrets(engine: IDataEngine | undefined): boolean { return typeof (engine as SecretResolvingEngine | undefined)?.resolveSecretField === 'function'; } +/** + * [#8022] Subscribe to the engine's crypto-provider registration. Returns an + * unsubscribe function, or `undefined` when the engine has no such channel. + * + * ## Why this exists + * Resolving a stored key stays fail-closed (#7799) — that is not what this + * changes. What it changes is how long a fail-closed READ is allowed to stand + * when the reason for it is about to disappear. "No CryptoProvider" is not only + * a misconfiguration: on every host it is also a *transient boot state*, because + * plugins run inside `kernel:ready` and the composition root injects the + * provider only after `runtime.start()` returns. So the enqueuer's FIRST cache + * build reliably precedes the capability it needs, drops every secret-bearing + * subscription (correctly, on what it could see), and — before this — stayed + * dropped until the next periodic refresh 60s later. + * + * Feature-detected rather than required, exactly like `resolveSecretField` + * above, because this package deliberately takes no dependency on + * `@objectstack/objectql`. An engine without the channel keeps the previous + * behaviour — the periodic refresh remains the backstop — rather than failing + * to start. + */ +export function onCryptoProviderChange( + engine: IDataEngine | undefined, + listener: () => void, +): (() => void) | undefined { + const observable = engine as SecretResolvingEngine | undefined; + if (typeof observable?.onCryptoProviderChange !== 'function') return undefined; + return observable.onCryptoProviderChange(listener); +} + /** * Recover a row's signing key. Returns `undefined` when the row has no stored * key — which is not an error: `secret` is optional on the authoring envelope,