From 35d1663c803fc036a8e2e510de641aa86731315a Mon Sep 17 00:00:00 2001 From: Brad Harris Date: Fri, 7 Aug 2026 16:00:51 -0600 Subject: [PATCH 1/4] Harden assisted update against legacy pinned-symlink Linux installs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On ≤v0.31.x Linux installs whose ExecStart resolves through a symlink hard-pinned to a versioned dist/bun binary, the managed-update-first ordering in migration 0010 extracted the target release but restarted back into the old runtime, whose boot-time release-binary pruning then deleted the new artifact — while validations false-greened because they trusted release.json rather than the running executable. - Reorder migration 0010 for Linux legacy shapes: inspect the entrypoint first and perform the fixed-runtime cutover (stage exact checksum- verified target, preserve .previous, activate fixed path, repoint ExecStart) BEFORE the first restart; macOS keeps the launchd bridge. Adds legacy bin/dispatch symlink + unit-backup hygiene and makes rollback prefer the fixed runtime .previous over legacy unit backups. - New running_version required check that proves the actually running executable via the X-Dispatch-Version header; the assisted framework enforces it implicitly at launch and at check-run time (manifests can't name it yet — pre-v0.33 parsers reject unknown check names). - expected_runtime_artifact now uses lstat and rejects a symlinked fixed runtime path. - Manifest requiredChecks schema accepts unknown well-formed names so future check additions can't silently drop a whole migration on older installs; unknown names fail closed at run time instead. A test guards that shipped manifests stay within the legacy-parser-safe set. - VM fixture script (scripts/vm-fixtures/legacy-pinned-symlink.sh) that converts a healthy Linux install into the pinned-symlink legacy shape for the documented VM validation row. Co-Authored-By: Claude Fable 5 --- apps/server/src/assisted-update-store.ts | 9 +- apps/server/src/assisted-update.ts | 28 +++-- apps/server/src/release-checks.ts | 102 ++++++++++++++-- apps/server/src/release-metadata.ts | 7 ++ apps/server/src/update-migrations.ts | 12 +- .../test/assisted-update-migrations.test.ts | 56 ++++++++- apps/server/test/assisted-update.test.ts | 5 +- apps/server/test/release-checks.test.ts | 115 +++++++++++++++++- apps/server/test/update-migrations.test.ts | 72 ++++++++++- docs/vm-release-validation.md | 4 + scripts/vm-fixtures/legacy-pinned-symlink.sh | 91 ++++++++++++++ .../0010-fixed-runtime-entrypoint.yaml | 95 ++++++++++++--- 12 files changed, 543 insertions(+), 53 deletions(-) create mode 100755 scripts/vm-fixtures/legacy-pinned-symlink.sh diff --git a/apps/server/src/assisted-update-store.ts b/apps/server/src/assisted-update-store.ts index 9801f17a3..2fd6a5078 100644 --- a/apps/server/src/assisted-update-store.ts +++ b/apps/server/src/assisted-update-store.ts @@ -2,10 +2,7 @@ import { randomBytes } from "node:crypto"; import os from "node:os"; import path from "node:path"; import { mkdir, readFile, rename, writeFile } from "node:fs/promises"; -import type { - AssistedUpdateMetadata, - RequiredCheckName, -} from "./release-metadata.js"; +import type { AssistedUpdateMetadata } from "./release-metadata.js"; import type { CheckResult } from "./release-checks.js"; import type { UpdateMigrationManifest } from "./update-migrations.js"; @@ -54,7 +51,9 @@ export type AssistedUpdateState = { * `metadata` for the legacy single-block flow. */ migrations: UpdateMigrationManifest[] | null; - requiredChecks: RequiredCheckName[]; + /** Usually RequiredCheckName values; manifests from a newer release may + * name checks this runtime doesn't know (they fail closed when run). */ + requiredChecks: string[]; phase: AssistedPhase; /** Random nonce; the launched agent uses this to authenticate phase POSTs. */ token: string; diff --git a/apps/server/src/assisted-update.ts b/apps/server/src/assisted-update.ts index fea66e24c..11958f09b 100644 --- a/apps/server/src/assisted-update.ts +++ b/apps/server/src/assisted-update.ts @@ -16,7 +16,6 @@ import { type AssistedUpdateState, } from "./assisted-update-store.js"; import { runRequiredChecks, type CheckContext } from "./release-checks.js"; -import type { RequiredCheckName } from "./release-metadata.js"; import type { UpdateMigrationManifest } from "./update-migrations.js"; import { markMigrationsApplied } from "./applied-migrations-store.js"; import { clearEvaluatorCache } from "./update-migrations-evaluator.js"; @@ -74,9 +73,17 @@ export async function buildAssistedUpdateContext( "buildAssistedUpdateContext requires either migrations or metadata" ); } - const requiredChecks: RequiredCheckName[] = migrations + const requiredChecks: string[] = migrations ? unionMigrationChecks(migrations) : normalizeRequiredChecks(metadata!); + // Always prove the running executable's version, even when no manifest + // asks for it. Manifests can't name this check yet (pre-v0.33 runtimes + // reject unknown check names at parse time), and release.json-based + // checks alone false-green when a pinned service entrypoint restarts back + // into the old binary. + if (!requiredChecks.includes("running_version")) { + requiredChecks.push("running_version"); + } const state: AssistedUpdateState = { tag: input.tag, @@ -111,11 +118,9 @@ export async function buildAssistedUpdateContext( return { state, prompt }; } -function unionMigrationChecks( - migrations: UpdateMigrationManifest[] -): RequiredCheckName[] { - const seen = new Set(); - const ordered: RequiredCheckName[] = []; +function unionMigrationChecks(migrations: UpdateMigrationManifest[]): string[] { + const seen = new Set(); + const ordered: string[] = []; for (const m of migrations) { for (const c of m.validation.requiredChecks) { if (seen.has(c)) continue; @@ -201,7 +206,14 @@ export async function runAndRecordChecks( state: AssistedUpdateState, ctx: CheckContext ): Promise { - const results = await runRequiredChecks(state.requiredChecks, ctx); + // Enforce running_version at check-run time as well as launch time: a run + // launched by an older server (whose state.requiredChecks predates the + // check) is often validated by the freshly restarted target binary — this + // code — so appending here closes the gap for in-flight upgrades. + const names = state.requiredChecks.includes("running_version") + ? state.requiredChecks + : [...state.requiredChecks, "running_version"]; + const results = await runRequiredChecks(names, ctx); state.checks = results; state.updatedAt = new Date().toISOString(); await writeAssistedUpdateState(state); diff --git a/apps/server/src/release-checks.ts b/apps/server/src/release-checks.ts index cba599b29..cada31629 100644 --- a/apps/server/src/release-checks.ts +++ b/apps/server/src/release-checks.ts @@ -1,4 +1,4 @@ -import { readFile, stat } from "node:fs/promises"; +import { lstat, readFile } from "node:fs/promises"; import https from "node:https"; import os from "node:os"; import path from "node:path"; @@ -14,13 +14,15 @@ export type CheckContext = { }; export type CheckResult = { - name: RequiredCheckName; + /** Usually a RequiredCheckName; a manifest authored for a newer runtime + * may name a check this build doesn't know (those fail closed). */ + name: string; ok: boolean; message: string; }; export async function runRequiredChecks( - names: RequiredCheckName[], + names: ReadonlyArray, ctx: CheckContext ): Promise { const results: CheckResult[] = []; @@ -38,11 +40,8 @@ export async function runRequiredChecks( return results; } -async function runCheck( - name: RequiredCheckName, - ctx: CheckContext -): Promise { - switch (name) { +async function runCheck(name: string, ctx: CheckContext): Promise { + switch (name as RequiredCheckName) { case "expected_runtime_artifact": return checkRuntimeArtifact(ctx); case "service_entrypoint": @@ -53,13 +52,34 @@ async function runCheck( return checkHealthEndpoint(ctx); case "version_converged": return checkVersionConverged(ctx); + case "running_version": + return checkRunningVersion(ctx); + default: + // Fail closed: a manifest authored for a newer runtime may require a + // check this build cannot evaluate. Passing it silently would defeat + // the point of a required check. + return { + name, + ok: false, + message: `unknown required check "${name}" — this runtime cannot evaluate it`, + }; } } async function checkRuntimeArtifact(ctx: CheckContext): Promise { const runtime = fixedRuntimePath(ctx.serverDir); try { - const info = await stat(runtime); + // lstat, not stat: a symlink at the fixed path (e.g. a legacy pin to a + // versioned dist/bun binary) must not pass as the activated runtime — + // atomic replacement and .previous rollback assume a regular file. + const info = await lstat(runtime); + if (info.isSymbolicLink()) { + return { + name: "expected_runtime_artifact", + ok: false, + message: `${runtime} is a symlink; the fixed runtime must be a regular executable`, + }; + } return info.isFile() ? { name: "expected_runtime_artifact", @@ -225,9 +245,13 @@ function isLoopbackHttps(url: string): boolean { async function fetchViaGlobal( url: string -): Promise<{ status: number; body: string }> { +): Promise<{ status: number; body: string; versionHeader: string | null }> { const res = await fetch(url, { signal: AbortSignal.timeout(5_000) }); - return { status: res.status, body: await res.text() }; + return { + status: res.status, + body: await res.text(), + versionHeader: res.headers.get("x-dispatch-version"), + }; } // The same-process self-check legitimately hits the local server's @@ -238,7 +262,7 @@ async function fetchViaGlobal( // future config points it at a remote host. function fetchLoopbackHttps( url: string -): Promise<{ status: number; body: string }> { +): Promise<{ status: number; body: string; versionHeader: string | null }> { return new Promise((resolve, reject) => { const parsed = new URL(url); const req = https.request( @@ -254,9 +278,13 @@ function fetchLoopbackHttps( const chunks: Buffer[] = []; res.on("data", (c: Buffer) => chunks.push(c)); res.on("end", () => { + const header = res.headers["x-dispatch-version"]; resolve({ status: res.statusCode ?? 0, body: Buffer.concat(chunks).toString("utf8"), + versionHeader: Array.isArray(header) + ? (header[0] ?? null) + : (header ?? null), }); }); res.on("error", reject); @@ -270,6 +298,56 @@ function fetchLoopbackHttps( }); } +/** + * Prove the executable that is actually serving requests is the target + * version. Every API response carries the build-time package version in the + * X-Dispatch-Version header, so this cannot be spoofed by an on-disk record + * — release.json only says what was *deployed*, not what is *running* (a + * version-pinned service entrypoint can restart straight back into the old + * binary while release.json claims the target). + */ +async function checkRunningVersion(ctx: CheckContext): Promise { + const url = ctx.healthUrl ?? "http://127.0.0.1:6767/api/v1/health"; + const expected = ctx.targetTag.replace(/^v/, ""); + try { + const { status, versionHeader } = isLoopbackHttps(url) + ? await fetchLoopbackHttps(url) + : await fetchViaGlobal(url); + if (status < 200 || status >= 300) { + return { + name: "running_version", + ok: false, + message: `${url} returned ${status}`, + }; + } + const running = versionHeader?.trim(); + if (!running) { + return { + name: "running_version", + ok: false, + message: `no X-Dispatch-Version header from ${url} — cannot prove the running executable's version (runtime predates version reporting or is not the target binary)`, + }; + } + return running === expected + ? { + name: "running_version", + ok: true, + message: `running executable reports ${running} (target ${ctx.targetTag})`, + } + : { + name: "running_version", + ok: false, + message: `running executable reports ${running}, expected ${expected} (target ${ctx.targetTag})`, + }; + } catch (err) { + return { + name: "running_version", + ok: false, + message: err instanceof Error ? err.message : String(err), + }; + } +} + async function checkVersionConverged(ctx: CheckContext): Promise { const record = await readReleaseStore().catch(() => null); if (!record) { diff --git a/apps/server/src/release-metadata.ts b/apps/server/src/release-metadata.ts index f15ed6f4b..7026aff5d 100644 --- a/apps/server/src/release-metadata.ts +++ b/apps/server/src/release-metadata.ts @@ -15,6 +15,13 @@ export const REQUIRED_CHECK_NAMES = [ "service_restarted", "health_endpoint", "version_converged", + // Proves the actual running executable's baked-in version (via the + // X-Dispatch-Version response header), not what release.json claims. + // Runtimes before v0.33 reject unknown check names when parsing update + // migration manifests, so keep this OUT of update-migrations/*.yaml + // requiredChecks until the pre-v0.33 population is gone — the assisted + // framework enforces it implicitly (see runAndRecordChecks). + "running_version", ] as const; export type RequiredCheckName = (typeof REQUIRED_CHECK_NAMES)[number]; diff --git a/apps/server/src/update-migrations.ts b/apps/server/src/update-migrations.ts index db51d23f1..9d6038afa 100644 --- a/apps/server/src/update-migrations.ts +++ b/apps/server/src/update-migrations.ts @@ -2,7 +2,6 @@ import { readdir, readFile } from "node:fs/promises"; import path from "node:path"; import { parse as parseYaml } from "yaml"; import { z } from "zod"; -import { REQUIRED_CHECK_NAMES } from "./release-metadata.js"; /** * Persistent install-update migration manifest. Migrations live in @@ -14,7 +13,16 @@ import { REQUIRED_CHECK_NAMES } from "./release-metadata.js"; const MANIFEST_FILE_RE = /^(\d+)-([a-z0-9][a-z0-9-]*)\.ya?ml$/i; -const RequiredCheckSchema = z.enum(REQUIRED_CHECK_NAMES); +// Deliberately NOT an enum of REQUIRED_CHECK_NAMES. Manifests ship inside +// the *target* release tarball but are parsed by the *currently installed* +// runtime — a strict enum here would make every older install reject a +// manifest the moment a new check name is introduced, silently dropping the +// whole migration from its pending set. Accept any well-formed name at +// parse time; `runRequiredChecks` fails closed on names this runtime +// doesn't implement. +const RequiredCheckSchema = z + .string() + .regex(/^[a-z0-9][a-z0-9_]*$/, "check name must be lowercase snake_case"); export const UpdateMigrationManifestSchema = z.object({ id: z diff --git a/apps/server/test/assisted-update-migrations.test.ts b/apps/server/test/assisted-update-migrations.test.ts index 841e06861..38b404872 100644 --- a/apps/server/test/assisted-update-migrations.test.ts +++ b/apps/server/test/assisted-update-migrations.test.ts @@ -75,9 +75,11 @@ describe("buildAssistedUpdateContext (migrations path)", () => { ); expect(ctx.state.migrations?.map((m) => m.id)).toEqual(["first", "second"]); expect(ctx.state.metadata).toBeNull(); - // Union: service_entrypoint from first + health_endpoint from second + // Union: service_entrypoint from first + health_endpoint from second, + // plus the framework-enforced running_version. expect(ctx.state.requiredChecks.sort()).toEqual([ "health_endpoint", + "running_version", "service_entrypoint", ]); expect(ctx.state.phase).toBe("inspect"); @@ -105,6 +107,7 @@ describe("buildAssistedUpdateContext (migrations path)", () => { expect(ctx.state.requiredChecks).toEqual([ "health_endpoint", "service_entrypoint", + "running_version", ]); }); @@ -154,7 +157,10 @@ describe("buildAssistedUpdateContext (migrations path)", () => { ); expect(ctx.state.migrations).toBeNull(); expect(ctx.state.metadata?.title).toBe("Legacy block"); - expect(ctx.state.requiredChecks).toEqual(["health_endpoint"]); + expect(ctx.state.requiredChecks).toEqual([ + "health_endpoint", + "running_version", + ]); expect(ctx.prompt).toContain("Legacy block"); // Legacy header, not the migration header expect(ctx.prompt).not.toContain("Pending migrations"); @@ -260,4 +266,50 @@ describe("runAndRecordChecks (migrations path)", () => { const state = await applied.readAppliedMigrationsState(); expect(state.appliedMigrations).toEqual({}); }); + + // A run launched by an older server has no running_version in its + // persisted requiredChecks — but the validating (newer) binary must still + // prove the running executable's version rather than trusting + // release.json. + it("enforces running_version even when the launched state predates it", async () => { + const { assisted } = await importEverything(); + const ctx = await assisted.buildAssistedUpdateContext( + { + tag: "v0.18.13", + fromTag: "v0.18.12", + migrations: [MIGRATION("first")], + serverDir: "/tmp/dispatch-test-server", + recovery: RECOVERY, + }, + "http://127.0.0.1:6767" + ); + // Simulate state written by an older server version. + ctx.state.requiredChecks = ctx.state.requiredChecks.filter( + (name) => name !== "running_version" + ); + + const releaseChecks = await import("../src/release-checks.js"); + let requestedNames: ReadonlyArray = []; + const spy = vi + .spyOn(releaseChecks, "runRequiredChecks") + .mockImplementation(async (names) => { + requestedNames = names; + return names.map((name) => ({ + name, + ok: true, + message: `${name} passed`, + })); + }); + + try { + await assisted.runAndRecordChecks(ctx.state, { + serverDir: "/tmp/dispatch-test-server", + targetTag: "v0.18.13", + }); + } finally { + spy.mockRestore(); + } + + expect(requestedNames).toContain("running_version"); + }); }); diff --git a/apps/server/test/assisted-update.test.ts b/apps/server/test/assisted-update.test.ts index a64e074a2..93ea6b1bc 100644 --- a/apps/server/test/assisted-update.test.ts +++ b/apps/server/test/assisted-update.test.ts @@ -161,6 +161,7 @@ describe("buildAssistedUpdateContext", () => { expect(ctx.state.requiredChecks).toEqual([ "service_restarted", "version_converged", + "running_version", ]); }); @@ -257,7 +258,9 @@ describe("buildAssistedUpdateContext", () => { ); expect(ctx.prompt).not.toContain("## Instructions"); expect(ctx.prompt).not.toContain("## Rollback guidance"); - expect(ctx.prompt).toContain("(none)"); + // Even with no metadata checks, the framework-enforced running_version + // check is always listed. + expect(ctx.prompt).toContain("- running_version"); }); }); diff --git a/apps/server/test/release-checks.test.ts b/apps/server/test/release-checks.test.ts index efca1ecc9..05b93d83a 100644 --- a/apps/server/test/release-checks.test.ts +++ b/apps/server/test/release-checks.test.ts @@ -1,7 +1,7 @@ import { spawnSync } from "node:child_process"; import https from "node:https"; import type { Server as HttpsServer } from "node:https"; -import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { mkdtemp, readFile, rm, symlink, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; @@ -87,6 +87,24 @@ describe("expected_runtime_artifact", () => { expect(result.ok).toBe(false); }); + + // Regression: a legacy install can leave the fixed path as a symlink + // pinned to a versioned dist/bun binary. stat() follows the link and saw + // a regular file, so the check false-greened on exactly the shape the + // fixed-runtime migration exists to eliminate. + it("fails when the fixed runtime is a symlink to a versioned binary", async () => { + const target = path.join(tmpServerDir, "dispatch-0.31.4-bun-linux-x64"); + await writeFile(target, "old binary"); + await symlink(target, path.join(tmpServerDir, "dispatch")); + + const [result] = await runRequiredChecks(["expected_runtime_artifact"], { + serverDir: tmpServerDir, + targetTag: "v0.19.0", + }); + + expect(result.ok).toBe(false); + expect(result.message).toMatch(/symlink/); + }); }); describe("service_entrypoint", () => { @@ -353,6 +371,101 @@ describe("health_endpoint with self-signed loopback HTTPS", () => { }); }); +describe("running_version", () => { + const respond = (headers: Record) => + vi.spyOn(globalThis, "fetch").mockResolvedValue( + new Response(JSON.stringify({ status: "ok" }), { + status: 200, + headers, + }) + ); + + it("passes when the X-Dispatch-Version header matches the target tag", async () => { + respond({ "X-Dispatch-Version": "0.19.0" }); + + const [result] = await runRequiredChecks(["running_version"], { + serverDir: tmpServerDir, + targetTag: "v0.19.0", + healthUrl: "http://test/health", + }); + + expect(result.ok).toBe(true); + expect(result.message).toMatch(/reports 0\.19\.0/); + }); + + it("fails when the running executable reports an older version", async () => { + respond({ "X-Dispatch-Version": "0.18.4" }); + + const [result] = await runRequiredChecks(["running_version"], { + serverDir: tmpServerDir, + targetTag: "v0.19.0", + healthUrl: "http://test/health", + }); + + expect(result.ok).toBe(false); + expect(result.message).toMatch(/reports 0\.18\.4, expected 0\.19\.0/); + }); + + // The false-green scenario: a pre-header runtime is serving after the + // restart. No header means we cannot prove the running version — fail. + it("fails when no version header is present", async () => { + respond({}); + + const [result] = await runRequiredChecks(["running_version"], { + serverDir: tmpServerDir, + targetTag: "v0.19.0", + healthUrl: "http://test/health", + }); + + expect(result.ok).toBe(false); + expect(result.message).toMatch(/no X-Dispatch-Version header/); + }); + + it("fails on non-2xx response", async () => { + vi.spyOn(globalThis, "fetch").mockResolvedValue( + new Response("oops", { status: 503 }) + ); + + const [result] = await runRequiredChecks(["running_version"], { + serverDir: tmpServerDir, + targetTag: "v0.19.0", + healthUrl: "http://test/health", + }); + + expect(result.ok).toBe(false); + expect(result.message).toMatch(/503/); + }); + + it("fails on fetch error", async () => { + vi.spyOn(globalThis, "fetch").mockRejectedValue(new Error("ECONNREFUSED")); + + const [result] = await runRequiredChecks(["running_version"], { + serverDir: tmpServerDir, + targetTag: "v0.19.0", + healthUrl: "http://test/health", + }); + + expect(result.ok).toBe(false); + expect(result.message).toMatch(/ECONNREFUSED/); + }); +}); + +describe("unknown check names", () => { + // Manifest schemas accept names this runtime may not implement (they come + // from the target release's tarball). They must fail closed, not pass or + // throw. + it("fails closed on a check name this runtime does not implement", async () => { + const [result] = await runRequiredChecks(["check_from_the_future"], { + serverDir: tmpServerDir, + targetTag: "v0.19.0", + }); + + expect(result.name).toBe("check_from_the_future"); + expect(result.ok).toBe(false); + expect(result.message).toMatch(/unknown required check/); + }); +}); + describe("version_converged", () => { it("passes when release.json matches the target tag", async () => { setReleaseRecord({ diff --git a/apps/server/test/update-migrations.test.ts b/apps/server/test/update-migrations.test.ts index ffab8034f..d4133ce48 100644 --- a/apps/server/test/update-migrations.test.ts +++ b/apps/server/test/update-migrations.test.ts @@ -47,7 +47,33 @@ describe("parseManifest", () => { expect(result.data.rollback.length).toBe(2); }); - it("rejects an unknown requiredChecks entry", () => { + // Manifests ship in the target tarball but are parsed by the currently + // installed runtime — an unknown (but well-formed) check name must parse + // so an older install doesn't drop the whole migration. It fails closed + // at run time instead (see release-checks runCheck default). + it("accepts an unknown but well-formed requiredChecks entry", () => { + const result = parseManifest(` +id: forward +title: t +summary: s +alreadySatisfied: + description: d +instructions: + - step +validation: + requiredChecks: + - check_from_the_future +rollback: [] +`); + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.validation.requiredChecks).toEqual([ + "check_from_the_future", + ]); + } + }); + + it("rejects a malformed requiredChecks entry", () => { const result = parseManifest(` id: bad title: t @@ -58,7 +84,7 @@ instructions: - step validation: requiredChecks: - - made_up_check + - "Not A Check!" rollback: [] `); expect(result.success).toBe(false); @@ -194,6 +220,48 @@ describe("filterPending", () => { }); }); +describe("shipped manifests (update-migrations/)", () => { + const shippedDir = path.join( + __dirname, + "..", + "..", + "..", + "update-migrations" + ); + + it("all parse cleanly", async () => { + const result = await loadUpdateMigrations(shippedDir); + expect(result.errors).toEqual([]); + expect(result.migrations.length).toBeGreaterThan(0); + }); + + // Compatibility guard: manifests ship in the target release tarball but + // are parsed by whatever runtime is currently installed. Runtimes before + // v0.33 use a strict check-name enum and DROP an entire manifest on an + // unknown name — so shipped manifests must stick to the original five + // names (in particular, no running_version; the framework enforces that + // one implicitly). Relax this only once installs older than v0.33 no + // longer need to parse new manifests. + it("only use check names pre-v0.33 runtimes can parse", async () => { + const legacyParserSafe = new Set([ + "expected_runtime_artifact", + "service_entrypoint", + "service_restarted", + "health_endpoint", + "version_converged", + ]); + const result = await loadUpdateMigrations(shippedDir); + for (const m of result.migrations) { + for (const check of m.manifest.validation.requiredChecks) { + expect( + legacyParserSafe.has(check), + `${m.filename} uses check "${check}", which pre-v0.33 runtimes cannot parse` + ).toBe(true); + } + } + }); +}); + function makeFile(filename: string, id: string) { return { filename, diff --git a/docs/vm-release-validation.md b/docs/vm-release-validation.md index 50b0a29a0..f121376a9 100644 --- a/docs/vm-release-validation.md +++ b/docs/vm-release-validation.md @@ -56,6 +56,10 @@ tmux child of the Dispatch user service. record its unit file, `MainPID`, current release record, and health result. 2. For a legacy fixture, make `ExecStart` resolve a version-pinned binary or symlink. Keep a backup of the fixture unit inside the VM. + `scripts/vm-fixtures/legacy-pinned-symlink.sh` converts a healthy + fixed-path install into this shape (pinned `bin/dispatch` symlink, no + fixed runtime file, no `KillMode=process`, fixed-runtime migrations + un-applied) — run it only inside the disposable VM. 3. Download the published target tarball, select the exact platform/arch member, reject unexpected archive members, and compare its SHA-256 to the tarball manifest. diff --git a/scripts/vm-fixtures/legacy-pinned-symlink.sh b/scripts/vm-fixtures/legacy-pinned-symlink.sh new file mode 100755 index 000000000..2fba55c8b --- /dev/null +++ b/scripts/vm-fixtures/legacy-pinned-symlink.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +# VM fixture: convert a healthy fixed-path Linux Dispatch install into the +# ≤v0.31.x legacy shape where the systemd ExecStart resolves through a +# bin/dispatch symlink hard-pinned to a versioned dist/bun binary. +# +# Use this ONLY inside a disposable VM (see docs/vm-release-validation.md, +# "Existing legacy Linux service" row). It models the failure mode where a +# managed update restarts back into the old pinned binary, whose boot-time +# release-binary pruning deletes the freshly extracted target artifact while +# release.json claims success. +# +# After running it, exercise the assisted update to the target release and +# verify the fixed-runtime cutover happens BEFORE the first restart and that +# the actually-running executable is the target version (X-Dispatch-Version +# header or /proc//exe), not just release.json. +set -euo pipefail + +if [ "$(uname -s)" != "Linux" ]; then + echo "error: this fixture models a Linux systemd install" >&2 + exit 1 +fi + +INSTALL_DIR="${DISPATCH_INSTALL_DIR:-$HOME/.dispatch/server}" +ENV_FILE="$INSTALL_DIR/.env" +UNIT="$HOME/.config/systemd/user/dispatch.service" +STATE_DIR="$HOME/.dispatch" +APPLIED_STORE="$STATE_DIR/applied-migrations.json" + +# Resolve the configured runtime path the same way the server does. +RUNTIME_PATH="$INSTALL_DIR/dispatch" +if [ -f "$ENV_FILE" ]; then + configured="$(sed -n 's/^DISPATCH_RUNTIME_PATH=//p' "$ENV_FILE" | tail -n 1)" + [ -n "$configured" ] && RUNTIME_PATH="$configured" +fi + +[ -f "$RUNTIME_PATH" ] || { echo "error: no fixed runtime at $RUNTIME_PATH — install Dispatch first" >&2; exit 1; } +[ -f "$UNIT" ] || { echo "error: no user unit at $UNIT" >&2; exit 1; } + +# The version the fixture pins to: the currently installed one. +PINNED_VERSION="$(sed -n 's/.*"tag": *"v\([^"]*\)".*/\1/p' "$STATE_DIR/release.json" | head -n 1)" +[ -n "$PINNED_VERSION" ] || { echo "error: could not read installed tag from $STATE_DIR/release.json" >&2; exit 1; } + +case "$(uname -m)" in + arm64|aarch64) ARCH=arm64 ;; + x86_64|amd64) ARCH=x64 ;; + *) echo "unsupported architecture" >&2; exit 1 ;; +esac +PINNED_BINARY="$INSTALL_DIR/dist/bun/dispatch-$PINNED_VERSION-bun-linux-$ARCH" + +echo "==> pinning service to dist/bun/dispatch-$PINNED_VERSION-bun-linux-$ARCH" + +# 1. Materialize the versioned binary the legacy install would have. +mkdir -p "$INSTALL_DIR/dist/bun" "$INSTALL_DIR/bin" +cp -p "$RUNTIME_PATH" "$PINNED_BINARY" + +# 2. Legacy entrypoint: bin/dispatch symlink hard-pinned to the versioned binary. +ln -sfn "$PINNED_BINARY" "$INSTALL_DIR/bin/dispatch" + +# 3. Repoint ExecStart at the pinned symlink and drop KillMode=process +# (legacy units predate it). Keep a copy of the fixture unit for the +# procedure's "backup of the fixture unit" step. +cp -p "$UNIT" "$UNIT.fixture-backup" +sed -i \ + -e "s|^ExecStart=.*|ExecStart=$INSTALL_DIR/bin/dispatch|" \ + -e '/^KillMode=process$/d' \ + "$UNIT" + +# 4. Remove fixed-path artifacts the legacy install never had. +rm -f "$RUNTIME_PATH" "$RUNTIME_PATH.previous" + +# 5. Mark the fixed-runtime migrations as never applied so the assisted +# update treats them as pending (fresh installs seed them as applied). +if [ -f "$APPLIED_STORE" ]; then + python3 - "$APPLIED_STORE" <<'PY' +import json, sys +path = sys.argv[1] +state = json.load(open(path)) +for mid in ("fixed-runtime-entrypoint", "agent-restart-safety"): + state.get("appliedMigrations", {}).pop(mid, None) +json.dump(state, open(path, "w"), indent=2) +PY +fi + +systemctl --user daemon-reload +systemctl --user restart dispatch + +echo "==> legacy pinned-symlink fixture in place:" +echo " ExecStart -> $INSTALL_DIR/bin/dispatch -> $PINNED_BINARY" +echo " fixed runtime removed: $RUNTIME_PATH" +echo " unit backup: $UNIT.fixture-backup" +systemctl --user --no-pager status dispatch || true diff --git a/update-migrations/0010-fixed-runtime-entrypoint.yaml b/update-migrations/0010-fixed-runtime-entrypoint.yaml index f327ae85a..a38b10213 100644 --- a/update-migrations/0010-fixed-runtime-entrypoint.yaml +++ b/update-migrations/0010-fixed-runtime-entrypoint.yaml @@ -8,8 +8,9 @@ summary: > alreadySatisfied: description: > The service manager invokes the configured fixed Dispatch runtime path, - that path is a regular executable, the service is healthy, and - release.json reports the target tag. + that path is a regular executable (not a symlink), the running process + resolves to that executable, the service is healthy, and release.json + reports the target tag. instructions: - > @@ -21,25 +22,60 @@ instructions: Docker setup, or custom wrapper as report-only; do not create a service or modify it. - > - This release is a compatibility bridge: invoke the managed update first. - Its retained launchd wrapper starts the exact binary for the checked-out - target tag, so do not change or restart a legacy service definition before - that update has completed and the service is healthy. The wrapper is only - for this transition; the fixed path below is the permanent entrypoint. + First inspect the service entrypoint shape before running any update or + restart. Resolve what ExecStart (Linux) or ProgramArguments (macOS) + actually executes, following symlinks to the final target. Classify it + as either the fixed runtime path already, or a legacy shape: a symlink + or wrapper pinned to a versioned binary such as + dist/bun/dispatch--bun--, or a legacy + bin/dispatch symlink. - > - After the managed update is healthy, determine the configured fixed - runtime path (DISPATCH_RUNTIME_PATH, or DISPATCH_SERVER_DIR/dispatch when - unset). Hardlink or copy the target platform binary extracted at + On Linux with a legacy version-pinned entrypoint, do NOT invoke the + managed update or restart the service first. A restart would relaunch + the old pinned binary, and old runtimes prune versioned release binaries + at boot — deleting the freshly extracted target artifact while + release.json still claims the update succeeded. Instead perform the + fixed-runtime cutover BEFORE the first restart, as described in the next + steps, and only then let the managed flow record state. + - > + Linux cutover, staging: download or reuse the cached target release + tarball, verify the target platform binary against dist/bun/SHA256SUMS.txt + from that tarball, and extract the exact member + dist/bun/dispatch--bun--. Select by exact + name; do not choose a glob match by modification time. + - > + Linux cutover, activation: determine the configured fixed runtime path + (DISPATCH_RUNTIME_PATH, or DISPATCH_SERVER_DIR/dispatch when unset). + Preserve the currently healthy executable that the service actually runs + (resolve the entrypoint symlink to its real file) as the fixed path's + adjacent .previous rollback file. Hardlink or copy the verified target + binary to the fixed runtime path, preserving executable mode, so the + fixed path is a regular file — never a symlink into dist/bun. Then + repoint ExecStart to the fixed runtime path, run systemctl --user + daemon-reload, and restart Dispatch. + - > + On macOS this release is a compatibility bridge: invoke the managed + update first. Its retained launchd wrapper starts the exact binary for + the checked-out target tag, so do not change or restart a legacy service + definition before that update has completed and the service is healthy. + The wrapper is only for this transition; the fixed path is the permanent + entrypoint. + - > + On macOS, after the managed update is healthy, determine the configured + fixed runtime path (DISPATCH_RUNTIME_PATH, or + DISPATCH_SERVER_DIR/dispatch when unset). Hardlink or copy the target + platform binary extracted at DISPATCH_SERVER_DIR/dist/bun/dispatch--bun-- to that path, preserving executable mode. Do not choose a glob match by modification time. - > - Before changing the service entrypoint, preserve the currently healthy - executable as the adjacent .previous rollback file. If its path is a - deleted inode, re-download and checksum-verify the last known healthy - release artifact to reconstruct that rollback file. If no healthy tag can - be established, stop and report that manual rollback is unavailable rather - than claiming the migration is reversible. + Before changing the service entrypoint, ensure the currently healthy + executable is preserved as the fixed path's adjacent .previous rollback + file. If its path is a deleted inode, re-download and checksum-verify + the last known healthy release artifact to reconstruct that rollback + file. If no healthy tag can be established, stop and report that manual + rollback is unavailable rather than claiming the migration is + reversible. - Update the systemd unit or launchd plist so ExecStart/ProgramArguments invokes the fixed runtime path, preserving environment, working directory, logging, user scope, and display configuration. @@ -47,12 +83,27 @@ instructions: Reload the service manager configuration and restart Dispatch. On macOS, if launchd is stuck watching a deleted old wrapper, use bootout followed by bootstrap for the user LaunchAgent; do not wait indefinitely on kickstart. - - Confirm the health endpoint is healthy and the newly running binary has - promoted the target tag into release.json. + - > + Hygiene: after the cutover is healthy, repoint any remaining legacy + bin/dispatch symlink at the fixed runtime path, or remove it if nothing + references it, so no dangling pin to an old versioned binary survives. + Also refresh any pre-update service-definition backup you created so it + does not reintroduce the legacy entrypoint if restored. + - > + Confirm the health endpoint is healthy and that the actual running + process is the target executable — verify the X-Dispatch-Version + response header (or /proc//exe on Linux) reports the target + version, and that the newly running binary has promoted the target tag + into release.json. Do not treat release.json alone as proof the running + binary changed. - If the supported definition is absent or this account cannot modify it, stop and report the exact operator action required; do not leave a half-updated service entrypoint. +# running_version is intentionally absent below: pre-v0.33 runtimes parse +# this file with a strict check-name enum and would drop the whole manifest +# on an unknown name. The assisted-update framework enforces running_version +# implicitly from v0.33 on. validation: requiredChecks: - expected_runtime_artifact @@ -62,7 +113,11 @@ validation: - version_converged rollback: - - Atomically replace the fixed runtime path with its adjacent .previous - executable and restore the former service entrypoint if necessary. + - > + Atomically replace the fixed runtime path with its adjacent .previous + executable and restart via the fixed-path service definition. Prefer + this over restoring a pre-update service-definition backup — such + backups can name the legacy version-pinned bin path, whose target binary + may no longer exist. - Reload the service manager configuration, restart Dispatch, and confirm the health endpoint and release.json return to the prior healthy release. From caa9c2659152c589ae0740ab032bbb521ddfcd0b Mon Sep 17 00:00:00 2001 From: Brad Harris Date: Fri, 7 Aug 2026 16:31:11 -0600 Subject: [PATCH 2/4] Ensure KillMode=process lands before 0010's Linux cutover restart MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On a legacy host both fixed-runtime-entrypoint (0010) and agent-restart-safety (0011) are pending and run in numeric order, so 0010's new cutover restart happened before 0011 added KillMode=process — terminating the tmux-backed assisted agent inside dispatch.service's control group at exactly that first restart. 0010 now adds KillMode=process + daemon-reload as a required pre-restart step (the only permitted unit change besides the ExecStart repoint), intentionally front-running 0011. 0011's alreadySatisfied is now config-based (systemctl show -p KillMode, no behavioral restart probe), non-systemd hosts are an explicit no-op terminal state, and its instructions acknowledge 0010 may have already applied the setting. Review feedback from the Ubuntu VM validation agent on PR #896. Co-Authored-By: Claude Fable 5 --- .../0010-fixed-runtime-entrypoint.yaml | 12 +++++++++ .../0011-agent-restart-safety.yaml | 27 +++++++++++++------ 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/update-migrations/0010-fixed-runtime-entrypoint.yaml b/update-migrations/0010-fixed-runtime-entrypoint.yaml index a38b10213..68176f788 100644 --- a/update-migrations/0010-fixed-runtime-entrypoint.yaml +++ b/update-migrations/0010-fixed-runtime-entrypoint.yaml @@ -43,6 +43,18 @@ instructions: from that tarball, and extract the exact member dist/bun/dispatch--bun--. Select by exact name; do not choose a glob match by modification time. + - > + Linux restart safety: before ANY restart this migration performs, ensure + the unit has KillMode=process under [Service] and run systemctl --user + daemon-reload, confirming the loaded value with + systemctl --user show dispatch.service -p KillMode. The assisted-update + agent runs in a tmux child inside dispatch.service's control group, and + the default control-group kill mode terminates it during the very + restart it must observe. This intentionally front-runs the + agent-restart-safety migration (which is ordered after this one and + would arrive too late); keep the setting in place afterwards. Make no + other unit changes before the restart beyond this and the ExecStart + repoint described below. - > Linux cutover, activation: determine the configured fixed runtime path (DISPATCH_RUNTIME_PATH, or DISPATCH_SERVER_DIR/dispatch when unset). diff --git a/update-migrations/0011-agent-restart-safety.yaml b/update-migrations/0011-agent-restart-safety.yaml index 84101d5e3..571250414 100644 --- a/update-migrations/0011-agent-restart-safety.yaml +++ b/update-migrations/0011-agent-restart-safety.yaml @@ -8,20 +8,31 @@ summary: > alreadySatisfied: description: > The current user's supported systemd Dispatch service has - KillMode=process under [Service], its fixed runtime entrypoint is healthy, - and a normal Dispatch restart does not terminate its tmux child processes. + KillMode=process under [Service] and `systemctl --user show + dispatch.service -p KillMode` reports the loaded value as process, and + its fixed runtime entrypoint is healthy. Verify by configuration only — + do not perform a restart just to probe kill behavior. On hosts without a + supported user systemd unit (launchd, system scope, root-owned file, + Docker, custom supervisor, or no definition), this migration is a no-op: + treat it as satisfied and report why. instructions: - > Only modify the current user's ~/.config/systemd/user/dispatch.service - managed through systemctl --user. Treat launchd, system-scope units, - root-owned files, Docker, custom supervisors, or a missing definition as - report-only; do not create or modify another service definition. + managed through systemctl --user. On launchd, system-scope units, + root-owned files, Docker, custom supervisors, or a missing definition, + this migration is an explicit no-op: report the situation and continue + with the remaining plan; do not create or modify another service + definition. - > Before invoking the managed update, add KillMode=process under [Service] - and run systemctl --user daemon-reload. Dispatch agents run in tmux child - processes; systemd's default control-group kill mode terminates the - assisted-update agent during the very restart it must observe. + and run systemctl --user daemon-reload, confirming the loaded value with + systemctl --user show dispatch.service -p KillMode. An earlier migration + in this run (fixed-runtime-entrypoint) may already have added it before + its cutover restart — in that case just confirm and continue. Dispatch + agents run in tmux child processes; systemd's default control-group kill + mode terminates the assisted-update agent during the very restart it + must observe. - > Keep KillMode=process in the unit after the update. It intentionally preserves running agents across future Dispatch service restarts; stop or From 6189ee49a81c256e264ab784d9f7ca6114b45987 Mon Sep 17 00:00:00 2001 From: Brad Harris Date: Fri, 7 Aug 2026 17:45:58 -0600 Subject: [PATCH 3/4] Preflight fixture dependencies before destructive conversion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit python3 was first invoked after the fixture had already pinned the symlink, rewritten the unit, and removed the fixed runtime — a VM without Python would be stranded half-converted with migrations still marked applied. Preflight python3/systemctl/sed/ln/cp and the user systemd manager before touching anything, and make the applied-migrations rewrite atomic (tmp + os.replace). infra-review #592 item #1262. Co-Authored-By: Claude Fable 5 --- scripts/vm-fixtures/legacy-pinned-symlink.sh | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/scripts/vm-fixtures/legacy-pinned-symlink.sh b/scripts/vm-fixtures/legacy-pinned-symlink.sh index 2fba55c8b..16f8d9b67 100755 --- a/scripts/vm-fixtures/legacy-pinned-symlink.sh +++ b/scripts/vm-fixtures/legacy-pinned-symlink.sh @@ -20,6 +20,14 @@ if [ "$(uname -s)" != "Linux" ]; then exit 1 fi +# Preflight every tool the conversion needs BEFORE touching the install, so +# a missing dependency can't strand a half-converted service. +for tool in python3 systemctl sed ln cp; do + command -v "$tool" >/dev/null || { echo "error: $tool is required" >&2; exit 1; } +done +systemctl --user show-environment >/dev/null 2>&1 \ + || { echo "error: user systemd manager is not reachable (systemctl --user)" >&2; exit 1; } + INSTALL_DIR="${DISPATCH_INSTALL_DIR:-$HOME/.dispatch/server}" ENV_FILE="$INSTALL_DIR/.env" UNIT="$HOME/.config/systemd/user/dispatch.service" @@ -72,12 +80,15 @@ rm -f "$RUNTIME_PATH" "$RUNTIME_PATH.previous" # update treats them as pending (fresh installs seed them as applied). if [ -f "$APPLIED_STORE" ]; then python3 - "$APPLIED_STORE" <<'PY' -import json, sys +import json, os, sys path = sys.argv[1] state = json.load(open(path)) for mid in ("fixed-runtime-entrypoint", "agent-restart-safety"): state.get("appliedMigrations", {}).pop(mid, None) -json.dump(state, open(path, "w"), indent=2) +tmp = path + ".tmp" +with open(tmp, "w") as f: + json.dump(state, f, indent=2) +os.replace(tmp, path) PY fi From c5004a62a10cde7aebfa8f3791f00fdff73b9085 Mon Sep 17 00:00:00 2001 From: Brad Harris Date: Fri, 7 Aug 2026 18:42:21 -0600 Subject: [PATCH 4/4] Make 0010's platform/shape branches explicit Architecture review flagged two literal-reading hazards: the Linux staging/activation steps were phrased unconditionally, so an already-fixed Linux entrypoint could be steered into a manual cutover that bypasses the managed update; and the platform-generic .previous preservation step, read after the Linux cutover, would overwrite the run's only rollback artifact with the target binary. Instructions are now explicit branches: Branch A (Linux already-fixed: managed update only), Branch B (Linux legacy pinned: staged cutover before first restart, with "never overwrite the .previous this step created"), Branch C (macOS bridge, including the deleted-inode .previous recovery and plist work), plus shared restart-safety, hygiene, and verification steps keyed to their branches. architecture-review #593 items #1263 #1264. Co-Authored-By: Claude Fable 5 --- .../0010-fixed-runtime-entrypoint.yaml | 113 ++++++++++-------- 1 file changed, 64 insertions(+), 49 deletions(-) diff --git a/update-migrations/0010-fixed-runtime-entrypoint.yaml b/update-migrations/0010-fixed-runtime-entrypoint.yaml index 68176f788..350d1d79f 100644 --- a/update-migrations/0010-fixed-runtime-entrypoint.yaml +++ b/update-migrations/0010-fixed-runtime-entrypoint.yaml @@ -28,35 +28,46 @@ instructions: as either the fixed runtime path already, or a legacy shape: a symlink or wrapper pinned to a versioned binary such as dist/bun/dispatch--bun--, or a legacy - bin/dispatch symlink. + bin/dispatch symlink. The steps below are explicit branches keyed on + platform and that classification — follow only the branch that matches; + do not apply steps from another branch. - > - On Linux with a legacy version-pinned entrypoint, do NOT invoke the + Branch A — Linux, entrypoint already invokes the fixed runtime path: + do not stage binaries or edit the entrypoint manually. Ensure + KillMode=process is loaded per the Linux restart-safety step below, then + invoke the managed update normally — it verifies the artifact, + atomically replaces the fixed executable, preserves the adjacent + .previous rollback file, records candidate state, and restarts. Then + continue at the verification step. + - > + Branch B — Linux, legacy version-pinned entrypoint: do NOT invoke the managed update or restart the service first. A restart would relaunch the old pinned binary, and old runtimes prune versioned release binaries at boot — deleting the freshly extracted target artifact while - release.json still claims the update succeeded. Instead perform the - fixed-runtime cutover BEFORE the first restart, as described in the next - steps, and only then let the managed flow record state. + release.json still claims the update succeeded. Perform the cutover in + the next steps BEFORE the first restart, and only afterwards let the + managed flow record state. - > - Linux cutover, staging: download or reuse the cached target release - tarball, verify the target platform binary against dist/bun/SHA256SUMS.txt - from that tarball, and extract the exact member - dist/bun/dispatch--bun--. Select by exact - name; do not choose a glob match by modification time. + Branch B staging: download or reuse the cached target release tarball, + verify the target platform binary against dist/bun/SHA256SUMS.txt from + that tarball, and extract the exact member + dist/bun/dispatch--bun--. Select by + exact name; do not choose a glob match by modification time. - > - Linux restart safety: before ANY restart this migration performs, ensure - the unit has KillMode=process under [Service] and run systemctl --user - daemon-reload, confirming the loaded value with + Linux restart safety (applies to every restart this migration performs + on Linux, in either branch): before restarting, ensure the unit has + KillMode=process under [Service] and run systemctl --user daemon-reload, + confirming the loaded value with systemctl --user show dispatch.service -p KillMode. The assisted-update agent runs in a tmux child inside dispatch.service's control group, and the default control-group kill mode terminates it during the very restart it must observe. This intentionally front-runs the agent-restart-safety migration (which is ordered after this one and - would arrive too late); keep the setting in place afterwards. Make no - other unit changes before the restart beyond this and the ExecStart - repoint described below. + would arrive too late); keep the setting in place afterwards. In Branch + B, make no other unit changes before the restart beyond this and the + ExecStart repoint described next. - > - Linux cutover, activation: determine the configured fixed runtime path + Branch B activation: determine the configured fixed runtime path (DISPATCH_RUNTIME_PATH, or DISPATCH_SERVER_DIR/dispatch when unset). Preserve the currently healthy executable that the service actually runs (resolve the entrypoint symlink to its real file) as the fixed path's @@ -64,16 +75,19 @@ instructions: binary to the fixed runtime path, preserving executable mode, so the fixed path is a regular file — never a symlink into dist/bun. Then repoint ExecStart to the fixed runtime path, run systemctl --user - daemon-reload, and restart Dispatch. + daemon-reload, and restart Dispatch. After this restart, never overwrite + the .previous file this step created — it is the run's only rollback + artifact, and the executable now live at the fixed path is the target, + not a rollback candidate. - > - On macOS this release is a compatibility bridge: invoke the managed - update first. Its retained launchd wrapper starts the exact binary for - the checked-out target tag, so do not change or restart a legacy service - definition before that update has completed and the service is healthy. - The wrapper is only for this transition; the fixed path is the permanent - entrypoint. + Branch C — macOS: this release is a compatibility bridge, so invoke the + managed update first. Its retained launchd wrapper starts the exact + binary for the checked-out target tag; do not change or restart a legacy + service definition before that update has completed and the service is + healthy. The wrapper is only for this transition; the fixed path is the + permanent entrypoint. - > - On macOS, after the managed update is healthy, determine the configured + Branch C, after the managed update is healthy: determine the configured fixed runtime path (DISPATCH_RUNTIME_PATH, or DISPATCH_SERVER_DIR/dispatch when unset). Hardlink or copy the target platform binary extracted at @@ -81,33 +95,34 @@ instructions: to that path, preserving executable mode. Do not choose a glob match by modification time. - > - Before changing the service entrypoint, ensure the currently healthy - executable is preserved as the fixed path's adjacent .previous rollback - file. If its path is a deleted inode, re-download and checksum-verify - the last known healthy release artifact to reconstruct that rollback - file. If no healthy tag can be established, stop and report that manual - rollback is unavailable rather than claiming the migration is - reversible. - - Update the systemd unit or launchd plist so ExecStart/ProgramArguments - invokes the fixed runtime path, preserving environment, working directory, - logging, user scope, and display configuration. + Branch C, before changing the service entrypoint: ensure the executable + that was healthy before this update is preserved as the fixed path's + adjacent .previous rollback file. If its path is a deleted inode, + re-download and checksum-verify the last known healthy release artifact + to reconstruct that rollback file. If no healthy tag can be established, + stop and report that manual rollback is unavailable rather than claiming + the migration is reversible. - > - Reload the service manager configuration and restart Dispatch. On macOS, - if launchd is stuck watching a deleted old wrapper, use bootout followed by - bootstrap for the user LaunchAgent; do not wait indefinitely on kickstart. + Branch C: update the launchd plist so ProgramArguments invokes the fixed + runtime path, preserving environment, working directory, logging, user + scope, and display configuration. Reload the service configuration and + restart Dispatch. If launchd is stuck watching a deleted old wrapper, + use bootout followed by bootstrap for the user LaunchAgent; do not wait + indefinitely on kickstart. - > - Hygiene: after the cutover is healthy, repoint any remaining legacy - bin/dispatch symlink at the fixed runtime path, or remove it if nothing - references it, so no dangling pin to an old versioned binary survives. - Also refresh any pre-update service-definition backup you created so it - does not reintroduce the legacy entrypoint if restored. + Hygiene (all branches): after the service is healthy on the fixed path, + repoint any remaining legacy bin/dispatch symlink at the fixed runtime + path, or remove it if nothing references it, so no dangling pin to an + old versioned binary survives. Also refresh any pre-update + service-definition backup you created so it does not reintroduce the + legacy entrypoint if restored. - > - Confirm the health endpoint is healthy and that the actual running - process is the target executable — verify the X-Dispatch-Version - response header (or /proc//exe on Linux) reports the target - version, and that the newly running binary has promoted the target tag - into release.json. Do not treat release.json alone as proof the running - binary changed. + Verification (all branches): confirm the health endpoint is healthy and + that the actual running process is the target executable — verify the + X-Dispatch-Version response header (or /proc//exe on Linux) + reports the target version, and that the newly running binary has + promoted the target tag into release.json. Do not treat release.json + alone as proof the running binary changed. - If the supported definition is absent or this account cannot modify it, stop and report the exact operator action required; do not leave a half-updated service entrypoint.