diff --git a/apps/server/src/assisted-update-store.ts b/apps/server/src/assisted-update-store.ts index 9801f17a..2fd6a507 100644 --- a/apps/server/src/assisted-update-store.ts +++ b/apps/server/src/assisted-update-store.ts @@ -2,10 +2,7 @@ import { randomBytes } from "node:crypto"; import os from "node:os"; import path from "node:path"; import { mkdir, readFile, rename, writeFile } from "node:fs/promises"; -import type { - AssistedUpdateMetadata, - RequiredCheckName, -} from "./release-metadata.js"; +import type { AssistedUpdateMetadata } from "./release-metadata.js"; import type { CheckResult } from "./release-checks.js"; import type { UpdateMigrationManifest } from "./update-migrations.js"; @@ -54,7 +51,9 @@ export type AssistedUpdateState = { * `metadata` for the legacy single-block flow. */ migrations: UpdateMigrationManifest[] | null; - requiredChecks: RequiredCheckName[]; + /** Usually RequiredCheckName values; manifests from a newer release may + * name checks this runtime doesn't know (they fail closed when run). */ + requiredChecks: string[]; phase: AssistedPhase; /** Random nonce; the launched agent uses this to authenticate phase POSTs. */ token: string; diff --git a/apps/server/src/assisted-update.ts b/apps/server/src/assisted-update.ts index fea66e24..11958f09 100644 --- a/apps/server/src/assisted-update.ts +++ b/apps/server/src/assisted-update.ts @@ -16,7 +16,6 @@ import { type AssistedUpdateState, } from "./assisted-update-store.js"; import { runRequiredChecks, type CheckContext } from "./release-checks.js"; -import type { RequiredCheckName } from "./release-metadata.js"; import type { UpdateMigrationManifest } from "./update-migrations.js"; import { markMigrationsApplied } from "./applied-migrations-store.js"; import { clearEvaluatorCache } from "./update-migrations-evaluator.js"; @@ -74,9 +73,17 @@ export async function buildAssistedUpdateContext( "buildAssistedUpdateContext requires either migrations or metadata" ); } - const requiredChecks: RequiredCheckName[] = migrations + const requiredChecks: string[] = migrations ? unionMigrationChecks(migrations) : normalizeRequiredChecks(metadata!); + // Always prove the running executable's version, even when no manifest + // asks for it. Manifests can't name this check yet (pre-v0.33 runtimes + // reject unknown check names at parse time), and release.json-based + // checks alone false-green when a pinned service entrypoint restarts back + // into the old binary. + if (!requiredChecks.includes("running_version")) { + requiredChecks.push("running_version"); + } const state: AssistedUpdateState = { tag: input.tag, @@ -111,11 +118,9 @@ export async function buildAssistedUpdateContext( return { state, prompt }; } -function unionMigrationChecks( - migrations: UpdateMigrationManifest[] -): RequiredCheckName[] { - const seen = new Set(); - const ordered: RequiredCheckName[] = []; +function unionMigrationChecks(migrations: UpdateMigrationManifest[]): string[] { + const seen = new Set(); + const ordered: string[] = []; for (const m of migrations) { for (const c of m.validation.requiredChecks) { if (seen.has(c)) continue; @@ -201,7 +206,14 @@ export async function runAndRecordChecks( state: AssistedUpdateState, ctx: CheckContext ): Promise { - const results = await runRequiredChecks(state.requiredChecks, ctx); + // Enforce running_version at check-run time as well as launch time: a run + // launched by an older server (whose state.requiredChecks predates the + // check) is often validated by the freshly restarted target binary — this + // code — so appending here closes the gap for in-flight upgrades. + const names = state.requiredChecks.includes("running_version") + ? state.requiredChecks + : [...state.requiredChecks, "running_version"]; + const results = await runRequiredChecks(names, ctx); state.checks = results; state.updatedAt = new Date().toISOString(); await writeAssistedUpdateState(state); diff --git a/apps/server/src/release-checks.ts b/apps/server/src/release-checks.ts index cba599b2..cada3162 100644 --- a/apps/server/src/release-checks.ts +++ b/apps/server/src/release-checks.ts @@ -1,4 +1,4 @@ -import { readFile, stat } from "node:fs/promises"; +import { lstat, readFile } from "node:fs/promises"; import https from "node:https"; import os from "node:os"; import path from "node:path"; @@ -14,13 +14,15 @@ export type CheckContext = { }; export type CheckResult = { - name: RequiredCheckName; + /** Usually a RequiredCheckName; a manifest authored for a newer runtime + * may name a check this build doesn't know (those fail closed). */ + name: string; ok: boolean; message: string; }; export async function runRequiredChecks( - names: RequiredCheckName[], + names: ReadonlyArray, ctx: CheckContext ): Promise { const results: CheckResult[] = []; @@ -38,11 +40,8 @@ export async function runRequiredChecks( return results; } -async function runCheck( - name: RequiredCheckName, - ctx: CheckContext -): Promise { - switch (name) { +async function runCheck(name: string, ctx: CheckContext): Promise { + switch (name as RequiredCheckName) { case "expected_runtime_artifact": return checkRuntimeArtifact(ctx); case "service_entrypoint": @@ -53,13 +52,34 @@ async function runCheck( return checkHealthEndpoint(ctx); case "version_converged": return checkVersionConverged(ctx); + case "running_version": + return checkRunningVersion(ctx); + default: + // Fail closed: a manifest authored for a newer runtime may require a + // check this build cannot evaluate. Passing it silently would defeat + // the point of a required check. + return { + name, + ok: false, + message: `unknown required check "${name}" — this runtime cannot evaluate it`, + }; } } async function checkRuntimeArtifact(ctx: CheckContext): Promise { const runtime = fixedRuntimePath(ctx.serverDir); try { - const info = await stat(runtime); + // lstat, not stat: a symlink at the fixed path (e.g. a legacy pin to a + // versioned dist/bun binary) must not pass as the activated runtime — + // atomic replacement and .previous rollback assume a regular file. + const info = await lstat(runtime); + if (info.isSymbolicLink()) { + return { + name: "expected_runtime_artifact", + ok: false, + message: `${runtime} is a symlink; the fixed runtime must be a regular executable`, + }; + } return info.isFile() ? { name: "expected_runtime_artifact", @@ -225,9 +245,13 @@ function isLoopbackHttps(url: string): boolean { async function fetchViaGlobal( url: string -): Promise<{ status: number; body: string }> { +): Promise<{ status: number; body: string; versionHeader: string | null }> { const res = await fetch(url, { signal: AbortSignal.timeout(5_000) }); - return { status: res.status, body: await res.text() }; + return { + status: res.status, + body: await res.text(), + versionHeader: res.headers.get("x-dispatch-version"), + }; } // The same-process self-check legitimately hits the local server's @@ -238,7 +262,7 @@ async function fetchViaGlobal( // future config points it at a remote host. function fetchLoopbackHttps( url: string -): Promise<{ status: number; body: string }> { +): Promise<{ status: number; body: string; versionHeader: string | null }> { return new Promise((resolve, reject) => { const parsed = new URL(url); const req = https.request( @@ -254,9 +278,13 @@ function fetchLoopbackHttps( const chunks: Buffer[] = []; res.on("data", (c: Buffer) => chunks.push(c)); res.on("end", () => { + const header = res.headers["x-dispatch-version"]; resolve({ status: res.statusCode ?? 0, body: Buffer.concat(chunks).toString("utf8"), + versionHeader: Array.isArray(header) + ? (header[0] ?? null) + : (header ?? null), }); }); res.on("error", reject); @@ -270,6 +298,56 @@ function fetchLoopbackHttps( }); } +/** + * Prove the executable that is actually serving requests is the target + * version. Every API response carries the build-time package version in the + * X-Dispatch-Version header, so this cannot be spoofed by an on-disk record + * — release.json only says what was *deployed*, not what is *running* (a + * version-pinned service entrypoint can restart straight back into the old + * binary while release.json claims the target). + */ +async function checkRunningVersion(ctx: CheckContext): Promise { + const url = ctx.healthUrl ?? "http://127.0.0.1:6767/api/v1/health"; + const expected = ctx.targetTag.replace(/^v/, ""); + try { + const { status, versionHeader } = isLoopbackHttps(url) + ? await fetchLoopbackHttps(url) + : await fetchViaGlobal(url); + if (status < 200 || status >= 300) { + return { + name: "running_version", + ok: false, + message: `${url} returned ${status}`, + }; + } + const running = versionHeader?.trim(); + if (!running) { + return { + name: "running_version", + ok: false, + message: `no X-Dispatch-Version header from ${url} — cannot prove the running executable's version (runtime predates version reporting or is not the target binary)`, + }; + } + return running === expected + ? { + name: "running_version", + ok: true, + message: `running executable reports ${running} (target ${ctx.targetTag})`, + } + : { + name: "running_version", + ok: false, + message: `running executable reports ${running}, expected ${expected} (target ${ctx.targetTag})`, + }; + } catch (err) { + return { + name: "running_version", + ok: false, + message: err instanceof Error ? err.message : String(err), + }; + } +} + async function checkVersionConverged(ctx: CheckContext): Promise { const record = await readReleaseStore().catch(() => null); if (!record) { diff --git a/apps/server/src/release-metadata.ts b/apps/server/src/release-metadata.ts index f15ed6f4..7026aff5 100644 --- a/apps/server/src/release-metadata.ts +++ b/apps/server/src/release-metadata.ts @@ -15,6 +15,13 @@ export const REQUIRED_CHECK_NAMES = [ "service_restarted", "health_endpoint", "version_converged", + // Proves the actual running executable's baked-in version (via the + // X-Dispatch-Version response header), not what release.json claims. + // Runtimes before v0.33 reject unknown check names when parsing update + // migration manifests, so keep this OUT of update-migrations/*.yaml + // requiredChecks until the pre-v0.33 population is gone — the assisted + // framework enforces it implicitly (see runAndRecordChecks). + "running_version", ] as const; export type RequiredCheckName = (typeof REQUIRED_CHECK_NAMES)[number]; diff --git a/apps/server/src/update-migrations.ts b/apps/server/src/update-migrations.ts index db51d23f..9d6038af 100644 --- a/apps/server/src/update-migrations.ts +++ b/apps/server/src/update-migrations.ts @@ -2,7 +2,6 @@ import { readdir, readFile } from "node:fs/promises"; import path from "node:path"; import { parse as parseYaml } from "yaml"; import { z } from "zod"; -import { REQUIRED_CHECK_NAMES } from "./release-metadata.js"; /** * Persistent install-update migration manifest. Migrations live in @@ -14,7 +13,16 @@ import { REQUIRED_CHECK_NAMES } from "./release-metadata.js"; const MANIFEST_FILE_RE = /^(\d+)-([a-z0-9][a-z0-9-]*)\.ya?ml$/i; -const RequiredCheckSchema = z.enum(REQUIRED_CHECK_NAMES); +// Deliberately NOT an enum of REQUIRED_CHECK_NAMES. Manifests ship inside +// the *target* release tarball but are parsed by the *currently installed* +// runtime — a strict enum here would make every older install reject a +// manifest the moment a new check name is introduced, silently dropping the +// whole migration from its pending set. Accept any well-formed name at +// parse time; `runRequiredChecks` fails closed on names this runtime +// doesn't implement. +const RequiredCheckSchema = z + .string() + .regex(/^[a-z0-9][a-z0-9_]*$/, "check name must be lowercase snake_case"); export const UpdateMigrationManifestSchema = z.object({ id: z diff --git a/apps/server/test/assisted-update-migrations.test.ts b/apps/server/test/assisted-update-migrations.test.ts index 841e0686..38b40487 100644 --- a/apps/server/test/assisted-update-migrations.test.ts +++ b/apps/server/test/assisted-update-migrations.test.ts @@ -75,9 +75,11 @@ describe("buildAssistedUpdateContext (migrations path)", () => { ); expect(ctx.state.migrations?.map((m) => m.id)).toEqual(["first", "second"]); expect(ctx.state.metadata).toBeNull(); - // Union: service_entrypoint from first + health_endpoint from second + // Union: service_entrypoint from first + health_endpoint from second, + // plus the framework-enforced running_version. expect(ctx.state.requiredChecks.sort()).toEqual([ "health_endpoint", + "running_version", "service_entrypoint", ]); expect(ctx.state.phase).toBe("inspect"); @@ -105,6 +107,7 @@ describe("buildAssistedUpdateContext (migrations path)", () => { expect(ctx.state.requiredChecks).toEqual([ "health_endpoint", "service_entrypoint", + "running_version", ]); }); @@ -154,7 +157,10 @@ describe("buildAssistedUpdateContext (migrations path)", () => { ); expect(ctx.state.migrations).toBeNull(); expect(ctx.state.metadata?.title).toBe("Legacy block"); - expect(ctx.state.requiredChecks).toEqual(["health_endpoint"]); + expect(ctx.state.requiredChecks).toEqual([ + "health_endpoint", + "running_version", + ]); expect(ctx.prompt).toContain("Legacy block"); // Legacy header, not the migration header expect(ctx.prompt).not.toContain("Pending migrations"); @@ -260,4 +266,50 @@ describe("runAndRecordChecks (migrations path)", () => { const state = await applied.readAppliedMigrationsState(); expect(state.appliedMigrations).toEqual({}); }); + + // A run launched by an older server has no running_version in its + // persisted requiredChecks — but the validating (newer) binary must still + // prove the running executable's version rather than trusting + // release.json. + it("enforces running_version even when the launched state predates it", async () => { + const { assisted } = await importEverything(); + const ctx = await assisted.buildAssistedUpdateContext( + { + tag: "v0.18.13", + fromTag: "v0.18.12", + migrations: [MIGRATION("first")], + serverDir: "/tmp/dispatch-test-server", + recovery: RECOVERY, + }, + "http://127.0.0.1:6767" + ); + // Simulate state written by an older server version. + ctx.state.requiredChecks = ctx.state.requiredChecks.filter( + (name) => name !== "running_version" + ); + + const releaseChecks = await import("../src/release-checks.js"); + let requestedNames: ReadonlyArray = []; + const spy = vi + .spyOn(releaseChecks, "runRequiredChecks") + .mockImplementation(async (names) => { + requestedNames = names; + return names.map((name) => ({ + name, + ok: true, + message: `${name} passed`, + })); + }); + + try { + await assisted.runAndRecordChecks(ctx.state, { + serverDir: "/tmp/dispatch-test-server", + targetTag: "v0.18.13", + }); + } finally { + spy.mockRestore(); + } + + expect(requestedNames).toContain("running_version"); + }); }); diff --git a/apps/server/test/assisted-update.test.ts b/apps/server/test/assisted-update.test.ts index a64e074a..93ea6b1b 100644 --- a/apps/server/test/assisted-update.test.ts +++ b/apps/server/test/assisted-update.test.ts @@ -161,6 +161,7 @@ describe("buildAssistedUpdateContext", () => { expect(ctx.state.requiredChecks).toEqual([ "service_restarted", "version_converged", + "running_version", ]); }); @@ -257,7 +258,9 @@ describe("buildAssistedUpdateContext", () => { ); expect(ctx.prompt).not.toContain("## Instructions"); expect(ctx.prompt).not.toContain("## Rollback guidance"); - expect(ctx.prompt).toContain("(none)"); + // Even with no metadata checks, the framework-enforced running_version + // check is always listed. + expect(ctx.prompt).toContain("- running_version"); }); }); diff --git a/apps/server/test/release-checks.test.ts b/apps/server/test/release-checks.test.ts index efca1ecc..05b93d83 100644 --- a/apps/server/test/release-checks.test.ts +++ b/apps/server/test/release-checks.test.ts @@ -1,7 +1,7 @@ import { spawnSync } from "node:child_process"; import https from "node:https"; import type { Server as HttpsServer } from "node:https"; -import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { mkdtemp, readFile, rm, symlink, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; @@ -87,6 +87,24 @@ describe("expected_runtime_artifact", () => { expect(result.ok).toBe(false); }); + + // Regression: a legacy install can leave the fixed path as a symlink + // pinned to a versioned dist/bun binary. stat() follows the link and saw + // a regular file, so the check false-greened on exactly the shape the + // fixed-runtime migration exists to eliminate. + it("fails when the fixed runtime is a symlink to a versioned binary", async () => { + const target = path.join(tmpServerDir, "dispatch-0.31.4-bun-linux-x64"); + await writeFile(target, "old binary"); + await symlink(target, path.join(tmpServerDir, "dispatch")); + + const [result] = await runRequiredChecks(["expected_runtime_artifact"], { + serverDir: tmpServerDir, + targetTag: "v0.19.0", + }); + + expect(result.ok).toBe(false); + expect(result.message).toMatch(/symlink/); + }); }); describe("service_entrypoint", () => { @@ -353,6 +371,101 @@ describe("health_endpoint with self-signed loopback HTTPS", () => { }); }); +describe("running_version", () => { + const respond = (headers: Record) => + vi.spyOn(globalThis, "fetch").mockResolvedValue( + new Response(JSON.stringify({ status: "ok" }), { + status: 200, + headers, + }) + ); + + it("passes when the X-Dispatch-Version header matches the target tag", async () => { + respond({ "X-Dispatch-Version": "0.19.0" }); + + const [result] = await runRequiredChecks(["running_version"], { + serverDir: tmpServerDir, + targetTag: "v0.19.0", + healthUrl: "http://test/health", + }); + + expect(result.ok).toBe(true); + expect(result.message).toMatch(/reports 0\.19\.0/); + }); + + it("fails when the running executable reports an older version", async () => { + respond({ "X-Dispatch-Version": "0.18.4" }); + + const [result] = await runRequiredChecks(["running_version"], { + serverDir: tmpServerDir, + targetTag: "v0.19.0", + healthUrl: "http://test/health", + }); + + expect(result.ok).toBe(false); + expect(result.message).toMatch(/reports 0\.18\.4, expected 0\.19\.0/); + }); + + // The false-green scenario: a pre-header runtime is serving after the + // restart. No header means we cannot prove the running version — fail. + it("fails when no version header is present", async () => { + respond({}); + + const [result] = await runRequiredChecks(["running_version"], { + serverDir: tmpServerDir, + targetTag: "v0.19.0", + healthUrl: "http://test/health", + }); + + expect(result.ok).toBe(false); + expect(result.message).toMatch(/no X-Dispatch-Version header/); + }); + + it("fails on non-2xx response", async () => { + vi.spyOn(globalThis, "fetch").mockResolvedValue( + new Response("oops", { status: 503 }) + ); + + const [result] = await runRequiredChecks(["running_version"], { + serverDir: tmpServerDir, + targetTag: "v0.19.0", + healthUrl: "http://test/health", + }); + + expect(result.ok).toBe(false); + expect(result.message).toMatch(/503/); + }); + + it("fails on fetch error", async () => { + vi.spyOn(globalThis, "fetch").mockRejectedValue(new Error("ECONNREFUSED")); + + const [result] = await runRequiredChecks(["running_version"], { + serverDir: tmpServerDir, + targetTag: "v0.19.0", + healthUrl: "http://test/health", + }); + + expect(result.ok).toBe(false); + expect(result.message).toMatch(/ECONNREFUSED/); + }); +}); + +describe("unknown check names", () => { + // Manifest schemas accept names this runtime may not implement (they come + // from the target release's tarball). They must fail closed, not pass or + // throw. + it("fails closed on a check name this runtime does not implement", async () => { + const [result] = await runRequiredChecks(["check_from_the_future"], { + serverDir: tmpServerDir, + targetTag: "v0.19.0", + }); + + expect(result.name).toBe("check_from_the_future"); + expect(result.ok).toBe(false); + expect(result.message).toMatch(/unknown required check/); + }); +}); + describe("version_converged", () => { it("passes when release.json matches the target tag", async () => { setReleaseRecord({ diff --git a/apps/server/test/update-migrations.test.ts b/apps/server/test/update-migrations.test.ts index ffab8034..d4133ce4 100644 --- a/apps/server/test/update-migrations.test.ts +++ b/apps/server/test/update-migrations.test.ts @@ -47,7 +47,33 @@ describe("parseManifest", () => { expect(result.data.rollback.length).toBe(2); }); - it("rejects an unknown requiredChecks entry", () => { + // Manifests ship in the target tarball but are parsed by the currently + // installed runtime — an unknown (but well-formed) check name must parse + // so an older install doesn't drop the whole migration. It fails closed + // at run time instead (see release-checks runCheck default). + it("accepts an unknown but well-formed requiredChecks entry", () => { + const result = parseManifest(` +id: forward +title: t +summary: s +alreadySatisfied: + description: d +instructions: + - step +validation: + requiredChecks: + - check_from_the_future +rollback: [] +`); + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.validation.requiredChecks).toEqual([ + "check_from_the_future", + ]); + } + }); + + it("rejects a malformed requiredChecks entry", () => { const result = parseManifest(` id: bad title: t @@ -58,7 +84,7 @@ instructions: - step validation: requiredChecks: - - made_up_check + - "Not A Check!" rollback: [] `); expect(result.success).toBe(false); @@ -194,6 +220,48 @@ describe("filterPending", () => { }); }); +describe("shipped manifests (update-migrations/)", () => { + const shippedDir = path.join( + __dirname, + "..", + "..", + "..", + "update-migrations" + ); + + it("all parse cleanly", async () => { + const result = await loadUpdateMigrations(shippedDir); + expect(result.errors).toEqual([]); + expect(result.migrations.length).toBeGreaterThan(0); + }); + + // Compatibility guard: manifests ship in the target release tarball but + // are parsed by whatever runtime is currently installed. Runtimes before + // v0.33 use a strict check-name enum and DROP an entire manifest on an + // unknown name — so shipped manifests must stick to the original five + // names (in particular, no running_version; the framework enforces that + // one implicitly). Relax this only once installs older than v0.33 no + // longer need to parse new manifests. + it("only use check names pre-v0.33 runtimes can parse", async () => { + const legacyParserSafe = new Set([ + "expected_runtime_artifact", + "service_entrypoint", + "service_restarted", + "health_endpoint", + "version_converged", + ]); + const result = await loadUpdateMigrations(shippedDir); + for (const m of result.migrations) { + for (const check of m.manifest.validation.requiredChecks) { + expect( + legacyParserSafe.has(check), + `${m.filename} uses check "${check}", which pre-v0.33 runtimes cannot parse` + ).toBe(true); + } + } + }); +}); + function makeFile(filename: string, id: string) { return { filename, diff --git a/docs/vm-release-validation.md b/docs/vm-release-validation.md index 50b0a29a..f121376a 100644 --- a/docs/vm-release-validation.md +++ b/docs/vm-release-validation.md @@ -56,6 +56,10 @@ tmux child of the Dispatch user service. record its unit file, `MainPID`, current release record, and health result. 2. For a legacy fixture, make `ExecStart` resolve a version-pinned binary or symlink. Keep a backup of the fixture unit inside the VM. + `scripts/vm-fixtures/legacy-pinned-symlink.sh` converts a healthy + fixed-path install into this shape (pinned `bin/dispatch` symlink, no + fixed runtime file, no `KillMode=process`, fixed-runtime migrations + un-applied) — run it only inside the disposable VM. 3. Download the published target tarball, select the exact platform/arch member, reject unexpected archive members, and compare its SHA-256 to the tarball manifest. diff --git a/scripts/vm-fixtures/legacy-pinned-symlink.sh b/scripts/vm-fixtures/legacy-pinned-symlink.sh new file mode 100755 index 00000000..16f8d9b6 --- /dev/null +++ b/scripts/vm-fixtures/legacy-pinned-symlink.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash +# VM fixture: convert a healthy fixed-path Linux Dispatch install into the +# ≤v0.31.x legacy shape where the systemd ExecStart resolves through a +# bin/dispatch symlink hard-pinned to a versioned dist/bun binary. +# +# Use this ONLY inside a disposable VM (see docs/vm-release-validation.md, +# "Existing legacy Linux service" row). It models the failure mode where a +# managed update restarts back into the old pinned binary, whose boot-time +# release-binary pruning deletes the freshly extracted target artifact while +# release.json claims success. +# +# After running it, exercise the assisted update to the target release and +# verify the fixed-runtime cutover happens BEFORE the first restart and that +# the actually-running executable is the target version (X-Dispatch-Version +# header or /proc//exe), not just release.json. +set -euo pipefail + +if [ "$(uname -s)" != "Linux" ]; then + echo "error: this fixture models a Linux systemd install" >&2 + exit 1 +fi + +# Preflight every tool the conversion needs BEFORE touching the install, so +# a missing dependency can't strand a half-converted service. +for tool in python3 systemctl sed ln cp; do + command -v "$tool" >/dev/null || { echo "error: $tool is required" >&2; exit 1; } +done +systemctl --user show-environment >/dev/null 2>&1 \ + || { echo "error: user systemd manager is not reachable (systemctl --user)" >&2; exit 1; } + +INSTALL_DIR="${DISPATCH_INSTALL_DIR:-$HOME/.dispatch/server}" +ENV_FILE="$INSTALL_DIR/.env" +UNIT="$HOME/.config/systemd/user/dispatch.service" +STATE_DIR="$HOME/.dispatch" +APPLIED_STORE="$STATE_DIR/applied-migrations.json" + +# Resolve the configured runtime path the same way the server does. +RUNTIME_PATH="$INSTALL_DIR/dispatch" +if [ -f "$ENV_FILE" ]; then + configured="$(sed -n 's/^DISPATCH_RUNTIME_PATH=//p' "$ENV_FILE" | tail -n 1)" + [ -n "$configured" ] && RUNTIME_PATH="$configured" +fi + +[ -f "$RUNTIME_PATH" ] || { echo "error: no fixed runtime at $RUNTIME_PATH — install Dispatch first" >&2; exit 1; } +[ -f "$UNIT" ] || { echo "error: no user unit at $UNIT" >&2; exit 1; } + +# The version the fixture pins to: the currently installed one. +PINNED_VERSION="$(sed -n 's/.*"tag": *"v\([^"]*\)".*/\1/p' "$STATE_DIR/release.json" | head -n 1)" +[ -n "$PINNED_VERSION" ] || { echo "error: could not read installed tag from $STATE_DIR/release.json" >&2; exit 1; } + +case "$(uname -m)" in + arm64|aarch64) ARCH=arm64 ;; + x86_64|amd64) ARCH=x64 ;; + *) echo "unsupported architecture" >&2; exit 1 ;; +esac +PINNED_BINARY="$INSTALL_DIR/dist/bun/dispatch-$PINNED_VERSION-bun-linux-$ARCH" + +echo "==> pinning service to dist/bun/dispatch-$PINNED_VERSION-bun-linux-$ARCH" + +# 1. Materialize the versioned binary the legacy install would have. +mkdir -p "$INSTALL_DIR/dist/bun" "$INSTALL_DIR/bin" +cp -p "$RUNTIME_PATH" "$PINNED_BINARY" + +# 2. Legacy entrypoint: bin/dispatch symlink hard-pinned to the versioned binary. +ln -sfn "$PINNED_BINARY" "$INSTALL_DIR/bin/dispatch" + +# 3. Repoint ExecStart at the pinned symlink and drop KillMode=process +# (legacy units predate it). Keep a copy of the fixture unit for the +# procedure's "backup of the fixture unit" step. +cp -p "$UNIT" "$UNIT.fixture-backup" +sed -i \ + -e "s|^ExecStart=.*|ExecStart=$INSTALL_DIR/bin/dispatch|" \ + -e '/^KillMode=process$/d' \ + "$UNIT" + +# 4. Remove fixed-path artifacts the legacy install never had. +rm -f "$RUNTIME_PATH" "$RUNTIME_PATH.previous" + +# 5. Mark the fixed-runtime migrations as never applied so the assisted +# update treats them as pending (fresh installs seed them as applied). +if [ -f "$APPLIED_STORE" ]; then + python3 - "$APPLIED_STORE" <<'PY' +import json, os, sys +path = sys.argv[1] +state = json.load(open(path)) +for mid in ("fixed-runtime-entrypoint", "agent-restart-safety"): + state.get("appliedMigrations", {}).pop(mid, None) +tmp = path + ".tmp" +with open(tmp, "w") as f: + json.dump(state, f, indent=2) +os.replace(tmp, path) +PY +fi + +systemctl --user daemon-reload +systemctl --user restart dispatch + +echo "==> legacy pinned-symlink fixture in place:" +echo " ExecStart -> $INSTALL_DIR/bin/dispatch -> $PINNED_BINARY" +echo " fixed runtime removed: $RUNTIME_PATH" +echo " unit backup: $UNIT.fixture-backup" +systemctl --user --no-pager status dispatch || true diff --git a/update-migrations/0010-fixed-runtime-entrypoint.yaml b/update-migrations/0010-fixed-runtime-entrypoint.yaml index f327ae85..350d1d79 100644 --- a/update-migrations/0010-fixed-runtime-entrypoint.yaml +++ b/update-migrations/0010-fixed-runtime-entrypoint.yaml @@ -8,8 +8,9 @@ summary: > alreadySatisfied: description: > The service manager invokes the configured fixed Dispatch runtime path, - that path is a regular executable, the service is healthy, and - release.json reports the target tag. + that path is a regular executable (not a symlink), the running process + resolves to that executable, the service is healthy, and release.json + reports the target tag. instructions: - > @@ -21,38 +22,115 @@ instructions: Docker setup, or custom wrapper as report-only; do not create a service or modify it. - > - This release is a compatibility bridge: invoke the managed update first. - Its retained launchd wrapper starts the exact binary for the checked-out - target tag, so do not change or restart a legacy service definition before - that update has completed and the service is healthy. The wrapper is only - for this transition; the fixed path below is the permanent entrypoint. + First inspect the service entrypoint shape before running any update or + restart. Resolve what ExecStart (Linux) or ProgramArguments (macOS) + actually executes, following symlinks to the final target. Classify it + as either the fixed runtime path already, or a legacy shape: a symlink + or wrapper pinned to a versioned binary such as + dist/bun/dispatch--bun--, or a legacy + bin/dispatch symlink. The steps below are explicit branches keyed on + platform and that classification — follow only the branch that matches; + do not apply steps from another branch. - > - After the managed update is healthy, determine the configured fixed - runtime path (DISPATCH_RUNTIME_PATH, or DISPATCH_SERVER_DIR/dispatch when - unset). Hardlink or copy the target platform binary extracted at + Branch A — Linux, entrypoint already invokes the fixed runtime path: + do not stage binaries or edit the entrypoint manually. Ensure + KillMode=process is loaded per the Linux restart-safety step below, then + invoke the managed update normally — it verifies the artifact, + atomically replaces the fixed executable, preserves the adjacent + .previous rollback file, records candidate state, and restarts. Then + continue at the verification step. + - > + Branch B — Linux, legacy version-pinned entrypoint: do NOT invoke the + managed update or restart the service first. A restart would relaunch + the old pinned binary, and old runtimes prune versioned release binaries + at boot — deleting the freshly extracted target artifact while + release.json still claims the update succeeded. Perform the cutover in + the next steps BEFORE the first restart, and only afterwards let the + managed flow record state. + - > + Branch B staging: download or reuse the cached target release tarball, + verify the target platform binary against dist/bun/SHA256SUMS.txt from + that tarball, and extract the exact member + dist/bun/dispatch--bun--. Select by + exact name; do not choose a glob match by modification time. + - > + Linux restart safety (applies to every restart this migration performs + on Linux, in either branch): before restarting, ensure the unit has + KillMode=process under [Service] and run systemctl --user daemon-reload, + confirming the loaded value with + systemctl --user show dispatch.service -p KillMode. The assisted-update + agent runs in a tmux child inside dispatch.service's control group, and + the default control-group kill mode terminates it during the very + restart it must observe. This intentionally front-runs the + agent-restart-safety migration (which is ordered after this one and + would arrive too late); keep the setting in place afterwards. In Branch + B, make no other unit changes before the restart beyond this and the + ExecStart repoint described next. + - > + Branch B activation: determine the configured fixed runtime path + (DISPATCH_RUNTIME_PATH, or DISPATCH_SERVER_DIR/dispatch when unset). + Preserve the currently healthy executable that the service actually runs + (resolve the entrypoint symlink to its real file) as the fixed path's + adjacent .previous rollback file. Hardlink or copy the verified target + binary to the fixed runtime path, preserving executable mode, so the + fixed path is a regular file — never a symlink into dist/bun. Then + repoint ExecStart to the fixed runtime path, run systemctl --user + daemon-reload, and restart Dispatch. After this restart, never overwrite + the .previous file this step created — it is the run's only rollback + artifact, and the executable now live at the fixed path is the target, + not a rollback candidate. + - > + Branch C — macOS: this release is a compatibility bridge, so invoke the + managed update first. Its retained launchd wrapper starts the exact + binary for the checked-out target tag; do not change or restart a legacy + service definition before that update has completed and the service is + healthy. The wrapper is only for this transition; the fixed path is the + permanent entrypoint. + - > + Branch C, after the managed update is healthy: determine the configured + fixed runtime path (DISPATCH_RUNTIME_PATH, or + DISPATCH_SERVER_DIR/dispatch when unset). Hardlink or copy the target + platform binary extracted at DISPATCH_SERVER_DIR/dist/bun/dispatch--bun-- to that path, preserving executable mode. Do not choose a glob match by modification time. - > - Before changing the service entrypoint, preserve the currently healthy - executable as the adjacent .previous rollback file. If its path is a - deleted inode, re-download and checksum-verify the last known healthy - release artifact to reconstruct that rollback file. If no healthy tag can - be established, stop and report that manual rollback is unavailable rather - than claiming the migration is reversible. - - Update the systemd unit or launchd plist so ExecStart/ProgramArguments - invokes the fixed runtime path, preserving environment, working directory, - logging, user scope, and display configuration. + Branch C, before changing the service entrypoint: ensure the executable + that was healthy before this update is preserved as the fixed path's + adjacent .previous rollback file. If its path is a deleted inode, + re-download and checksum-verify the last known healthy release artifact + to reconstruct that rollback file. If no healthy tag can be established, + stop and report that manual rollback is unavailable rather than claiming + the migration is reversible. + - > + Branch C: update the launchd plist so ProgramArguments invokes the fixed + runtime path, preserving environment, working directory, logging, user + scope, and display configuration. Reload the service configuration and + restart Dispatch. If launchd is stuck watching a deleted old wrapper, + use bootout followed by bootstrap for the user LaunchAgent; do not wait + indefinitely on kickstart. - > - Reload the service manager configuration and restart Dispatch. On macOS, - if launchd is stuck watching a deleted old wrapper, use bootout followed by - bootstrap for the user LaunchAgent; do not wait indefinitely on kickstart. - - Confirm the health endpoint is healthy and the newly running binary has - promoted the target tag into release.json. + Hygiene (all branches): after the service is healthy on the fixed path, + repoint any remaining legacy bin/dispatch symlink at the fixed runtime + path, or remove it if nothing references it, so no dangling pin to an + old versioned binary survives. Also refresh any pre-update + service-definition backup you created so it does not reintroduce the + legacy entrypoint if restored. + - > + Verification (all branches): confirm the health endpoint is healthy and + that the actual running process is the target executable — verify the + X-Dispatch-Version response header (or /proc//exe on Linux) + reports the target version, and that the newly running binary has + promoted the target tag into release.json. Do not treat release.json + alone as proof the running binary changed. - If the supported definition is absent or this account cannot modify it, stop and report the exact operator action required; do not leave a half-updated service entrypoint. +# running_version is intentionally absent below: pre-v0.33 runtimes parse +# this file with a strict check-name enum and would drop the whole manifest +# on an unknown name. The assisted-update framework enforces running_version +# implicitly from v0.33 on. validation: requiredChecks: - expected_runtime_artifact @@ -62,7 +140,11 @@ validation: - version_converged rollback: - - Atomically replace the fixed runtime path with its adjacent .previous - executable and restore the former service entrypoint if necessary. + - > + Atomically replace the fixed runtime path with its adjacent .previous + executable and restart via the fixed-path service definition. Prefer + this over restoring a pre-update service-definition backup — such + backups can name the legacy version-pinned bin path, whose target binary + may no longer exist. - Reload the service manager configuration, restart Dispatch, and confirm the health endpoint and release.json return to the prior healthy release. diff --git a/update-migrations/0011-agent-restart-safety.yaml b/update-migrations/0011-agent-restart-safety.yaml index 84101d5e..57125041 100644 --- a/update-migrations/0011-agent-restart-safety.yaml +++ b/update-migrations/0011-agent-restart-safety.yaml @@ -8,20 +8,31 @@ summary: > alreadySatisfied: description: > The current user's supported systemd Dispatch service has - KillMode=process under [Service], its fixed runtime entrypoint is healthy, - and a normal Dispatch restart does not terminate its tmux child processes. + KillMode=process under [Service] and `systemctl --user show + dispatch.service -p KillMode` reports the loaded value as process, and + its fixed runtime entrypoint is healthy. Verify by configuration only — + do not perform a restart just to probe kill behavior. On hosts without a + supported user systemd unit (launchd, system scope, root-owned file, + Docker, custom supervisor, or no definition), this migration is a no-op: + treat it as satisfied and report why. instructions: - > Only modify the current user's ~/.config/systemd/user/dispatch.service - managed through systemctl --user. Treat launchd, system-scope units, - root-owned files, Docker, custom supervisors, or a missing definition as - report-only; do not create or modify another service definition. + managed through systemctl --user. On launchd, system-scope units, + root-owned files, Docker, custom supervisors, or a missing definition, + this migration is an explicit no-op: report the situation and continue + with the remaining plan; do not create or modify another service + definition. - > Before invoking the managed update, add KillMode=process under [Service] - and run systemctl --user daemon-reload. Dispatch agents run in tmux child - processes; systemd's default control-group kill mode terminates the - assisted-update agent during the very restart it must observe. + and run systemctl --user daemon-reload, confirming the loaded value with + systemctl --user show dispatch.service -p KillMode. An earlier migration + in this run (fixed-runtime-entrypoint) may already have added it before + its cutover restart — in that case just confirm and continue. Dispatch + agents run in tmux child processes; systemd's default control-group kill + mode terminates the assisted-update agent during the very restart it + must observe. - > Keep KillMode=process in the unit after the update. It intentionally preserves running agents across future Dispatch service restarts; stop or