From 005f1a1121b8df3f1c4bddce1c0fa63eb829b150 Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Thu, 6 Aug 2026 15:55:11 -0700 Subject: [PATCH 1/3] Speed up selfhost e2e: tunable sandbox deadline + 3-way sharding --- .github/workflows/ci.yml | 10 +++- apps/host-selfhost/src/config.ts | 25 +++++++++ apps/host-selfhost/src/execution.ts | 7 ++- .../resume-after-sandbox-deadline.test.ts | 55 ++++++++++++------- e2e/setup/sandbox-timeout.ts | 28 ++++++++++ e2e/setup/selfhost.boot.ts | 6 ++ e2e/setup/selfhost.globalsetup.ts | 6 ++ 7 files changed, 114 insertions(+), 23 deletions(-) create mode 100644 e2e/setup/sandbox-timeout.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9eb2ca5852..6db740b719 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -199,7 +199,13 @@ jobs: - { target: cloud, shard: 2/4, shard-name: 2of4 } - { target: cloud, shard: 3/4, shard-name: 3of4 } - { target: cloud, shard: 4/4, shard-name: 4of4 } - - target: selfhost + # Selfhost shards the same way: each shard is its own runner booting + # its own fresh instance (own port block + data dir), so the + # project's shared-bootstrap-admin assumption stays intact per shard + # and `fileParallelism: false` still serializes within a shard. + - { target: selfhost, shard: 1/3, shard-name: 1of3 } + - { target: selfhost, shard: 2/3, shard-name: 2of3 } + - { target: selfhost, shard: 3/3, shard-name: 3of3 } runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 30 steps: @@ -254,7 +260,7 @@ jobs: - name: Run selfhost scenarios if: matrix.target == 'selfhost' - run: bunx vitest run --project selfhost --retry=2 + run: bunx vitest run --project selfhost --retry=2 ${{ matrix.shard && format('--shard={0}', matrix.shard) || '' }} working-directory: e2e # Failed runs keep their trace.zip / session.mp4 / step screenshots in diff --git a/apps/host-selfhost/src/config.ts b/apps/host-selfhost/src/config.ts index a435b117eb..20728fabff 100644 --- a/apps/host-selfhost/src/config.ts +++ b/apps/host-selfhost/src/config.ts @@ -45,6 +45,14 @@ export interface SelfHostConfig { readonly organizationName: string; /** URL slug for org-prefixed console paths (`//policies`). */ readonly orgSlug: string; + /** + * Sandbox execution budget passed to the QuickJS runtime, or undefined for + * the runtime's own default (5 minutes). An operator knob in principle, but + * its real consumer is the e2e harness, which shrinks it to seconds so the + * sandbox-deadline scenario proves its race without waiting out real + * minutes (the same pattern as MCP_PAUSED_SESSION_IDLE_TIMEOUT_MS on cloud). + */ + readonly sandboxTimeoutMs: number | undefined; } export const resolveDataDir = (): string => @@ -151,9 +159,26 @@ export const loadConfig = (): SelfHostConfig => { bootstrapAdminName: process.env.EXECUTOR_BOOTSTRAP_ADMIN_NAME ?? "Admin", organizationName: process.env.EXECUTOR_ORG_NAME ?? "Default", orgSlug: resolveOrgSlug(), + sandboxTimeoutMs: resolveSandboxTimeoutMs(), }; }; +// A malformed value is refused rather than silently ignored: an operator who +// sets the knob and typos it should find out at boot, not by watching a +// runaway execution use the 5-minute default. +const resolveSandboxTimeoutMs = (): number | undefined => { + const raw = process.env.EXECUTOR_SANDBOX_TIMEOUT_MS; + if (!raw) return undefined; + const parsed = Number(raw); + if (!Number.isFinite(parsed) || parsed <= 0) { + // oxlint-disable-next-line executor/no-try-catch-or-throw, executor/no-error-constructor -- boundary: refuse to boot on a malformed operator knob + throw new Error( + `EXECUTOR_SANDBOX_TIMEOUT_MS ${JSON.stringify(raw)} is not a positive number of milliseconds`, + ); + } + return Math.floor(parsed); +}; + // The org slug doubles as a URL segment (`//policies`), so an // operator-set value must fit the shared grammar and avoid reserved root // segments (api, mcp, login, …) — a colliding slug would shadow real routes. diff --git a/apps/host-selfhost/src/execution.ts b/apps/host-selfhost/src/execution.ts index 270ffc4f8e..aa2ffee536 100644 --- a/apps/host-selfhost/src/execution.ts +++ b/apps/host-selfhost/src/execution.ts @@ -65,7 +65,12 @@ export const SelfHostHostConfig: Layer.Layer = Layer.sync(HostConfig export const SelfHostCodeExecutorProvider: Layer.Layer = Layer.sync( CodeExecutorProvider, - () => makeQuickJsExecutor(), + () => { + const { sandboxTimeoutMs } = loadConfig(); + return makeQuickJsExecutor( + sandboxTimeoutMs === undefined ? {} : { timeoutMs: sandboxTimeoutMs }, + ); + }, ); /** diff --git a/e2e/scenarios/resume-after-sandbox-deadline.test.ts b/e2e/scenarios/resume-after-sandbox-deadline.test.ts index 42c9154424..47bccc6235 100644 --- a/e2e/scenarios/resume-after-sandbox-deadline.test.ts +++ b/e2e/scenarios/resume-after-sandbox-deadline.test.ts @@ -8,14 +8,18 @@ // unknown execution. // // The journey drives exactly that shape: ONE execution with TWO approval -// gates. The first approval is granted late in its window (~3.5 min), so the -// second pause's window reaches well past the sandbox's 5-minute mark. The -// second approval arrives ~5.75 min after execution start — inside its OWN -// advertised window, but past the old absolute deadline. Deliberately slow -// (~6 min): the elapsed time IS the subject under test. A single-pause -// variant cannot express this cross-target — hosts that advertise a -// 4-minute window would expire it legitimately before the sandbox clock -// even matters. +// gates. The first approval is granted late (70% of the sandbox budget in), +// so the second pause's window reaches well past the budget. The second +// approval arrives at ~115% of the budget after execution start — inside its +// OWN advertised window, but past the old absolute deadline. The subject is +// that RATIO, not any absolute duration, so the delays scale off the budget +// the target was booted with: selfhost boots with a seconds-long +// EXECUTOR_SANDBOX_TIMEOUT_MS (setup/sandbox-timeout.ts) and proves the race +// in ~25s; a target on the production 5-minute budget runs the original +// ~6-minute journey (the elapsed time IS the subject — nothing is mocked). A +// single-pause variant cannot express this cross-target — hosts that +// advertise a 4-minute window would expire it legitimately before the +// sandbox clock even matters. // // The gate is `policies.create`'s own `requiresApproval` annotation // (hermetic, same device as policy-tool-approval.test.ts); both approvals @@ -29,15 +33,23 @@ import { composePluginApi } from "@executor-js/api/server"; import { scenario } from "../src/scenario"; import { Api, Mcp, Target } from "../src/services"; import { configuredMcpPausedSessionIdleTimeoutMs } from "../setup/mcp-session-timeouts"; +import { configuredSandboxTimeoutMs } from "../setup/sandbox-timeout"; const coreApi = composePluginApi([] as const); -// Grant the first approval at 3.5 min — late but inside its 4-minute window. -// The second pause then opens a fresh window reaching ~7.5 min. -const FIRST_APPROVAL_DELAY_MS = 3.5 * 60_000; -// Grant the second approval 2.25 min later: ~5.75 min after execution start, -// past the sandbox's 5-minute budget but inside the second window. -const SECOND_APPROVAL_DELAY_MS = 2.25 * 60_000; +const SANDBOX_BUDGET_MS = configuredSandboxTimeoutMs(); + +// Grant the first approval at 70% of the budget — late but inside its window +// (was 3.5 of 5 min). The second pause then opens a fresh window reaching +// past the budget. +const FIRST_APPROVAL_DELAY_MS = 0.7 * SANDBOX_BUDGET_MS; +// Grant the second approval 45% of the budget later: ~115% of the budget +// after execution start, past the sandbox clock but inside the second window +// (was 2.25 of 5 min → ~5.75 min total). +const SECOND_APPROVAL_DELAY_MS = 0.45 * SANDBOX_BUDGET_MS; +// The whole journey plus scheduling slack, for the idle-window guard and the +// vitest timeout. +const JOURNEY_MS = FIRST_APPROVAL_DELAY_MS + SECOND_APPROVAL_DELAY_MS; /** Sandbox code that creates two policies through the approval-gated core * tool. Patterns are unique-per-run and match no real tool, so the rules are @@ -56,19 +68,22 @@ const second = await tools.executor.coreTools.policies.create({ return JSON.stringify({ first: first.ok, second: second.ok }); `; -// The journey spans ~6 real minutes of paused waiting, so the host must keep -// the paused session alive that long. The suite's default e2e override shrinks +// The journey spans the whole paused waiting time, so the host must keep the +// paused session alive that long. The suite's default e2e override shrinks // the paused-session idle teardown to seconds (to keep teardown tests fast), // which would evict the session mid-scenario for reasons unrelated to the -// clock under test — require the production-like window instead. +// clock under test — require a window that outlasts the journey instead. +// With a shrunken sandbox budget the journey shrinks too, so even the short +// e2e idle window can suffice; the guard compares the two rather than +// hardcoding either. const PAUSED_IDLE_WINDOW_TOO_SHORT = - configuredMcpPausedSessionIdleTimeoutMs() < 8 * 60_000 - ? `the target's paused-session idle teardown (${configuredMcpPausedSessionIdleTimeoutMs()}ms) evicts the session before this ~6-minute journey completes; boot the target with MCP_PAUSED_SESSION_IDLE_TIMEOUT_MS >= 480000 to run it` + configuredMcpPausedSessionIdleTimeoutMs() < JOURNEY_MS + 60_000 + ? `the target's paused-session idle teardown (${configuredMcpPausedSessionIdleTimeoutMs()}ms) evicts the session before this ${Math.round(JOURNEY_MS / 1000)}s journey completes; boot the target with MCP_PAUSED_SESSION_IDLE_TIMEOUT_MS >= ${JOURNEY_MS + 60_000} or a smaller E2E_SANDBOX_TIMEOUT_MS to run it` : undefined; scenario( "MCP · chained approvals granted within their windows survive the sandbox clock", - { timeout: 480_000, skip: PAUSED_IDLE_WINDOW_TOO_SHORT }, + { timeout: Math.max(120_000, JOURNEY_MS + 120_000), skip: PAUSED_IDLE_WINDOW_TOO_SHORT }, Effect.gen(function* () { const target = yield* Target; const apiSurface = yield* Api; diff --git a/e2e/setup/sandbox-timeout.ts b/e2e/setup/sandbox-timeout.ts new file mode 100644 index 0000000000..51a85df37d --- /dev/null +++ b/e2e/setup/sandbox-timeout.ts @@ -0,0 +1,28 @@ +// The sandbox execution budget shared between a target's boot env and the +// sandbox-deadline scenario, so they cannot drift apart (same pattern as +// execution-limits.ts). The scenario proves a RATIO — approvals granted +// inside their own windows survive an execution that outlives the sandbox's +// absolute budget — so the budget's magnitude is free to shrink: on selfhost +// the boot recipe passes E2E_SANDBOX_TIMEOUT_MS through to the server as +// EXECUTOR_SANDBOX_TIMEOUT_MS and the scenario scales its approval delays to +// match, turning a ~6-minute real-time wait into seconds. Targets that cannot +// shrink the budget (cloud's dynamic-worker deadline is not env-tunable) run +// against the production default and skip via their paused-session window +// guard instead. +export const E2E_SANDBOX_TIMEOUT_MS = 20_000; + +export const SANDBOX_TIMEOUT_ENV = "E2E_SANDBOX_TIMEOUT_MS"; + +const PRODUCTION_SANDBOX_TIMEOUT_MS = 5 * 60_000; + +const positiveMilliseconds = (raw: string | undefined): number | undefined => { + if (!raw) return undefined; + const parsed = Number(raw); + if (!Number.isFinite(parsed) || parsed <= 0) return undefined; + return Math.floor(parsed); +}; + +/** The sandbox budget the current target enforces: the harness override when + * the target was booted with one, else the production default. */ +export const configuredSandboxTimeoutMs = (): number => + positiveMilliseconds(process.env[SANDBOX_TIMEOUT_ENV]) ?? PRODUCTION_SANDBOX_TIMEOUT_MS; diff --git a/e2e/setup/selfhost.boot.ts b/e2e/setup/selfhost.boot.ts index 6f0d743d7b..a705e4e543 100644 --- a/e2e/setup/selfhost.boot.ts +++ b/e2e/setup/selfhost.boot.ts @@ -21,6 +21,9 @@ export interface SelfhostBootOptions { /** vite --host (e.g. "0.0.0.0" to be tailnet-reachable). */ readonly host?: string; readonly logFile?: string; + /** Shrink the sandbox execution budget (EXECUTOR_SANDBOX_TIMEOUT_MS) so + * deadline scenarios prove their race in seconds. Omit for production. */ + readonly sandboxTimeoutMs?: number; } export const bootSelfhost = async (options: SelfhostBootOptions): Promise => { @@ -51,6 +54,9 @@ export const bootSelfhost = async (options: SelfhostBootOptions): Promise Promise) | void> { [{ envVar: "E2E_SELFHOST_PORT", offset: 4, label: "selfhost vite dev" }], async (ports) => { const port = ports.E2E_SELFHOST_PORT!; + // Shrink the sandbox execution budget and publish the value to the test + // workers (spawned after this globalsetup, so they inherit the env): the + // sandbox-deadline scenario reads it to scale its approval delays. + process.env[SANDBOX_TIMEOUT_ENV] = String(E2E_SANDBOX_TIMEOUT_MS); // Fresh data dir per suite run — hermetic; in-suite isolation comes from // fresh identities, not resets (bootSelfhost wipes it). const procs = await bootSelfhost({ @@ -44,6 +49,7 @@ export default async function setup(): Promise<(() => Promise) | void> { webBaseUrl: `http://localhost:${port}`, admin: SELFHOST_ADMIN, logFile: bootLogFile, + sandboxTimeoutMs: E2E_SANDBOX_TIMEOUT_MS, }); return { teardown: procs.teardown, value: procs }; }, From 03db468be5f3c00b7ae50e97633ee5d6786443cf Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Fri, 7 Aug 2026 18:19:17 -0700 Subject: [PATCH 2/3] Fix the local e2e suite: toolkit MCP DB lock, Bun-only spawn, stale auth selector --- apps/local/src/executor.ts | 55 ++++++++---- apps/local/src/main.ts | 5 ++ e2e/local/auth.test.ts | 8 +- .../cli-mcp-daemon-attach-stress.test.ts | 87 +++++++++---------- 4 files changed, 90 insertions(+), 65 deletions(-) diff --git a/apps/local/src/executor.ts b/apps/local/src/executor.ts index dd91838f35..1ec7ebbf68 100644 --- a/apps/local/src/executor.ts +++ b/apps/local/src/executor.ts @@ -19,7 +19,7 @@ import type { McpPluginExtension } from "@executor-js/plugin-mcp"; import executorConfig from "../executor.config"; import { localAnalytics } from "./analytics"; import { localDataMigrations } from "./db/data-migrations"; -import { openOwnedLocalDatabase } from "./db/owned-database"; +import { openOwnedLocalDatabase, type OwnedLocalDatabase } from "./db/owned-database"; interface ResolvedStorage { readonly dataDir: string; @@ -56,6 +56,16 @@ type LocalPlugins = readonly AnyPlugin[]; export interface LocalExecutorOptions { readonly activeToolkitSlug?: string; + /** + * Reuse an already-open owned database instead of opening (and locking) the + * data dir again. A toolkit-scoped MCP session differs from the default one + * only in its plugin set, so it must ride the running server's DB handle: + * `openOwnedLocalDatabase` takes an EXCLUSIVE lock, and a second open from + * inside the same process contends with the lock this process already holds. + * The borrowed handle is NOT closed when the derived executor disposes — + * whoever opened it still owns its lifetime. + */ + readonly borrowedDb?: OwnedLocalDatabase; } const loadLocalPlugins = (options: LocalExecutorOptions = {}) => @@ -92,6 +102,10 @@ const loadLocalPlugins = (options: LocalExecutorOptions = {}) => interface LocalExecutorBundle { readonly executor: Executor; readonly plugins: LocalPlugins; + /** The owned DB this bundle opened (or borrowed). Surfaced so a + * toolkit-scoped executor can ride the SAME handle instead of contending + * with this process's own exclusive data-dir lock. */ + readonly db: OwnedLocalDatabase; /** Where this daemon's web UI is reachable, resolved once at boot. Surfaced * so callers building user-facing links (MCP artifact deep links) use the * same origin the executor itself was configured with. */ @@ -151,23 +165,27 @@ const createLocalExecutorLayer = (options: LocalExecutorOptions = {}) => { const tenantId = makeTenantId(cwd); const tables = collectTables(); - const owned = yield* Effect.acquireRelease( - Effect.tryPromise({ - try: () => - openOwnedLocalDatabase({ - dataDir: storage.dataDir, - tables, - namespace: localNamespace, - tenantId, + // A borrowed handle is owned by its opener, so it is used as-is and left + // open on release; only a handle opened here is closed here. + const owned = options.borrowedDb + ? options.borrowedDb + : yield* Effect.acquireRelease( + Effect.tryPromise({ + try: () => + openOwnedLocalDatabase({ + dataDir: storage.dataDir, + tables, + namespace: localNamespace, + tenantId, + }), + catch: (cause) => + new LocalExecutorCreateError({ + message: CREATE_SQLITE_ERROR_MESSAGE, + cause, + }), }), - catch: (cause) => - new LocalExecutorCreateError({ - message: CREATE_SQLITE_ERROR_MESSAGE, - cause, - }), - }), - (database) => Effect.promise(() => database.close()).pipe(Effect.ignore), - ); + (database) => Effect.promise(() => database.close()).pipe(Effect.ignore), + ); const sqlite = owned.db; const migration = owned.migration; @@ -243,7 +261,7 @@ const createLocalExecutorLayer = (options: LocalExecutorOptions = {}) => { ); } - return { executor, plugins, webBaseUrl }; + return { executor, plugins, webBaseUrl, db: owned }; }), ); }; @@ -257,6 +275,7 @@ export const createExecutorHandle = async (options: LocalExecutorOptions = {}) = executor: bundle.executor, plugins: bundle.plugins, webBaseUrl: bundle.webBaseUrl, + db: bundle.db, dispose: async () => { await Effect.runPromise(Effect.ignore(bundle.executor.close())); await ignorePromiseFailure("disposeRuntime", () => runtime.dispose()); diff --git a/apps/local/src/main.ts b/apps/local/src/main.ts index 14e272f37a..0e74cec5d8 100644 --- a/apps/local/src/main.ts +++ b/apps/local/src/main.ts @@ -137,8 +137,13 @@ export const createServerHandlers = async (token: string): Promise localStorage.getItem("executor.authToken")); @@ -70,7 +72,7 @@ scenario( await page.getByRole("button", { name: "Connect" }).click(); await page.getByRole("link", { name: "Secrets" }).first().waitFor({ timeout: 30_000 }); // The reconnect fully restores — integrations LOAD, not a stale 401. - await page.getByText("built-in").first().waitFor({ timeout: 30_000 }); + await page.getByTestId("integration-entry-executor").first().waitFor({ timeout: 30_000 }); }); }), ); diff --git a/e2e/local/cli-mcp-daemon-attach-stress.test.ts b/e2e/local/cli-mcp-daemon-attach-stress.test.ts index 95daad7cfe..57ebd5fac5 100644 --- a/e2e/local/cli-mcp-daemon-attach-stress.test.ts +++ b/e2e/local/cli-mcp-daemon-attach-stress.test.ts @@ -21,11 +21,11 @@ import { expect } from "@effect/vitest"; import { Client } from "@modelcontextprotocol/sdk/client/index.js"; import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js"; import { Effect } from "effect"; +import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process"; import { mkdtempSync, readdirSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { fileURLToPath } from "node:url"; -import type { Subprocess } from "bun"; import { scenario } from "../src/scenario"; @@ -34,7 +34,10 @@ const testScope = join(repoRoot, "apps/local"); // Generous: a dev-mode daemon boots a Vite dev server, slow under machine load. const readyTimeoutMs = 150_000; -type DaemonProc = Subprocess<"ignore", "pipe", "pipe">; +// vitest runs this suite under NODE, not bun, so the daemon is spawned with +// node:child_process (the rest of the e2e harness does the same). `Bun.spawn` +// here threw `ReferenceError: Bun is not defined` on every run. +type DaemonProc = ChildProcessWithoutNullStreams; const waitForDaemonReady = ( proc: DaemonProc, @@ -44,50 +47,38 @@ const waitForDaemonReady = ( let stdoutBuffer = ""; let stderrBuffer = ""; let settled = false; - const decoder = new TextDecoder(); - const stdout = proc.stdout.getReader(); - const stderr = proc.stderr.getReader(); const deadline = setTimeout(() => { if (settled) return; settled = true; // oxlint-disable-next-line executor/no-promise-reject, executor/no-error-constructor -- boundary: captured daemon stderr rejectReady(new Error(`daemon did not announce ready: ${stderrBuffer}`)); }, readyTimeoutMs); - void (async () => { - while (true) { - const { value, done } = await stderr.read(); - if (done) return; - stderrBuffer += decoder.decode(value); - } - })(); - void (async () => { - while (true) { - const { value, done } = await stdout.read(); - if (done) { - if (!settled) { - settled = true; - clearTimeout(deadline); - // oxlint-disable-next-line executor/no-promise-reject, executor/no-error-constructor -- boundary: captured daemon stderr - rejectReady(new Error(`daemon stdout closed before ready: ${stderrBuffer}`)); - } - return; - } - stdoutBuffer += decoder.decode(value); - const match = /Daemon ready on http:\/\/(?:\[[^\]]+\]|[^:\s]+):(\d+)/.exec(stdoutBuffer); - if (match) { - settled = true; - clearTimeout(deadline); - resolveReady({ port: Number(match[1]), stderr: () => stderrBuffer }); - return; - } + proc.stderr.on("data", (chunk: Buffer) => { + stderrBuffer += chunk.toString(); + }); + proc.stdout.on("data", (chunk: Buffer) => { + if (settled) return; + stdoutBuffer += chunk.toString(); + const match = /Daemon ready on http:\/\/(?:\[[^\]]+\]|[^:\s]+):(\d+)/.exec(stdoutBuffer); + if (match) { + settled = true; + clearTimeout(deadline); + resolveReady({ port: Number(match[1]), stderr: () => stderrBuffer }); } - })(); + }); + proc.stdout.on("close", () => { + if (settled) return; + settled = true; + clearTimeout(deadline); + // oxlint-disable-next-line executor/no-promise-reject, executor/no-error-constructor -- boundary: captured daemon stderr + rejectReady(new Error(`daemon stdout closed before ready: ${stderrBuffer}`)); + }); }); const spawnDaemon = (dataDir: string): DaemonProc => - Bun.spawn( + spawn( + "bun", [ - "bun", "run", "dev:cli", "daemon", @@ -103,17 +94,20 @@ const spawnDaemon = (dataDir: string): DaemonProc => { cwd: repoRoot, env: { ...process.env, EXECUTOR_DATA_DIR: dataDir }, - stdin: "ignore", - stdout: "pipe", - stderr: "pipe", + stdio: ["ignore", "pipe", "pipe"], }, - ); + ) as DaemonProc; + +const exited = (proc: DaemonProc): Promise => + proc.exitCode !== null || proc.signalCode !== null + ? Promise.resolve() + : new Promise((resolve) => proc.once("exit", () => resolve())); const stopProc = async (proc: DaemonProc): Promise => { - if (proc.exitCode !== null) return; + if (proc.exitCode !== null || proc.signalCode !== null) return; proc.kill("SIGTERM"); - await Promise.race([proc.exited, Bun.sleep(3000)]); - if (proc.exitCode === null) proc.kill("SIGKILL"); + await Promise.race([exited(proc), new Promise((resolve) => setTimeout(resolve, 3000))]); + if (proc.exitCode === null && proc.signalCode === null) proc.kill("SIGKILL"); }; const startForegroundDaemon = (dataDir: string) => @@ -322,7 +316,12 @@ scenario( ); daemon.proc.kill("SIGKILL"); - yield* Effect.promise(() => Promise.race([daemon.proc.exited, Bun.sleep(3000)])); + yield* Effect.promise(() => + Promise.race([ + exited(daemon.proc), + new Promise((resolve) => setTimeout(resolve, 3000)), + ]), + ); // The next call must settle (reject) quickly — a 10s bound well under the // scenario timeout catches a hang. @@ -332,7 +331,7 @@ scenario( .callTool({ name: "execute", arguments: { code: "return 3" } }) .then(() => "resolved" as const) .catch(() => "rejected" as const), - Bun.sleep(10_000).then(() => "timeout" as const), + new Promise<"timeout">((resolve) => setTimeout(() => resolve("timeout"), 10_000)), ]), ); // eslint-disable-next-line no-console From c9cb8f060b00e386e9503e29795cf393d1706317 Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Mon, 17 Aug 2026 16:50:30 -0700 Subject: [PATCH 3/3] Stabilize CI process lifecycle --- .github/workflows/ci.yml | 47 +++--- apps/cli/src/main.ts | 2 + apps/local/src/serve.ts | 50 +++++-- e2e/local/boot-process.test.ts | 101 +++++++++++++ .../cli-mcp-daemon-attach-stress.test.ts | 29 ++-- e2e/local/cli-mcp-protocol.test.ts | 21 +-- e2e/local/daemon-process.ts | 55 +++++++ e2e/local/local-server.ts | 10 +- e2e/local/vite-dev-routing.test.ts | 5 +- e2e/selfhost/posthog-mcp-oauth.test.ts | 63 +++++--- e2e/setup/boot.ts | 135 ++++++++++++++++-- e2e/setup/cloud.boot.ts | 8 +- e2e/setup/cloud.globalsetup.ts | 4 +- e2e/setup/cloudflare.boot.ts | 9 +- e2e/setup/motel.ts | 6 +- e2e/setup/selfhost.boot.ts | 6 +- e2e/setup/selfhost.globalsetup.ts | 4 +- e2e/src/ports.ts | 28 +++- .../hosts/mcp/src/stdio-integration.test.ts | 60 +++++++- 19 files changed, 512 insertions(+), 131 deletions(-) create mode 100644 e2e/local/boot-process.test.ts create mode 100644 e2e/local/daemon-process.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6db740b719..dc003182b3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -192,13 +192,18 @@ jobs: fail-fast: false matrix: include: - # Each cloud shard boots its own fresh dev stack. On 4 vCPU runners, - # four fatter shards keep the longest shard below selfhost while saving - # four runner boots and four warm cache restores. - - { target: cloud, shard: 1/4, shard-name: 1of4 } - - { target: cloud, shard: 2/4, shard-name: 2of4 } - - { target: cloud, shard: 3/4, shard-name: 3of4 } - - { target: cloud, shard: 4/4, shard-name: 4of4 } + # PGlite is deliberately single-connection, and under a sustained + # multi-minute shard it can stop accepting postgres sockets. Keep + # every hermetic dev stack short: eight serial shards remove that + # lifetime-dependent failure and put cloud below the selfhost lane. + - { target: cloud, shard: 1/8, shard-name: 1of8 } + - { target: cloud, shard: 2/8, shard-name: 2of8 } + - { target: cloud, shard: 3/8, shard-name: 3of8 } + - { target: cloud, shard: 4/8, shard-name: 4of8 } + - { target: cloud, shard: 5/8, shard-name: 5of8 } + - { target: cloud, shard: 6/8, shard-name: 6of8 } + - { target: cloud, shard: 7/8, shard-name: 7of8 } + - { target: cloud, shard: 8/8, shard-name: 8of8 } # Selfhost shards the same way: each shard is its own runner booting # its own fresh instance (own port block + data dir), so the # project's shared-bootstrap-admin assumption stays intact per shard @@ -247,20 +252,20 @@ jobs: # The globalsetup boots the target's own dev server (ports are claimed # per checkout, so this is hermetic) and tears it down after the run. - # --retry=2: browser scenarios can still hit isolated waitFor timeouts - # (single-test waitFor timeouts, not systemic failures); a retry on the - # same booted stack clears them. + # Do not retry scenarios: retries hide flakes and multiply slow timeout + # failures. The fixtures and process lifecycle are deterministic enough + # that the first result is the result. - name: Run cloud scenarios if: matrix.target == 'cloud' env: MCP_SESSION_TIMEOUT_MS: "3000" MCP_PAUSED_SESSION_IDLE_TIMEOUT_MS: "6000" - run: bunx vitest run --project cloud --retry=2 ${{ matrix.shard && format('--shard={0}', matrix.shard) || '' }} + run: bunx vitest run --project cloud ${{ matrix.shard && format('--shard={0}', matrix.shard) || '' }} working-directory: e2e - name: Run selfhost scenarios if: matrix.target == 'selfhost' - run: bunx vitest run --project selfhost --retry=2 ${{ matrix.shard && format('--shard={0}', matrix.shard) || '' }} + run: bunx vitest run --project selfhost ${{ matrix.shard && format('--shard={0}', matrix.shard) || '' }} working-directory: e2e # Failed runs keep their trace.zip / session.mp4 / step screenshots in @@ -274,10 +279,7 @@ jobs: retention-days: 7 e2e-local: - name: E2E (stdio MCP) - # Skipped on pull_request: the local scenario boots a real `executor web` - # plus a browser and is currently flaky on PRs. Still runs on push to main. - if: github.event_name != 'pull_request' + name: E2E (local) runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 20 steps: @@ -320,15 +322,10 @@ jobs: run: bunx playwright install --with-deps chromium chromium-headless-shell working-directory: e2e - # The `local` project is excluded from the default `test` chain (each - # scenario boots its own `executor web`). Run just the stdio MCP scenario - # here: it is the auto-connect / env-as-secret regression guard, and - # running it alone avoids the boot-resource accumulation and the - # pre-existing browser flakiness of the rest of the local suite. Expanding - # to the full `local` project (bun run test:local) is a follow-up once - # those are stabilized. - - name: Run the stdio MCP scenario - run: bunx vitest run --project local local/stdio-mcp.test.ts + # Each scenario owns its server, browser, data directory, and descendants; + # run the complete hermetic suite on PRs without scenario retries. + - name: Run local scenarios + run: bunx vitest run --project local working-directory: e2e desktop-smoke: diff --git a/apps/cli/src/main.ts b/apps/cli/src/main.ts index 2b59190897..03df64876c 100644 --- a/apps/cli/src/main.ts +++ b/apps/cli/src/main.ts @@ -215,9 +215,11 @@ const waitForShutdownSignal = () => const shutdown = () => resume(Effect.void); process.once("SIGINT", shutdown); process.once("SIGTERM", shutdown); + process.once("SIGHUP", shutdown); return Effect.sync(() => { process.off("SIGINT", shutdown); process.off("SIGTERM", shutdown); + process.off("SIGHUP", shutdown); }); }); diff --git a/apps/local/src/serve.ts b/apps/local/src/serve.ts index fd3ca6ac4a..95e8b7705d 100644 --- a/apps/local/src/serve.ts +++ b/apps/local/src/serve.ts @@ -113,6 +113,8 @@ interface ViteChild { readonly stop: () => Promise; } +const viteChildSignals = ["SIGINT", "SIGTERM", "SIGHUP"] as const; + async function allocatePort(): Promise { const probe = Bun.serve({ port: 0, @@ -127,15 +129,15 @@ async function allocatePort(): Promise { async function startViteChild(): Promise { const vitePort = await allocatePort(); const cwd = resolve(import.meta.dirname, ".."); + const viteEntrypoint = resolve(cwd, "node_modules/vite/bin/vite.js"); const env = { ...process.env }; delete env.PORT; - // `bunx --bun vite` runs vite under Bun, matching the `dev:vite` script - // already in apps/local. --strictPort keeps the URL we hand back stable. + // Run Vite directly under Bun, matching the `dev:vite` script without a + // bunx wrapper that can outlive its child. --strictPort keeps the URL stable. const child: Subprocess = Bun.spawn( [ - "bunx", - "--bun", - "vite", + process.execPath, + viteEntrypoint, "dev", "--port", String(vitePort), @@ -158,20 +160,45 @@ async function startViteChild(): Promise { }, ); + let stopping = false; + const stop = async (): Promise => { + if (stopping) { + await child.exited; + return; + } + stopping = true; + for (const signal of viteChildSignals) process.off(signal, stopOnParentSignal); + if (child.exitCode === null) child.kill(); + await Promise.race([child.exited, Bun.sleep(5_000)]); + if (child.exitCode === null) child.kill("SIGKILL"); + await child.exited; + }; + const stopOnParentSignal = (): void => { + // A PTY/session teardown can signal the CLI while Vite is still optimizing + // dependencies, before the server's normal stop handle exists. Reap the + // owned child immediately; the CLI's signal waiter performs full cleanup + // once startup has completed. + void stop(); + }; + for (const signal of viteChildSignals) process.once(signal, stopOnParentSignal); + const url = `http://127.0.0.1:${vitePort}`; const deadline = Date.now() + 30_000; while (Date.now() < deadline) { // oxlint-disable-next-line executor/no-try-catch-or-throw -- boundary: probing a child process that may not be listening yet try { - const r = await fetch(`${url}/`, { redirect: "manual" }); + const r = await fetch(`${url}/`, { + redirect: "manual", + // A listening socket is not proof that Vite can answer. Bound each + // probe so one accepted-but-stalled request cannot defeat the 30s boot + // deadline and wedge the entire local e2e suite. + signal: AbortSignal.timeout(5_000), + }); if (r.status < 500) { await r.body?.cancel(); return { url, - stop: async () => { - child.kill(); - await child.exited; - }, + stop, }; } await r.body?.cancel(); @@ -179,12 +206,13 @@ async function startViteChild(): Promise { // not up yet } if (child.exitCode !== null) { + await stop(); // oxlint-disable-next-line executor/no-try-catch-or-throw, executor/no-error-constructor -- boundary: child process aborted before becoming ready throw new Error(`vite dev exited with code ${child.exitCode} before becoming ready`); } await Bun.sleep(150); } - child.kill(); + await stop(); // oxlint-disable-next-line executor/no-try-catch-or-throw, executor/no-error-constructor -- boundary: vite never became reachable throw new Error(`vite dev did not become reachable on ${url} within 30s`); } diff --git a/e2e/local/boot-process.test.ts b/e2e/local/boot-process.test.ts new file mode 100644 index 0000000000..f5c43cca20 --- /dev/null +++ b/e2e/local/boot-process.test.ts @@ -0,0 +1,101 @@ +import { describe, expect, it } from "@effect/vitest"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { + BootProcessExitError, + BootReadinessTimeoutError, + bootProcesses, + isBootReadinessTimeout, + waitForBoot, +} from "../setup/boot"; +import { claimAndBoot, isAddrInUse } from "../src/ports"; + +describe("e2e boot process lifecycle", () => { + it("fails immediately with the boot log when a child exits", async () => { + const tempDir = mkdtempSync(join(tmpdir(), "executor-e2e-boot-")); + const logFile = join(tempDir, "boot.log"); + let readinessProbeAborted = false; + + try { + const processes = bootProcesses( + [ + { + cmd: process.execPath, + args: [ + "-e", + 'console.error("Error: Port 44550 is already in use (EADDRINUSE)"); process.exit(17)', + ], + cwd: tempDir, + logFile, + }, + ], + { label: "lifecycle-test" }, + ); + + const startedAt = Date.now(); + let failure: unknown; + try { + await waitForBoot( + processes, + (signal) => + new Promise((_resolve, reject) => { + signal.addEventListener( + "abort", + () => { + readinessProbeAborted = true; + // oxlint-disable-next-line executor/no-promise-reject -- boundary: the fixture models an abort-aware readiness promise + reject(signal.reason); + }, + { once: true }, + ); + }), + ); + } catch (error) { + failure = error; + } + + expect(failure).toBeInstanceOf(BootProcessExitError); + expect(Date.now() - startedAt, "child exit beats the readiness timeout").toBeLessThan(5_000); + expect(readinessProbeAborted, "the losing readiness probe is cancelled").toBe(true); + expect(isAddrInUse(failure), "the port claimer can retry this boot failure").toBe(true); + const bootFailure = failure as BootProcessExitError; + expect(bootFailure.exitCode).toBe(17); + expect(bootFailure.logTail).toContain("Port 44550 is already in use"); + + await processes.teardown(); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } + }); + + it("releases a failed claim before retrying a readiness timeout", async () => { + const envVar = "E2E_BOOT_LIFECYCLE_TEST_PORT"; + let attempts = 0; + const claimedPorts: number[] = []; + + try { + const booted = await claimAndBoot( + [{ envVar, offset: 8, label: "boot lifecycle test" }], + async (ports) => { + attempts += 1; + claimedPorts.push(ports[envVar]!); + if (attempts === 1) { + throw new BootReadinessTimeoutError("http://127.0.0.1:1", 10, "fixture timeout"); + } + return { teardown: async () => {}, value: "ready" }; + }, + { maxAttempts: 2, label: "lifecycle-test", retryWhen: isBootReadinessTimeout }, + ); + + expect(booted.value).toBe("ready"); + expect(attempts).toBe(2); + expect(claimedPorts).toHaveLength(2); + await booted.teardown(); + expect(process.env[envVar], "the successful claim is cleared at teardown").toBeUndefined(); + } finally { + delete process.env[envVar]; + } + }); +}); diff --git a/e2e/local/cli-mcp-daemon-attach-stress.test.ts b/e2e/local/cli-mcp-daemon-attach-stress.test.ts index 57ebd5fac5..488c7e13fa 100644 --- a/e2e/local/cli-mcp-daemon-attach-stress.test.ts +++ b/e2e/local/cli-mcp-daemon-attach-stress.test.ts @@ -28,6 +28,7 @@ import { join } from "node:path"; import { fileURLToPath } from "node:url"; import { scenario } from "../src/scenario"; +import { stopAutoSpawnedDaemon } from "./daemon-process"; const repoRoot = fileURLToPath(new URL("../../", import.meta.url)); const testScope = join(repoRoot, "apps/local"); @@ -95,6 +96,7 @@ const spawnDaemon = (dataDir: string): DaemonProc => cwd: repoRoot, env: { ...process.env, EXECUTOR_DATA_DIR: dataDir }, stdio: ["ignore", "pipe", "pipe"], + detached: true, }, ) as DaemonProc; @@ -105,9 +107,12 @@ const exited = (proc: DaemonProc): Promise => const stopProc = async (proc: DaemonProc): Promise => { if (proc.exitCode !== null || proc.signalCode !== null) return; - proc.kill("SIGTERM"); + if (proc.pid) process.kill(-proc.pid, "SIGTERM"); await Promise.race([exited(proc), new Promise((resolve) => setTimeout(resolve, 3000))]); - if (proc.exitCode === null && proc.signalCode === null) proc.kill("SIGKILL"); + if (proc.exitCode === null && proc.signalCode === null && proc.pid) { + process.kill(-proc.pid, "SIGKILL"); + await exited(proc); + } }; const startForegroundDaemon = (dataDir: string) => @@ -181,28 +186,14 @@ const runOneClient = async ( } }; -/** `executor mcp` now ensures a DURABLE (detached) daemon and bridges to it, so a - * cold-start scenario leaves that daemon running. Stop it before removing the - * data dir so the test never leaks an orphan daemon. */ -const stopAutoSpawnedDaemon = (dataDir: string): void => { - try { - const manifest = JSON.parse( - readFileSync(join(dataDir, "server-control", "server.json"), "utf8"), - ) as { pid?: number }; - if (manifest.pid) process.kill(manifest.pid, "SIGTERM"); - } catch { - // no manifest (no daemon spawned) — nothing to stop. - } -}; - const withTempData = Effect.acquireRelease( Effect.sync(() => { const root = mkdtempSync(join(tmpdir(), "executor-mcp-stress-")); return join(root, "data"); }), (dataDir) => - Effect.sync(() => { - stopAutoSpawnedDaemon(dataDir); + Effect.promise(async () => { + await stopAutoSpawnedDaemon(dataDir); rmSync(join(dataDir, ".."), { recursive: true, force: true }); }), ); @@ -315,7 +306,7 @@ scenario( "2", ); - daemon.proc.kill("SIGKILL"); + if (daemon.proc.pid) process.kill(-daemon.proc.pid, "SIGKILL"); yield* Effect.promise(() => Promise.race([ exited(daemon.proc), diff --git a/e2e/local/cli-mcp-protocol.test.ts b/e2e/local/cli-mcp-protocol.test.ts index 144883d1fc..0d002eefc7 100644 --- a/e2e/local/cli-mcp-protocol.test.ts +++ b/e2e/local/cli-mcp-protocol.test.ts @@ -11,12 +11,13 @@ import { StdioClientTransport as ModernStdioClientTransport } from "@modelcontex import { Client as LegacyClient } from "@modelcontextprotocol/sdk/client/index.js"; import { StdioClientTransport as LegacyStdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js"; import { Effect } from "effect"; -import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { fileURLToPath } from "node:url"; import { scenario } from "../src/scenario"; +import { stopAutoSpawnedDaemon } from "./daemon-process"; const repoRoot = fileURLToPath(new URL("../../", import.meta.url)); const testScope = join(repoRoot, "apps/local"); @@ -33,28 +34,14 @@ const bridgeCommand = (dataDir: string) => ({ stderr: "pipe" as const, }); -const stopAutoSpawnedDaemon = (dataDir: string): void => { - // The bridge is transient, while its auto-started daemon is detached. Reap - // that owner before deleting this scenario's private data directory. - // oxlint-disable-next-line executor/no-try-catch-or-throw -- cleanup tolerates a bridge that failed before writing its manifest - try { - const manifest = JSON.parse( - readFileSync(join(dataDir, "server-control", "server.json"), "utf8"), - ) as { readonly pid?: number }; - if (manifest.pid) process.kill(manifest.pid, "SIGTERM"); - } catch { - // No manifest means there is no auto-started daemon to stop. - } -}; - const withTempData = Effect.acquireRelease( Effect.sync(() => { const root = mkdtempSync(join(tmpdir(), "executor-mcp-protocol-")); return { root, dataDir: join(root, "data") }; }), ({ root, dataDir }) => - Effect.sync(() => { - stopAutoSpawnedDaemon(dataDir); + Effect.promise(async () => { + await stopAutoSpawnedDaemon(dataDir); rmSync(root, { recursive: true, force: true }); }), ); diff --git a/e2e/local/daemon-process.ts b/e2e/local/daemon-process.ts new file mode 100644 index 0000000000..d7eddb09b4 --- /dev/null +++ b/e2e/local/daemon-process.ts @@ -0,0 +1,55 @@ +import { readFile } from "node:fs/promises"; +import { join } from "node:path"; + +const isAlive = (pid: number): boolean => { + // oxlint-disable-next-line executor/no-try-catch-or-throw -- boundary: process liveness probing reports false for an already-reaped test daemon + try { + process.kill(process.platform === "win32" ? pid : -pid, 0); + return true; + } catch { + return false; + } +}; + +const signal = (pid: number, name: NodeJS.Signals): void => { + // Auto-started daemons are detached process-group leaders. Signal the whole + // private group so their Vite child cannot survive a failed test. + // oxlint-disable-next-line executor/no-try-catch-or-throw -- boundary: Windows and pre-detach failures require a direct-pid fallback + try { + process.kill(process.platform === "win32" ? pid : -pid, name); + } catch { + // oxlint-disable-next-line executor/no-try-catch-or-throw -- cleanup tolerates a process that exited between the liveness check and signal + try { + process.kill(pid, name); + } catch {} + } +}; + +const waitUntilStopped = async (pid: number, timeoutMs: number): Promise => { + const deadline = performance.now() + timeoutMs; + while (performance.now() < deadline) { + if (!isAlive(pid)) return true; + await new Promise((resolve) => setTimeout(resolve, 50)); + } + return !isAlive(pid); +}; + +/** Stop the detached daemon elected by an `executor mcp` cold start. */ +export const stopAutoSpawnedDaemon = async (dataDir: string): Promise => { + // oxlint-disable-next-line executor/no-try-catch-or-throw -- cleanup tolerates a bridge that failed before writing its manifest + try { + const manifest = JSON.parse( + await readFile(join(dataDir, "server-control", "server.json"), "utf8"), + ) as { readonly pid?: unknown }; + if (!Number.isSafeInteger(manifest.pid) || (manifest.pid as number) <= 0) return; + + const pid = manifest.pid as number; + signal(pid, "SIGTERM"); + if (await waitUntilStopped(pid, 10_000)) return; + + signal(pid, "SIGKILL"); + await waitUntilStopped(pid, 2_000); + } catch { + // No manifest means there is no auto-started daemon to stop. + } +}; diff --git a/e2e/local/local-server.ts b/e2e/local/local-server.ts index 10a1fe27a2..798a62cc9f 100644 --- a/e2e/local/local-server.ts +++ b/e2e/local/local-server.ts @@ -68,7 +68,7 @@ export const withLocalServer = ( yield* Effect.all( [ cli.session( - ["bun", "run", "dev:cli", "web", "--foreground", "--port", "0"], + ["bun", "run", "apps/cli/src/main.ts", "web", "--foreground", "--port", "0"], async (term) => { markRecordingStart(runDir, "terminal"); markFocus(runDir, "terminal"); @@ -107,10 +107,16 @@ export const withLocalServer = ( // otherwise the orphaned child wedges the terminal teardown. markFocus(runDir, "terminal"); await term.keyboard.press("Control+C"); + await term.waitForExit({ timeoutMs: 15_000 }); }, { cwd: repoRoot, - env: { EXECUTOR_DATA_DIR: dataDir, EXECUTOR_SCOPE_DIR: dataDir, ...options?.env }, + env: { + EXECUTOR_DEV: "1", + EXECUTOR_DATA_DIR: dataDir, + EXECUTOR_SCOPE_DIR: dataDir, + ...options?.env, + }, record: join(runDir, options?.castName ?? "terminal.cast"), viewport: { cols: 120, rows: 40 }, }, diff --git a/e2e/local/vite-dev-routing.test.ts b/e2e/local/vite-dev-routing.test.ts index 287eb3f8c6..3e0c490311 100644 --- a/e2e/local/vite-dev-routing.test.ts +++ b/e2e/local/vite-dev-routing.test.ts @@ -80,10 +80,11 @@ const startPlainViteDev = async (): Promise => { const dataDir = mkdtempSync(join(tmpdir(), "executor-local-vite-e2e-")); const port = await freePort(); const origin = `http://127.0.0.1:${port}`; + const viteEntrypoint = join(localAppDir, "node_modules", "vite", "bin", "vite.js"); let logs = ""; const child = spawn( - "bunx", - ["--bun", "vite", "dev", "--host", "127.0.0.1", "--port", String(port), "--strictPort"], + "bun", + [viteEntrypoint, "dev", "--host", "127.0.0.1", "--port", String(port), "--strictPort"], { cwd: localAppDir, env: { ...process.env, EXECUTOR_DATA_DIR: dataDir, PORT: String(port) }, diff --git a/e2e/selfhost/posthog-mcp-oauth.test.ts b/e2e/selfhost/posthog-mcp-oauth.test.ts index 9163831649..49d20349a9 100644 --- a/e2e/selfhost/posthog-mcp-oauth.test.ts +++ b/e2e/selfhost/posthog-mcp-oauth.test.ts @@ -1,8 +1,14 @@ -// Selfhost browser regression for the reported PostHog MCP OAuth dead-end. A -// real Executor instance adds https://mcp.posthog.com/mcp, then starts the -// connection flow. The product guarantee: clicking Connect opens PostHog's -// OAuth authorization page through dynamic client registration, not the -// bring-your-own OAuth app picker with "Automatic setup unavailable". +// Hermetic selfhost browser regression for the reported PostHog MCP OAuth +// dead-end. A real Executor instance adds a wire-level OAuth-protected MCP +// server, then starts the connection flow. The product guarantee: clicking +// Connect reaches the authorization page through dynamic client registration, +// not the bring-your-own OAuth app picker with "Automatic setup unavailable". +// +// This used to call PostHog's production MCP and OAuth sites directly. That +// made Executor CI depend on a third party's availability, metadata, and popup +// response time. The local fixtures implement the same RFC 9728 discovery, +// RFC 8414 metadata, RFC 7591 registration, and authorization redirect over +// real HTTP, while keeping the assertion deterministic. import { randomBytes } from "node:crypto"; import { expect } from "@effect/vitest"; @@ -10,22 +16,28 @@ import { Effect } from "effect"; import { composePluginApi } from "@executor-js/api/server"; import { deriveMcpNamespace } from "@executor-js/plugin-mcp"; import { mcpHttpPlugin } from "@executor-js/plugin-mcp/api"; +import { makeGreetingMcpServer, serveMcpServerWithOAuth } from "@executor-js/plugin-mcp/testing"; import { IntegrationSlug } from "@executor-js/sdk/shared"; +import { OAuthTestServer } from "@executor-js/sdk/testing"; import { scenario } from "../src/scenario"; import { Api, Browser, Target } from "../src/services"; -const POSTHOG_MCP_URL = "https://mcp.posthog.com/mcp"; const api = composePluginApi([mcpHttpPlugin()] as const); scenario( - "MCP OAuth · PostHog starts OAuth from Add connection", + "MCP OAuth · dynamic registration opens the discovered authorization server", { timeout: 180_000 }, Effect.scoped( Effect.gen(function* () { const target = yield* Target; const browser = yield* Browser; const { client: makeApiClient } = yield* Api; + const oauth = yield* OAuthTestServer; + const server = yield* serveMcpServerWithOAuth( + () => makeGreetingMcpServer({ name: "dcr-regression-mcp" }), + { path: "/mcp" }, + ); const identity = yield* target.newIdentity(); const client = yield* makeApiClient(api, identity); const displayName = `PostHog MCP ${randomBytes(3).toString("hex")}`; @@ -33,9 +45,9 @@ scenario( yield* Effect.gen(function* () { yield* browser.session(identity, async ({ page, step }) => { - await step("Open the add-MCP flow pointed at PostHog", async () => { + await step("Open the add-MCP flow pointed at the OAuth server", async () => { const addUrl = new URL("/integrations/add/mcp", target.baseUrl); - addUrl.searchParams.set("url", POSTHOG_MCP_URL); + addUrl.searchParams.set("url", server.endpoint); await page.goto(addUrl.toString(), { waitUntil: "networkidle" }); await page.getByText("How does this server authenticate?").waitFor({ timeout: 30_000 }); await page.getByText("Method 1 · Detected").waitFor(); @@ -59,26 +71,37 @@ scenario( const popupPromise = page.waitForEvent("popup", { timeout: 30_000 }); await page.getByRole("button", { name: "Connect", exact: true }).click(); const popup = await popupPromise; - await popup.waitForURL(/^https:\/\/oauth\.posthog\.com\/oauth\/authorize\//, { + await popup.waitForURL((url) => url.origin === new URL(oauth.issuerUrl).origin, { timeout: 30_000, }); await popup.waitForLoadState("domcontentloaded", { timeout: 30_000 }); const authorizeUrl = new URL(popup.url()); - expect(authorizeUrl.origin, "OAuth opened PostHog's authorization host").toBe( - "https://oauth.posthog.com", + expect(authorizeUrl.origin, "OAuth opened the discovered authorization host").toBe( + new URL(oauth.authorizationEndpoint).origin, ); - expect(authorizeUrl.pathname, "OAuth opened the authorize endpoint").toBe( - "/oauth/authorize/", - ); - expect( - authorizeUrl.searchParams.get("resource"), - "resource targets the MCP endpoint", - ).toBe(POSTHOG_MCP_URL); await popup.close(); }); }); + + const oauthRequests = yield* oauth.requests; + expect( + oauthRequests.some( + (request) => request.method === "POST" && request.path === "/register", + ), + "the connection flow dynamically registered its OAuth client", + ).toBe(true); + const authorizeRequest = oauthRequests.find( + (request) => request.method === "GET" && request.path === "/authorize", + ); + expect( + authorizeRequest, + "the popup reached the discovered authorize endpoint", + ).toBeDefined(); + expect(authorizeRequest?.query.resource, "resource targets the MCP endpoint").toBe( + server.endpoint, + ); }).pipe(Effect.ensuring(client.mcp.removeServer({ params: { slug } }).pipe(Effect.ignore))); }), - ), + ).pipe(Effect.provide(OAuthTestServer.layer())), ); diff --git a/e2e/setup/boot.ts b/e2e/setup/boot.ts index 2b6b137313..bec463023d 100644 --- a/e2e/setup/boot.ts +++ b/e2e/setup/boot.ts @@ -3,7 +3,46 @@ // what runs (their dev stack, their stub flags); this file only owns process // lifecycle, so it stays target-agnostic. import { spawn, type ChildProcess } from "node:child_process"; -import { openSync } from "node:fs"; +import { closeSync, openSync, readFileSync } from "node:fs"; +import { setTimeout as sleep } from "node:timers/promises"; + +const BOOT_LOG_TAIL_BYTES = 8_000; +const HTTP_PROBE_TIMEOUT_MS = 5_000; + +/** A long-lived boot process exited before its service became ready. */ +export class BootProcessExitError extends Error { + readonly _tag = "BootProcessExitError"; + + constructor( + readonly command: string, + readonly exitCode: number | null, + readonly signal: NodeJS.Signals | null, + readonly logTail: string, + ) { + const outcome = exitCode === null ? `signal ${signal ?? "unknown"}` : `code ${exitCode}`; + const diagnostic = logTail ? `\nLast boot log output:\n${logTail}` : ""; + super(`Boot process ${JSON.stringify(command)} exited with ${outcome}${diagnostic}`); + this.name = "BootProcessExitError"; + } +} + +/** A service process stayed alive but did not become HTTP-ready in time. */ +export class BootReadinessTimeoutError extends Error { + readonly _tag = "BootReadinessTimeoutError"; + + constructor( + readonly url: string, + readonly timeoutMs: number, + readonly lastError: unknown, + ) { + super(`Timed out after ${timeoutMs}ms waiting for ${url}: ${String(lastError)}`); + this.name = "BootReadinessTimeoutError"; + } +} + +/** Whether an acquisition failed because a live process never became ready. */ +export const isBootReadinessTimeout = (error: unknown): boolean => + error instanceof BootReadinessTimeoutError; export interface BootedProcesses { readonly teardown: () => Promise; @@ -11,6 +50,21 @@ export interface BootedProcesses { readonly pids: ReadonlyArray; } +/** A spawned process tree that can report an exit during startup. */ +export interface MonitoredBootedProcesses extends BootedProcesses { + /** Rejects as soon as any child exits before readiness/teardown. */ + readonly unexpectedExit: Promise; +} + +const readLogTail = (logFile: string | undefined): string => { + if (!logFile) return ""; + try { + return readFileSync(logFile, "utf8").slice(-BOOT_LOG_TAIL_BYTES).trim(); + } catch { + return ""; + } +}; + export const bootProcesses = ( procs: ReadonlyArray<{ readonly cmd: string; @@ -21,9 +75,10 @@ export const bootProcesses = ( readonly logFile?: string; }>, options: { readonly label: string }, -): BootedProcesses => { +): MonitoredBootedProcesses => { const children: ChildProcess[] = []; let tearingDown = false; + const unexpectedExits: Array> = []; for (const proc of procs) { const log = proc.logFile ? openSync(proc.logFile, "a") : undefined; const child = spawn(proc.cmd, [...proc.args], { @@ -36,11 +91,35 @@ export const bootProcesses = ( // kill and squat the port into the NEXT invocation's waitForHttp. detached: true, }); - child.on("exit", (code) => { - if (code !== 0 && code !== null && !tearingDown) { - console.error(`[e2e:${options.label}] ${proc.cmd} exited with ${code}`); - } - }); + if (log !== undefined) closeSync(log); + unexpectedExits.push( + new Promise((_resolve, reject) => { + child.once("error", (cause) => { + if (tearingDown) return; + // oxlint-disable-next-line executor/no-promise-reject -- boundary: adapt the child-process error event to the startup race + reject( + new BootProcessExitError( + [proc.cmd, ...proc.args].join(" "), + child.exitCode, + child.signalCode, + `${readLogTail(proc.logFile)}\n${String(cause)}`.trim(), + ), + ); + }); + child.once("exit", (code, signal) => { + if (tearingDown) return; + const error = new BootProcessExitError( + [proc.cmd, ...proc.args].join(" "), + code, + signal, + readLogTail(proc.logFile), + ); + console.error(`[e2e:${options.label}] ${error.message}`); + // oxlint-disable-next-line executor/no-promise-reject -- boundary: adapt the child-process exit event to the startup race + reject(error); + }); + }), + ); children.push(child); } @@ -77,28 +156,58 @@ export const bootProcesses = ( } }, pids: children.flatMap((child) => (child.pid === undefined ? [] : [child.pid])), + unexpectedExit: Promise.race(unexpectedExits), }; }; +/** + * Wait for a boot probe while also observing the spawned process tree. The + * losing readiness probe is aborted, so an early process exit neither hides + * behind the full HTTP timeout nor leaves a polling timer alive. + */ +export const waitForBoot = async ( + processes: MonitoredBootedProcesses, + ready: (signal: AbortSignal) => Promise, +): Promise => { + const controller = new AbortController(); + try { + return await Promise.race([ready(controller.signal), processes.unexpectedExit]); + } finally { + controller.abort(); + } +}; + export const waitForHttp = async ( url: string, - options: { readonly timeoutMs?: number; readonly expectRedirect?: boolean } = {}, + options: { + readonly timeoutMs?: number; + readonly expectRedirect?: boolean; + readonly signal?: AbortSignal; + } = {}, ): Promise => { - const deadline = Date.now() + (options.timeoutMs ?? 90_000); + const timeoutMs = options.timeoutMs ?? 90_000; + const deadline = performance.now() + timeoutMs; let lastError: unknown; - while (Date.now() < deadline) { + while (performance.now() < deadline) { + options.signal?.throwIfAborted(); try { - const response = await fetch(url, { redirect: "manual" }); + const remainingMs = Math.max(1, deadline - performance.now()); + const probeTimeout = AbortSignal.timeout(Math.min(HTTP_PROBE_TIMEOUT_MS, remainingMs)); + const signal = options.signal + ? AbortSignal.any([options.signal, probeTimeout]) + : probeTimeout; + const response = await fetch(url, { redirect: "manual", signal }); // During a cold vite compile /api/* falls back to the SPA's 200 HTML — // expectRedirect waits for the real handler (302) instead. if (options.expectRedirect ? response.status === 302 : response.status < 500) return; lastError = new Error(`status ${response.status}`); } catch (error) { + options.signal?.throwIfAborted(); lastError = error; } - await new Promise((resolve) => setTimeout(resolve, 400)); + await sleep(400, undefined, { signal: options.signal }); } - throw new Error(`timed out waiting for ${url}: ${String(lastError)}`); + throw new BootReadinessTimeoutError(url, timeoutMs, lastError); }; /** diff --git a/e2e/setup/cloud.boot.ts b/e2e/setup/cloud.boot.ts index 3544afabe0..191fdebde3 100644 --- a/e2e/setup/cloud.boot.ts +++ b/e2e/setup/cloud.boot.ts @@ -10,7 +10,7 @@ import { fileURLToPath } from "node:url"; // Vendored fork import (same pattern as mcporter). import { createEmulator } from "@executor-js/emulate"; -import { bootProcesses, waitForHttp } from "./boot"; +import { bootProcesses, waitForBoot, waitForHttp } from "./boot"; import { AUTUMN_PLAN_SEED } from "./autumn-plans"; import { E2E_EXECUTION_RATE_LIMIT } from "./execution-limits"; @@ -168,9 +168,11 @@ export const bootCloud = async (options: CloudBootOptions): Promise try { const local = `http://127.0.0.1:${options.cloudPort}`; - await waitForHttp(local); + await waitForBoot(procs, (signal) => waitForHttp(local, { signal })); // The API plane is ready when login actually redirects to AuthKit. - await waitForHttp(`${local}/api/auth/login`, { expectRedirect: true }); + await waitForBoot(procs, (signal) => + waitForHttp(`${local}/api/auth/login`, { expectRedirect: true, signal }), + ); } catch (error) { await teardown(); throw error; diff --git a/e2e/setup/cloud.globalsetup.ts b/e2e/setup/cloud.globalsetup.ts index 648abe568d..98433ca835 100644 --- a/e2e/setup/cloud.globalsetup.ts +++ b/e2e/setup/cloud.globalsetup.ts @@ -8,7 +8,7 @@ import { resolve } from "node:path"; import { claimAndBoot } from "../src/ports"; import { E2E_COOKIE_PASSWORD, E2E_WORKOS_CLIENT_ID } from "../targets/cloud"; -import { waitForHttp } from "./boot"; +import { isBootReadinessTimeout, waitForHttp } from "./boot"; import { bootCloud } from "./cloud.boot"; import { ensureE2eMcpSessionTimeoutEnv } from "./mcp-session-timeouts"; import { bootMotel, motelExporterEnv } from "./motel"; @@ -93,7 +93,7 @@ export default async function setup(): Promise<(() => Promise) | void> { }); return { teardown: cloud.teardown, value: cloud }; }, - { label: "cloud" }, + { label: "cloud", retryWhen: isBootReadinessTimeout }, ); // Publish the Autumn emulator URL to the test workers (they inherit this // process's env): scenarios that assert on tracked usage yield the Autumn diff --git a/e2e/setup/cloudflare.boot.ts b/e2e/setup/cloudflare.boot.ts index b883084f28..fa0eb70ce1 100644 --- a/e2e/setup/cloudflare.boot.ts +++ b/e2e/setup/cloudflare.boot.ts @@ -9,7 +9,7 @@ import { execFile } from "node:child_process"; import { fileURLToPath } from "node:url"; import { promisify } from "node:util"; -import { bootProcesses, waitForHttp, type BootedProcesses } from "./boot"; +import { bootProcesses, waitForBoot, waitForHttp, type BootedProcesses } from "./boot"; export const cloudflareDir = fileURLToPath(new URL("../../apps/host-cloudflare/", import.meta.url)); const wranglerBin = fileURLToPath( @@ -63,7 +63,12 @@ export const bootCloudflare = async (options: CloudflareBootOptions): Promise + waitForHttp(`http://127.0.0.1:${options.port}/api/account/me`, { + timeoutMs: 120_000, + signal, + }), + ); } catch (error) { await procs.teardown(); throw error; diff --git a/e2e/setup/motel.ts b/e2e/setup/motel.ts index ad14e8ec4c..1c10bfa441 100644 --- a/e2e/setup/motel.ts +++ b/e2e/setup/motel.ts @@ -8,7 +8,7 @@ import { mkdirSync, rmSync } from "node:fs"; import { join } from "node:path"; import { fileURLToPath } from "node:url"; -import { bootProcesses, waitForHttp, type BootedProcesses } from "./boot"; +import { bootProcesses, waitForBoot, waitForHttp, type MonitoredBootedProcesses } from "./boot"; export const MOTEL_PORT = 4796; export const MOTEL_URL = `http://127.0.0.1:${MOTEL_PORT}`; @@ -28,7 +28,7 @@ export const bootMotel = async (): Promise => { rmSync(dataDir, { recursive: true, force: true }); mkdirSync(dataDir, { recursive: true }); - let procs: BootedProcesses | null = null; + let procs: MonitoredBootedProcesses | null = null; // oxlint-disable-next-line executor/no-try-catch-or-throw -- boundary: optional infrastructure; a motel-less host still runs the suite try { procs = bootProcesses( @@ -45,7 +45,7 @@ export const bootMotel = async (): Promise => { ], { label: "motel" }, ); - await waitForHttp(`${MOTEL_URL}/api/health`); + await waitForBoot(procs, (signal) => waitForHttp(`${MOTEL_URL}/api/health`, { signal })); console.log(`[e2e] traces → suite motel at ${MOTEL_URL}`); return { url: MOTEL_URL, teardown: procs.teardown }; } catch (error) { diff --git a/e2e/setup/selfhost.boot.ts b/e2e/setup/selfhost.boot.ts index a705e4e543..90b29bc87d 100644 --- a/e2e/setup/selfhost.boot.ts +++ b/e2e/setup/selfhost.boot.ts @@ -5,7 +5,7 @@ import { rmSync } from "node:fs"; import { resolve } from "node:path"; import { fileURLToPath } from "node:url"; -import { bootProcesses, waitForHttp, type BootedProcesses } from "./boot"; +import { bootProcesses, waitForBoot, waitForHttp, type BootedProcesses } from "./boot"; export const selfhostDir = fileURLToPath(new URL("../../apps/host-selfhost/", import.meta.url)); @@ -68,7 +68,9 @@ export const bootSelfhost = async (options: SelfhostBootOptions): Promise + waitForHttp(`http://localhost:${options.port}`, { signal }), + ); } catch (error) { await procs.teardown(); throw error; diff --git a/e2e/setup/selfhost.globalsetup.ts b/e2e/setup/selfhost.globalsetup.ts index 5775a61cba..39bd59081f 100644 --- a/e2e/setup/selfhost.globalsetup.ts +++ b/e2e/setup/selfhost.globalsetup.ts @@ -7,7 +7,7 @@ import { resolve } from "node:path"; import { claimAndBoot } from "../src/ports"; import { SELFHOST_ADMIN } from "../targets/selfhost"; -import { waitForHttp } from "./boot"; +import { isBootReadinessTimeout, waitForHttp } from "./boot"; import { E2E_SANDBOX_TIMEOUT_MS, SANDBOX_TIMEOUT_ENV } from "./sandbox-timeout"; import { bootSelfhost } from "./selfhost.boot"; import { RUNS_DIR } from "../src/scenario"; @@ -53,7 +53,7 @@ export default async function setup(): Promise<(() => Promise) | void> { }); return { teardown: procs.teardown, value: procs }; }, - { label: "selfhost" }, + { label: "selfhost", retryWhen: isBootReadinessTimeout }, ); return teardown; } diff --git a/e2e/src/ports.ts b/e2e/src/ports.ts index 7176465e41..811f7f4fc1 100644 --- a/e2e/src/ports.ts +++ b/e2e/src/ports.ts @@ -207,6 +207,16 @@ export const claimPorts = async (claims: ReadonlyArray): Promise { + // Values published by this claim are not operator pins. Clear them + // when the acquisition is released so a failed `claimAndBoot` attempt + // can genuinely probe and claim again instead of treating its own + // stale E2E_* value as an explicit override on every retry. + for (const claim of unpinned) { + const published = ports[claim.envVar]; + if (published !== undefined && process.env[claim.envVar] === String(published)) { + delete process.env[claim.envVar]; + } + } const held = heldLocks.get(block); if (!held) return; heldLocks.delete(block); @@ -238,8 +248,9 @@ export const isAddrInUse = (error: unknown): boolean => { * ephemeral) an outbound socket can still grab a just-released probe port before * the service binds it. When that happens the boot throws EADDRINUSE; we release * the block (freeing its lock so `claimPorts` walks past it) and re-claim + retry - * up to `maxAttempts` times. Any non-EADDRINUSE boot failure — or exhausting the - * retries — releases and rethrows, so a genuinely broken boot still surfaces. + * up to `maxAttempts` times. Callers may also classify another idempotent + * acquisition failure with `retryWhen` (for example, a bounded Vite readiness + * timeout). Unclassified failures and exhausted retries surface unchanged. * * `boot` receives the freshly claimed ports and must return its teardown; the * returned `teardown` chains the caller's teardown then releases the block. @@ -247,7 +258,12 @@ export const isAddrInUse = (error: unknown): boolean => { export const claimAndBoot = async ( claims: ReadonlyArray, boot: (ports: Record) => Promise<{ teardown: () => Promise; value: T }>, - options: { readonly maxAttempts?: number; readonly label?: string } = {}, + options: { + readonly maxAttempts?: number; + readonly label?: string; + /** Additional acquisition failures that are safe to retry from scratch. */ + readonly retryWhen?: (error: unknown) => boolean; + } = {}, ): Promise<{ ports: Record; teardown: () => Promise; value: T }> => { const maxAttempts = options.maxAttempts ?? 3; const label = options.label ?? "boot"; @@ -267,13 +283,15 @@ export const claimAndBoot = async ( } catch (error) { await release(); lastError = error; - if (!isAddrInUse(error) || attempt === maxAttempts) throw error; + const retryable = isAddrInUse(error) || options.retryWhen?.(error) === true; + if (!retryable || attempt === maxAttempts) throw error; const collided = claims .map((claim) => ports[claim.envVar]) .filter((port): port is number => port !== undefined) .join(", "); + const reason = isAddrInUse(error) ? `hit EADDRINUSE on port(s) ${collided}` : String(error); console.warn( - `[e2e] ${label} hit EADDRINUSE on port(s) ${collided} (attempt ${attempt}/${maxAttempts}); re-claiming a fresh block and retrying`, + `[e2e] ${label} acquisition failed (${reason}, attempt ${attempt}/${maxAttempts}); re-claiming a fresh block and retrying`, ); } } diff --git a/packages/hosts/mcp/src/stdio-integration.test.ts b/packages/hosts/mcp/src/stdio-integration.test.ts index 5f6e03482f..8b831b1b4b 100644 --- a/packages/hosts/mcp/src/stdio-integration.test.ts +++ b/packages/hosts/mcp/src/stdio-integration.test.ts @@ -4,8 +4,8 @@ import { StdioClientTransport as ModernStdioClientTransport } from "@modelcontex import { Client } from "@modelcontextprotocol/sdk/client/index.js"; import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js"; import { ElicitRequestSchema } from "@modelcontextprotocol/sdk/types.js"; -import { Effect } from "effect"; -import { mkdtempSync } from "node:fs"; +import { Effect, Schema } from "effect"; +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; @@ -17,6 +17,60 @@ const stdioServer = { command: "bun", args: ["run", stdioServerEntry], }; +const decodeDaemonManifest = Schema.decodeUnknownSync( + Schema.fromJsonString(Schema.Struct({ pid: Schema.Number })), +); + +const isProcessGroupAlive = (pid: number): boolean => { + // oxlint-disable-next-line executor/no-try-catch-or-throw -- boundary: process liveness probing reports false after the test daemon is reaped + try { + process.kill(process.platform === "win32" ? pid : -pid, 0); + return true; + } catch { + return false; + } +}; + +const signalProcessGroup = (pid: number, signal: NodeJS.Signals): void => { + // oxlint-disable-next-line executor/no-try-catch-or-throw -- boundary: Windows and pre-detach failures require a direct-pid fallback + try { + process.kill(process.platform === "win32" ? pid : -pid, signal); + } catch { + // oxlint-disable-next-line executor/no-try-catch-or-throw -- cleanup tolerates a daemon that exited between liveness check and signal + try { + process.kill(pid, signal); + } catch {} + } +}; + +const stopAutoSpawnedDaemon = async (dataDir: string): Promise => { + // oxlint-disable-next-line executor/no-try-catch-or-throw -- cleanup tolerates startup that failed before publishing a manifest + try { + const manifest = decodeDaemonManifest( + readFileSync(join(dataDir, "server-control", "server.json"), "utf8"), + ); + if (!Number.isSafeInteger(manifest.pid) || manifest.pid <= 0) return; + + const pid = manifest.pid; + signalProcessGroup(pid, "SIGTERM"); + const deadline = performance.now() + 10_000; + while (isProcessGroupAlive(pid) && performance.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, 50)); + } + if (isProcessGroupAlive(pid)) signalProcessGroup(pid, "SIGKILL"); + } catch { + // No manifest means there is no auto-started daemon to stop. + } +}; + +const withTempData = Effect.acquireRelease( + Effect.sync(() => mkdtempSync(join(tmpdir(), "executor-mcp-test-"))), + (dataDir) => + Effect.promise(async () => { + await stopAutoSpawnedDaemon(dataDir); + rmSync(dataDir, { recursive: true, force: true }); + }), +); describe("MCP stdio integration", () => { it.effect( @@ -25,7 +79,7 @@ describe("MCP stdio integration", () => { Effect.gen(function* () { // Fresh temp dir so the test doesn't migrate against the developer's // real ~/.executor/data.db. - const dataDir = mkdtempSync(join(tmpdir(), "executor-mcp-test-")); + const dataDir = yield* withTempData; const transport = new StdioClientTransport({ command: "bun", args: ["run", cliEntry, "mcp", "--scope", testScope],