diff --git a/apps/cli/docs/go-cli-divergences.md b/apps/cli/docs/go-cli-divergences.md index 228fc2af93..bb5fb67f45 100644 --- a/apps/cli/docs/go-cli-divergences.md +++ b/apps/cli/docs/go-cli-divergences.md @@ -230,3 +230,17 @@ These commands exist in the TS CLI today but have no direct top-level equivalent pull."). An in-sync database is a finding, not a failure to troubleshoot, so the debug hint sent users chasing a non-existent bug. Message text and exit code — the parts scripts depend on — are unchanged. +- Edge Runtime's Docker container `--ulimit nofile` value (`functions serve` and `start`): Go + hardcodes `nofile=65536:65536`, raised from the daemon default to accommodate FD usage from + many concurrent Deno isolates (supabase/cli#5151). TS clamps that value to the host's own hard + nofile limit on Linux (`@supabase/stack`'s `edgeRuntimeNofileUlimit`, via + `process.report`'s `userLimits`), so a constrained sandbox (hard cap below 65536) can still start the + container instead of failing outright (CLI-2220). The CLI process's own limit is used as a + proxy for the daemon's — exact in the sandboxes this targets, where both share the cap; a + Linux client more constrained than its daemon (remote `DOCKER_HOST`, mounted socket) just + gets a smaller fd budget, never a failed start. When the clamp lowers the request, the legacy + `functions serve`/`start` bring-up warns with the reduced limit. The `@supabase/stack` service + builder (next-shell `stack start`) applies the same clamp silently: its defs are built without + an output channel, and in managed mode inside the daemon process, so a user-visible warning + there needs a diagnostics channel on `BuildResult` first; the applied value stays visible via + `docker inspect`. diff --git a/apps/cli/src/legacy/commands/start/services/edge-runtime.service.integration.test.ts b/apps/cli/src/legacy/commands/start/services/edge-runtime.service.integration.test.ts index 79ae42c489..913582d235 100644 --- a/apps/cli/src/legacy/commands/start/services/edge-runtime.service.integration.test.ts +++ b/apps/cli/src/legacy/commands/start/services/edge-runtime.service.integration.test.ts @@ -2,6 +2,7 @@ import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { describe, expect, it } from "@effect/vitest"; +import { edgeRuntimeNofileUlimit } from "@supabase/stack/effect"; import { Deferred, Effect, Exit, Sink, Stream } from "effect"; import { type ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; import { beforeEach } from "vitest"; @@ -189,20 +190,24 @@ describe("legacyStartEdgeRuntimeContainer", () => { }), ); - it.effect("sets --ulimit nofile=65536:65536, matching Go's Ulimits container.Config", () => - Effect.gen(function* () { - const mock = mockDockerSpawner(); - const out = mockOutput(); + it.effect( + "sets --ulimit nofile, capped at Go's 65536 and clamped to the host hard limit (CLI-2220)", + () => + Effect.gen(function* () { + const mock = mockDockerSpawner(); + const out = mockOutput(); - yield* legacyStartEdgeRuntimeContainer(baseInput(tempWorkdir.current)).pipe( - Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, mock.spawner), - Effect.provide(out.layer), - ); + yield* legacyStartEdgeRuntimeContainer(baseInput(tempWorkdir.current)).pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, mock.spawner), + Effect.provide(out.layer), + ); - const runCall = mock.runCall!; - const ulimitIndex = runCall.args.indexOf("--ulimit"); - expect(runCall.args[ulimitIndex + 1]).toBe("nofile=65536:65536"); - }), + const runCall = mock.runCall!; + const ulimitIndex = runCall.args.indexOf("--ulimit"); + expect(runCall.args[ulimitIndex + 1]).toBe(edgeRuntimeNofileUlimit("darwin").arg); + // Off Linux the raise is never clamped, so no clamp warning is emitted. + expect(out.messages.filter((message) => message.type === "warn")).toEqual([]); + }), ); it.effect("sets --workdir once an enabled function mounts the project root (#6035)", () => diff --git a/apps/cli/src/legacy/commands/start/services/edge-runtime.service.ts b/apps/cli/src/legacy/commands/start/services/edge-runtime.service.ts index e283f0c822..2d85b4d64c 100644 --- a/apps/cli/src/legacy/commands/start/services/edge-runtime.service.ts +++ b/apps/cli/src/legacy/commands/start/services/edge-runtime.service.ts @@ -13,7 +13,7 @@ * - `docker-create-args.ts`'s own header explicitly excludes `WorkingDir` * and `Ulimits` from `LegacyStartContainerSpec` ("none of the 13 [other] * call sites... set them") — Edge Runtime needs BOTH (`--workdir`, - * `--ulimit nofile=65536:65536`), so "mapping cleanly" would mean + * `--ulimit nofile`, host-clamped), so "mapping cleanly" would mean * extending the shared spec for a single caller. * - Every other service's env travels as bare `-e KEY` flags whose values * come from the spawned `docker create` process's own environment diff --git a/apps/cli/src/shared/functions/serve.ts b/apps/cli/src/shared/functions/serve.ts index 0db4d04c47..479889d723 100644 --- a/apps/cli/src/shared/functions/serve.ts +++ b/apps/cli/src/shared/functions/serve.ts @@ -10,7 +10,12 @@ import { type ResolvedProjectValue, type ResolvedFunctionConfig as ManifestFunctionConfig, } from "@supabase/config"; -import { defaultJwtSecret, defaultPublishableKey, defaultSecretKey } from "@supabase/stack/effect"; +import { + defaultJwtSecret, + defaultPublishableKey, + defaultSecretKey, + edgeRuntimeNofileUlimit, +} from "@supabase/stack/effect"; import { createHmac, createPrivateKey, @@ -1744,6 +1749,10 @@ export const startEdgeRuntimeContainer = Effect.fn("functions.startEdgeRuntimeCo catch: (cause) => (cause instanceof Error ? cause : new Error(String(cause))), }); const containerProjectRoot = toDockerPath(input.projectRoot); + const nofile = edgeRuntimeNofileUlimit(input.platform); + if (nofile.clampWarning !== undefined) { + yield* output.warn(nofile.clampWarning); + } const command = [ "create", "--name", @@ -1754,7 +1763,7 @@ export const startEdgeRuntimeContainer = Effect.fn("functions.startEdgeRuntimeCo "edge_runtime", ...(hasBindUnder(binds, containerProjectRoot) ? ["--workdir", containerProjectRoot] : []), "--ulimit", - "nofile=65536:65536", + nofile.arg, "--label", `com.supabase.cli.project=${labels["com.supabase.cli.project"]}`, "--label", diff --git a/packages/stack/src/effect.ts b/packages/stack/src/effect.ts index 56ca2752cf..79f2e1542c 100644 --- a/packages/stack/src/effect.ts +++ b/packages/stack/src/effect.ts @@ -37,6 +37,8 @@ export { generateJwt, } from "./JwtGenerator.ts"; +export { edgeRuntimeNofileUlimit } from "./services/nofile-limit.ts"; + export type { AllocatedPorts, ConfigPortKey, diff --git a/packages/stack/src/services/edge-runtime.ts b/packages/stack/src/services/edge-runtime.ts index 168a14cd26..3938b7d5c0 100644 --- a/packages/stack/src/services/edge-runtime.ts +++ b/packages/stack/src/services/edge-runtime.ts @@ -6,6 +6,7 @@ import type { StackIdentity } from "../StackIdentity.ts"; import { dockerRunService, hostHttpHealthCheck, type ServiceDependency } from "./service-utils.ts"; import bootstrapSource from "./edge-runtime-main.ts" with { type: "text" }; import { stackHealthBudgets } from "./health-budgets.ts"; +import { edgeRuntimeNofileUlimit } from "./nofile-limit.ts"; interface EdgeRuntimeOptions { readonly runtimeRoot: string; @@ -94,7 +95,7 @@ export const makeEdgeRuntimeServiceDocker = (opts: DockerEdgeRuntimeOptions): Se `${bootstrapDir}:${bootstrapMountDir}:ro`, ...(opts.projectDir === undefined ? [] : [`${opts.projectDir}:${opts.projectDir}:ro`]), ], - args: ["--ulimit", "nofile=65536:65536"], + args: ["--ulimit", edgeRuntimeNofileUlimit(opts.platformOs).arg], env: { ...edgeRuntimeEnv(opts), FUNCTIONS_RUNTIME_CONFIG_PATH: `${bootstrapMountDir}/functions-runtime-config.json`, diff --git a/packages/stack/src/services/nofile-limit.ts b/packages/stack/src/services/nofile-limit.ts new file mode 100644 index 0000000000..2c2b8df9c6 --- /dev/null +++ b/packages/stack/src/services/nofile-limit.ts @@ -0,0 +1,59 @@ +// Raised from the daemon default so many concurrent Deno isolates can run +// (supabase/cli#5151). +const desiredNofile = 65536; + +// `userLimits.open_files.hard` from a `process.report.getReport()` diagnostic +// report — a number or "unlimited". Narrowed structurally because getReport() +// is typed as a bare `object`. +export const hardNofileLimitFromReport = (report: unknown): number | undefined => { + if (typeof report !== "object" || report === null || !("userLimits" in report)) return undefined; + const userLimits = report.userLimits; + if (typeof userLimits !== "object" || userLimits === null || !("open_files" in userLimits)) { + return undefined; + } + const openFiles = userLimits.open_files; + if (typeof openFiles !== "object" || openFiles === null || !("hard" in openFiles)) { + return undefined; + } + const hard = openFiles.hard; + return typeof hard === "number" && Number.isSafeInteger(hard) && hard > 0 ? hard : undefined; +}; + +const hostHardNofileLimit = (platformOs: string): number | undefined => + platformOs === "linux" ? hardNofileLimitFromReport(process.report?.getReport()) : undefined; + +// Never request more than the host's own hard cap: sandboxed hosts cap it +// below 65536, their docker daemon shares the cap, and exceeding it fails the +// container start (CLI-2220). The process's limit is a proxy for the daemon's +// — exact only when they share a kernel and limits, so only Linux is clamped +// (elsewhere the daemon runs in a VM), and only downward: a client more +// constrained than its daemon yields a smaller fd budget, never a failed start. +export const clampNofileLimit = (hardLimit: number | undefined): number => + hardLimit === undefined ? desiredNofile : Math.min(desiredNofile, hardLimit); + +interface EdgeRuntimeNofileUlimit { + /** The docker `--ulimit` value, `nofile=:`. */ + readonly arg: string; + readonly limit: number; + /** Present only when the host's hard cap forced the request below the 65536 raise. */ + readonly clampWarning?: string; +} + +// `hostHardLimit` defaults to the real host probe and exists as a parameter so +// callers with no host dependence (tests) can pin the clamp decision. +export const edgeRuntimeNofileUlimit = ( + platformOs: string, + hostHardLimit: number | undefined = hostHardNofileLimit(platformOs), +): EdgeRuntimeNofileUlimit => { + const limit = clampNofileLimit(hostHardLimit); + return { + arg: `nofile=${limit}:${limit}`, + limit, + ...(limit < desiredNofile && { + clampWarning: + `Edge Runtime file descriptor limit lowered to ${limit}: ` + + `the host's hard limit (ulimit -Hn) is below the default ${desiredNofile}. ` + + `Heavy Edge Function workloads may exhaust file descriptors.`, + }), + }; +}; diff --git a/packages/stack/src/services/nofile-limit.unit.test.ts b/packages/stack/src/services/nofile-limit.unit.test.ts new file mode 100644 index 0000000000..acda910c26 --- /dev/null +++ b/packages/stack/src/services/nofile-limit.unit.test.ts @@ -0,0 +1,74 @@ +import { describe, expect, it } from "vitest"; +import { + clampNofileLimit, + edgeRuntimeNofileUlimit, + hardNofileLimitFromReport, +} from "./nofile-limit.ts"; + +const reportWithHard = (hard: number | string) => ({ + header: { reportVersion: 5 }, + userLimits: { + open_files: { soft: 1024, hard }, + }, +}); + +describe("hardNofileLimitFromReport", () => { + it("reads the hard limit from a diagnostic report", () => { + expect(hardNofileLimitFromReport(reportWithHard(1048576))).toBe(1048576); + expect(hardNofileLimitFromReport(reportWithHard(20000))).toBe(20000); + }); + + it("returns undefined when the hard limit is unlimited", () => { + expect(hardNofileLimitFromReport(reportWithHard("unlimited"))).toBeUndefined(); + }); + + it("returns undefined for missing or malformed report shapes", () => { + expect(hardNofileLimitFromReport(undefined)).toBeUndefined(); + expect(hardNofileLimitFromReport(null)).toBeUndefined(); + expect(hardNofileLimitFromReport({})).toBeUndefined(); + expect(hardNofileLimitFromReport({ userLimits: {} })).toBeUndefined(); + expect(hardNofileLimitFromReport({ userLimits: { open_files: {} } })).toBeUndefined(); + expect(hardNofileLimitFromReport(reportWithHard(-1))).toBeUndefined(); + }); +}); + +describe("clampNofileLimit", () => { + it("keeps the 65536 raise when the hard limit is unknown or higher", () => { + expect(clampNofileLimit(undefined)).toBe(65536); + expect(clampNofileLimit(1048576)).toBe(65536); + expect(clampNofileLimit(65536)).toBe(65536); + }); + + it("clamps down to a lower hard limit (CLI-2220's 20000-cap sandbox)", () => { + expect(clampNofileLimit(20000)).toBe(20000); + }); +}); + +describe("edgeRuntimeNofileUlimit", () => { + it("keeps the full 65536 raise off Linux, where the daemon runs in a VM", () => { + expect(edgeRuntimeNofileUlimit("darwin")).toEqual({ arg: "nofile=65536:65536", limit: 65536 }); + expect(edgeRuntimeNofileUlimit("win32")).toEqual({ arg: "nofile=65536:65536", limit: 65536 }); + }); + + it("produces a matched soft:hard arg within Go's 65536 raise on Linux", () => { + const { arg, limit, clampWarning } = edgeRuntimeNofileUlimit("linux"); + expect(arg).toBe(`nofile=${limit}:${limit}`); + expect(limit).toBeGreaterThan(0); + expect(limit).toBeLessThanOrEqual(65536); + // The warning exists exactly when this host's cap forced a reduction. + expect(clampWarning !== undefined).toBe(limit < 65536); + }); + + it("carries a warning when a lower host hard limit forces a clamp (CLI-2220's 20000-cap sandbox)", () => { + const clamped = edgeRuntimeNofileUlimit("linux", 20000); + expect(clamped.arg).toBe("nofile=20000:20000"); + expect(clamped.limit).toBe(20000); + expect(clamped.clampWarning).toContain("lowered to 20000"); + expect(clamped.clampWarning).toContain("65536"); + }); + + it("omits the warning when the host limit does not constrain the raise", () => { + expect(edgeRuntimeNofileUlimit("linux", 1048576).clampWarning).toBeUndefined(); + expect(edgeRuntimeNofileUlimit("linux", 65536).clampWarning).toBeUndefined(); + }); +}); diff --git a/packages/stack/src/services/services.unit.test.ts b/packages/stack/src/services/services.unit.test.ts index 6259462de5..ca84c3b279 100644 --- a/packages/stack/src/services/services.unit.test.ts +++ b/packages/stack/src/services/services.unit.test.ts @@ -5,6 +5,7 @@ import { describe, expect, it } from "vitest"; import { analyticsDockerRuntimeNetwork, makeAnalyticsServiceDocker } from "./analytics.ts"; import { makeAuthServiceNative, makeAuthServiceDocker } from "./auth.ts"; import { makeEdgeRuntimeServiceDocker, makeEdgeRuntimeServiceNative } from "./edge-runtime.ts"; +import { edgeRuntimeNofileUlimit } from "./nofile-limit.ts"; import { makeImgproxyServiceDocker } from "./imgproxy.ts"; import { makeMailpitServiceDocker } from "./mailpit.ts"; import { makePgmetaServiceDocker } from "./pgmeta.ts"; @@ -407,7 +408,7 @@ describe("makeEdgeRuntimeServiceDocker", () => { expect(def.args).toContain(`--policy=per_worker`); expect(def.args).toContain(`${bootstrapDir}:/workspace:ro`); expect(def.args).toContain("--ulimit"); - expect(def.args).toContain("nofile=65536:65536"); + expect(def.args).toContain(edgeRuntimeNofileUlimit("linux").arg); expect(def.dependencies).toEqual([{ service: "postgres", condition: "healthy" }]); expect(def.healthCheck?.probe).toEqual({ _tag: "Http",