diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 117ec33..2adb545 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,6 +36,9 @@ jobs: # R3: every numeric literal in a tuning position carries its derivation (AC-0.2). - name: Literal check run: cargo xtask literals + # The daemon provisions from usable memory and degrades honestly when the OS grants little + # lockable RAM (§4.2 D-12; docs/bugs/2026-09-10-provisioning-budget-from-mlock-limit.md), so the + # tests run under the runner's default mlock limit — no `ulimit` bump, which the runners ignore. - name: Test run: cargo test --workspace - name: AC-1.1 model suite over 10^6 generated operations @@ -130,9 +133,13 @@ jobs: components: clippy - uses: Swatinem/rust-cache@v2 # WinFsp: the mount host links `winfsp-x64.dll`'s import library and the live mount test needs the - # kernel FSD installed. The MSI ships the SDK (inc/, lib/) under `C:\Program Files (x86)\WinFsp`. + # kernel FSD installed. The MSI ships the SDK (inc/, lib/) under `C:\Program Files (x86)\WinFsp` + # and the DLL under `bin/`; put `bin/` on PATH so the test executable's loader finds the DLL at + # run time (linking uses the lib via WINFSP_LIB; running needs the DLL, else STATUS_DLL_NOT_FOUND). - name: Install WinFsp - run: choco install winfsp -y --no-progress + run: | + choco install winfsp -y --no-progress + "C:\Program Files (x86)\WinFsp\bin" | Out-File -FilePath $env:GITHUB_PATH -Append -Encoding utf8 # `--workspace` cannot build here: the NFS-mount and FUSE/FSKit crates (bridge-nfs, cli, # bridge-fuse, bridge-fskit) are macOS/Linux-only — Windows mounts through WinFsp, and the fleet # transport is QUIC over UDP, not the NFS-over-TCP loopback. So clippy the crates that carry @@ -190,7 +197,8 @@ jobs: - name: Benches run: cargo bench --workspace --bench callgrind - on the unit tests of the crates that hold unsafe code. Miri ships only with the + # Miri (the memory-model interpreter) and loom (the concurrency-permutation checker) run + # on the unit tests of the crates that hold unsafe code. Miri ships only with the # nightly toolchain, which CI installs for this job alone; the pinned stable toolchain builds # everything else. loom runs under stable with `--cfg loom`. miri-and-loom: @@ -207,13 +215,13 @@ jobs: - uses: dtolnay/rust-toolchain@nightly with: components: miri - - name: Miri (memory and wire: leak check on) + - name: "Miri (memory and wire: leak check on)" run: cargo +nightly miri test -p slates-mem -p slates-wire --lib # The runtime leaks its shard contexts, rings and registry entries on purpose (they live # for the process; the leak is what makes every reference to them a plain `&'static`), so # Miri's leak check is off for it; every other check stays on. The OS-driver tests are # marked ignored under Miri, which cannot model kqueue or eventfd. - - name: Miri (runtime: simulation tests, intentional leaks ignored) + - name: "Miri (runtime: simulation tests, intentional leaks ignored)" run: MIRIFLAGS="-Zmiri-ignore-leaks" cargo +nightly miri test -p slates-rt --lib --test differential windows-nightly: diff --git a/crates/mem/src/budget.rs b/crates/mem/src/budget.rs index 177d223..a4bb100 100644 --- a/crates/mem/src/budget.rs +++ b/crates/mem/src/budget.rs @@ -257,12 +257,14 @@ pub fn slab_bytes(page: u64, burst_p99_slots: u64, slot_bytes: u64) -> Derived Derived { +/// The region size for a shard: its share of a memory `capacity` divided among the size classes. The +/// caller decides what the capacity is — the daemon's default reserve derives it from usable memory +/// (§4.2 D-12 honest degradation), a locked reserve from the OS lock capacity. +pub fn region_bytes(capacity: u64, shards: u64, classes: u64) -> Derived { derived!( - lock_capacity / shards.max(1) / classes.max(1), - "lock capacity / shards / classes", - ["lock.bytes", "rt.shards", "mem.classes"] + capacity / shards.max(1) / classes.max(1), + "memory capacity / shards / classes", + ["mem.capacity", "rt.shards", "mem.classes"] ) } diff --git a/crates/server/src/config.rs b/crates/server/src/config.rs index fc0a703..e3ffacd 100644 --- a/crates/server/src/config.rs +++ b/crates/server/src/config.rs @@ -123,7 +123,16 @@ impl DaemonConfig { let mut runtime = RuntimeConfig::from_profile(profile, admission.get(), admission.get(), LATENCY_BUDGET_NS); let shards = u64::from(runtime.shards.max(1)); - let reserve = region_bytes(profile.lock.bytes, shards, MEMORY_CLASSES); + // §4.2 D-12 "honest degradation": the default provisioning reserve is USABLE memory, not the OS + // lock LIMIT. A volume's arena is locked only when a client asks (`require_locked`, verbs.rs); by + // default it is a normal, usable — if unlocked — mapping (crate mem's lock sequence keeps unmapped + // regions "usable, unlocked, and counted"). Deriving the reserve from the mlock limit refused every + // volume where the OS grants little lockable RAM — the design's own failure case, "a locked-down CI + // container refuses mlock; lock capacity 0" (§4.2 boot order). Usable memory lets the daemon + // provision there (degraded: those pages may swap), while a `require_locked` volume still respects + // the lock capacity best-effort. On a machine that can lock freely the two are the same — the lock + // probe already records `memory.available` when `RLIMIT_MEMLOCK` is unlimited (`probes.rs`). + let reserve = region_bytes(profile.facts.memory.available, shards, MEMORY_CLASSES); derivations.push(note("reserve_per_shard", &reserve)); let page = profile.facts.page.base; let tables: Derived = derived!( diff --git a/docs/bugs/2026-09-10-provisioning-budget-from-mlock-limit.md b/docs/bugs/2026-09-10-provisioning-budget-from-mlock-limit.md new file mode 100644 index 0000000..c27839f --- /dev/null +++ b/docs/bugs/2026-09-10-provisioning-budget-from-mlock-limit.md @@ -0,0 +1,72 @@ +# The daemon refuses all provisioning when the OS grants little lockable memory + +Status: **fixed** — the default provisioning reserve is derived from usable memory (`memory.available`), +not the OS lock limit; the lock limit governs only the `require_locked` path, best-effort (§4.2 D-12 +honest degradation). Date: 2026-09-10. + +## Description + +On a machine where the OS grants little lockable memory — a locked-down container, an old default +`RLIMIT_MEMLOCK` (64 KiB), a CI runner — the daemon refuses **every** volume provision, even a 1 MiB +one, with `BudgetExceeded { available: 0 }`. Reproduced locally, matching the CI failure exactly: + +``` +$ ( ulimit -l 64; cargo test -p slates-client --test client ) +thread 'the_typed_verbs_drive_the_lifecycle_and_refusals_are_typed' panicked at crates/client/tests/client.rs:93: + called `Result::unwrap()` on an `Err` value: Refused(BudgetExceeded { available: 0 }) +``` + +Surfaced when CI was restored (the workflow YAML had been unparseable, so no CI had run): the macOS and +Linux gates both failed here, as did the `slates-anchor` supervised-child test — every test that spawns a +real daemon. + +## Root cause + +The per-shard provisioning reserve was derived from the OS **lock limit**: + +```rust +// crates/server/src/config.rs +let reserve = region_bytes(profile.lock.bytes, shards, MEMORY_CLASSES); // lock.bytes / shards / classes +``` + +`profile.lock.bytes` is what the OS will let the process `mlock` (`RLIMIT_MEMLOCK`, the macOS wire limit, +or 0 when locking is refused — `crates/machine/src/probes.rs`). The reserve feeds the store's byte budget +and the arena size (`crates/server/src/daemon.rs`), so a tiny lock limit rounds the reserve to ~0 and the +budget admits nothing. + +But **the arena is not locked by default.** Locking happens only when a client asks for it +(`require_locked`, `crates/server/src/verbs.rs`: `if require_locked && … arena_mut().lock()`), and the +memory crate's lock sequence is explicit that an unlocked region "stays mapped and **usable**, unlocked, +and counted" (`crates/mem/src/lock.rs`; D-12 "honest degradation"). The design's own §4.2 boot-order +failure matrix names this exact case: *"a locked-down CI container refuses `mlock`; the profile records +lock capacity 0, the diagnostic surface reports why residency cannot be established"* — i.e. **degrade and +keep serving**, not refuse. So capping the *default* provisioning budget at the lock *limit* contradicts +the design: default volumes come from usable RAM, whether or not it can be locked. + +## Impact + +A daemon on any machine with a small mlock limit — a container without `CAP_IPC_LOCK`, an old Linux +default, a CI runner — could not provision a single volume, defeating "laptop ≡ fleet, one code path" +(R8) and sub-50 µs provisioning (R9) on those hosts. It is not a data or safety bug (nothing is written; +the RAM is real), but a robustness/availability one. It was invisible until CI was fixed, because CI had +not been running. + +## Fix (applied) + +`crates/server/src/config.rs`: derive the default reserve from usable memory — +`region_bytes(profile.facts.memory.available, shards, MEMORY_CLASSES)` — with a comment citing D-12 and +the §4.2 failure case. `crates/mem/src/budget.rs`: `region_bytes`'s parameter and derivation note are +generalized from "lock capacity" to "memory capacity" (the caller decides the basis). The lock limit is +unchanged where it belongs: a `require_locked` volume still calls `arena.lock()`, which respects the mlock +limit best-effort and refuses if it cannot lock. On a machine that can lock freely the two bases are the +same — the lock probe already records `memory.available` when `RLIMIT_MEMLOCK` is unlimited. + +Verified by reproducing under `ulimit -l 64` before and after: the client, anchor and server-`nfs_mount` +suites go from failing (`BudgetExceeded { available: 0 }`) to passing; the budget unit tests and normal +(high-mlock) runs are unchanged; `cargo xtask check` (structural/literals/unsafe) clean. + +## Sibling sweep + +`region_bytes` (the budget helper) had exactly one real caller (this reserve derivation); the other +`region_bytes` name is an unrelated `SharedObject`/buddy method. No other site derives a *default* budget +from the lock limit. The `require_locked` path is intentionally lock-bounded and left as is.