From f36459b1552cb73fd41fb7b39b72808e71b1faa0 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 19 Aug 2026 07:42:36 +0000 Subject: [PATCH 1/2] feat(scripts): mechanize verify-lock acquisition into a capped, FIFO, self-reporting entry point Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01AeA3nU1B5Q2pgxqxgUrexd --- scripts/pm/os-verify-lock.sh | 627 +++++++++++++++++++++++++++++++++++ 1 file changed, 627 insertions(+) create mode 100755 scripts/pm/os-verify-lock.sh diff --git a/scripts/pm/os-verify-lock.sh b/scripts/pm/os-verify-lock.sh new file mode 100755 index 0000000000..b8657ae741 --- /dev/null +++ b/scripts/pm/os-verify-lock.sh @@ -0,0 +1,627 @@ +#!/usr/bin/env bash +# os-verify-lock.sh — the single entry point for this container's shared +# heavy-verify lock (`/tmp/os-heavy-verify.lock`). +# +# scripts/pm/os-verify-lock.sh -c 'pnpm --filter @objectstack/core test' +# scripts/pm/os-verify-lock.sh -- pnpm --filter @objectstack/core test +# scripts/pm/os-verify-lock.sh --status # holder, how long it has held, the queue +# scripts/pm/os-verify-lock.sh --show-budget # the acquisition budget this call would use +# scripts/pm/os-verify-lock.sh --self-test # verify this script +# +# Exit codes: 99 means THIS CALL NEVER ACQUIRED the lock (the same code the +# free-hand `flock -E 99` convention this replaces used, so callers migrate +# without changing how they branch). Anything else is the wrapped command's own +# exit code — including a command that itself exits 99, which is why every run +# ends with a VERDICT line naming which of the two happened. Read the verdict +# line, never a bare `$?`. +# +# --------------------------------------------------------------------------- +# WHY AN ENTRY POINT AND NOT A CONVENTION +# +# The convention it replaces was: `flock -E 99 -w 540 /tmp/os-heavy-verify.lock +# -c ''`, with the 540 explained in prose. Three mechanisms were measured +# in this container while that was the whole mechanism: +# +# 1. WAITER ASYMMETRY — the one that makes obeying the rule a losing strategy. +# `flock(2)` is not FIFO: it grants to whichever waiter happens to be +# blocked when the lock frees, so the DUTY CYCLE of a waiter decides who +# wins, not its arrival. A compliant waiter (`-w 540`) is present for nine +# minutes, times out, goes off to do lock-free work, and comes back — it is +# absent from the queue for part of every cycle. A waiter that wrote +# `-w 3000` is continuously resident for fifty. Measured, one container, +# five live waiters: three exceeded the declared cap, by up to 6x, and the +# two compliant ones were the ones not verifying. One dev burned 68 minutes +# over 9 attempts, every one returning 99, and correctly declared its +# ablation unrun rather than running unlocked. +# +# 2. LONG HOLDERS — a single run held the lock 28+ minutes straight, which +# lengthens every cycle underneath (1). Nothing made that holder visible +# except another agent going and looking with `fuser`. +# +# 3. ORPHANED FD HOLDERS — a backgrounded child inherited the caller's lock +# fd and kept the lock long after the caller was gone (see the lifecycle +# block in scripts/gen-sdui-manifest.sh, where that was diagnosed). +# +# A declared cap that nothing enforces is not a convention when its violators +# win. So the cap moves from prose into the call site: this entry point takes no +# `-w` at all, and the only knob (`OS_VERIFY_LOCK_WAIT`) can lower the budget, +# never raise it. That is (a). Grants are ordered by a ticket file, so presence +# stops deciding winners — (b). And every run reports how long it held, loudly +# past a threshold, so the next long holder names itself instead of waiting to +# be found — (c). +# +# --------------------------------------------------------------------------- +# HOW THE ORDERING WORKS, AND WHAT IT DELIBERATELY DOES NOT DO +# +# `flock` remains the ONLY mutual-exclusion primitive. The ticket queue is +# ADVISORY ORDER layered on top of it and holds no exclusion of its own. That +# split is the whole coexistence story, so it is stated rather than implied: +# +# - Every entry-point call drops a ticket file named by arrival time into +# `.q/`. Only the ticket at the HEAD of the live queue ever calls +# `flock`; everyone else polls. Head-only means no thundering herd, and +# headship is stable — tickets sort by arrival and the ones ahead of you can +# only disappear. +# +# - A LEGACY free-hand `flock` user (an agent still on the old line, or any +# script that locks this file directly) contends on the same file with the +# same primitive. Mutual exclusion is unaffected — it cannot corrupt the +# queue, deadlock it, or run concurrently with an entry-point holder. What +# it can do is win a grant ahead of the queue head, because it never took a +# ticket. So during rollout the guarantee degrades to: entry-point callers +# are FIFO AMONG THEMSELVES, legacy callers behave exactly as they do today, +# and nobody loses exclusion. `--status` still names a legacy holder (via +# `fuser`/`lsof`), it just cannot report its duration — it never registered. +# +# - A waiter that dies, is killed, or times out leaves at most one stale +# ticket, and a stale ticket cannot wedge the queue: tickets are pruned by +# liveness (pid present AND its `/proc` start time unchanged, so a reused +# pid does not resurrect a dead ticket) and by an absolute age bound. If the +# queue directory cannot be used at all, acquisition FALLS BACK to a plain +# capped `flock` with a warning: the ordering layer is best-effort, the cap +# and the exclusion are not. +# +# The lock fd is closed in the wrapped command's child (`{LFD}>&-`), so a +# process the command leaves behind cannot inherit the lock — mechanism 3 above. +# Release still belongs to the fd: this script never hand-rolls a lockfile, and +# a kill -9 at any point releases the lock the moment the process dies. +# --------------------------------------------------------------------------- + +set -uo pipefail + +# The declared invariant, in one place: an acquisition wait must fit inside a +# single foreground agent call (harness ceiling: 10 minutes). Everything above +# this is unrepresentable through this entry point — that is the point of the +# entry point, so it is a constant and not an option. +readonly HARD_CAP_S=540 +readonly DEFAULT_WAIT_S=540 + +# Past this, a holder is loud about itself on release. 15 minutes: the measured +# long holder was 28+, an ordinary targeted package build is well under. +readonly LONG_HOLD_WARN_S="${OS_VERIFY_LOCK_LONG_HOLD_WARN:-900}" + +readonly POLL_S=1 # queue poll while not head +readonly SLICE_S=30 # flock slice while head, so waiting still reports +readonly PROGRESS_EVERY_S=30 + +# `OS_VERIFY_LOCK_FILE` exists so --self-test can run real two-process +# contention without touching the shared lock. Pointing real verification at a +# private lock defeats the serialisation the lock exists for; don't. +LOCK_FILE="${OS_VERIFY_LOCK_FILE:-/tmp/os-heavy-verify.lock}" +QUEUE_DIR="${LOCK_FILE}.q" +HOLDER_FILE="${LOCK_FILE}.holder" +readonly TICKET_MAX_AGE_S=$((HARD_CAP_S + 300)) + +SELF="${BASH_SOURCE[0]}" +TICKET="" +HOLDING=0 +BUDGET_NOTE="" + +log() { printf 'os-verify-lock: %s\n' "$*" >&2; } + +now_s() { printf '%s' "${EPOCHSECONDS}"; } + +# Microsecond arrival stamp, zero-padded so plain lexical (glob) order IS +# arrival order. `EPOCHREALTIME` renders its separator per locale, hence the +# character class; `date` is the fallback if the shell ever stops providing it. +now_stamp() { + local raw="${EPOCHREALTIME:-}" + raw="${raw/[.,]/}" + case "$raw" in '' | *[!0-9]*) raw="$(date +%s%N 2> /dev/null || echo 0)" ;; esac + printf '%020d' "$raw" +} + +# Field 22 of /proc//stat (process start time). Strips through the LAST +# ') ' first: field 2 is the comm, which may itself contain spaces and parens. +proc_starttime() { + local pid="$1" stat rest + [[ -r "/proc/${pid}/stat" ]] || return 1 + stat="$(< "/proc/${pid}/stat")" || return 1 + rest="${stat##*) }" + [[ "$rest" != "$stat" ]] || return 1 + awk '{ print $20 }' <<< "$rest" +} + +# --- budget ----------------------------------------------------------------- + +# Sets BUDGET (seconds) and BUDGET_NOTE. The clamp is the enforcement half: +# a caller asking for more than the cap gets the cap, and gets told. +effective_budget() { + local want="${OS_VERIFY_LOCK_WAIT:-$DEFAULT_WAIT_S}" + BUDGET_NOTE="" + case "$want" in + '' | *[!0-9]*) + BUDGET_NOTE="OS_VERIFY_LOCK_WAIT='${want}' is not a number — using the default ${DEFAULT_WAIT_S}s" + want="$DEFAULT_WAIT_S" + ;; + esac + if ((want > HARD_CAP_S)); then + BUDGET_NOTE="OS_VERIFY_LOCK_WAIT=${want} exceeds the declared cap — clamped to ${HARD_CAP_S}s (an acquisition wait must fit inside one foreground call)" + want="$HARD_CAP_S" + fi + ((want < 1)) && want=1 + BUDGET="$want" +} + +# --- ticket queue ----------------------------------------------------------- + +queue_usable() { + mkdir -p "$QUEUE_DIR" 2> /dev/null || return 1 + chmod 1777 "$QUEUE_DIR" 2> /dev/null || true + [[ -w "$QUEUE_DIR" ]] +} + +# ticket file: "