diff --git a/.github/workflows/monitor-collector-host.yml b/.github/workflows/monitor-collector-host.yml index 92dc53537..9915a8d57 100644 --- a/.github/workflows/monitor-collector-host.yml +++ b/.github/workflows/monitor-collector-host.yml @@ -193,6 +193,7 @@ jobs: set -euo pipefail breaches=$(jq -r '.breaches[]' /tmp/collector-health.json) checked_at=$(jq -r '.checked_at' /tmp/collector-health.json) + warnings=$(jq -r '(.warnings // [])[]' /tmp/collector-health.json) { printf 'Host: monday-trade-data-26 (Aliyun Tokyo ap-northeast-1, instance %s)\n' "${{ env.INSTANCE_ID }}" printf 'Checked at: %s\n' "$checked_at" @@ -201,6 +202,10 @@ jobs: printf 'Primary alert channel is the Cloud Monitor disk alarm; this issue is the repository fallback and needs human triage.\n\n' printf 'Breaches:\n' printf '%s\n' "$breaches" | sed 's/^/- /' + if [ -n "$warnings" ]; then + printf '\nWarnings (not blocking):\n' + printf '%s\n' "$warnings" | sed 's/^/- /' + fi } > /tmp/collector-health-issue.md existing=$(gh issue list --state open --search 'in:title "Durable monitor breach"' \ diff --git a/agent-worktree.yml b/agent-worktree.yml index 6fcbc9746..45e029dfb 100644 --- a/agent-worktree.yml +++ b/agent-worktree.yml @@ -1,10 +1,13 @@ owner: codex -contract: actionlint-macos-15-intel -worktree: /Users/proerror/Documents/monday/.worktrees/codex/actionlint-macos-15-intel -branch: codex/actionlint-macos-15-intel -base_sha: 161b761dff56acb758dd4a1787fd0b3b8130196a +contract: collector-health-hard-gates +worktree: /Users/proerror/Documents/monday/.worktrees/codex/collector-health-hard-gates +branch: codex/collector-health-hard-gates +base_sha: be9e746a1ddf7e6f26264c4811c4a3511c3caa87 allowed_files: - - .github/workflows/ploy-ci.yml - agent-worktree.yml -dependency: none + - deployment/aliyun/monday-collector-health.sh + - deployment/aliyun/test-monday-collector-health.sh + - .github/workflows/monitor-collector-host.yml + - deployment/aliyun/README.md +dependency: upload-status last_success_at for the fee uploader lands in a parallel PR; this contract treats a missing last_success_at as a breach, so it must merge after or together with that PR no_external_runtime_mutation: true diff --git a/deployment/aliyun/README.md b/deployment/aliyun/README.md index b46a387a0..eb4123afe 100644 --- a/deployment/aliyun/README.md +++ b/deployment/aliyun/README.md @@ -162,19 +162,34 @@ tape files or `upload-status.json`. It emits one JSON snapshot (or a human to journald tag `monday-collector-health`. Run with `--json` for machine output and `--dry-run` to avoid reading or writing the persistent delta state. -| Check | Breach condition | +The monitor has exactly four hard gates. Each is a breach: it fails closed into +the `monitor-collector-host` workflow issue and blocks `ok:true`. + +| Hard gate | Breach condition | +| --- | --- | +| 1. Status file | `upload-status.json` missing, a symlink, or unparseable on the mandated lanes (`binance-lob` spot/usdm, `binance-fee`) | +| 2. Upload freshness | `last_success_at` missing/unparseable, or older than the lane bound (LOB 7200s, fee 600s, usdm-reference 1200s, polymarket 7200s, bybit 5400s; each just above the lane's upload cadence — the polymarket lanes rotate tapes hourly, so the 5-minute upload timer is not a heartbeat) | +| 3. Pending backlog | pending count over the lane limit, or oldest pending artifact older than the lane age bound, using each collector's own pending definition (LOB `*.manifest.json`, fee/usdm-reference `lake/raw/**/batch=*`, polymarket rotated `market-updates.*.ndjson` tapes, bybit marked `.ndjson` without `.uploaded.json`) | +| 4. Upload failures | `last_error_at`/`last_error` present, or a `failure_count` increase since the previous poll (prior counts live under `/var/lib/monday-collector-health`) | + +Two legacy guards stay breaches: `polymarket-raw-ops-gate@.service` must remain +disabled/masked/not-found (`static` proves an uncleaned host installation), and +state-persistence failures (gate 4 delta detection depends on that state). + +Every other check is a warning — reported in the JSON `warnings` array and as +`warning:` lines, never blocking `ok:true`: + +| Warning | Condition | | --- | --- | | `/data` disk | free < 25% (warn) or < 10% (critical) via `df -Pk /data` | -| Governed services | `binance-lob-archiver-production@spot/usdm` and `binance-usdm-reference-collector` active AND enabled AND `Result==success`, plus a restart-rate delta > 1 since the last poll | -| Upload lanes | `polymarket-market-tape-upload.timer` and `polymarket-reference-upload.timer` active AND enabled; their oneshot services' last `Result==success` | -| Watchdog | `polymarket-market-tape-upload-watchdog.timer` active AND enabled; the watchdog service's last `Result==success` | -| Incident fill source | `bybit-options-archiver.service` and `polymarket-raw-ops-gate@.service` must stay disabled/masked/not-found | +| Governed services | `binance-lob-archiver-production@spot/usdm`, `binance-usdm-reference-collector`, `bybit-options-archiver` active AND enabled AND `Result==success`, plus a restart-rate delta > 1 since the last poll | +| Upload lane units | upload/watchdog/fee timers active AND enabled; their oneshot services' last `Result==success` | | `health.json` | missing/unparseable, wall-clock age of `updated_at_ns` > 300s, or `sequence_gaps` > 0 (spot + usdm spools) | -| `upload-status.json` | `last_error_at`/`last_error` present, or a `failure_count` delta since the previous poll (prior counts live under `/var/lib/monday-collector-health`) | | Delay-gate trips | > 0 journald `source-to-receive delay exceeds the governed limit` lines per Binance unit in the last 15 minutes | +| Fee snapshot failures | > 0 `Failed with result` journald lines per fee snapshot unit in the last 10 minutes | | `/data` mount | `mountpoint -q /data` fails (the monitor must DETECT a missing mount, not gate on it) | -The persistent-service check deliberately does not breach on `NRestarts > 0`: +The persistent-service check deliberately does not warn on `NRestarts > 0`: both Binance archivers restart every six hours by design (`RuntimeMaxSec=21600`). Crash loops are detected through `Result != success` or an `NRestarts` delta greater than one between consecutive five-minute polls. diff --git a/deployment/aliyun/monday-collector-health.sh b/deployment/aliyun/monday-collector-health.sh index 46060c531..d387c0cff 100755 --- a/deployment/aliyun/monday-collector-health.sh +++ b/deployment/aliyun/monday-collector-health.sh @@ -8,6 +8,39 @@ # accumulated while the only on-host monitor (polymarket-market-tape-upload- # watchdog.sh) self-healed without ever alerting a human. # +# Health contract: exactly FOUR hard gates, each a breach that fails closed +# into the monitor-collector-host workflow issue, plus the expected-disabled +# installation gates: +# 1. upload-status.json exists and parses for the mandated lanes +# (binance-lob spot/usdm and binance-fee); missing or malformed = breach. +# 2. last_success_at is present and fresh on every upload lane. Thresholds +# sit just above each lane's upload cadence (see the *_SUCCESS_MAX_AGE +# constants). A missing/unparseable last_success_at is a breach: the +# delivery loop is unproven. The fee uploader gains last_success_at in a +# parallel change; until that deploys, the fee lane breaches by design. +# 3. Pending upload backlog is bounded: the pending count stays under the +# lane limit AND the oldest pending artifact stays younger than the lane +# age bound. Pending artifacts are discovered with the same rules the +# collectors themselves use: LOB lanes count top-level *.manifest.json +# (the archiver's own pending_upload_segments definition), the fee and +# usdm-reference lanes count lake/raw/**/batch=* directories (removed +# after each verified upload), the polymarket lanes count rotated +# market-updates.[.frac][.uuid].ndjson tapes, and the bybit lane +# counts .ndjson segments carrying manifest+_SUCCESS markers but no +# .uploaded.json readback marker. +# 4. failure_count must not grow between polls and last_error must be empty, +# uniformly across all upload lanes. +# polymarket-raw-ops-gate stays a breach when is-enabled is anything but +# disabled/masked/not-found (e.g. 'static'): it proves an uncleaned host +# installation. State-persistence failures also stay breaches because gate 4 +# delta detection depends on the persisted state. +# +# Everything else is a WARNING: unit/timer active+enabled state, systemd +# Result, restart-rate deltas, health.json freshness/gaps, journald delay-gate +# trips, fee snapshot journal failures, disk space, and the /data mount. +# Warnings are reported in the JSON warnings array (and as warning: lines in +# text mode) but never block ok:true. +# # The script is READ-ONLY: it never starts, stops, enables, or disables a unit # and never modifies tape files or upload-status.json. It emits one JSON # snapshot (or a human ok:/breach: summary) and exits nonzero when any breach @@ -36,13 +69,48 @@ DISK_CRIT_PERCENT=10 RESTART_MAX_DELTA=1 # journalctl --since value; must be a timestamp journalctl can parse # ("15min" is rejected with "Failed to parse timestamp" and would read as a -# permanent journald-query breach). +# permanent journald-query warning). DELAY_GATE_WINDOW='15 min ago' FEE_FAILURE_WINDOW='10 min ago' -# Governed units. Persistent services must be active + enabled + Result=success -# and are monitored for restart-rate deltas. Upload lanes are driven by timers -# (active + enabled) whose oneshot services must have a successful last Result. +# Gate 2: last_success_at freshness per lane, set just above the lane's upload +# cadence: +# - LOB segments rotate every SEGMENT_SECONDS (default 3600s) and the +# in-process upload loop runs every 300s, so a healthy lane uploads at least +# once per rotation; allow two full rotations. +# - fee snapshots publish every 60s and binance-fee-upload.timer retries every +# 60s (mirrors the FEE_FAILURE_WINDOW='10 min ago' journal window). +# - usdm-reference runs on a 5-minute upload timer over hourly reference +# batches. +# - both polymarket lanes rotate tapes hourly +# (record_market_updates_rotate_seconds = 3600; the reference writer rotates +# at UTC-hour boundaries) and last_success_at only advances when a rotated +# tape actually uploads, so the 5-minute upload timer is not a heartbeat; +# allow two full rotations, same as LOB. +# - bybit options segments finalize on the hour and the upload timer sweeps +# them at :23, so 90 minutes covers one full finalize+sweep cycle. +LOB_SUCCESS_MAX_AGE=7200 +FEE_SUCCESS_MAX_AGE=600 +REF_SUCCESS_MAX_AGE=1200 +POLY_SUCCESS_MAX_AGE=7200 +BYBIT_SUCCESS_MAX_AGE=5400 + +# Gate 3: pending backlog bounds per lane (count limit, oldest-artifact age). +LOB_PENDING_MAX=4 +LOB_PENDING_MAX_AGE=10800 +FEE_PENDING_MAX=120 +FEE_PENDING_MAX_AGE=7200 +REF_PENDING_MAX=24 +REF_PENDING_MAX_AGE=10800 +POLY_PENDING_MAX=100 +POLY_PENDING_MAX_AGE=86400 +BYBIT_PENDING_MAX=48 +BYBIT_PENDING_MAX_AGE=7200 + +# Governed units. Persistent services are observed for active + enabled + +# Result=success and restart-rate deltas (all warnings). Upload lanes are +# driven by timers whose oneshot services' last Result is observed (warning); +# their delivery is hard-gated through upload-status.json instead. ARCHIVER_SPOT=binance-lob-archiver-production@spot.service ARCHIVER_USDM=binance-lob-archiver-production@usdm.service REFERENCE_COLLECTOR=binance-usdm-reference-collector.service @@ -87,6 +155,7 @@ fi breach_count=0 breaches="" +warnings="" units_json='{}' health_json='{}' uploads_json='{}' @@ -115,6 +184,17 @@ $msg" log err "$msg" } +record_warning() { + msg=$1 + if [ -n "$warnings" ]; then + warnings="$warnings +$msg" + else + warnings=$msg + fi + log warning "$msg" +} + read_prior() { # $1 = state key (e.g. "nrestarts|"); prints prior value or empty if [ -f "$STATE_FILE" ]; then @@ -122,6 +202,11 @@ read_prior() { fi } +file_mtime() { + # Portable mtime: GNU stat on the host, BSD stat under the macOS test stubs. + stat -c %Y "$1" 2>/dev/null || stat -f %m "$1" 2>/dev/null +} + unit_is_active() { systemctl is-active "$1" 2>/dev/null || true } @@ -145,7 +230,7 @@ check_mount() { data_mounted=1 fi if [ "$data_mounted" -eq 0 ]; then - record_breach "mount: /data is not mounted" + record_warning "mount: /data is not mounted" fi mount_json=$(jq -n --argjson m "$(bool_json "$data_mounted")" '{data_mounted: $m}') } @@ -163,7 +248,7 @@ check_disk() { disk_free_gb=0 disk_critical=1 disk_warning=1 - record_breach "disk: cannot determine /data free space (df unavailable)" + record_warning "disk: cannot determine /data free space (df unavailable)" else disk_free_percent=$((disk_avail_kib * 100 / disk_total_kib)) disk_free_gb=$((disk_avail_kib / 1048576)) @@ -171,10 +256,10 @@ check_disk() { disk_warning=0 if [ "$disk_free_percent" -lt "$DISK_CRIT_PERCENT" ]; then disk_critical=1 - record_breach "disk: /data free ${disk_free_percent}% (${disk_free_gb}GiB) below critical ${DISK_CRIT_PERCENT}%" + record_warning "disk: /data free ${disk_free_percent}% (${disk_free_gb}GiB) below critical ${DISK_CRIT_PERCENT}%" elif [ "$disk_free_percent" -lt "$DISK_WARN_PERCENT" ]; then disk_warning=1 - record_breach "disk: /data free ${disk_free_percent}% (${disk_free_gb}GiB) below warning ${DISK_WARN_PERCENT}%" + record_warning "disk: /data free ${disk_free_percent}% (${disk_free_gb}GiB) below warning ${DISK_WARN_PERCENT}%" fi fi disk_json=$(jq -n --argjson p "$disk_free_percent" --argjson g "$disk_free_gb" \ @@ -183,7 +268,8 @@ check_disk() { } check_service() { - # Persistent service: active AND enabled AND Result=success AND restart-rate delta. + # Persistent service: active AND enabled AND Result=success AND restart-rate + # delta. All soft signals: the hard delivery gates run on upload-status.json. unit=$1 label=$2 active=$(unit_is_active "$unit") @@ -191,16 +277,16 @@ check_service() { result=$(unit_result "$unit") nrestarts=$(unit_nrestarts "$unit") case "$nrestarts" in (*[!0-9]*|'') nrestarts=0;; esac - [ "$active" = "active" ] || record_breach "$label: not active (is-active='$active')" - [ "$enabled" = "enabled" ] || record_breach "$label: not enabled (is-enabled='$enabled')" - [ "$result" = "success" ] || record_breach "$label: last systemd Result='$result'" + [ "$active" = "active" ] || record_warning "$label: not active (is-active='$active')" + [ "$enabled" = "enabled" ] || record_warning "$label: not enabled (is-enabled='$enabled')" + [ "$result" = "success" ] || record_warning "$label: last systemd Result='$result'" prior=$(read_prior "nrestarts|$unit") if [ "$DRY_RUN" -eq 0 ] && [ -n "$prior" ]; then case "$prior" in (*[!0-9]*|'') prior="" ;; esac if [ -n "$prior" ]; then delta=$((nrestarts - prior)) if [ "$delta" -gt "$RESTART_MAX_DELTA" ]; then - record_breach "$label: restart rate high (NRestarts $prior -> $nrestarts)" + record_warning "$label: restart rate high (NRestarts $prior -> $nrestarts)" fi fi fi @@ -217,8 +303,8 @@ check_timer() { label=$2 active=$(unit_is_active "$unit") enabled=$(unit_is_enabled "$unit") - [ "$active" = "active" ] || record_breach "$label: timer not active (is-active='$active')" - [ "$enabled" = "enabled" ] || record_breach "$label: timer not enabled (is-enabled='$enabled')" + [ "$active" = "active" ] || record_warning "$label: timer not active (is-active='$active')" + [ "$enabled" = "enabled" ] || record_warning "$label: timer not enabled (is-enabled='$enabled')" obj=$(jq -n --arg a "$active" --arg e "$enabled" \ '{active: ($a == "active"), enabled: ($e == "enabled")}') units_json=$(jq -n --argjson base "$units_json" --arg k "$unit" --argjson v "$obj" \ @@ -226,12 +312,12 @@ check_timer() { } check_oneshot_result() { - # Oneshot upload/watchdog service: last Result must be success. + # Oneshot upload/watchdog service: last Result observed as a warning. unit=$1 label=$2 result=$(unit_result "$unit") if [ -n "$result" ] && [ "$result" != "success" ]; then - record_breach "$label: last systemd Result='$result'" + record_warning "$label: last systemd Result='$result'" fi obj=$(jq -n --arg r "$result" '{result: $r}') units_json=$(jq -n --argjson base "$units_json" --arg k "$unit" --argjson v "$obj" \ @@ -239,9 +325,10 @@ check_oneshot_result() { } check_disabled_unit() { - # The incident fill source must remain DISABLED. Breach on any is-enabled + # The raw-ops gate template must remain DISABLED. Breach on any is-enabled # state that is not explicitly disabled/masked/not-found (enabled, static, - # indirect, ... all mean the unit can be activated). + # indirect, ... all mean the unit can be activated and the host still carries + # an uncleaned installation). unit=$1 label=$2 enabled=$(unit_is_enabled "$unit") @@ -255,7 +342,9 @@ check_disabled_unit() { } check_binance_health() { - # health.json at /data/monday/spool/binance-lob//: fresh + gap-free. + # health.json at /data/monday/spool/binance-lob//: freshness and + # sequence gaps are soft signals (warnings); the hard LOB gates run on + # upload-status.json and the pending segment backlog. label=$1 spool_dir=$2 health_file="$spool_dir/health.json" @@ -264,10 +353,10 @@ check_binance_health() { hwarn=false hstatus=unknown if [ ! -f "$health_file" ] || [ -L "$health_file" ]; then - record_breach "$label: health.json missing or a symbolic link ($health_file)" + record_warning "$label: health.json missing or a symbolic link ($health_file)" age=999999 elif ! updated_ns=$(jq -r '.updated_at_ns // 0' "$health_file" 2>/dev/null); then - record_breach "$label: health.json unparseable ($health_file)" + record_warning "$label: health.json unparseable ($health_file)" age=999999 else gaps=$(jq -r '.sequence_gaps // 0' "$health_file" 2>/dev/null || printf '0') @@ -279,10 +368,10 @@ check_binance_health() { age=$((NOW_SEC - updated_sec)) [ "$age" -lt 0 ] && age=0 if [ "$age" -gt "$HEALTH_SILENCE_SECONDS" ]; then - record_breach "$label: health.json stale (age ${age}s > ${HEALTH_SILENCE_SECONDS}s)" + record_warning "$label: health.json stale (age ${age}s > ${HEALTH_SILENCE_SECONDS}s)" fi if [ "$gaps" -gt 0 ]; then - record_breach "$label: sequence_gaps=$gaps" + record_warning "$label: sequence_gaps=$gaps" fi fi hobj=$(jq -n --argjson age "$age" --argjson gaps "$gaps" --arg hw "$hwarn" --arg s "$hstatus" \ @@ -291,38 +380,195 @@ check_binance_health() { '$base + {($k): $v}') } -check_upload() { - # upload-status.json: breach on a present last_error_at/last_error, or on a - # failure_count delta since the previous poll (failure_count is cumulative - # and never reset, so a delta means an upload failed between polls). +# Pending-backlog scanners. Each sets pending_count and pending_oldest (epoch +# mtime of the oldest pending artifact, 0 when none), mirroring the collector's +# own pending definition for that lane. +scan_pending_glob() { + # $1 = glob for pending artifacts (LOB manifests, polymarket rotated tapes) + pending_count=0 + pending_oldest=0 + for entry in $1; do + [ -f "$entry" ] || continue + pending_count=$((pending_count + 1)) + mtime=$(file_mtime "$entry") + case "$mtime" in (*[!0-9]*|'') continue ;; esac + if [ "$pending_oldest" -eq 0 ] || [ "$mtime" -lt "$pending_oldest" ]; then + pending_oldest=$mtime + fi + done +} + +scan_pending_lake() { + # $1 = output root; pending batches are lake/raw/**/batch=* directories. The + # fee and usdm-reference uploaders remove each batch directory after a + # verified upload, so a surviving batch directory is exactly the backlog. + pending_count=0 + pending_oldest=0 + [ -d "$1/lake/raw" ] || return 0 + # Capture find's exit status: a traversal error (permissions, I/O) must set + # pending_scan_failed so the caller can breach instead of reading a partial + # scan as an empty backlog. + scan_out=$(find "$1/lake/raw" -type d -name 'batch=*' 2>/dev/null) || pending_scan_failed=1 + # Batch directory names come from the governed lake layout (no whitespace). + for dir in $scan_out; do + pending_count=$((pending_count + 1)) + mtime=$(file_mtime "$dir") + case "$mtime" in (*[!0-9]*|'') continue ;; esac + if [ "$pending_oldest" -eq 0 ] || [ "$mtime" -lt "$pending_oldest" ]; then + pending_oldest=$mtime + fi + done +} + +scan_pending_bybit_raw() { + # $1 = spool dir; pending = rotated .ndjson segments whose manifest+_SUCCESS + # markers exist (rotation finished publishing) and whose .uploaded.json + # readback marker is absent (mirrors upload_pending in + # bybit-options-archiver.rs). + pending_count=0 + pending_oldest=0 + for entry in "$1"/*.ndjson; do + [ -f "$entry" ] || continue + [ -f "$entry.manifest.json" ] || continue + [ -f "$entry._SUCCESS" ] || continue + [ ! -e "$entry.uploaded.json" ] || continue + pending_count=$((pending_count + 1)) + mtime=$(file_mtime "$entry") + case "$mtime" in (*[!0-9]*|'') continue ;; esac + if [ "$pending_oldest" -eq 0 ] || [ "$mtime" -lt "$pending_oldest" ]; then + pending_oldest=$mtime + fi + done +} + +check_upload_lane() { + # The four hard gates, applied per upload lane: + # gate 1 (mandated lanes only): upload-status.json exists and parses. + # gate 2: last_success_at present and younger than $success_max_age. + # gate 3: pending backlog count/age under $pending_max/$pending_max_age. + # gate 4: last_error empty and failure_count not growing (cumulative + # counter vs the previous poll; a first observation with a + # nonzero count is a breach because the failures were never + # acknowledged by this monitor). label=$1 spool_dir=$2 + required=$3 + success_max_age=$4 + pending_max=$5 + pending_max_age=$6 + pending_kind=$7 upload_file="$spool_dir/upload-status.json" - err_at=null - err_msg=null - failure_count=0 - failure_delta=0 - if [ -f "$upload_file" ]; then - if [ "$label" = "binance-fee-upload" ]; then - upload_filter='if type == "object" - and has("failure_count") - and (.failure_count != null) - and (.failure_count | type == "number") - and (.failure_count | floor == .) - and (.failure_count >= 0) - then . else error("invalid upload status") end' + + emit_lane_json() { + uploads_json=$(jq -n --argjson base "$uploads_json" --arg k "$label" --argjson v "$1" \ + '$base + {($k): $v}') + } + + # Gate 3 runs first so a missing status file on a non-mandated lane cannot + # hide a real backlog: the pending artifacts live on disk independently of + # the uploader's status reporting. A scan that cannot inspect the spool is + # a breach (fail closed), never a silent zero backlog. + pending_count=0 + pending_oldest=0 + pending_scan_failed=0 + if [ ! -d "$spool_dir" ] || [ ! -r "$spool_dir" ]; then + pending_scan_failed=1 + else + case "$pending_kind" in + manifests) scan_pending_glob "$spool_dir/*.manifest.json" ;; + lake) scan_pending_lake "$spool_dir" ;; + tapes) scan_pending_glob "$spool_dir/market-updates.*.ndjson" ;; + bybit-raw) scan_pending_bybit_raw "$spool_dir" ;; + esac + fi + if [ "$pending_scan_failed" -eq 1 ]; then + record_breach "$label: pending upload backlog scan failed ($spool_dir)" + fi + pending_age=0 + if [ "$pending_oldest" -gt 0 ]; then + pending_age=$((NOW_SEC - pending_oldest)) + [ "$pending_age" -lt 0 ] && pending_age=0 + fi + if [ "$pending_count" -gt "$pending_max" ]; then + record_breach "$label: pending upload backlog $pending_count over limit $pending_max" + fi + if [ "$pending_age" -gt "$pending_max_age" ]; then + record_breach "$label: oldest pending upload backlog age ${pending_age}s over ${pending_max_age}s" + fi + + if [ ! -f "$upload_file" ] || [ -L "$upload_file" ]; then + if [ "$required" = 1 ]; then + record_breach "$label: upload-status.json missing or a symbolic link" else - upload_filter='if type == "object" then . else error("invalid upload status") end' + record_warning "$label: upload-status.json missing or a symbolic link" fi - if upload_json=$(jq -ce "$upload_filter" "$upload_file" 2>/dev/null); then - err_at=$(printf '%s' "$upload_json" | jq -r '(.last_error_at // null)') - err_msg=$(printf '%s' "$upload_json" | jq -r '(.last_error // null)') - failure_count=$(printf '%s' "$upload_json" | jq -r '(.failure_count // 0)') - else - record_breach "$label: upload-status.json is malformed" + uobj=$(jq -n --argjson pc "$pending_count" --argjson pa "$pending_age" \ + '{last_success_at: null, last_success_age_seconds: null, last_error_at: null, last_error: null, failure_count: 0, failure_delta: false, pending_count: $pc, oldest_pending_age_seconds: $pa}') + emit_lane_json "$uobj" + return 0 + fi + + if [ "$required" = 1 ]; then + # Mandated lanes must carry a usable cumulative failure_count. + upload_filter='if type == "object" + and has("failure_count") + and (.failure_count != null) + and (.failure_count | type == "number") + and (.failure_count | floor == .) + and (.failure_count >= 0) + then . else error("invalid upload status") end' + else + upload_filter='if type == "object" then . else error("invalid upload status") end' + fi + if ! upload_json=$(jq -ce "$upload_filter" "$upload_file" 2>/dev/null); then + record_breach "$label: upload-status.json is malformed" + uobj=$(jq -n --argjson pc "$pending_count" --argjson pa "$pending_age" \ + '{last_success_at: null, last_success_age_seconds: null, last_error_at: null, last_error: null, failure_count: 0, failure_delta: false, pending_count: $pc, oldest_pending_age_seconds: $pa}') + emit_lane_json "$uobj" + return 0 + fi + + # Gate 2: last_success_at presence + freshness. The real emitters produce + # RFC3339 with six fractional digits and a Z suffix (polymarket_upload:: + # utc_now, reused by the fee and usdm-reference uploaders) or Chrono + # to_rfc3339() with a +00:00 offset and optional fractional seconds (LOB); + # the bybit lane writes epoch milliseconds. jq's fromdateiso8601 only + # accepts the whole-second Z form, so normalize first; non-UTC offsets are + # refused rather than silently reinterpreted. + success_raw=$(printf '%s' "$upload_json" | jq -c '(.last_success_at // null)') + success_epoch=$(printf '%s' "$upload_json" | jq -r ' + (.last_success_at // null) + | if . == null then empty + elif type == "number" then (if . > 100000000000 then (. / 1000) else . end | floor) + elif type == "string" then + if test("[+-](0[1-9]|[1-9][0-9]):[0-9]{2}$") then empty + else ( sub("([Zz]|[+-]00:00)$"; "") + | sub("\\.[0-9]+$"; "") + | . + "Z" + | try fromdateiso8601 catch empty ) + end + else empty + end' 2>/dev/null) + case "$success_epoch" in (*[!0-9]*|'') success_epoch="" ;; esac + success_age_json=null + if [ -z "$success_epoch" ]; then + record_breach "$label: upload-status.json missing a parseable last_success_at" + else + success_age=$((NOW_SEC - success_epoch)) + [ "$success_age" -lt 0 ] && success_age=0 + success_age_json=$success_age + if [ "$success_age" -gt "$success_max_age" ]; then + record_breach "$label: last upload success stale (age ${success_age}s > ${success_max_age}s)" fi fi + + # Gate 4: last_error must be empty and the cumulative failure_count must not + # grow between polls. + err_at=$(printf '%s' "$upload_json" | jq -r '(.last_error_at // null)') + err_msg=$(printf '%s' "$upload_json" | jq -r '(.last_error // null)') + failure_count=$(printf '%s' "$upload_json" | jq -r '(.failure_count // 0)') case "$failure_count" in (*[!0-9]*|'') failure_count=0 ;; esac + failure_delta=0 if [ -n "$err_at" ] && [ "$err_at" != "null" ]; then record_breach "$label: upload last_error_at=$err_at" fi @@ -332,7 +578,7 @@ check_upload() { prior=$(read_prior "failure_count|$label") if [ "$DRY_RUN" -eq 0 ]; then case "$prior" in (*[!0-9]*|'') prior="" ;; esac - if [ -z "$prior" ] && [ "$label" = "binance-fee-upload" ] && [ "$failure_count" -gt 0 ]; then + if [ -z "$prior" ] && [ "$failure_count" -gt 0 ]; then failure_delta=1 record_breach "$label: initial upload failure_count=$failure_count" elif [ -n "$prior" ] && [ "$failure_count" -gt "$prior" ]; then @@ -341,28 +587,19 @@ check_upload() { fi fi state_lines="$state_lines failure_count|$label=$failure_count" - uobj=$(jq -n --arg e "$err_at" --arg m "$err_msg" --argjson f "$failure_count" \ - --argjson d "$failure_delta" \ - '{last_error_at: $e, last_error: $m, failure_count: $f, failure_delta: ($d == 1)}') - uploads_json=$(jq -n --argjson base "$uploads_json" --arg k "$label" --argjson v "$uobj" \ - '$base + {($k): $v}') -} -check_upload_required() { - label=$1 - spool_dir=$2 - if [ ! -f "$spool_dir/upload-status.json" ] || [ -L "$spool_dir/upload-status.json" ]; then - record_breach "$label: upload-status.json missing or a symbolic link" - fi - check_upload "$label" "$spool_dir" + uobj=$(jq -n --argjson s "$success_raw" --argjson sa "$success_age_json" \ + --arg e "$err_at" --arg m "$err_msg" --argjson f "$failure_count" \ + --argjson d "$failure_delta" --argjson pc "$pending_count" --argjson pa "$pending_age" \ + '{last_success_at: $s, last_success_age_seconds: $sa, last_error_at: $e, last_error: $m, failure_count: $f, failure_delta: ($d == 1), pending_count: $pc, oldest_pending_age_seconds: $pa}') + emit_lane_json "$uobj" } check_delay_gate() { - # Journald delay-gate trips (the fail-closed reconnect path) in the last 15m. - # Capture journalctl's own exit status separately: a successful no-match query - # is trips=0, but a failed query means the delay-gate evidence could not be - # inspected and must itself be reported as a breach so the monitor cannot - # emit ok:true without inspectable delay-gate evidence. + # Journald delay-gate trips (the fail-closed reconnect path) in the last 15m, + # observed as warnings. Capture journalctl's own exit status separately: a + # successful no-match query is trips=0, while a failed query means the + # delay-gate evidence could not be inspected. unit=$1 label=$2 journal_out=$(journalctl -u "$unit" --since "$DELAY_GATE_WINDOW" --no-pager 2>/dev/null) @@ -371,9 +608,9 @@ check_delay_gate() { | grep -c 'source-to-receive delay exceeds the governed limit' || true) case "$trips" in (*[!0-9]*|'') trips=0 ;; esac if [ "$journal_rc" -ne 0 ]; then - record_breach "$label: journald query failed (exit $journal_rc)" + record_warning "$label: journald query failed (exit $journal_rc)" elif [ "$trips" -gt 0 ]; then - record_breach "$label: $trips delay-gate trip(s) in last 15 minutes" + record_warning "$label: $trips delay-gate trip(s) in last 15 minutes" fi dobj=$(jq -n --argjson t "$trips" '{trips_15m: $t}') delay_gate_json=$(jq -n --argjson base "$delay_gate_json" --arg k "$unit" --argjson v "$dobj" \ @@ -388,9 +625,9 @@ check_recent_snapshot_failures() { failures=$(printf '%s\n' "$journal_out" | grep -c 'Failed with result' || true) case "$failures" in (*[!0-9]*|'') failures=0 ;; esac if [ "$journal_rc" -ne 0 ]; then - record_breach "$label: snapshot failure journal query failed (exit $journal_rc)" + record_warning "$label: snapshot failure journal query failed (exit $journal_rc)" elif [ "$failures" -gt 0 ]; then - record_breach "$label: $failures recent snapshot failure(s)" + record_warning "$label: $failures recent snapshot failure(s)" fi } @@ -398,7 +635,7 @@ write_state() { [ "$DRY_RUN" -eq 1 ] && return 0 # A state-persistence failure means the next poll can lose restart/upload # deltas while the monitor reports healthy, so record it as a breach rather - # than only logging. + # than only logging: gate 4 delta detection depends on this state. if ! mkdir -p "$STATE_DIR" 2>/dev/null; then record_breach "state: state directory unavailable: $STATE_DIR" return 0 @@ -452,13 +689,20 @@ check_disabled_unit "$POLY_RAW_OPS_GATE" "polymarket-raw-ops-gate" check_binance_health "binance-lob-archiver-production@spot" "$SPOOL_ROOT/binance-lob/spot" check_binance_health "binance-lob-archiver-production@usdm" "$SPOOL_ROOT/binance-lob/usdm" -check_upload "binance-lob-archiver-production@spot" "$SPOOL_ROOT/binance-lob/spot" -check_upload "binance-lob-archiver-production@usdm" "$SPOOL_ROOT/binance-lob/usdm" -check_upload "binance-usdm-reference-collector" "$SPOOL_ROOT/binance-usdm-reference" -check_upload "bybit-options-upload" "$SPOOL_ROOT/bybit-options" -check_upload "polymarket-market-tape-upload" "$SPOOL_ROOT/polymarket" -check_upload "polymarket-reference-upload" "$SPOOL_ROOT/polymarket-reference" -check_upload_required "binance-fee-upload" "$SPOOL_ROOT/binance-fee" +check_upload_lane "binance-lob-archiver-production@spot" "$SPOOL_ROOT/binance-lob/spot" \ + 1 "$LOB_SUCCESS_MAX_AGE" "$LOB_PENDING_MAX" "$LOB_PENDING_MAX_AGE" manifests +check_upload_lane "binance-lob-archiver-production@usdm" "$SPOOL_ROOT/binance-lob/usdm" \ + 1 "$LOB_SUCCESS_MAX_AGE" "$LOB_PENDING_MAX" "$LOB_PENDING_MAX_AGE" manifests +check_upload_lane "binance-usdm-reference-collector" "$SPOOL_ROOT/binance-usdm-reference" \ + 0 "$REF_SUCCESS_MAX_AGE" "$REF_PENDING_MAX" "$REF_PENDING_MAX_AGE" lake +check_upload_lane "bybit-options-upload" "$SPOOL_ROOT/bybit-options" \ + 0 "$BYBIT_SUCCESS_MAX_AGE" "$BYBIT_PENDING_MAX" "$BYBIT_PENDING_MAX_AGE" bybit-raw +check_upload_lane "polymarket-market-tape-upload" "$SPOOL_ROOT/polymarket" \ + 0 "$POLY_SUCCESS_MAX_AGE" "$POLY_PENDING_MAX" "$POLY_PENDING_MAX_AGE" tapes +check_upload_lane "polymarket-reference-upload" "$SPOOL_ROOT/polymarket-reference" \ + 0 "$POLY_SUCCESS_MAX_AGE" "$POLY_PENDING_MAX" "$POLY_PENDING_MAX_AGE" tapes +check_upload_lane "binance-fee-upload" "$SPOOL_ROOT/binance-fee" \ + 1 "$FEE_SUCCESS_MAX_AGE" "$FEE_PENDING_MAX" "$FEE_PENDING_MAX_AGE" lake check_delay_gate "$ARCHIVER_SPOT" "binance-lob-archiver-production@spot" check_delay_gate "$ARCHIVER_USDM" "binance-lob-archiver-production@usdm" @@ -473,13 +717,15 @@ fi if [ "$JSON_MODE" -eq 1 ]; then breaches_json=$(printf '%s' "$breaches" | jq -Rs 'split("\n") | map(select(length > 0))') + warnings_json=$(printf '%s' "$warnings" | jq -Rs 'split("\n") | map(select(length > 0))') checks_json=$(jq -n --argjson disk "$disk_json" --argjson mount "$mount_json" \ --argjson units "$units_json" --argjson health "$health_json" \ --argjson uploads "$uploads_json" --argjson delay "$delay_gate_json" \ '{disk: $disk, mount: $mount, units: $units, health: $health, uploads: $uploads, delay_gate: $delay}') jq -n --argjson ok "$ok_str" --arg checked "$CHECKED_AT" \ - --argjson breaches "$breaches_json" --argjson checks "$checks_json" \ - '{ok: $ok, checked_at: $checked, breaches: $breaches, checks: $checks}' + --argjson breaches "$breaches_json" --argjson warnings "$warnings_json" \ + --argjson checks "$checks_json" \ + '{ok: $ok, checked_at: $checked, breaches: $breaches, warnings: $warnings, checks: $checks}' else if [ "$breach_count" -gt 0 ]; then printf 'ok:false\n' @@ -487,6 +733,9 @@ else else printf 'ok:true\n' fi + if [ -n "$warnings" ]; then + printf '%s\n' "$warnings" | sed 's/^/warning: /' + fi fi if [ "$breach_count" -gt 0 ]; then diff --git a/deployment/aliyun/test-monday-collector-health.sh b/deployment/aliyun/test-monday-collector-health.sh index 99a570e4e..caba64bfd 100755 --- a/deployment/aliyun/test-monday-collector-health.sh +++ b/deployment/aliyun/test-monday-collector-health.sh @@ -4,6 +4,16 @@ # /tmp fixture trees with stubbed systemctl/df/journalctl/mountpoint/logger. # The script is read-only toward units, so the stubs never mutate anything. # +# Contract under test: exactly four hard gates (breaches) — +# 1. upload-status.json exists and parses (binance-lob spot/usdm, binance-fee) +# 2. last_success_at present and fresh per upload lane +# 3. pending upload backlog bounded (count + oldest pending age) +# 4. failure_count not growing, last_error empty (all lanes) +# plus the expected-disabled installation gate (polymarket-raw-ops-gate) and +# state-persistence failures. Everything else (units, timers, restarts, +# health.json, delay-gate journal, disk, mount, fee snapshot journal) is a +# warning: reported, never blocking ok:true. +# # Usage: ./test-monday-collector-health.sh set -euo pipefail @@ -22,7 +32,6 @@ err_file="$test_root/err" DF_TOTAL=196000000 # KiB, ~187 GiB (matches the ~196G host disk) DF_AVAIL_HEALTHY=117600000 # 60% free -DF_AVAIL_WARN=39200000 # 20% free (<25% warn, >=10% crit) DF_AVAIL_CRIT=9800000 # 5% free (<10% crit) pass_count=0 @@ -94,6 +103,11 @@ reset_state() { mkdir -p "$state_dir" } +reset_spool() { + rm -rf "$spool_root" + make_spools +} + write_scenario() { cat > "$scenario" } @@ -133,23 +147,47 @@ EOF } write_upload() { - # $1 path, $2 last_error_at JSON, $3 last_error JSON, $4 failure_count + # $1 path, $2 last_error_at JSON, $3 last_error JSON, $4 failure_count, + # $5 last_success age in seconds (default 60). Emits six fractional digits + # and a Z suffix, exactly what polymarket_upload::utc_now (reused by the fee + # and usdm-reference uploaders) produces, so the fixtures cannot hide a + # parser gap behind jq's whole-second todate form. jq computes the time + # portably (BSD date and GNU date disagree on epoch formatting flags). + age=${5:-60} + success_at=$(jq -rn --argjson age "$age" ' + (now - $age) as $t + | ($t | floor | gmtime | strftime("%Y-%m-%dT%H:%M:%S")) + + "." + + (("000000" + ((($t - ($t | floor)) * 1000000 | floor) | tostring))[-6:]) + + "Z"') + cat > "$1" < "$1" < "$spool_root/polymarket/market-updates.ndjson" + : > "$spool_root/polymarket-reference/market-updates.ndjson" } healthy_scenario() { @@ -280,136 +318,276 @@ run_health expect "healthy: exit 0" "$(rc_is 0; echo $?)" expect "healthy: ok:true" "$(grep_out '^ok:true$'; echo $?)" expect "healthy: no breach lines" "$(grep_not_out '^breach:'; echo $?)" -expect "healthy: state file written" "$([ -f "$state_dir/state.json" ]; echo $?)" +expect "healthy: no warning lines" "$(grep_not_out '^warning:'; echo $?)" +expect "healthy: state file written" "$(if [ -f "$state_dir/state.json" ]; then echo 0; else echo 1; fi)" expect "healthy: state records nrestarts" "$(grep -q '^nrestarts|binance-lob-archiver-production@spot.service=4$' "$state_dir/state.json"; echo $?)" expect "healthy: state records failure_count" "$(grep -q '^failure_count|polymarket-market-tape-upload=0$' "$state_dir/state.json"; echo $?)" # --------------------------------------------------------------------------- -# 2. Disk critical +# 2. Gate 1: missing upload-status.json on a mandated lane is a breach +# (replaces the old 'LOB missing status is healthy' expectation) # --------------------------------------------------------------------------- reset_env reset_state healthy_scenario healthy_fixtures -STUB_DF_AVAIL_KIB=$DF_AVAIL_CRIT +rm -f "$spool_root/binance-lob/spot/upload-status.json" run_health -expect "disk critical: exit 1" "$(rc_is 1; echo $?)" -expect "disk critical: ok:false" "$(grep_out '^ok:false$'; echo $?)" -expect "disk critical: breach message" "$(grep_out 'below critical 10%'; echo $?)" +expect "gate1 lob missing: exit 1" "$(rc_is 1; echo $?)" +expect "gate1 lob missing: breach message" "$(grep_out 'binance-lob-archiver-production@spot: upload-status.json missing'; echo $?)" -# --------------------------------------------------------------------------- -# 3. Disk warning -# --------------------------------------------------------------------------- reset_env reset_state healthy_scenario healthy_fixtures -STUB_DF_AVAIL_KIB=$DF_AVAIL_WARN +rm -f "$spool_root/binance-fee/upload-status.json" run_health -expect "disk warning: exit 1" "$(rc_is 1; echo $?)" -expect "disk warning: breach message" "$(grep_out 'below warning 25%'; echo $?)" +expect "gate1 fee missing: exit 1" "$(rc_is 1; echo $?)" +expect "gate1 fee missing: breach message" "$(grep_out 'binance-fee-upload: upload-status.json missing'; echo $?)" # --------------------------------------------------------------------------- -# 4. Unit inactive +# 3. Gate 1: malformed upload-status.json is a breach # --------------------------------------------------------------------------- reset_env reset_state healthy_scenario healthy_fixtures -rewrite_scenario 's|^binance-lob-archiver-production@spot.service active|binance-lob-archiver-production@spot.service inactive|' +printf '{not json\n' > "$spool_root/binance-fee/upload-status.json" run_health -expect "unit inactive: exit 1" "$(rc_is 1; echo $?)" -expect "unit inactive: breach message" "$(grep_out 'not active'; echo $?)" +expect "gate1 fee malformed: exit 1" "$(rc_is 1; echo $?)" +expect "gate1 fee malformed: breach message" "$(grep_out 'binance-fee-upload: upload-status.json is malformed'; echo $?)" -# --------------------------------------------------------------------------- -# 5. Timer not enabled -# --------------------------------------------------------------------------- reset_env reset_state healthy_scenario healthy_fixtures -rewrite_scenario 's|^polymarket-market-tape-upload-watchdog.timer active enabled|polymarket-market-tape-upload-watchdog.timer active disabled|' +printf '{"last_success_at":"2026-08-07T00:00:00Z","last_error_at":null,"last_error":null}\n' > "$spool_root/binance-fee/upload-status.json" run_health -expect "timer not enabled: exit 1" "$(rc_is 1; echo $?)" -expect "timer not enabled: breach message" "$(grep_out 'timer not enabled'; echo $?)" +expect "gate1 fee missing count: exit 1" "$(rc_is 1; echo $?)" +expect "gate1 fee missing count: breach message" "$(grep_out 'binance-fee-upload: upload-status.json is malformed'; echo $?)" # --------------------------------------------------------------------------- -# 6. Unit result failure +# 4. Gate 1: missing upload-status.json on a non-mandated lane is only a warning # --------------------------------------------------------------------------- reset_env reset_state healthy_scenario healthy_fixtures -rewrite_scenario 's|^binance-lob-archiver-production@usdm.service active enabled success|binance-lob-archiver-production@usdm.service active enabled exit-code|' +rm -f "$spool_root/polymarket/upload-status.json" run_health -expect "unit result failure: exit 1" "$(rc_is 1; echo $?)" -expect "unit result failure: breach message" "$(grep_out "Result='exit-code'"; echo $?)" +expect "gate1 polymarket missing: exit 0" "$(rc_is 0; echo $?)" +expect "gate1 polymarket missing: ok:true" "$(grep_out '^ok:true$'; echo $?)" +expect "gate1 polymarket missing: warning message" "$(grep_out '^warning: polymarket-market-tape-upload: upload-status.json missing'; echo $?)" +expect "gate1 polymarket missing: no breach lines" "$(grep_not_out '^breach:'; echo $?)" # --------------------------------------------------------------------------- -# 7. Restart-rate delta (two runs) +# 5. Gate 2: stale last_success_at is a breach # --------------------------------------------------------------------------- reset_env reset_state healthy_scenario healthy_fixtures +write_upload "$spool_root/binance-lob/spot/upload-status.json" null null 0 7300 run_health -expect "restart delta: baseline healthy" "$(rc_is 0; echo $?)" -rewrite_scenario 's|^binance-lob-archiver-production@spot.service active enabled success 4|binance-lob-archiver-production@spot.service active enabled success 7|' +expect "gate2 lob stale: exit 1" "$(rc_is 1; echo $?)" +expect "gate2 lob stale: breach message" "$(grep_out 'binance-lob-archiver-production@spot: last upload success stale'; echo $?)" + +reset_env +reset_state +healthy_scenario +healthy_fixtures +write_upload "$spool_root/binance-fee/upload-status.json" null null 0 700 run_health -expect "restart delta: exit 1" "$(rc_is 1; echo $?)" -expect "restart delta: breach message" "$(grep_out 'restart rate high'; echo $?)" +expect "gate2 fee stale: exit 1" "$(rc_is 1; echo $?)" +expect "gate2 fee stale: breach message" "$(grep_out 'binance-fee-upload: last upload success stale'; echo $?)" + +reset_env +reset_state +healthy_scenario +healthy_fixtures +write_upload_ms "$spool_root/bybit-options/upload-status.json" null null 0 5500 +run_health +expect "gate2 bybit stale (epoch ms): exit 1" "$(rc_is 1; echo $?)" +expect "gate2 bybit stale (epoch ms): breach message" "$(grep_out 'bybit-options-upload: last upload success stale'; echo $?)" # --------------------------------------------------------------------------- -# 8. Health stale +# 6. Gate 2: missing last_success_at is a breach (fee lane before the parallel +# uploader change deploys) # --------------------------------------------------------------------------- reset_env reset_state healthy_scenario healthy_fixtures -write_health spot 600 0 false synced +printf '{"last_error_at":null,"last_error":null,"failure_count":0}\n' > "$spool_root/binance-fee/upload-status.json" run_health -expect "health stale: exit 1" "$(rc_is 1; echo $?)" -expect "health stale: breach message" "$(grep_out 'health.json stale'; echo $?)" +expect "gate2 fee missing success: exit 1" "$(rc_is 1; echo $?)" +expect "gate2 fee missing success: breach message" "$(grep_out 'binance-fee-upload: upload-status.json missing a parseable last_success_at'; echo $?)" # --------------------------------------------------------------------------- -# 9. Health sequence gap +# 6b. Gate 2: real uploader timestamp formats parse (green) +# - whole-second Z (legacy form) +# - six fractional digits + Z (polymarket_upload::utc_now; fee, reference) +# - +00:00 offset without/with fractional seconds (Chrono to_rfc3339, LOB) # --------------------------------------------------------------------------- reset_env reset_state healthy_scenario healthy_fixtures -write_health usdm 45 5 false synced +fresh_offset=$(jq -rn '(now - 60) | floor | gmtime | strftime("%Y-%m-%dT%H:%M:%S") + "+00:00"') +printf '{"last_success_at":"%s","last_error_at":null,"last_error":null,"failure_count":0}\n' "$fresh_offset" \ + > "$spool_root/binance-lob/spot/upload-status.json" +run_health +expect "gate2 offset format: exit 0" "$(rc_is 0; echo $?)" + +reset_env +reset_state +healthy_scenario +healthy_fixtures +fresh_frac_offset=$(jq -rn '(now - 60) | floor | gmtime | strftime("%Y-%m-%dT%H:%M:%S") + ".123456+00:00"') +printf '{"last_success_at":"%s","last_error_at":null,"last_error":null,"failure_count":0}\n' "$fresh_frac_offset" \ + > "$spool_root/binance-lob/spot/upload-status.json" +run_health +expect "gate2 frac+offset format: exit 0" "$(rc_is 0; echo $?)" + +reset_env +reset_state +healthy_scenario +healthy_fixtures +fresh_whole=$(jq -rn '(now - 60) | floor | gmtime | strftime("%Y-%m-%dT%H:%M:%S") + "Z"') +printf '{"last_success_at":"%s","last_error_at":null,"last_error":null,"failure_count":0}\n' "$fresh_whole" \ + > "$spool_root/binance-lob/spot/upload-status.json" +run_health +expect "gate2 whole-second format: exit 0" "$(rc_is 0; echo $?)" + +# A non-UTC offset is refused (breach) rather than silently read as UTC. +reset_env +reset_state +healthy_scenario +healthy_fixtures +fresh_non_utc=$(jq -rn '(now - 60) | floor | gmtime | strftime("%Y-%m-%dT%H:%M:%S") + "+09:00"') +printf '{"last_success_at":"%s","last_error_at":null,"last_error":null,"failure_count":0}\n' "$fresh_non_utc" \ + > "$spool_root/binance-lob/spot/upload-status.json" run_health -expect "health gap: exit 1" "$(rc_is 1; echo $?)" -expect "health gap: breach message" "$(grep_out 'sequence_gaps=5'; echo $?)" +expect "gate2 non-UTC offset: exit 1" "$(rc_is 1; echo $?)" +expect "gate2 non-UTC offset: breach message" "$(grep_out 'binance-lob-archiver-production@spot: upload-status.json missing a parseable last_success_at'; echo $?)" # --------------------------------------------------------------------------- -# 10. Health missing +# 7. Gate 3: pending backlog count over the limit is a breach # --------------------------------------------------------------------------- reset_env reset_state healthy_scenario healthy_fixtures -rm -f "$spool_root/binance-lob/spot/health.json" +for i in 1 2 3 4 5; do + : > "$spool_root/binance-lob/spot/segment-$i.manifest.json" +done run_health -expect "health missing: exit 1" "$(rc_is 1; echo $?)" -expect "health missing: breach message" "$(grep_out 'health.json missing'; echo $?)" +expect "gate3 lob count: exit 1" "$(rc_is 1; echo $?)" +expect "gate3 lob count: breach message" "$(grep_out 'binance-lob-archiver-production@spot: pending upload backlog 5 over limit 4'; echo $?)" + +reset_env +reset_state +healthy_scenario +healthy_fixtures +i=0 +while [ "$i" -lt 25 ]; do + mkdir -p "$spool_root/binance-usdm-reference/lake/raw/venue=binance_usdm/dataset=reference/date=2099-01-01/hour=00/batch=$i" + i=$((i + 1)) +done +run_health +expect "gate3 reference count: exit 1" "$(rc_is 1; echo $?)" +expect "gate3 reference count: breach message" "$(grep_out 'binance-usdm-reference-collector: pending upload backlog 25 over limit 24'; echo $?)" # --------------------------------------------------------------------------- -# 10b. Health.json is a symbolic link +# 8. Gate 3: oldest pending backlog age over the bound is a breach # --------------------------------------------------------------------------- reset_env reset_state healthy_scenario healthy_fixtures -rm -f "$spool_root/binance-lob/spot/health.json" -ln -s /dev/null "$spool_root/binance-lob/spot/health.json" +: > "$spool_root/binance-lob/spot/old.manifest.json" +touch -t 202001010000 "$spool_root/binance-lob/spot/old.manifest.json" +run_health +expect "gate3 lob age: exit 1" "$(rc_is 1; echo $?)" +expect "gate3 lob age: breach message" "$(grep_out 'binance-lob-archiver-production@spot: oldest pending upload backlog age'; echo $?)" + +reset_env +reset_state +healthy_scenario +healthy_fixtures +tape="$spool_root/polymarket/market-updates.20200101T000000.ndjson" +: > "$tape" +touch -t 202001010000 "$tape" +run_health +expect "gate3 polymarket tape age: exit 1" "$(rc_is 1; echo $?)" +expect "gate3 polymarket tape age: breach message" "$(grep_out 'polymarket-market-tape-upload: oldest pending upload backlog age'; echo $?)" + +reset_env +reset_state +healthy_scenario +healthy_fixtures +mkdir -p "$spool_root/binance-fee/lake/raw/venue=binance_usdm/dataset=fee/account=abc/date=2020-01-01/hour=00/batch=1" +touch -t 202001010000 "$spool_root/binance-fee/lake/raw/venue=binance_usdm/dataset=fee/account=abc/date=2020-01-01/hour=00/batch=1" +run_health +expect "gate3 fee batch age: exit 1" "$(rc_is 1; echo $?)" +expect "gate3 fee batch age: breach message" "$(grep_out 'binance-fee-upload: oldest pending upload backlog age'; echo $?)" + +reset_env +reset_state +healthy_scenario +healthy_fixtures +raw="$spool_root/bybit-options/quotes.ndjson" +: > "$raw" +: > "$raw.manifest.json" +: > "$raw._SUCCESS" +touch -t 202001010000 "$raw" +run_health +expect "gate3 bybit raw age: exit 1" "$(rc_is 1; echo $?)" +expect "gate3 bybit raw age: breach message" "$(grep_out 'bybit-options-upload: oldest pending upload backlog age'; echo $?)" + +# An uploaded bybit segment (readback marker present) is not backlog. +reset_env +reset_state +healthy_scenario +healthy_fixtures +raw="$spool_root/bybit-options/quotes.ndjson" +: > "$raw" +: > "$raw.manifest.json" +: > "$raw._SUCCESS" +: > "$raw.uploaded.json" +touch -t 202001010000 "$raw" +run_health +expect "gate3 bybit uploaded: exit 0" "$(rc_is 0; echo $?)" + +# Gate 3 still fails closed when a non-mandated lane has no status file: the +# backlog lives on disk independently of the uploader's status reporting. +reset_env +reset_state +healthy_scenario +healthy_fixtures +rm -f "$spool_root/polymarket/upload-status.json" +tape="$spool_root/polymarket/market-updates.20200101T000000.ndjson" +: > "$tape" +touch -t 202001010000 "$tape" run_health -expect "health symlink: exit 1" "$(rc_is 1; echo $?)" -expect "health symlink: breach message" "$(grep_out 'health.json missing or a symbolic link'; echo $?)" +expect "gate3 no-status backlog: exit 1" "$(rc_is 1; echo $?)" +expect "gate3 no-status backlog: breach message" "$(grep_out 'polymarket-market-tape-upload: oldest pending upload backlog age'; echo $?)" +expect "gate3 no-status backlog: missing status still a warning" "$(grep_out '^warning: polymarket-market-tape-upload: upload-status.json missing'; echo $?)" + +# A pending-backlog scan that cannot inspect the spool is a breach, never a +# silent zero backlog. +reset_env +reset_state +healthy_scenario +healthy_fixtures +chmod 000 "$spool_root/binance-fee" +run_health +chmod 755 "$spool_root/binance-fee" +expect "gate3 scan fail: exit 1" "$(rc_is 1; echo $?)" +expect "gate3 scan fail: breach message" "$(grep_out 'binance-fee-upload: pending upload backlog scan failed'; echo $?)" # --------------------------------------------------------------------------- -# 11. Upload error present +# 9. Gate 4: last_error present is a breach # --------------------------------------------------------------------------- reset_env reset_state @@ -417,225 +595,284 @@ healthy_scenario healthy_fixtures write_upload "$spool_root/binance-lob/spot/upload-status.json" '"2026-08-07T01:00:00Z"' '"oss upload readback mismatch"' 3 run_health -expect "upload error: exit 1" "$(rc_is 1; echo $?)" -expect "upload error: breach message" "$(grep_out 'upload last_error'; echo $?)" +expect "gate4 upload error: exit 1" "$(rc_is 1; echo $?)" +expect "gate4 upload error: breach message" "$(grep_out 'upload last_error'; echo $?)" # --------------------------------------------------------------------------- -# 12. Upload failure-count delta (two runs) +# 10. Gate 4: failure_count growth is a breach (two runs) # --------------------------------------------------------------------------- reset_env reset_state healthy_scenario healthy_fixtures -write_upload "$spool_root/binance-lob/spot/upload-status.json" null null 3 run_health -expect "upload delta: baseline healthy" "$(rc_is 0; echo $?)" +expect "gate4 delta: baseline healthy" "$(rc_is 0; echo $?)" write_upload "$spool_root/binance-lob/spot/upload-status.json" null null 5 run_health -expect "upload delta: exit 1" "$(rc_is 1; echo $?)" -expect "upload delta: breach message" "$(grep_out 'failure_count increased 3 -> 5'; echo $?)" +expect "gate4 delta: exit 1" "$(rc_is 1; echo $?)" +expect "gate4 delta: breach message" "$(grep_out 'failure_count increased 0 -> 5'; echo $?)" # --------------------------------------------------------------------------- -# 13. Delay-gate trips +# 11. Gate 4: a first observation with a nonzero failure_count is a breach, +# uniformly across lanes # --------------------------------------------------------------------------- reset_env reset_state healthy_scenario healthy_fixtures -STUB_JOURNAL_TRIPS=2 +write_upload "$spool_root/polymarket/upload-status.json" null null 2 run_health -expect "delay gate: exit 1" "$(rc_is 1; echo $?)" -expect "delay gate: breach message" "$(grep_out 'delay-gate trip'; echo $?)" +expect "gate4 initial count: exit 1" "$(rc_is 1; echo $?)" +expect "gate4 initial count: breach message" "$(grep_out 'polymarket-market-tape-upload: initial upload failure_count=2'; echo $?)" # --------------------------------------------------------------------------- -# 13b. Delay-gate evidence uninspectable (journalctl failure) +# 12. Demoted: disk critical is a warning, not a breach # --------------------------------------------------------------------------- reset_env reset_state healthy_scenario healthy_fixtures -STUB_JOURNAL_FAIL=1 +STUB_DF_AVAIL_KIB=$DF_AVAIL_CRIT run_health -expect "delay gate journald fail: exit 1" "$(rc_is 1; echo $?)" -expect "delay gate journald fail: breach message" "$(grep_out 'journald query failed'; echo $?)" +expect "disk critical: exit 0" "$(rc_is 0; echo $?)" +expect "disk critical: ok:true" "$(grep_out '^ok:true$'; echo $?)" +expect "disk critical: warning message" "$(grep_out '^warning: disk: /data free .* below critical 10%'; echo $?)" # --------------------------------------------------------------------------- -# 13c. State persistence failure is a breach +# 13. Demoted: unit inactive / timer disabled / result failure are warnings # --------------------------------------------------------------------------- reset_env reset_state healthy_scenario healthy_fixtures -# Make the state directory uncreatable so write_state must record a breach. -: > "$test_root/blocked" -MONDAY_COLLECTOR_STATE_DIR="$test_root/blocked/state" +rewrite_scenario 's|^binance-lob-archiver-production@spot.service active|binance-lob-archiver-production@spot.service inactive|' run_health -expect "state persist fail: exit 1" "$(rc_is 1; echo $?)" -expect "state persist fail: breach message" "$(grep_out 'state: state directory unavailable'; echo $?)" +expect "unit inactive: exit 0" "$(rc_is 0; echo $?)" +expect "unit inactive: warning message" "$(grep_out '^warning: binance-lob-archiver-production@spot: not active'; echo $?)" -# --------------------------------------------------------------------------- -# 14. /data unmounted -# --------------------------------------------------------------------------- reset_env reset_state healthy_scenario healthy_fixtures -STUB_MOUNTED=0 +rewrite_scenario 's|^polymarket-market-tape-upload-watchdog.timer active enabled|polymarket-market-tape-upload-watchdog.timer active disabled|' run_health -expect "mount: exit 1" "$(rc_is 1; echo $?)" -expect "mount: breach message" "$(grep_out '/data is not mounted'; echo $?)" +expect "timer not enabled: exit 0" "$(rc_is 0; echo $?)" +expect "timer not enabled: warning message" "$(grep_out '^warning: .*timer not enabled'; echo $?)" -# --------------------------------------------------------------------------- -# 15. bybit-options-archiver inactive (governed production lane must run) -# --------------------------------------------------------------------------- reset_env reset_state healthy_scenario healthy_fixtures -rewrite_scenario 's|^bybit-options-archiver.service active enabled|bybit-options-archiver.service inactive enabled|' +rewrite_scenario 's|^binance-lob-archiver-production@usdm.service active enabled success|binance-lob-archiver-production@usdm.service active enabled exit-code|' run_health -expect "bybit inactive: exit 1" "$(rc_is 1; echo $?)" -expect "bybit inactive: breach message" "$(grep_out 'bybit-options-archiver: not active'; echo $?)" +expect "unit result failure: exit 0" "$(rc_is 0; echo $?)" +expect "unit result failure: warning message" "$(grep_out "^warning: .*Result='exit-code'"; echo $?)" # --------------------------------------------------------------------------- -# 16. polymarket-raw-ops-gate indirect (an instance enabled) +# 14. Demoted: restart-rate delta is a warning (two runs) # --------------------------------------------------------------------------- reset_env reset_state healthy_scenario healthy_fixtures -rewrite_scenario 's|^polymarket-raw-ops-gate@.service - disabled|polymarket-raw-ops-gate@.service - indirect|' run_health -expect "poly gate indirect: exit 1" "$(rc_is 1; echo $?)" -expect "poly gate indirect: breach message" "$(grep_out 'polymarket-raw-ops-gate: expected disabled'; echo $?)" +expect "restart delta: baseline healthy" "$(rc_is 0; echo $?)" +rewrite_scenario 's|^binance-lob-archiver-production@spot.service active enabled success 4|binance-lob-archiver-production@spot.service active enabled success 7|' +run_health +expect "restart delta: exit 0" "$(rc_is 0; echo $?)" +expect "restart delta: warning message" "$(grep_out '^warning: .*restart rate high'; echo $?)" # --------------------------------------------------------------------------- -# 17. Missing upload-status.json is not a breach +# 15. Demoted: health.json stale/gap/missing/symlink are warnings # --------------------------------------------------------------------------- reset_env reset_state healthy_scenario -make_spools -write_health spot 45 0 false synced -write_health usdm 45 0 false synced -write_upload "$spool_root/binance-fee/upload-status.json" null null 0 +healthy_fixtures +write_health spot 600 0 false synced +run_health +expect "health stale: exit 0" "$(rc_is 0; echo $?)" +expect "health stale: warning message" "$(grep_out '^warning: .*health.json stale'; echo $?)" + +reset_env +reset_state +healthy_scenario +healthy_fixtures +write_health usdm 45 5 false synced run_health -expect "missing upload-status: exit 0" "$(rc_is 0; echo $?)" -expect "missing upload-status: ok:true" "$(grep_out '^ok:true$'; echo $?)" +expect "health gap: exit 0" "$(rc_is 0; echo $?)" +expect "health gap: warning message" "$(grep_out '^warning: .*sequence_gaps=5'; echo $?)" + +reset_env +reset_state +healthy_scenario +healthy_fixtures +rm -f "$spool_root/binance-lob/spot/health.json" +run_health +expect "health missing: exit 0" "$(rc_is 0; echo $?)" +expect "health missing: warning message" "$(grep_out '^warning: .*health.json missing'; echo $?)" + +reset_env +reset_state +healthy_scenario +healthy_fixtures +rm -f "$spool_root/binance-lob/spot/health.json" +ln -s /dev/null "$spool_root/binance-lob/spot/health.json" +run_health +expect "health symlink: exit 0" "$(rc_is 0; echo $?)" +expect "health symlink: warning message" "$(grep_out '^warning: .*health.json missing or a symbolic link'; echo $?)" # --------------------------------------------------------------------------- -# 18. JSON output shape (healthy) +# 16. Demoted: delay-gate trips / journald failure / fee snapshot failures are +# warnings # --------------------------------------------------------------------------- reset_env reset_state healthy_scenario healthy_fixtures -run_health --json -expect "json healthy: exit 0" "$(rc_is 0; echo $?)" -expect "json healthy: parses and shape valid" "$(json_query ' - .ok == true - and (.checked_at | type) == "string" - and (.breaches | type) == "array" - and (.breaches | length) == 0 - and (.checks.disk.free_percent | type) == "number" - and .checks.mount.data_mounted == true - and .checks.units["binance-lob-archiver-production@spot.service"].active == true - and .checks.units["bybit-options-archiver.service"].active == true - and .checks.units["bybit-options-archiver.service"].enabled == true - and .checks.units["binance-fee-snapshot-spot.timer"].active == true - and .checks.units["binance-fee-snapshot-usdm.timer"].active == true - and .checks.units["binance-fee-upload.timer"].enabled == true - and (.checks.health["binance-lob-archiver-production@spot"].age_seconds | type) == "number" - and .checks.health["binance-lob-archiver-production@spot"].status == "synced" - and (.checks.uploads["binance-lob-archiver-production@spot"].failure_count | type) == "number" - and .checks.uploads["binance-lob-archiver-production@spot"].last_error_at == "null" - and .checks.uploads["binance-fee-upload"].last_error == "null" - and (.checks.delay_gate["binance-lob-archiver-production@spot.service"].trips_15m | type) == "number" -'; echo $?)" +STUB_JOURNAL_TRIPS=2 +run_health +expect "delay gate: exit 0" "$(rc_is 0; echo $?)" +expect "delay gate: warning message" "$(grep_out '^warning: .*delay-gate trip'; echo $?)" + +reset_env +reset_state +healthy_scenario +healthy_fixtures +STUB_JOURNAL_FAIL=1 +run_health +expect "journald fail: exit 0" "$(rc_is 0; echo $?)" +expect "journald fail: warning message" "$(grep_out '^warning: .*journald query failed'; echo $?)" + +reset_env +reset_state +healthy_scenario +healthy_fixtures +STUB_JOURNAL_FEE_FAILURES=1 +run_health +expect "fee snapshot failure: exit 0" "$(rc_is 0; echo $?)" +expect "fee snapshot failure: warning message" "$(grep_out '^warning: .*recent snapshot failure'; echo $?)" # --------------------------------------------------------------------------- -# 19. JSON output shape (breaching) +# 17. Demoted: /data unmounted is a warning # --------------------------------------------------------------------------- reset_env reset_state healthy_scenario healthy_fixtures -STUB_DF_AVAIL_KIB=$DF_AVAIL_CRIT -run_health --json -expect "json breach: exit 1" "$(rc_is 1; echo $?)" -expect "json breach: ok:false and breaches non-empty" "$(json_query '.ok == false and (.breaches | length) >= 1'; echo $?)" +STUB_MOUNTED=0 +run_health +expect "mount: exit 0" "$(rc_is 0; echo $?)" +expect "mount: warning message" "$(grep_out '^warning: mount: /data is not mounted'; echo $?)" # --------------------------------------------------------------------------- -# 20. --dry-run does not touch state +# 18. State persistence failure stays a breach (gate 4 delta evidence) # --------------------------------------------------------------------------- reset_env reset_state healthy_scenario healthy_fixtures -run_health --dry-run -expect "dry-run: exit 0" "$(rc_is 0; echo $?)" -expect "dry-run: state file not created" "$([ ! -e "$state_dir/state.json" ]; echo $?)" +# Make the state directory uncreatable so write_state must record a breach. +: > "$test_root/blocked" +MONDAY_COLLECTOR_STATE_DIR="$test_root/blocked/state" +run_health +expect "state persist fail: exit 1" "$(rc_is 1; echo $?)" +expect "state persist fail: breach message" "$(grep_out 'state: state directory unavailable'; echo $?)" # --------------------------------------------------------------------------- -# 21. Binance fee upload status is mandatory +# 19. polymarket-raw-ops-gate stays a breach (indirect and static) # --------------------------------------------------------------------------- reset_env reset_state healthy_scenario healthy_fixtures -rm -f "$spool_root/binance-fee/upload-status.json" +rewrite_scenario 's|^polymarket-raw-ops-gate@.service - disabled|polymarket-raw-ops-gate@.service - indirect|' run_health -expect "fee status missing: exit 1" "$(rc_is 1; echo $?)" -expect "fee status missing: breach message" "$(grep_out 'binance-fee-upload: upload-status.json missing'; echo $?)" +expect "poly gate indirect: exit 1" "$(rc_is 1; echo $?)" +expect "poly gate indirect: breach message" "$(grep_out 'polymarket-raw-ops-gate: expected disabled'; echo $?)" -# --------------------------------------------------------------------------- reset_env reset_state healthy_scenario healthy_fixtures -printf '{not json\n' > "$spool_root/binance-fee/upload-status.json" +rewrite_scenario 's|^polymarket-raw-ops-gate@.service - disabled|polymarket-raw-ops-gate@.service - static|' run_health -expect "fee status malformed: exit 1" "$(rc_is 1; echo $?)" -expect "fee status malformed: breach message" "$(grep_out 'binance-fee-upload: upload-status.json is malformed'; echo $?)" +expect "poly gate static: exit 1" "$(rc_is 1; echo $?)" +expect "poly gate static: breach message" "$(grep_out "polymarket-raw-ops-gate: expected disabled but is-enabled='static'"; echo $?)" +# --------------------------------------------------------------------------- +# 20. JSON output shape (healthy, includes the warnings array) # --------------------------------------------------------------------------- reset_env reset_state healthy_scenario healthy_fixtures -printf '{"last_success_at":"2026-08-07T00:00:00Z","last_error_at":null,"last_error":null}\n' > "$spool_root/binance-fee/upload-status.json" -run_health -expect "fee status missing count: exit 1" "$(rc_is 1; echo $?)" -expect "fee status missing count: breach message" "$(grep_out 'binance-fee-upload: upload-status.json is malformed'; echo $?)" +run_health --json +expect "json healthy: exit 0" "$(rc_is 0; echo $?)" +expect "json healthy: parses and shape valid" "$(json_query ' + .ok == true + and (.checked_at | type) == "string" + and (.breaches | type) == "array" + and (.breaches | length) == 0 + and (.warnings | type) == "array" + and (.warnings | length) == 0 + and (.checks.disk.free_percent | type) == "number" + and .checks.mount.data_mounted == true + and .checks.units["binance-lob-archiver-production@spot.service"].active == true + and .checks.units["bybit-options-archiver.service"].active == true + and .checks.units["polymarket-raw-ops-gate@.service"].is_enabled == "disabled" + and (.checks.health["binance-lob-archiver-production@spot"].age_seconds | type) == "number" + and .checks.health["binance-lob-archiver-production@spot"].status == "synced" + and (.checks.uploads["binance-lob-archiver-production@spot"].failure_count | type) == "number" + and .checks.uploads["binance-lob-archiver-production@spot"].last_error_at == "null" + and (.checks.uploads["binance-lob-archiver-production@spot"].last_success_age_seconds | type) == "number" + and (.checks.uploads["binance-lob-archiver-production@spot"].pending_count | type) == "number" + and .checks.uploads["binance-fee-upload"].last_error == "null" + and (.checks.uploads["bybit-options-upload"].last_success_age_seconds | type) == "number" + and (.checks.delay_gate["binance-lob-archiver-production@spot.service"].trips_15m | type) == "number" +'; echo $?)" +# --------------------------------------------------------------------------- +# 21. JSON output shape (warnings do not block ok:true) # --------------------------------------------------------------------------- reset_env reset_state healthy_scenario healthy_fixtures -printf '{"last_success_at":"2026-08-07T00:00:00Z","last_error_at":null,"last_error":null}\n' > "$spool_root/binance-usdm-reference/upload-status.json" -run_health -expect "reference status missing count: exit 0" "$(rc_is 0; echo $?)" +STUB_DF_AVAIL_KIB=$DF_AVAIL_CRIT +STUB_JOURNAL_TRIPS=1 +run_health --json +expect "json warnings: exit 0" "$(rc_is 0; echo $?)" +expect "json warnings: ok:true with warnings" "$(json_query ' + .ok == true + and (.breaches | length) == 0 + and (.warnings | length) >= 2 + and (.warnings | any(contains("below critical"))) + and (.warnings | any(contains("delay-gate trip"))) +'; echo $?)" +# --------------------------------------------------------------------------- +# 22. JSON output shape (breaching) # --------------------------------------------------------------------------- reset_env reset_state healthy_scenario healthy_fixtures -write_upload "$spool_root/binance-fee/upload-status.json" null null 1 -run_health -expect "fee initial failure: exit 1" "$(rc_is 1; echo $?)" -expect "fee initial failure: breach message" "$(grep_out 'binance-fee-upload: initial upload failure_count=1'; echo $?)" +rm -f "$spool_root/binance-lob/usdm/upload-status.json" +run_health --json +expect "json breach: exit 1" "$(rc_is 1; echo $?)" +expect "json breach: ok:false and breaches non-empty" "$(json_query '.ok == false and (.breaches | length) >= 1'; echo $?)" +# --------------------------------------------------------------------------- +# 23. --dry-run does not touch state # --------------------------------------------------------------------------- reset_env reset_state healthy_scenario healthy_fixtures -STUB_JOURNAL_FEE_FAILURES=1 -run_health -expect "fee recent snapshot failure: exit 1" "$(rc_is 1; echo $?)" -expect "fee recent snapshot failure: breach message" "$(grep_out 'recent snapshot failure'; echo $?)" +run_health --dry-run +expect "dry-run: exit 0" "$(rc_is 0; echo $?)" +expect "dry-run: state file not created" "$([ ! -e "$state_dir/state.json" ]; echo $?)" # --------------------------------------------------------------------------- printf '\n%d passed, %d failed\n' "$pass_count" "$fail_count"