From aa1149782c486ba09e66e3ac3321ee5d3b48eece Mon Sep 17 00:00:00 2001 From: Sonic Shih Date: Fri, 24 Jul 2026 00:25:05 +0800 Subject: [PATCH] fix(deployment): tolerate one health sampling interval (#224) --- deployment/aliyun/README.md | 2 +- .../aliyun/host-rust-lob-shadow-gate.sh | 2 +- .../aliyun/rust-lob-shadow-gate-policy.jq | 4 +- .../aliyun/test-rust-lob-control-plane.sh | 54 +++++++++++++------ 4 files changed, 43 insertions(+), 19 deletions(-) diff --git a/deployment/aliyun/README.md b/deployment/aliyun/README.md index cfead6cd4..4a229e85e 100644 --- a/deployment/aliyun/README.md +++ b/deployment/aliyun/README.md @@ -484,7 +484,7 @@ these are true for the entire candidate run: - Spot has at least 1,000 symbols and USD-M at least 400; - every discovered symbol has a ready snapshot and sequence gaps remain zero; - neither session nor catalog membership changes, health never stops advancing - for more than 90 seconds, and the persistent upload-failure count is unchanged; + for more than 120 seconds, and the persistent upload-failure count is unchanged; - queue, disk, and upload warnings are false, while the persistent upload-failure count does not increase during normal segment rotations; - CPU accounting and peak memory stay inside the systemd limits; diff --git a/deployment/aliyun/host-rust-lob-shadow-gate.sh b/deployment/aliyun/host-rust-lob-shadow-gate.sh index a0a604df4..a36f01a6f 100755 --- a/deployment/aliyun/host-rust-lob-shadow-gate.sh +++ b/deployment/aliyun/host-rust-lob-shadow-gate.sh @@ -6,7 +6,7 @@ export LC_ALL=C readonly REQUIRED_DURATION_SECONDS=3600 readonly HEALTH_SETTLE_SECONDS=360 -readonly MAX_HEALTH_SILENCE_SECONDS=90 +readonly MAX_HEALTH_SILENCE_SECONDS=120 readonly MAX_SEGMENT_GAP_NS=90000000000 readonly SHADOW_BINARY=/opt/monday/bin/binance-lob-archiver-shadow readonly RELEASE_ROOT=/opt/monday/releases/binance-lob-archiver diff --git a/deployment/aliyun/rust-lob-shadow-gate-policy.jq b/deployment/aliyun/rust-lob-shadow-gate-policy.jq index 6f203d7ce..0bb7cff95 100644 --- a/deployment/aliyun/rust-lob-shadow-gate-policy.jq +++ b/deployment/aliyun/rust-lob-shadow-gate-policy.jq @@ -22,7 +22,7 @@ and .markets.spot.health_samples == (.markets.spot.health_samples | floor) and .markets.spot.health_samples >= 40 and (.markets.spot.max_health_silence_seconds | type) == "number" and .markets.spot.max_health_silence_seconds >= 0 -and .markets.spot.max_health_silence_seconds <= 90 +and .markets.spot.max_health_silence_seconds <= 120 and (.markets.spot.catalog_sha256 | type) == "string" and (.markets.spot.catalog_sha256 | test("^[a-f0-9]{64}$")) and (.markets.spot.session_id | type) == "string" @@ -116,7 +116,7 @@ and .markets.usdm.health_samples == (.markets.usdm.health_samples | floor) and .markets.usdm.health_samples >= 40 and (.markets.usdm.max_health_silence_seconds | type) == "number" and .markets.usdm.max_health_silence_seconds >= 0 -and .markets.usdm.max_health_silence_seconds <= 90 +and .markets.usdm.max_health_silence_seconds <= 120 and (.markets.usdm.catalog_sha256 | type) == "string" and (.markets.usdm.catalog_sha256 | test("^[a-f0-9]{64}$")) and (.markets.usdm.session_id | type) == "string" diff --git a/deployment/aliyun/test-rust-lob-control-plane.sh b/deployment/aliyun/test-rust-lob-control-plane.sh index 4b71b52d7..f82a895ae 100755 --- a/deployment/aliyun/test-rust-lob-control-plane.sh +++ b/deployment/aliyun/test-rust-lob-control-plane.sh @@ -36,6 +36,7 @@ if grep -Fq -- '--argjson lob_continuity' "$GATE"; then fi grep -Fq 'manifest changed between discovery and readback' "$GATE" grep -Fq 'manifest_sha256:$manifest_sha256' "$GATE" +grep -Fq 'readonly MAX_HEALTH_SILENCE_SECONDS=120' "$GATE" tmp_dir=$(mktemp -d) trap 'rm -rf "$tmp_dir"' EXIT @@ -49,18 +50,26 @@ for current_mono in $(seq 30 30 3600); do read -r last_updated_ns last_advance_mono max_gap sample_increment < <( monday_observe_health_freshness \ "$last_updated_ns" "$last_advance_mono" "$max_gap" \ - "$current_updated_ns" "$current_mono" 90 + "$current_updated_ns" "$current_mono" 120 ) health_sample_increments=$((health_sample_increments + sample_increment)) done -((health_sample_increments == 120 && max_gap <= 90)) || { +((health_sample_increments == 120 && max_gap <= 120)) || { printf 'fresh one-hour health sequence did not pass the monotonic observer\n' >&2 exit 1 } +read -r jitter_updated_ns jitter_advance_mono jitter_max_gap jitter_increment < <( + monday_observe_health_freshness 1 0 0 2 91 120 +) +[[ $jitter_updated_ns == 2 && $jitter_advance_mono == 91 \ + && $jitter_max_gap == 91 && $jitter_increment == 1 ]] || { + printf 'monotonic observer rejected an advancing 91-second jitter sample\n' >&2 + exit 1 +} if monday_observe_health_freshness \ - "$last_updated_ns" "$last_advance_mono" "$max_gap" \ - "$last_updated_ns" "$((last_advance_mono + 91))" 90 >/dev/null; then - printf 'monotonic observer accepted a 91-second health freeze\n' >&2 + "$jitter_updated_ns" "$jitter_advance_mono" "$jitter_max_gap" \ + "$jitter_updated_ns" "$((jitter_advance_mono + 121))" 120 >/dev/null; then + printf 'monotonic observer accepted a 121-second health freeze\n' >&2 exit 1 fi @@ -207,16 +216,31 @@ if jq -e \ exit 1 fi -jq '.markets.usdm.max_health_silence_seconds = 91' \ - "$tmp_dir/gate.json" >"$tmp_dir/stale-health.json" -if jq -e \ - --arg candidate_sha256 "$artifact" \ - --arg deployment_bundle_sha256 "$bundle" \ - --arg deployment_source_revision "$source_revision" \ - -f "$POLICY" "$tmp_dir/stale-health.json" >/dev/null; then - printf 'gate policy accepted a health freshness gap over 90 seconds\n' >&2 - exit 1 -fi +for market in spot usdm; do + jq --arg market "$market" \ + '.markets[$market].max_health_silence_seconds = 91' \ + "$tmp_dir/gate.json" >"$tmp_dir/rotation-jitter-health-$market.json" + jq -e \ + --arg candidate_sha256 "$artifact" \ + --arg deployment_bundle_sha256 "$bundle" \ + --arg deployment_source_revision "$source_revision" \ + -f "$POLICY" "$tmp_dir/rotation-jitter-health-$market.json" >/dev/null || { + printf 'gate policy rejected a 91-second %s rotation jitter gap\n' "$market" >&2 + exit 1 + } + + jq --arg market "$market" \ + '.markets[$market].max_health_silence_seconds = 121' \ + "$tmp_dir/gate.json" >"$tmp_dir/stale-health-$market.json" + if jq -e \ + --arg candidate_sha256 "$artifact" \ + --arg deployment_bundle_sha256 "$bundle" \ + --arg deployment_source_revision "$source_revision" \ + -f "$POLICY" "$tmp_dir/stale-health-$market.json" >/dev/null; then + printf 'gate policy accepted a %s health freshness gap over 120 seconds\n' "$market" >&2 + exit 1 + fi +done jq '.markets.spot.agg_trade_count = 0' \ "$tmp_dir/gate.json" >"$tmp_dir/zero-agg-trades.json"