From d32c7832278197d772b6ee0af719d659acacf7ca Mon Sep 17 00:00:00 2001 From: Sonic Shih Date: Thu, 30 Jul 2026 08:36:35 +0800 Subject: [PATCH 1/2] fix(polymarket): permit expedited shadow gate --- .../aliyun/polymarket-raw-ops-shadow-gate.sh | 12 +++-- .../aliyun/polymarket-shadow-gate-policy.jq | 53 +++++++++++-------- .../test-polymarket-raw-ops-control-plane.sh | 40 +++++++++++--- 3 files changed, 75 insertions(+), 30 deletions(-) diff --git a/deployment/aliyun/polymarket-raw-ops-shadow-gate.sh b/deployment/aliyun/polymarket-raw-ops-shadow-gate.sh index 7ff724606..6b053a581 100755 --- a/deployment/aliyun/polymarket-raw-ops-shadow-gate.sh +++ b/deployment/aliyun/polymarket-raw-ops-shadow-gate.sh @@ -5,7 +5,7 @@ umask 027 export LC_ALL=C export TZ=UTC -readonly REQUIRED_DURATION_SECONDS=3600 +readonly REQUIRED_DURATION_SECONDS=900 # The verifier subtracts a 600-second trade maturity lag and requires a # non-empty event window, so the deployment tail must be strictly longer. readonly PARITY_TAIL_SECONDS=601 @@ -16,6 +16,7 @@ readonly HEALTH_SETTLE_SECONDS=$((MAX_ACCEPTED_CYCLE_SECONDS + INITIAL_HEALTH_GR readonly MAX_HEALTH_SILENCE_SECONDS=240 # Legacy full-catalog cycles observed 35–58 minutes; bound admission at 65. readonly LEGACY_HEALTH_START_WAIT_SECONDS=3900 +readonly LEGACY_HEALTH_COMPLETION_REQUIRED=false # One real-segment upload can spend one 300-second compression timeout, fifteen # candidate OSS operations, four independent Gate readbacks, and 300 seconds # of local processing reserve. @@ -73,7 +74,7 @@ usage() { printf '%s\n' \ 'Usage: polymarket-raw-ops-shadow-gate.sh ' \ '' \ - 'A production-eligible gate observes for 3600 seconds plus a 601-second current-hour parity tail.' + 'A production-eligible gate observes for 900 seconds plus a 601-second current-hour parity tail.' } valid_parity_window() { @@ -1463,6 +1464,8 @@ while :; do fi ((now_uptime - last_legacy_health_change <= MAX_HEALTH_SILENCE_SECONDS)) \ || die "$baseline_label health stopped advancing during shadow" + elif [[ $LEGACY_HEALTH_COMPLETION_REQUIRED == false ]]; then + legacy_health_decision='advance' else legacy_health="$LEGACY_SPOOL/health.json" [[ -f $legacy_health && ! -L $legacy_health ]] \ @@ -1554,7 +1557,7 @@ while :; do && now_epoch - legacy_success_epoch <= MAX_HEALTH_SILENCE_SECONDS)) \ || die "$baseline_label last_success_at is stale or from the future" ((legacy_success_epoch < common_cutoff)) && common_cutoff=$legacy_success_epoch - else + elif [[ $LEGACY_HEALTH_COMPLETION_REQUIRED == true ]]; then [[ $baseline_health_cutoff_unix =~ ^[1-9][0-9]*$ ]] \ || die 'no post-start legacy collector completion cutoff was observed' ((baseline_health_cutoff_unix < common_cutoff)) \ @@ -1572,6 +1575,7 @@ while :; do if ((elapsed >= gate_seconds)) \ && ! baseline_health_requires_continuous_freshness "$baseline_mode" \ + && [[ $LEGACY_HEALTH_COMPLETION_REQUIRED == true ]] \ && [[ $legacy_health_decision != advance ]]; then die 'legacy collector did not complete a clean post-start cycle during the gate' fi @@ -1744,6 +1748,7 @@ jq \ --argjson parity_window_started_at_unix "$parity_window_started_at" \ --argjson parity_window_ended_at_unix "$common_cutoff" \ --argjson production_eligible "$production_eligible" \ + --argjson baseline_health_completion_required "$LEGACY_HEALTH_COMPLETION_REQUIRED" \ --argjson baseline_health_snapshot "$baseline_health_snapshot" \ --argjson baseline_health_completion_snapshot "$baseline_health_completion_snapshot" \ --argjson baseline_health_start_success_unix "$baseline_health_start_success_unix" \ @@ -1777,6 +1782,7 @@ jq \ parity_window_started_at_unix:$parity_window_started_at_unix, parity_window_ended_at_unix:$parity_window_ended_at_unix, production_eligible:$production_eligible, + baseline_health_completion_required:$baseline_health_completion_required, baseline_health_snapshot:$baseline_health_snapshot, baseline_health_completion_snapshot:$baseline_health_completion_snapshot, baseline_health_start_success_unix:$baseline_health_start_success_unix, diff --git a/deployment/aliyun/polymarket-shadow-gate-policy.jq b/deployment/aliyun/polymarket-shadow-gate-policy.jq index 15f7b237a..03f7dc779 100644 --- a/deployment/aliyun/polymarket-shadow-gate-policy.jq +++ b/deployment/aliyun/polymarket-shadow-gate-policy.jq @@ -111,7 +111,7 @@ and ((.real_market_preflight.started_at | utc_iso8601_unix) <= (.real_market_preflight.completed_at | utc_iso8601_unix)) and ((.real_market_preflight.completed_at | utc_iso8601_unix) <= (.started_at | utc_iso8601_unix)) -and (.duration_seconds | positive_integer and . >= 4201) +and (.duration_seconds | positive_integer and . >= 1501) and (.started_at | utc_iso8601_unix | type == "number") and (.parity_window_started_at_unix | positive_integer) and (.parity_window_ended_at_unix | positive_integer) @@ -123,9 +123,6 @@ and ( ( .baseline_mode == "legacy_python" and (.baseline_health_snapshot | legacy_health_snapshot) - and (.baseline_health_completion_snapshot | legacy_health_snapshot) - and (.baseline_health_completion_snapshot.updated_at - != .baseline_health_snapshot.updated_at) and (.baseline_health_start_success_unix | positive_integer) and ((.baseline_health_snapshot.last_success_at | utc_iso8601_unix) == .baseline_health_start_success_unix) @@ -137,23 +134,36 @@ and ( <= (.started_at | utc_iso8601_unix) and ((.started_at | utc_iso8601_unix) - .baseline_health_start_written_at_unix <= 240) - and (.baseline_health_completion_snapshot.last_success_at - != .baseline_health_snapshot.last_success_at) - and (.baseline_health_cutoff_unix | positive_integer) - and ((.baseline_health_completion_snapshot.last_success_at | utc_iso8601_unix) - == .baseline_health_cutoff_unix) - and .baseline_health_cutoff_unix > .baseline_health_start_success_unix - and (.baseline_health_completion_written_at_unix | positive_integer) - and (.baseline_health_completion_file_identity | file_identity) - and .baseline_health_completion_file_identity - != .baseline_health_start_file_identity - and .baseline_health_cutoff_unix - <= .baseline_health_completion_written_at_unix - and .baseline_health_completion_written_at_unix - >= (.started_at | utc_iso8601_unix) - and .baseline_health_completion_written_at_unix - <= (.completed_at | utc_iso8601_unix) - and .parity_window_ended_at_unix <= .baseline_health_cutoff_unix + and (.baseline_health_completion_required | type == "boolean") + and ( + if .baseline_health_completion_required then + (.baseline_health_completion_snapshot | legacy_health_snapshot) + and (.baseline_health_completion_snapshot.updated_at + != .baseline_health_snapshot.updated_at) + and (.baseline_health_completion_snapshot.last_success_at + != .baseline_health_snapshot.last_success_at) + and (.baseline_health_cutoff_unix | positive_integer) + and ((.baseline_health_completion_snapshot.last_success_at | utc_iso8601_unix) + == .baseline_health_cutoff_unix) + and .baseline_health_cutoff_unix > .baseline_health_start_success_unix + and (.baseline_health_completion_written_at_unix | positive_integer) + and (.baseline_health_completion_file_identity | file_identity) + and .baseline_health_completion_file_identity + != .baseline_health_start_file_identity + and .baseline_health_cutoff_unix + <= .baseline_health_completion_written_at_unix + and .baseline_health_completion_written_at_unix + >= (.started_at | utc_iso8601_unix) + and .baseline_health_completion_written_at_unix + <= (.completed_at | utc_iso8601_unix) + and .parity_window_ended_at_unix <= .baseline_health_cutoff_unix + else + .baseline_health_completion_snapshot == null + and .baseline_health_cutoff_unix == null + and .baseline_health_completion_written_at_unix == null + and .baseline_health_completion_file_identity == null + end + ) and (.legacy_runtime | runtime_identity("/usr/bin/python3 /opt/monday/bin/polymarket_reference_collector.py"; "dffeb118d105e9312898460249f514eb982c20433cd20840ffb2107c64bbca4a") @@ -162,6 +172,7 @@ and ( or ( .baseline_mode == "rust_release" + and .baseline_health_completion_required == false and .baseline_health_snapshot == null and .baseline_health_completion_snapshot == null and .baseline_health_start_success_unix == null diff --git a/deployment/aliyun/test-polymarket-raw-ops-control-plane.sh b/deployment/aliyun/test-polymarket-raw-ops-control-plane.sh index de6a3632f..28067ba4d 100755 --- a/deployment/aliyun/test-polymarket-raw-ops-control-plane.sh +++ b/deployment/aliyun/test-polymarket-raw-ops-control-plane.sh @@ -2393,13 +2393,14 @@ jq \ upload_summary:{uploaded_segments:1,canonical_uploaded_segments:1, pending_segments:0,failed_segments:[],last_error:null} }, - duration_seconds:4201, + duration_seconds:1501, started_at:"1970-01-01T00:02:00Z", parity_window_started_at_unix:100, parity_window_ended_at_unix:1000, completed_at:"1970-01-01T01:12:01Z", shadow_run_id:"run-1", production_eligible:true, + baseline_health_completion_required:true, baseline_health_snapshot:{ updated_at:"1970-01-01T00:01:40.123456Z", last_success_at:"1970-01-01T00:01:40.123456Z", @@ -2451,6 +2452,29 @@ jq \ }) } | .passed = true' "$parity" >"$tmp_dir/gate.json" jq -e -f "$POLICY" "$tmp_dir/gate.json" >/dev/null +jq '.baseline_health_completion_required = false + | .baseline_health_completion_snapshot = null + | .baseline_health_cutoff_unix = null + | .baseline_health_completion_written_at_unix = null + | .baseline_health_completion_file_identity = null' \ + "$tmp_dir/gate.json" >"$tmp_dir/expedited-legacy-gate.json" +jq -e -f "$POLICY" "$tmp_dir/expedited-legacy-gate.json" >/dev/null || { + printf 'gate policy rejected approved expedited legacy baseline evidence\n' >&2 + exit 1 +} +for mutation in \ + 'del(.baseline_health_snapshot)' \ + '.baseline_health_completion_snapshot = .baseline_health_snapshot' \ + '.baseline_health_cutoff_unix = 1000' \ + '.baseline_health_completion_written_at_unix = 1301' \ + '.baseline_health_completion_file_identity = "1:11"'; do + jq "$mutation" "$tmp_dir/expedited-legacy-gate.json" \ + >"$tmp_dir/forged-expedited-legacy-gate.json" + if jq -e -f "$POLICY" "$tmp_dir/forged-expedited-legacy-gate.json" >/dev/null; then + printf 'gate policy accepted forged post-start legacy completion in expedited evidence\n' >&2 + exit 1 + fi +done jq 'del(.real_market_preflight)' "$tmp_dir/gate.json" \ >"$tmp_dir/missing-real-market-preflight.json" if jq -e -f "$POLICY" "$tmp_dir/missing-real-market-preflight.json" >/dev/null; then @@ -2758,9 +2782,9 @@ if jq -e -f "$POLICY" "$tmp_dir/unbound-settlement-end.json" >/dev/null; then printf 'gate policy accepted an unbound settlement end window\n' >&2 exit 1 fi -jq '.duration_seconds = 4200' "$tmp_dir/gate.json" >"$tmp_dir/short.json" +jq '.duration_seconds = 1500' "$tmp_dir/gate.json" >"$tmp_dir/short.json" if jq -e -f "$POLICY" "$tmp_dir/short.json" >/dev/null; then - printf 'gate policy accepted a shadow shorter than one hour plus its maturity tail\n' >&2 + printf 'gate policy accepted a shadow shorter than 15 minutes plus its maturity tail\n' >&2 exit 1 fi jq '.production_eligible = false' "$tmp_dir/gate.json" >"$tmp_dir/test-only.json" @@ -2918,6 +2942,7 @@ if jq -e -f "$POLICY" "$tmp_dir/shadow-once-cmdline.json" >/dev/null; then fi baseline_sha=$(printf '9%.0s' {1..64}) jq --arg baseline "$baseline_sha" '.baseline_mode = "rust_release" + | .baseline_health_completion_required = false | .baseline_health_snapshot = null | .baseline_health_completion_snapshot = null | .baseline_health_start_success_unix = null @@ -3068,12 +3093,14 @@ grep -Fq '[[ $zstd_timeout_seconds == 300 && $oss_copy_timeout_seconds == 300 ]] printf 'Gate runtime rejection does not report remaining and required seconds\n' >&2 exit 1 } - monotonic_uptime_seconds() { printf '7200\n'; } + unreserved_remaining=$((required - LEGACY_RUNTIME_RESERVE_SECONDS)) + unreserved_uptime=$((LEGACY_RUNTIME_MAX_SECONDS - unreserved_remaining + 1)) + monotonic_uptime_seconds() { printf '%s\n' "$unreserved_uptime"; } if observation=$(legacy_runtime_budget_observation "$required"); then printf 'Gate admitted the unreserved exact runtime boundary\n' >&2 exit 1 fi - [[ $observation == "remaining=14401 required=$required" ]] || { + [[ $observation == "remaining=$unreserved_remaining required=$required" ]] || { printf 'Gate reserve-boundary evidence is not exact\n' >&2 exit 1 } @@ -3447,8 +3474,9 @@ for mutation in \ fi done -grep -Fq 'readonly REQUIRED_DURATION_SECONDS=3600' "$GATE" +grep -Fq 'readonly REQUIRED_DURATION_SECONDS=900' "$GATE" grep -Fq 'readonly PARITY_TAIL_SECONDS=601' "$GATE" +grep -Fq 'readonly LEGACY_HEALTH_COMPLETION_REQUIRED=false' "$GATE" grep -Fq 'readonly SETTLEMENT_EVENT_LOOKBACK_SECONDS=900' "$GATE" grep -Fq 'bounded_parity_window_start' "$GATE" grep -Fq 'readonly MAX_ACCEPTED_CYCLE_SECONDS=180' "$GATE" From f7b8c657dc29c03ace23f4da215c1bae696817ad Mon Sep 17 00:00:00 2001 From: Sonic Shih Date: Thu, 30 Jul 2026 11:37:59 +0800 Subject: [PATCH 2/2] fix(polymarket): finalize expedited gate tape --- .../aliyun/polymarket-raw-ops-shadow-gate.sh | 16 +++++++- .../test-polymarket-raw-ops-control-plane.sh | 40 +++++++++++++++++-- 2 files changed, 51 insertions(+), 5 deletions(-) diff --git a/deployment/aliyun/polymarket-raw-ops-shadow-gate.sh b/deployment/aliyun/polymarket-raw-ops-shadow-gate.sh index 6b053a581..b63c8d030 100755 --- a/deployment/aliyun/polymarket-raw-ops-shadow-gate.sh +++ b/deployment/aliyun/polymarket-raw-ops-shadow-gate.sh @@ -153,6 +153,15 @@ valid_absolute_path() { && $path != */. && $path != */.. ]] } +valid_finalized_reference_tape_path() { + local path=$1 spool_dir=$2 name + valid_absolute_path "$path" || return 1 + [[ ${path%/*} == "$spool_dir" ]] || return 1 + name=${path##*/} + [[ $name =~ ^market-updates\.[0-9]{8}T[0-9]{12}\.ndjson$ \ + && -f $path && ! -L $path && $(readlink -f -- "$path") == "$path" ]] +} + secure_root_chain() { local path=$1 remainder component current= valid_absolute_path "$path" || return 1 @@ -939,7 +948,7 @@ run_budgeted_real_market_preflight() { } legacy_runtime_budget_required=$((REAL_MARKET_PREFLIGHT_BUDGET_SECONDS \ + LEGACY_HEALTH_START_WAIT_SECONDS + gate_seconds \ - + LEGACY_RUNTIME_RESERVE_SECONDS)) + + PARITY_CUTOFF_LAG_SECONDS + LEGACY_RUNTIME_RESERVE_SECONDS)) if observation=$(legacy_runtime_budget_observation \ "$legacy_runtime_budget_required"); then : @@ -1661,6 +1670,11 @@ verify_no_restart_after_cursor "$shadow_unit" "$shadow_stop_cursor" "$shadow_inv stopped_shadow_restarts=$(systemctl show --property=NRestarts --value "$shadow_unit") [[ $stopped_shadow_restarts == 0 ]] \ || die 'Rust shadow restarted between final verification and stop' +finalized_reference_tape=$(runuser -u hftcollector -- env HOME=/var/lib/hft-collector \ + "$release_binary" finalize-reference-tape --spool-dir "$shadow_spool") \ + || die 'could not finalize the stopped Rust shadow tape' +valid_finalized_reference_tape_path "$finalized_reference_tape" "$shadow_spool" \ + || die 'Rust shadow finalizer returned an invalid closed tape path' parity_json="$evidence_dir/parity.json" "$release_binary" verify-shadow-parity \ diff --git a/deployment/aliyun/test-polymarket-raw-ops-control-plane.sh b/deployment/aliyun/test-polymarket-raw-ops-control-plane.sh index 28067ba4d..63e55f23f 100755 --- a/deployment/aliyun/test-polymarket-raw-ops-control-plane.sh +++ b/deployment/aliyun/test-polymarket-raw-ops-control-plane.sh @@ -3040,6 +3040,7 @@ sed -n \ -e '/^readonly LEGACY_HEALTH_START_WAIT_SECONDS=/p' \ -e '/^readonly LEGACY_RUNTIME_MAX_SECONDS=/p' \ -e '/^readonly LEGACY_RUNTIME_RESERVE_SECONDS=/p' \ + -e '/^readonly PARITY_CUTOFF_LAG_SECONDS=/p' \ -e '/^readonly LEGACY_UNIT=/p' \ -e '/^monotonic_uptime_seconds() {$/,/^}$/p' \ -e '/^legacy_runtime_budget_observation() {$/,/^}$/p' \ @@ -3069,7 +3070,8 @@ grep -Fq '[[ $zstd_timeout_seconds == 300 && $oss_copy_timeout_seconds == 300 ]] source "$legacy_runtime_budget_contract" [[ $REAL_MARKET_PREFLIGHT_BUDGET_SECONDS -eq 6300 \ && $LEGACY_RUNTIME_MAX_SECONDS -eq 21600 \ - && $LEGACY_RUNTIME_RESERVE_SECONDS -eq 60 ]] || { + && $LEGACY_RUNTIME_RESERVE_SECONDS -eq 60 \ + && $PARITY_CUTOFF_LAG_SECONDS -eq 60 ]] || { printf 'Gate runtime budget does not bind the reviewed preflight and unit limits\n' >&2 exit 1 } @@ -3083,7 +3085,7 @@ grep -Fq '[[ $zstd_timeout_seconds == 300 && $oss_copy_timeout_seconds == 300 ]] monotonic_uptime_seconds() { printf '20906\n'; } required=$((REAL_MARKET_PREFLIGHT_BUDGET_SECONDS \ + LEGACY_HEALTH_START_WAIT_SECONDS + MINIMUM_GATE_SECONDS \ - + LEGACY_RUNTIME_RESERVE_SECONDS)) + + PARITY_CUTOFF_LAG_SECONDS + LEGACY_RUNTIME_RESERVE_SECONDS)) if observation=$(legacy_runtime_budget_observation "$required"); then printf 'Gate accepted 695 seconds of remaining runtime for a %s-second gate\n' \ "$required" >&2 @@ -3606,6 +3608,10 @@ final_thaw_line=$(grep -n '^systemctl thaw "$shadow_unit"' "$GATE" \ thawed_state_line=$(grep -n '^shadow_thawed_state=.*FreezerState' "$GATE" \ | cut -d: -f1 || true) final_stop_line=$(grep -n '^systemctl stop "$shadow_unit"$' "$GATE" | tail -1 | cut -d: -f1) +finalize_line=$(grep -n '"$release_binary" finalize-reference-tape' "$GATE" \ + | tail -1 | cut -d: -f1 || true) +parity_line=$(grep -n '"$release_binary" verify-shadow-parity' "$GATE" \ + | tail -1 | cut -d: -f1) [[ $freeze_line =~ ^[1-9][0-9]*$ \ && $freezer_state_line =~ ^[1-9][0-9]*$ \ && $final_memory_line =~ ^[1-9][0-9]*$ \ @@ -3613,17 +3619,43 @@ final_stop_line=$(grep -n '^systemctl stop "$shadow_unit"$' "$GATE" | tail -1 | && $final_thaw_line =~ ^[1-9][0-9]*$ \ && $thawed_state_line =~ ^[1-9][0-9]*$ \ && $final_stop_line =~ ^[1-9][0-9]*$ \ + && $finalize_line =~ ^[1-9][0-9]*$ \ + && $parity_line =~ ^[1-9][0-9]*$ \ && $freeze_line -lt $freezer_state_line \ && $freezer_state_line -lt $final_memory_line \ && $final_memory_line -lt $kill_line \ && $kill_line -lt $final_thaw_line \ && $final_thaw_line -lt $thawed_state_line \ && $thawed_state_line -lt $final_stop_line \ - && $kill_line -lt $final_stop_line ]] || { - printf 'shadow final freeze/snapshot/kill/thaw/stop sequence is unsafe\n' >&2 + && $kill_line -lt $final_stop_line \ + && $final_stop_line -lt $finalize_line \ + && $finalize_line -lt $parity_line ]] || { + printf 'shadow final stop/finalize/parity sequence is unsafe\n' >&2 exit 1 } grep -Fq '[[ $shadow_thawed_state == running ]]' "$GATE" +grep -Fq 'runuser -u hftcollector -- env HOME=/var/lib/hft-collector' "$GATE" +finalizer_path_contract="$tmp_dir/finalizer-path-contract.sh" +sed -n \ + -e '/^valid_absolute_path() {$/,/^}$/p' \ + -e '/^valid_finalized_reference_tape_path() {$/,/^}$/p' "$GATE" \ + >"$finalizer_path_contract" +# shellcheck disable=SC1090 +source "$finalizer_path_contract" +finalizer_spool="$tmp_dir/finalizer-spool" +mkdir -p "$finalizer_spool/market-updates.bad" +direct_finalized="$finalizer_spool/market-updates.20260730T120000000000.ndjson" +nested_finalized="$finalizer_spool/market-updates.bad/market-updates.20260730T120000000000.ndjson" +: >"$direct_finalized" +: >"$nested_finalized" +valid_finalized_reference_tape_path "$direct_finalized" "$finalizer_spool" || { + printf 'Gate rejected a direct finalized reference tape\n' >&2 + exit 1 +} +if valid_finalized_reference_tape_path "$nested_finalized" "$finalizer_spool"; then + printf 'Gate accepted a nested finalized reference tape\n' >&2 + exit 1 +fi validator_functions="$tmp_dir/control-group-validator.sh" sed -n '/^valid_absolute_path() {$/,/^}$/p' "$GATE" >"$validator_functions"