Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
165 changes: 126 additions & 39 deletions deployment/aliyun/polymarket-raw-ops-shadow-gate.sh
Original file line number Diff line number Diff line change
Expand Up @@ -377,17 +377,28 @@ verify_cutover_target_preflight() {
fi
}

verify_fresh_baseline_health() {
local health=$1 policy=${2:-$LEGACY_HEALTH_POLICY} field timestamp epoch now
fresh_baseline_health_snapshot() {
local health=$1 policy=${2:-$LEGACY_HEALTH_POLICY}
local snapshot field timestamp epoch now
[[ -f $health && ! -L $health ]] || return 1
jq -e -f "$policy" "$health" >/dev/null || return 1
snapshot=$(jq -cS . "$health") || return 1
jq -e -f "$policy" <<<"$snapshot" >/dev/null || return 1
now=$(date -u +%s) || return 1
for field in updated_at last_success_at; do
timestamp=$(jq -er --arg field "$field" \
'.[$field] | select(type == "string" and length > 0)' "$health") || return 1
'.[$field] | select(type == "string" and length > 0)' <<<"$snapshot") || return 1
epoch=$(date -u -d "$timestamp" +%s) || return 1
((epoch <= now && now - epoch <= MAX_HEALTH_SILENCE_SECONDS)) || return 1
done
printf '%s\n' "$snapshot"
}

verify_fresh_baseline_health() {
fresh_baseline_health_snapshot "$@" >/dev/null
}

baseline_health_requires_continuous_freshness() {
[[ $1 == rust_release ]]
}

legacy_health_sample_state() {
Expand Down Expand Up @@ -692,7 +703,8 @@ legacy_invocation_id=$(systemctl show --property=InvocationID --value "$LEGACY_U
|| die 'active legacy collector has no verifiable systemd invocation ID'
verify_baseline_identity \
|| die 'active reference collector identity or restart counter is not exact'
[[ $baseline_mode != rust_release ]] || verify_fresh_baseline_health "$LEGACY_SPOOL/health.json" \
! baseline_health_requires_continuous_freshness "$baseline_mode" \
|| verify_fresh_baseline_health "$LEGACY_SPOOL/health.json" \
|| die 'active Rust collector health is not fresh and fail-closed clean'
verify_cutover_target_preflight "$baseline_mode" "$RUST_ACTIVE_BINARY" \
"$CONTROL_DIR" "${RELEASE_MANIFEST##*/}" secure_control_file \
Expand Down Expand Up @@ -817,6 +829,26 @@ install -m 0644 "$release_control_dir/${SERVICE_TEMPLATE##*/}" \
/etc/systemd/system/polymarket-reference-collector-shadow@.service
systemctl daemon-reload

baseline_health_snapshot=null
baseline_health_started_at=
baseline_health_completion_snapshot=null
baseline_health_completion_updated_at=
baseline_health_start_success_unix=null
baseline_health_cutoff_unix=null
if ! baseline_health_requires_continuous_freshness "$baseline_mode"; then
baseline_health_snapshot=$(fresh_baseline_health_snapshot \
"$LEGACY_SPOOL/health.json" \
"$release_control_dir/${LEGACY_HEALTH_POLICY##*/}") \
|| die 'active legacy collector health is not fresh and fail-closed clean'
baseline_health_started_at=$(jq -er '.updated_at' <<<"$baseline_health_snapshot") \
|| die 'frozen legacy collector health has no updated_at'
baseline_health_start_success_at=$(jq -er '.last_success_at' \
<<<"$baseline_health_snapshot") \
|| die 'frozen legacy collector health has no last_success_at'
baseline_health_start_success_unix=$(date -u -d \
"$baseline_health_start_success_at" +%s) \
|| die 'frozen legacy collector last_success_at is invalid'
fi
started_at_unix=$(date -u +%s)
started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)
start_uptime=$SECONDS
Expand Down Expand Up @@ -855,34 +887,66 @@ while :; do
elapsed=$((now_uptime - start_uptime))
verify_baseline_identity \
|| die 'baseline collector PID, restart count, or effective unit identity changed during gate'
legacy_health="$LEGACY_SPOOL/health.json"
[[ -f $legacy_health && ! -L $legacy_health ]] \
|| die "$baseline_label health is missing"
legacy_health_state=$(legacy_health_sample_state \
"$legacy_health" "$release_control_dir/${LEGACY_HEALTH_POLICY##*/}" \
"$baseline_mode")
legacy_health_result=$(legacy_health_transition \
"$legacy_health_state" "$legacy_api_error_started_at" \
"$now_uptime" "$MAX_HEALTH_SILENCE_SECONDS")
legacy_health_decision=${legacy_health_result%%:*}
legacy_api_error_started_at=${legacy_health_result#*:}
case "$legacy_health_decision" in
advance|wait)
;;
expired)
die "$baseline_label API errors did not recover within the health budget"
;;
*)
die "$baseline_label health is not fail-closed clean during shadow"
;;
esac
current_legacy_health=$(jq -r '.updated_at' "$legacy_health")
if [[ $current_legacy_health != "$last_legacy_health" ]]; then
last_legacy_health=$current_legacy_health
last_legacy_health_change=$now_uptime
if baseline_health_requires_continuous_freshness "$baseline_mode"; then
legacy_health="$LEGACY_SPOOL/health.json"
[[ -f $legacy_health && ! -L $legacy_health ]] \
|| die "$baseline_label health is missing"
legacy_health_state=$(legacy_health_sample_state \
"$legacy_health" "$release_control_dir/${LEGACY_HEALTH_POLICY##*/}" \
"$baseline_mode")
legacy_health_result=$(legacy_health_transition \
"$legacy_health_state" "$legacy_api_error_started_at" \
"$now_uptime" "$MAX_HEALTH_SILENCE_SECONDS")
legacy_health_decision=${legacy_health_result%%:*}
legacy_api_error_started_at=${legacy_health_result#*:}
case "$legacy_health_decision" in
advance|wait)
;;
expired)
die "$baseline_label API errors did not recover within the health budget"
;;
*)
die "$baseline_label health is not fail-closed clean during shadow"
;;
esac
current_legacy_health=$(jq -r '.updated_at' "$legacy_health")
if [[ $current_legacy_health != "$last_legacy_health" ]]; then
last_legacy_health=$current_legacy_health
last_legacy_health_change=$now_uptime
fi
((now_uptime - last_legacy_health_change <= MAX_HEALTH_SILENCE_SECONDS)) \
|| die "$baseline_label health stopped advancing during shadow"
else
legacy_health="$LEGACY_SPOOL/health.json"
[[ -f $legacy_health && ! -L $legacy_health ]] \
|| die "$baseline_label health is missing"
current_legacy_health=$(jq -er '.updated_at' "$legacy_health") \
|| die "$baseline_label health has no updated_at"
if [[ $current_legacy_health != "$baseline_health_started_at" \
&& $current_legacy_health != "$baseline_health_completion_updated_at" ]]; then
baseline_health_completion_snapshot=$(fresh_baseline_health_snapshot \
"$legacy_health" "$release_control_dir/${LEGACY_HEALTH_POLICY##*/}") \
|| die 'post-start legacy collector health is not fresh and fail-closed clean'
current_legacy_health=$(jq -er '.updated_at' \
<<<"$baseline_health_completion_snapshot") \
|| die 'post-start legacy collector health has no updated_at'
[[ $current_legacy_health != "$baseline_health_started_at" ]] \
|| die 'post-start legacy collector health did not advance'
baseline_health_completion_updated_at=$current_legacy_health
legacy_success_at=$(jq -er '.last_success_at' \
<<<"$baseline_health_completion_snapshot") \
|| die 'post-start legacy collector health has no last_success_at'
baseline_health_cutoff_unix=$(date -u -d "$legacy_success_at" +%s) \
|| die 'post-start legacy collector last_success_at is invalid'
((baseline_health_cutoff_unix > baseline_health_start_success_unix)) \
|| die 'post-start legacy collector last_success_at did not advance'
legacy_health_decision='advance'
elif [[ $baseline_health_completion_snapshot != null ]]; then
legacy_health_decision='advance'
else
legacy_health_decision='wait'
fi
fi
((now_uptime - last_legacy_health_change <= MAX_HEALTH_SILENCE_SECONDS)) \
|| die "$baseline_label health stopped advancing during shadow"
shadow_pid=$(systemctl show --property=MainPID --value "$shadow_unit")
[[ $shadow_pid =~ ^[1-9][0-9]*$ ]] || die 'Rust shadow has no MainPID'
[[ $shadow_pid == "$initial_shadow_pid" ]] || die 'Rust shadow MainPID changed during gate'
Expand All @@ -909,20 +973,29 @@ while :; do

rust_success_at=$(jq -er '.last_success_at | select(type == "string" and length > 0)' \
"$health") || die 'Rust health has no last_success_at'
legacy_success_at=$(jq -er '.last_success_at | select(type == "string" and length > 0)' \
"$legacy_health") || die "$baseline_label health has no last_success_at"
rust_success_epoch=$(date -u -d "$rust_success_at" +%s) \
|| die 'Rust last_success_at is invalid'
legacy_success_epoch=$(date -u -d "$legacy_success_at" +%s) \
|| die 'Python last_success_at is invalid'
now_epoch=$(date -u +%s)
((rust_success_epoch <= now_epoch && now_epoch - rust_success_epoch <= MAX_HEALTH_SILENCE_SECONDS)) \
|| die 'Rust last_success_at is stale or from the future'
((legacy_success_epoch <= now_epoch && now_epoch - legacy_success_epoch <= MAX_HEALTH_SILENCE_SECONDS)) \
|| die "$baseline_label last_success_at is stale or from the future"
if [[ $legacy_health_decision == advance ]]; then
common_cutoff=$rust_success_epoch
((legacy_success_epoch < common_cutoff)) && common_cutoff=$legacy_success_epoch
if baseline_health_requires_continuous_freshness "$baseline_mode"; then
Comment thread
proerror77 marked this conversation as resolved.
legacy_success_at=$(jq -er \
'.last_success_at | select(type == "string" and length > 0)' \
"$legacy_health") || die "$baseline_label health has no last_success_at"
legacy_success_epoch=$(date -u -d "$legacy_success_at" +%s) \
|| die "$baseline_label last_success_at is invalid"
((legacy_success_epoch <= now_epoch \
&& now_epoch - legacy_success_epoch <= MAX_HEALTH_SILENCE_SECONDS)) \
|| die "$baseline_label last_success_at is stale or from the future"
((legacy_success_epoch < common_cutoff)) && common_cutoff=$legacy_success_epoch
else
[[ $baseline_health_cutoff_unix =~ ^[1-9][0-9]*$ ]] \
|| die 'no post-start legacy collector completion cutoff was observed'
((baseline_health_cutoff_unix < common_cutoff)) \
&& common_cutoff=$baseline_health_cutoff_unix
fi
if [[ $test_only == false ]]; then
common_cutoff=$((common_cutoff - PARITY_CUTOFF_LAG_SECONDS))
fi
Expand All @@ -933,6 +1006,12 @@ while :; do
fi
fi

if ((elapsed >= gate_seconds)) \
&& ! baseline_health_requires_continuous_freshness "$baseline_mode" \
&& [[ $legacy_health_decision != advance ]]; then
die 'legacy collector did not complete a clean post-start cycle during the gate'
fi

if ((elapsed >= gate_seconds)) && [[ -n $common_cutoff ]] \
&& [[ $legacy_health_decision == advance ]]; then
if valid_parity_window "$parity_window_started_at" "$common_cutoff"; then
Expand Down Expand Up @@ -1160,6 +1239,10 @@ jq \
--argjson parity_window_started_at_unix "$parity_window_started_at" \
--argjson parity_window_ended_at_unix "$common_cutoff" \
--argjson production_eligible "$production_eligible" \
--argjson baseline_health_snapshot "$baseline_health_snapshot" \
--argjson baseline_health_completion_snapshot "$baseline_health_completion_snapshot" \
--argjson baseline_health_start_success_unix "$baseline_health_start_success_unix" \
--argjson baseline_health_cutoff_unix "$baseline_health_cutoff_unix" \
--argjson uploaded_segments "$uploaded_segments" \
--argjson canonical_uploaded_segments "$canonical_uploaded_segments" \
--argjson market_uploaded_segments "$market_uploaded_segments" \
Expand All @@ -1180,6 +1263,10 @@ jq \
parity_window_started_at_unix:$parity_window_started_at_unix,
parity_window_ended_at_unix:$parity_window_ended_at_unix,
production_eligible:$production_eligible,
baseline_health_snapshot:$baseline_health_snapshot,
baseline_health_completion_snapshot:$baseline_health_completion_snapshot,
baseline_health_start_success_unix:$baseline_health_start_success_unix,
baseline_health_cutoff_unix:$baseline_health_cutoff_unix,
legacy_runtime:({exec_start:$legacy_exec,cmdline:$legacy_cmdline,
cmdline_sha256:$legacy_cmdline_sha256,
fragment_path:$legacy_fragment_path,drop_in_paths:$legacy_drop_in_paths,
Expand Down
29 changes: 29 additions & 0 deletions deployment/aliyun/polymarket-shadow-gate-policy.jq
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,17 @@ def runtime_identity($exec; $digest):
and (.invocation_id | type == "string" and test("^[a-f0-9]{32}$"));
def nonnegative_sub($left; $right):
if $left < $right then 0 else ($left - $right) end;
def legacy_health_snapshot:
(.updated_at | type == "string" and length > 0)
and (.last_success_at | type == "string" and length > 0)
and (.target_markets | positive_integer)
and .api_errors == []
and .malformed_trade_rows == 0
and .truncated_trade_markets == []
and .stale_trade_markets == []
and .stale_settlement_markets == []
and (.overdue_unresolved_markets
| type == "array" and all(.[]; type == "string" and length > 0));

.schema == "monday.polymarket_shadow_gate.v1"
and (.candidate_sha256 | sha256)
Expand All @@ -27,6 +38,20 @@ and .passed == true
and (
(
.baseline_mode == "legacy_python"
and (.baseline_health_snapshot | legacy_health_snapshot)
and (.baseline_health_completion_snapshot | legacy_health_snapshot)
and (.baseline_health_completion_snapshot.updated_at
!= .baseline_health_snapshot.updated_at)
and (.baseline_health_start_success_unix | positive_integer)
and ((.baseline_health_snapshot.last_success_at | fromdateiso8601?)
== .baseline_health_start_success_unix)
and (.baseline_health_completion_snapshot.last_success_at
!= .baseline_health_snapshot.last_success_at)
and (.baseline_health_cutoff_unix | positive_integer)
and ((.baseline_health_completion_snapshot.last_success_at | fromdateiso8601?)
== .baseline_health_cutoff_unix)
and .baseline_health_cutoff_unix > .baseline_health_start_success_unix
and .parity_window_ended_at_unix <= .baseline_health_cutoff_unix
and (.legacy_runtime |
runtime_identity("/usr/bin/python3 /opt/monday/bin/polymarket_reference_collector.py";
"dffeb118d105e9312898460249f514eb982c20433cd20840ffb2107c64bbca4a")
Expand All @@ -35,6 +60,10 @@ and (
or
(
.baseline_mode == "rust_release"
and .baseline_health_snapshot == null
and .baseline_health_completion_snapshot == null
and .baseline_health_start_success_unix == null
and .baseline_health_cutoff_unix == null
and (.legacy_runtime |
runtime_identity("/opt/monday/bin/polymarket-raw-ops collect-reference";
"7b06db4beb374f013a090e023289f8b026f39c324ee527f194b706656f6a1f94"))
Expand Down
67 changes: 67 additions & 0 deletions deployment/aliyun/test-polymarket-raw-ops-control-plane.sh
Original file line number Diff line number Diff line change
Expand Up @@ -1395,6 +1395,22 @@ jq \
completed_at:"2026-07-15T00:00:00Z",
shadow_run_id:"run-1",
production_eligible:true,
baseline_health_snapshot:{
updated_at:"1970-01-01T00:01:40Z",
last_success_at:"1970-01-01T00:01:40Z",
target_markets:14,api_errors:[],malformed_trade_rows:0,
truncated_trade_markets:[],stale_trade_markets:[],
stale_settlement_markets:[],overdue_unresolved_markets:[]
},
baseline_health_completion_snapshot:{
updated_at:"1970-01-01T00:16:40Z",
last_success_at:"1970-01-01T00:16:40Z",
target_markets:14,api_errors:[],malformed_trade_rows:0,
truncated_trade_markets:[],stale_trade_markets:[],
stale_settlement_markets:[],overdue_unresolved_markets:[]
},
baseline_health_start_success_unix:100,
baseline_health_cutoff_unix:1000,
legacy_runtime:{
exec_start:"/usr/bin/python3 /opt/monday/bin/polymarket_reference_collector.py",
cmdline:"/usr/bin/python3 /opt/monday/bin/polymarket_reference_collector.py",
Expand Down Expand Up @@ -1426,6 +1442,37 @@ jq \
})
} | .passed = true' "$parity" >"$tmp_dir/gate.json"
jq -e -f "$POLICY" "$tmp_dir/gate.json" >/dev/null
jq 'del(.baseline_health_snapshot)' "$tmp_dir/gate.json" \
>"$tmp_dir/missing-baseline-health-snapshot.json"
if jq -e -f "$POLICY" "$tmp_dir/missing-baseline-health-snapshot.json" >/dev/null; then
printf 'gate policy accepted legacy evidence without its frozen health snapshot\n' >&2
exit 1
fi
jq 'del(.baseline_health_completion_snapshot)' "$tmp_dir/gate.json" \
>"$tmp_dir/missing-baseline-health-completion.json"
if jq -e -f "$POLICY" "$tmp_dir/missing-baseline-health-completion.json" >/dev/null; then
printf 'gate policy accepted legacy evidence without a post-start clean cycle\n' >&2
exit 1
fi
jq '.baseline_health_completion_snapshot.last_success_at =
.baseline_health_snapshot.last_success_at' "$tmp_dir/gate.json" \
>"$tmp_dir/stale-baseline-health-completion.json"
if jq -e -f "$POLICY" "$tmp_dir/stale-baseline-health-completion.json" >/dev/null; then
printf 'gate policy accepted updated_at progress without a newer completed legacy cycle\n' >&2
exit 1
fi
jq '.baseline_health_start_success_unix = 99' "$tmp_dir/gate.json" \
>"$tmp_dir/unbound-baseline-health-start.json"
if jq -e -f "$POLICY" "$tmp_dir/unbound-baseline-health-start.json" >/dev/null; then
printf 'gate policy accepted an unbound legacy startup success epoch\n' >&2
exit 1
fi
jq '.baseline_health_cutoff_unix = 1001' "$tmp_dir/gate.json" \
>"$tmp_dir/unbound-baseline-health-cutoff.json"
if jq -e -f "$POLICY" "$tmp_dir/unbound-baseline-health-cutoff.json" >/dev/null; then
printf 'gate policy accepted a cutoff after legacy completed-cycle evidence\n' >&2
exit 1
fi
jq '.shadow_runtime.memory_events.start.high = 1
| .shadow_runtime.memory_events.end.high = 1' \
"$tmp_dir/gate.json" >"$tmp_dir/nonzero-memory-high-baseline.json"
Expand Down Expand Up @@ -1680,6 +1727,10 @@ if jq -e -f "$POLICY" "$tmp_dir/shadow-once-cmdline.json" >/dev/null; then
fi
baseline_sha=$(printf '9%.0s' {1..64})
jq --arg baseline "$baseline_sha" '.baseline_mode = "rust_release"
| .baseline_health_snapshot = null
| .baseline_health_completion_snapshot = null
| .baseline_health_start_success_unix = null
| .baseline_health_cutoff_unix = null
| .legacy_runtime += {
exec_start:"/opt/monday/bin/polymarket-raw-ops collect-reference",
cmdline:"/opt/monday/bin/polymarket-raw-ops collect-reference",
Expand Down Expand Up @@ -1750,11 +1801,27 @@ for mutation in \
done
legacy_health_classifier="$tmp_dir/legacy-health-classifier.sh"
sed -n \
-e '/^baseline_health_requires_continuous_freshness()/,/^}/p' \
-e '/^legacy_health_sample_state()/,/^}/p' \
-e '/^legacy_health_transition()/,/^}/p' "$GATE" \
>"$legacy_health_classifier"
# shellcheck source=/dev/null
source "$legacy_health_classifier"
if baseline_health_requires_continuous_freshness legacy_python; then
printf 'legacy collector health incorrectly requires continuous 240-second freshness\n' >&2
exit 1
fi
baseline_health_requires_continuous_freshness rust_release
daemon_reload_line=$(grep -nF 'systemctl daemon-reload' "$GATE" | tail -1 | cut -d: -f1)
snapshot_line=$(grep -nF 'baseline_health_snapshot=$(fresh_baseline_health_snapshot' \
"$GATE" | cut -d: -f1)
gate_start_line=$(grep -nF 'started_at_unix=$(date -u +%s)' "$GATE" | cut -d: -f1)
shadow_start_line=$(grep -nF 'systemctl start "$shadow_unit"' "$GATE" | cut -d: -f1)
if ! ((daemon_reload_line < snapshot_line && snapshot_line < gate_start_line \
&& gate_start_line < shadow_start_line)); then
printf 'legacy health is not frozen immediately at the shadow Gate start boundary\n' >&2
exit 1
fi
[[ $(legacy_health_sample_state \
"$tmp_dir/legacy-health.json" "$LEGACY_HEALTH_POLICY" legacy_python) == clean ]]
jq '.api_errors = ["trades condition-1: The read operation timed out"]' \
Expand Down
Loading