From f1f0cdea5d0c788ba4b773eb5c4a2923a5e9fa60 Mon Sep 17 00:00:00 2001 From: proerror Date: Wed, 15 Jul 2026 18:40:02 +0800 Subject: [PATCH 1/2] feat(collector): replace Polymarket raw ops with Rust --- .github/workflows/acr-publish.yml | 12 +- .github/workflows/ci.yml | 1 + deployment/aliyun/README.md | 123 +- .../aliyun/polymarket-legacy-health-policy.jq | 9 + .../polymarket-market-tape-upload.service | 3 +- .../aliyun/polymarket-raw-ops-cutover.sh | 722 +++++ .../aliyun/polymarket-raw-ops-shadow-gate.sh | 640 ++++ ...market-reference-collector-shadow@.service | 32 + .../polymarket-reference-collector.service | 2 +- .../polymarket-reference-upload.service | 3 +- .../aliyun/polymarket-rust-health-policy.jq | 13 + .../aliyun/polymarket-shadow-gate-policy.jq | 64 + .../aliyun/polymarket_market_tape_upload.py | 671 ----- .../polymarket_reference_canonicalize.py | 86 - .../aliyun/polymarket_reference_collector.py | 813 ----- .../test-polymarket-raw-ops-control-plane.sh | 458 +++ .../test_polymarket_market_tape_upload.py | 319 -- .../test_polymarket_reference_canonicalize.py | 59 - .../test_polymarket_reference_collector.py | 388 --- products/ploy/tasks/todo.md | 1 + .../tests/workspace_runtime_retirement.rs | 35 +- .../docker/Dockerfile.binance-lob-archiver | 6 +- rust_hft/tools/collector/Cargo.toml | 4 + .../collector/src/bin/polymarket-raw-ops.rs | 217 ++ rust_hft/tools/collector/src/lib.rs | 3 + rust_hft/tools/collector/src/lob_archiver.rs | 67 +- .../tools/collector/src/polymarket_parity.rs | 924 ++++++ .../tools/collector/src/polymarket_raw.rs | 2384 +++++++++++++++ .../tools/collector/src/polymarket_upload.rs | 2673 +++++++++++++++++ 29 files changed, 8343 insertions(+), 2389 deletions(-) create mode 100644 deployment/aliyun/polymarket-legacy-health-policy.jq create mode 100755 deployment/aliyun/polymarket-raw-ops-cutover.sh create mode 100755 deployment/aliyun/polymarket-raw-ops-shadow-gate.sh create mode 100644 deployment/aliyun/polymarket-reference-collector-shadow@.service create mode 100644 deployment/aliyun/polymarket-rust-health-policy.jq create mode 100644 deployment/aliyun/polymarket-shadow-gate-policy.jq delete mode 100644 deployment/aliyun/polymarket_market_tape_upload.py delete mode 100644 deployment/aliyun/polymarket_reference_canonicalize.py delete mode 100644 deployment/aliyun/polymarket_reference_collector.py create mode 100755 deployment/aliyun/test-polymarket-raw-ops-control-plane.sh delete mode 100644 deployment/aliyun/test_polymarket_market_tape_upload.py delete mode 100644 deployment/aliyun/test_polymarket_reference_canonicalize.py delete mode 100644 deployment/aliyun/test_polymarket_reference_collector.py create mode 100644 rust_hft/tools/collector/src/bin/polymarket-raw-ops.rs create mode 100644 rust_hft/tools/collector/src/polymarket_parity.rs create mode 100644 rust_hft/tools/collector/src/polymarket_raw.rs create mode 100644 rust_hft/tools/collector/src/polymarket_upload.rs diff --git a/.github/workflows/acr-publish.yml b/.github/workflows/acr-publish.yml index 460de80c4..101a859d4 100644 --- a/.github/workflows/acr-publish.yml +++ b/.github/workflows/acr-publish.yml @@ -54,7 +54,7 @@ jobs: run: | echo '${{ matrix.repository }}=${{ vars.ACR_REGISTRY }}/wildcard0923/${{ matrix.repository }}@${{ steps.build.outputs.digest }}' >> "$GITHUB_STEP_SUMMARY" - - name: Extract collector binary for bare-metal ECS + - name: Extract collector binaries for bare-metal ECS if: matrix.repository == 'binance-lob-archiver' env: IMAGE: ${{ vars.ACR_REGISTRY }}/wildcard0923/${{ matrix.repository }}:${{ github.sha }} @@ -64,10 +64,16 @@ jobs: container_id=$(docker create "$IMAGE") trap 'docker rm -f "$container_id" >/dev/null 2>&1 || true' EXIT docker cp "$container_id:/usr/local/bin/binance-lob-archiver" artifact/binance-lob-archiver + docker cp "$container_id:/usr/local/bin/polymarket-raw-ops" artifact/polymarket-raw-ops chmod 0755 artifact/binance-lob-archiver - sha256sum artifact/binance-lob-archiver | tee artifact/binance-lob-archiver.sha256 + chmod 0755 artifact/polymarket-raw-ops + ( + cd artifact + sha256sum binance-lob-archiver | tee binance-lob-archiver.sha256 + sha256sum polymarket-raw-ops | tee polymarket-raw-ops.sha256 + ) - - name: Upload collector binary artifact + - name: Upload collector binary artifact bundle if: matrix.repository == 'binance-lob-archiver' uses: actions/upload-artifact@v4 with: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3b5590f4b..1b864862a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -122,6 +122,7 @@ jobs: shellcheck ../deployment/aliyun/*.sh ../deployment/aliyun/test-rust-lob-control-plane.sh ../deployment/aliyun/test-rust-lob-release-adoption.sh + ../deployment/aliyun/test-polymarket-raw-ops-control-plane.sh - name: Build focused packages run: cargo build -p hft-live -p hft-paper -p hft-all-in-one -p alpha-harness -p hft-harnessctl --locked diff --git a/deployment/aliyun/README.md b/deployment/aliyun/README.md index 214edf1fc..19b284abb 100644 --- a/deployment/aliyun/README.md +++ b/deployment/aliyun/README.md @@ -55,8 +55,10 @@ token-to-event context remains independently replayable. Closed Polymarket sessions are validated, compressed, and uploaded every five minutes by `polymarket-market-tape-upload.timer`. The uploader ignores the active `market-updates.ndjson`, requires contiguous sequence numbers and monotonic record -timestamps, then writes `.ndjson.zst`, `manifest.json`, and `_SUCCESS` under -`lake/raw/venue=polymarket/dataset=crypto_expiry/`. Sessions crossing UTC-hour +timestamps, then writes `.ndjson.zst`, `manifest.json`, and `_SUCCESS` under the +immutable `lake/raw/venue=polymarket/dataset=crypto_expiry/date=.../hour=.../sha256=/` +prefix. Uploads are no-clobber, and an existing triplet must match byte for byte before +it is treated as a successful retry. Sessions crossing UTC-hour boundaries are split into hour partitions. Before deleting a closed source tape, the uploader reads all three OSS objects back and verifies the compressed byte count, SHA-256, manifest, and success marker. A bad tape does not block later @@ -74,53 +76,112 @@ Data APIs every 30 seconds. It writes complete Gamma market payloads (including volume, tick size, minimum order size, fee fields, token/outcome mappings, and status), all public taker and maker trade prints, and closed-market settlement payloads to `/data/monday/spool/polymarket-reference/`. Its independent uploader publishes the -same hash-bound artifact triplet under +same content-addressed artifact triplet under `lake/raw/venue=polymarket/dataset=crypto_expiry_reference/`. Stable trade IDs and a persisted overlap state prevent duplicate trade prints across polls and restarts. An in-place v1-to-v2 migration locally isolates the old active tape, reopens completed -markets, and emits a complete v2 overlap; the canonicalizer builds a v2 union before -historical v1 artifacts receive supersession markers. After settlement, +markets, and emits a complete v2 overlap. Historical v1 objects remain readable only +under their explicit supersession markers; there is no long-lived canonicalizer +process. After settlement, trade polling continues for at least 30 minutes after the latest observed change and requires three additional stable polls before a market is marked complete. Malformed trade rows are isolated and counted by reason in `health.json` instead of blocking valid rows. Each append batch rolls back to its starting offset if write or fsync fails, so a retry -cannot duplicate a durable prefix or leave a partial record behind. +cannot duplicate a durable prefix, suppress a required hourly metadata seed, or leave +a partial record behind. A durable per-hour seed marker also forces Rust metadata when +cutover inherits a current-hour Python tape. Discovery is fail-closed unless every +configured asset is present; `health.json.missing_target_symbols` must remain empty. Neither companion unit contains private keys or an execution command. -The Python companion is a transitional parity lane: the existing Rust -`collect-pm-trades` command requires PostgreSQL and cannot yet emit the stateless raw -OSS contract used by this data-only ECS. Replace it only after a Rust shadow service -produces byte/field, deduplication, settlement, rotation, and OSS-readback parity. +After cutover, both companion units use the same `polymarket-raw-ops` Rust binary. Its +`collect-reference` subcommand owns metadata/trade/settlement collection and its +`upload` subcommand owns validation, compression, OSS upload, and remote readback. +The former Python collector and uploader remain installed, but inactive, until the +rollback retention window closes. The one-off canonicalizer is not a runtime service. -Install the uploader beside the existing tape service: +Obtain `polymarket-raw-ops` and its SHA-256 from the immutable collector build +artifact and verify the digest from the extracted artifact directory. Do not install +it over the active runtime or replace any production unit manually: ```bash -sudo install -m 0755 deployment/aliyun/polymarket_market_tape_upload.py \ - /opt/monday/bin/polymarket_market_tape_upload.py -sudo install -m 0640 deployment/aliyun/polymarket-market-tape-upload.env \ - /etc/monday/polymarket-market-tape-upload.env -sudo install -m 0644 deployment/aliyun/polymarket-market-tape-upload.service \ - /etc/systemd/system/polymarket-market-tape-upload.service -sudo install -m 0644 deployment/aliyun/polymarket-market-tape-upload.timer \ - /etc/systemd/system/polymarket-market-tape-upload.timer -sudo systemctl daemon-reload -sudo systemctl enable --now polymarket-market-tape-upload.timer +sha256sum -c polymarket-raw-ops.sha256 +candidate_sha=$(awk '{print $1}' polymarket-raw-ops.sha256) +source_revision=$(git rev-parse HEAD) ``` -Install the companion reference lane: +Install the control bundle without changing the active Python units: ```bash -sudo install -m 0755 deployment/aliyun/polymarket_reference_collector.py \ - /opt/monday/bin/polymarket_reference_collector.py -sudo install -m 0644 deployment/aliyun/polymarket-reference-{collector,upload}.service \ - /etc/systemd/system/ -sudo install -m 0644 deployment/aliyun/polymarket-reference-upload.timer \ - /etc/systemd/system/ -sudo systemctl daemon-reload -sudo systemctl enable --now polymarket-reference-collector.service \ - polymarket-reference-upload.timer +sudo install -d -m 0755 /opt/monday/control/polymarket-raw-ops +sudo install -m 0755 \ + deployment/aliyun/polymarket-raw-ops-{shadow-gate,cutover}.sh \ + /opt/monday/control/polymarket-raw-ops/ +sudo install -m 0644 \ + deployment/aliyun/polymarket-legacy-health-policy.jq \ + deployment/aliyun/polymarket-rust-health-policy.jq \ + deployment/aliyun/polymarket-shadow-gate-policy.jq \ + deployment/aliyun/polymarket-reference-collector-shadow@.service \ + deployment/aliyun/polymarket-reference-{collector,upload}.service \ + deployment/aliyun/polymarket-reference-upload.timer \ + deployment/aliyun/polymarket-market-tape-upload.{service,timer} \ + /opt/monday/control/polymarket-raw-ops/ ``` +Run the isolated Rust shadow while the active unit is still the Python collector. +The gate cannot produce production-eligible evidence before 3600 continuous seconds +plus a verified five-minute comparison tail inside one UTC hour. Both lanes are bounded +to the same successful-poll cutoff (with a safety lag), so new Python rows written after +the Rust shadow stops cannot create a false mismatch. Every invocation gets a unique +run spool beneath the candidate digest, so a failed or expired gate can be rerun without +reusing prior tape data. +Evidence binds both the candidate binary digest and the exact control-bundle digest; +cutover refuses either identity if the bundle changes after shadowing. Control assets +and evidence must remain root-owned and non-writable by the service account, and a +gate older than 24 hours must be rerun. +The gate also writes a content-addressed, root-owned snapshot of the six non-secret +OSS uploader settings beside the candidate binary. Cutover renders both Rust upload +units against that immutable snapshot, so a later edit to the live legacy env file +cannot change the destination represented by the gate evidence. +It fails unless the seven assets have field and stable metadata-contract value parity, +identical non-duplicated in-window trade IDs, settlement parity, an hourly rotation, +fresh fail-closed health, and exact candidate process identity. The candidate must also +upload and read back both a closed reference segment and a deterministic market-tape +fixture under the isolated `crypto_expiry_reference_rust_shadow` and +`crypto_expiry_market_rust_shadow` datasets. Parity comparison itself runs through the +candidate's Rust `verify-shadow-parity` subcommand; the control bundle contains no +separate Python verifier: + +```bash +gate_json=$(sudo /opt/monday/control/polymarket-raw-ops/polymarket-raw-ops-shadow-gate.sh \ + ./polymarket-raw-ops "$candidate_sha" "$source_revision") +``` + +Only the cutover command may replace the active unit. It verifies the immutable gate, +snapshots the installed Python units and scripts, drains with the Python uploader, +stops the Python collector, installs the content-addressed Rust release, and performs +an explicit `systemctl restart`. Before enabling either upload timer it verifies the +new MainPID, `/proc//exe` digest, command line, fresh fail-closed health, journal, +and both one-shot upload services. Any failed step automatically restores and restarts +the snapshotted Python runtime: + +```bash +cutover_json=$(sudo /opt/monday/control/polymarket-raw-ops/polymarket-raw-ops-cutover.sh \ + cutover "$candidate_sha" "$gate_json") +``` + +Keep the returned cutover evidence directory for the rollback window. A later manual +rollback uses the same checksum-verified snapshot and confirms the Python PID and +command line after restart: + +```bash +cutover_dir=$(dirname "$cutover_json") +sudo /opt/monday/control/polymarket-raw-ops/polymarket-raw-ops-cutover.sh \ + rollback "$cutover_dir" +``` + +Never use `enable --now` as a substitute for this path: it does not prove that an +already-active Python process was replaced by the gated Rust artifact. + Each service opens bounded WebSocket shards, records every diff, fetches a REST Top-100 snapshot, validates sequence continuity, writes replay checkpoints, compresses hourly segments, and uploads `.jsonl.zst`, `manifest.json`, and `_SUCCESS` to OSS. diff --git a/deployment/aliyun/polymarket-legacy-health-policy.jq b/deployment/aliyun/polymarket-legacy-health-policy.jq new file mode 100644 index 000000000..f71d772d0 --- /dev/null +++ b/deployment/aliyun/polymarket-legacy-health-policy.jq @@ -0,0 +1,9 @@ +(.updated_at | type == "string" and length > 0) +and (.last_success_at | type == "string" and length > 0) +and (.target_markets | type == "number" and floor == . and . > 0) +and .api_errors == [] +and .malformed_trade_rows == 0 +and .truncated_trade_markets == [] +and .stale_trade_markets == [] +and .stale_settlement_markets == [] +and .overdue_unresolved_markets == [] diff --git a/deployment/aliyun/polymarket-market-tape-upload.service b/deployment/aliyun/polymarket-market-tape-upload.service index 7fe785fff..fcfde8b82 100644 --- a/deployment/aliyun/polymarket-market-tape-upload.service +++ b/deployment/aliyun/polymarket-market-tape-upload.service @@ -11,7 +11,8 @@ User=hftcollector Group=hftcollector Environment=HOME=/var/lib/hft-collector EnvironmentFile=/etc/monday/polymarket-market-tape-upload.env -ExecStart=/usr/bin/python3 /opt/monday/bin/polymarket_market_tape_upload.py --quote-depth-levels 0 --quote-sample-ms 1000 +ExecStart=/opt/monday/bin/polymarket-raw-ops upload --quote-depth-levels 0 --quote-sample-ms 1000 +TimeoutStartSec=0 NoNewPrivileges=true PrivateTmp=true ProtectSystem=strict diff --git a/deployment/aliyun/polymarket-raw-ops-cutover.sh b/deployment/aliyun/polymarket-raw-ops-cutover.sh new file mode 100755 index 000000000..078a2fea3 --- /dev/null +++ b/deployment/aliyun/polymarket-raw-ops-cutover.sh @@ -0,0 +1,722 @@ +#!/usr/bin/env bash +set -euo pipefail + +umask 027 +export LC_ALL=C + +SCRIPT_DIR=$(cd -- "$(dirname -- "$0")" && pwd) +readonly SCRIPT_DIR +readonly POLICY="$SCRIPT_DIR/polymarket-shadow-gate-policy.jq" +readonly LEGACY_HEALTH_POLICY="$SCRIPT_DIR/polymarket-legacy-health-policy.jq" +readonly RUST_HEALTH_POLICY="$SCRIPT_DIR/polymarket-rust-health-policy.jq" +readonly RELEASE_ROOT=/opt/monday/releases/polymarket-raw-ops +readonly ACTIVE_BINARY=/opt/monday/bin/polymarket-raw-ops +readonly EVIDENCE_ROOT=/data/monday/evidence/polymarket-cutovers +readonly GATE_EVIDENCE_ROOT=/data/monday/evidence/polymarket-shadow-gates +readonly MAX_GATE_AGE_SECONDS=86400 +readonly LOCK_FILE=/run/lock/monday-polymarket-raw-ops.lock +readonly COLLECTOR_UNIT=polymarket-reference-collector.service +readonly REFERENCE_UPLOAD_UNIT=polymarket-reference-upload.service +readonly REFERENCE_UPLOAD_TIMER=polymarket-reference-upload.timer +readonly MARKET_UPLOAD_UNIT=polymarket-market-tape-upload.service +readonly MARKET_UPLOAD_TIMER=polymarket-market-tape-upload.timer +readonly HEALTH=/data/monday/spool/polymarket-reference/health.json +readonly LEGACY_COLLECTOR=/opt/monday/bin/polymarket_reference_collector.py +readonly LEGACY_UPLOADER=/opt/monday/bin/polymarket_market_tape_upload.py +readonly LEGACY_EXEC="/usr/bin/python3 $LEGACY_COLLECTOR" +readonly RUST_EXEC="$ACTIVE_BINARY collect-reference" +readonly COLLECTOR_FRAGMENT="/etc/systemd/system/$COLLECTOR_UNIT" +readonly LEGACY_REFERENCE_UPLOAD_EXEC="/usr/bin/python3 $LEGACY_UPLOADER --spool-dir /data/monday/spool/polymarket-reference --dataset crypto_expiry_reference --quote-depth-levels 0 --quote-sample-ms 0" +readonly REFERENCE_UPLOAD_EXEC="$ACTIVE_BINARY upload --spool-dir /data/monday/spool/polymarket-reference --dataset crypto_expiry_reference --quote-depth-levels 0 --quote-sample-ms 0" +readonly MARKET_UPLOAD_EXEC="$ACTIVE_BINARY upload --quote-depth-levels 0 --quote-sample-ms 1000" +readonly UPLOAD_ENV=/etc/monday/polymarket-market-tape-upload.env +readonly MAX_HEALTH_SILENCE_SECONDS=90 +readonly -a UNIT_ASSETS=( + polymarket-reference-collector.service + polymarket-reference-upload.service + polymarket-reference-upload.timer + polymarket-market-tape-upload.service + polymarket-market-tape-upload.timer +) +readonly -a PYTHON_ASSETS=( + polymarket_reference_collector.py + polymarket_market_tape_upload.py +) +readonly -a BUNDLE_ASSETS=( + polymarket-raw-ops-shadow-gate.sh + polymarket-raw-ops-cutover.sh + polymarket-shadow-gate-policy.jq + polymarket-legacy-health-policy.jq + polymarket-rust-health-policy.jq + polymarket-reference-collector-shadow@.service + polymarket-reference-collector.service + polymarket-reference-upload.service + polymarket-reference-upload.timer + polymarket-market-tape-upload.service + polymarket-market-tape-upload.timer +) + +die() { + printf 'Polymarket cutover failed: %s\n' "$*" >&2 + exit 1 +} + +usage() { + printf '%s\n' \ + 'Usage:' \ + ' polymarket-raw-ops-cutover.sh cutover ' \ + ' polymarket-raw-ops-cutover.sh rollback ' +} + +bundle_sha256() { + ( + cd "$SCRIPT_DIR" + sha256sum "${BUNDLE_ASSETS[@]}" | sha256sum | awk '{print $1}' + ) +} + +direct_directory() { + local path=$1 + [[ -d $path && ! -L $path && $(readlink -f -- "$path") == "$path" ]] +} + +direct_directory_or_absent() { + local path=$1 + [[ ! -e $path && ! -L $path ]] || direct_directory "$path" +} + +secure_release_directory() { + local path=$1 owner mode + direct_directory "$path" || return 1 + owner=$(stat -c %u -- "$path") || return 1 + mode=$(stat -c %a -- "$path") || return 1 + [[ $owner == 0 && $mode == 755 ]] +} + +secure_regular_file() { + local path=$1 mode owner + [[ -f $path && ! -L $path ]] || die "required direct regular file is missing: $path" + owner=$(stat -c %u -- "$path") + mode=$(stat -c %a -- "$path") + [[ $owner == 0 ]] || die "required file is not root-owned: $path" + (( (8#$mode & 022) == 0 )) \ + || die "required file is group/world writable: $path" +} + +verify_gate_marker() { + local gate_dir=$1 marker expected actual line_count + marker="$gate_dir/PASSED.sha256" + [[ -f $marker && ! -L $marker ]] || return 1 + line_count=$(wc -l <"$marker") || return 1 + [[ $line_count -eq 1 ]] || return 1 + expected=$(cd "$gate_dir" && sha256sum gate.json) || return 1 + actual=$(<"$marker") || return 1 + [[ $actual == "$expected" ]] || return 1 + ( + cd "$gate_dir" + sha256sum --check --strict PASSED.sha256 >/dev/null + ) +} + +effective_exec_argv() { + local unit=$1 raw argv + raw=$(systemctl show --property=ExecStart --value "$unit") || return 1 + argv=$(sed -nE 's/^.*argv\[\]=([^;]+);.*$/\1/p' <<<"$raw" \ + | sed -E 's/[[:space:]]+$//') + [[ -n $argv ]] || return 1 + printf '%s\n' "$argv" +} + +proc_cmdline() { + local pid=$1 + [[ $pid =~ ^[1-9][0-9]*$ && -r /proc/$pid/cmdline ]] || return 1 + tr '\0' ' ' <"/proc/$pid/cmdline" +} + +verify_effective_unit() { + local unit=$1 expected_fragment=$2 expected_exec=$3 fragment drop_ins exec_argv + fragment=$(systemctl show --property=FragmentPath --value "$unit") || return 1 + [[ $fragment == "$expected_fragment" ]] || return 1 + drop_ins=$(systemctl show --property=DropInPaths --value "$unit") || return 1 + [[ -z $drop_ins ]] || return 1 + exec_argv=$(effective_exec_argv "$unit") || return 1 + [[ $exec_argv == "$expected_exec" ]] +} + +env_value() { + local key=$1 file=${2:-$UPLOAD_ENV} count value + count=$(grep -c "^${key}=" "$file" || true) + [[ $count -eq 1 ]] || die "$file must contain exactly one $key" + value=$(sed -n "s/^${key}=//p" "$file") + [[ -n $value ]] || die "$file has an empty $key" + printf '%s\n' "$value" +} + +oss_config_sha256() { + local file=${1:-$UPLOAD_ENV} key + for key in OSS_BUCKET OSS_ENDPOINT OSS_REGION ALIYUN_PROFILE \ + ZSTD_TIMEOUT_SECONDS OSS_COPY_TIMEOUT_SECONDS; do + printf '%s=%s\n' "$key" "$(env_value "$key" "$file")" + done | sha256sum | awk '{print $1}' +} + +atomic_install() { + local mode=$1 source=$2 destination=$3 temporary + temporary="${destination}.new.$$" + install -m "$mode" "$source" "$temporary" + mv -Tf "$temporary" "$destination" +} + +unit_active() { + systemctl is-active --quiet "$1" +} + +unit_enabled() { + systemctl is-enabled --quiet "$1" +} + +verify_oneshot_success() { + local unit=$1 result status + result=$(systemctl show --property=Result --value "$unit") + status=$(systemctl show --property=ExecMainStatus --value "$unit") + [[ $result == success && $status == 0 ]] +} + +verify_upload_units() { + local pinned_upload_env=$1 unit_file + verify_effective_unit "$REFERENCE_UPLOAD_UNIT" \ + "/etc/systemd/system/$REFERENCE_UPLOAD_UNIT" "$REFERENCE_UPLOAD_EXEC" || return 1 + verify_effective_unit "$MARKET_UPLOAD_UNIT" \ + "/etc/systemd/system/$MARKET_UPLOAD_UNIT" "$MARKET_UPLOAD_EXEC" || return 1 + for unit_file in "/etc/systemd/system/$REFERENCE_UPLOAD_UNIT" \ + "/etc/systemd/system/$MARKET_UPLOAD_UNIT"; do + secure_regular_file "$unit_file" + [[ $(grep -c '^EnvironmentFile=' "$unit_file" || true) -eq 1 ]] || return 1 + grep -Fxq "EnvironmentFile=$pinned_upload_env" "$unit_file" || return 1 + done + local timer drop_ins fragment + for timer in "$REFERENCE_UPLOAD_TIMER" "$MARKET_UPLOAD_TIMER"; do + fragment=$(systemctl show --property=FragmentPath --value "$timer") || return 1 + [[ $fragment == "/etc/systemd/system/$timer" ]] || return 1 + drop_ins=$(systemctl show --property=DropInPaths --value "$timer") || return 1 + [[ -z $drop_ins ]] || return 1 + done +} + +render_upload_unit() { + local source=$1 destination=$2 pinned_upload_env=$3 temporary + temporary="${destination}.new.$$" + sed "s|^EnvironmentFile=.*$|EnvironmentFile=$pinned_upload_env|" \ + "$source" >"$temporary" + chmod 0644 "$temporary" + mv -Tf "$temporary" "$destination" +} + +verify_saved_unit_state() { + local state_json=$1 unit expected_enabled expected_active + for unit in "$COLLECTOR_UNIT" "$REFERENCE_UPLOAD_TIMER" "$MARKET_UPLOAD_TIMER"; do + expected_enabled=$(jq -er --arg unit "$unit" '.units[$unit].enabled' "$state_json") \ + || return 1 + expected_active=$(jq -er --arg unit "$unit" '.units[$unit].active' "$state_json") \ + || return 1 + if [[ $expected_enabled == true ]]; then + unit_enabled "$unit" || return 1 + elif unit_enabled "$unit"; then + return 1 + fi + if [[ $expected_active == true ]]; then + unit_active "$unit" || return 1 + elif unit_active "$unit"; then + return 1 + fi + done +} + +verify_legacy_runtime() { + local expected_pid=$1 pid cmdline restarts + unit_active "$COLLECTOR_UNIT" || return 1 + verify_effective_unit "$COLLECTOR_UNIT" "$COLLECTOR_FRAGMENT" "$LEGACY_EXEC" || return 1 + pid=$(systemctl show --property=MainPID --value "$COLLECTOR_UNIT") + [[ $pid == "$expected_pid" ]] || return 1 + restarts=$(systemctl show --property=NRestarts --value "$COLLECTOR_UNIT") || return 1 + [[ $restarts == 0 ]] || return 1 + cmdline=$(proc_cmdline "$pid") || return 1 + [[ $cmdline == "$LEGACY_EXEC " ]] +} + +verify_legacy_health() { + local started_epoch=$1 health_policy=${2:-$LEGACY_HEALTH_POLICY} + local updated_at last_success_at updated_epoch success_epoch now_epoch + [[ -f $HEALTH && ! -L $HEALTH ]] || return 1 + (( $(stat -c %Y "$HEALTH") >= started_epoch )) || return 1 + jq -e -f "$health_policy" "$HEALTH" >/dev/null || return 1 + updated_at=$(jq -er '.updated_at' "$HEALTH") || return 1 + last_success_at=$(jq -er '.last_success_at' "$HEALTH") || return 1 + updated_epoch=$(date -u -d "$updated_at" +%s) || return 1 + success_epoch=$(date -u -d "$last_success_at" +%s) || return 1 + now_epoch=$(date -u +%s) + ((updated_epoch >= started_epoch && updated_epoch <= now_epoch)) || return 1 + ((success_epoch >= started_epoch && success_epoch <= now_epoch)) || return 1 + ((now_epoch - updated_epoch <= MAX_HEALTH_SILENCE_SECONDS)) || return 1 + ((now_epoch - success_epoch <= MAX_HEALTH_SILENCE_SECONDS)) +} + +verify_fresh_legacy_runtime() { + local started_epoch=$1 expected_pid=$2 health_policy=${3:-$LEGACY_HEALTH_POLICY} + verify_legacy_runtime "$expected_pid" || return 1 + verify_legacy_health "$started_epoch" "$health_policy" +} + +clear_health_before_restart() { + local evidence_dir=$1 label=$2 snapshot + [[ ! -e $HEALTH && ! -L $HEALTH ]] && return 0 + [[ -f $HEALTH && ! -L $HEALTH ]] || die 'health path is not a direct regular file' + snapshot="$evidence_dir/${label}-health.json" + [[ ! -e $snapshot && ! -L $snapshot ]] || die 'health snapshot path already exists' + install -m 0640 "$HEALTH" "$snapshot" + rm -f "$HEALTH" + sync "$snapshot" +} + +verify_rust_health_file() { + local health_file=$1 started_epoch=$2 health_mtime updated_at last_success_at + local updated_epoch success_epoch now_epoch + [[ -f $health_file && ! -L $health_file ]] || return 1 + health_mtime=$(stat -c %Y "$health_file") + ((health_mtime >= started_epoch)) || return 1 + jq -e -f "$RUST_HEALTH_POLICY" "$health_file" >/dev/null || return 1 + updated_at=$(jq -er '.updated_at' "$health_file") || return 1 + last_success_at=$(jq -er '.last_success_at' "$health_file") || return 1 + updated_epoch=$(date -u -d "$updated_at" +%s) || return 1 + success_epoch=$(date -u -d "$last_success_at" +%s) || return 1 + now_epoch=$(date -u +%s) + ((updated_epoch >= started_epoch && updated_epoch <= now_epoch)) || return 1 + ((success_epoch >= started_epoch && success_epoch <= now_epoch)) || return 1 + ((now_epoch - updated_epoch <= MAX_HEALTH_SILENCE_SECONDS)) || return 1 + ((now_epoch - success_epoch <= MAX_HEALTH_SILENCE_SECONDS)) +} + +verify_rust_runtime() { + local expected_binary=$1 started_epoch=$2 expected_pid=$3 + local pid cmdline restarts + unit_active "$COLLECTOR_UNIT" || return 1 + verify_effective_unit "$COLLECTOR_UNIT" "$COLLECTOR_FRAGMENT" "$RUST_EXEC" || return 1 + pid=$(systemctl show --property=MainPID --value "$COLLECTOR_UNIT") + [[ $pid == "$expected_pid" ]] || return 1 + restarts=$(systemctl show --property=NRestarts --value "$COLLECTOR_UNIT") || return 1 + [[ $restarts == 0 ]] || return 1 + [[ $(readlink -f "/proc/$pid/exe") == "$expected_binary" ]] || return 1 + cmdline=$(proc_cmdline "$pid") || return 1 + [[ $cmdline == "$RUST_EXEC " ]] || return 1 + verify_rust_health_file "$HEALTH" "$started_epoch" +} + +snapshot_legacy() { + local rollback_dir=$1 + local state_json=$rollback_dir/state.json asset enabled active + install -d -m 0750 "$rollback_dir/systemd" "$rollback_dir/bin" \ + "$rollback_dir/config" "$rollback_dir/control" + for asset in "${UNIT_ASSETS[@]}"; do + secure_regular_file "/etc/systemd/system/$asset" + install -m 0644 "/etc/systemd/system/$asset" "$rollback_dir/systemd/$asset" + done + secure_regular_file "$LEGACY_COLLECTOR" + secure_regular_file "$LEGACY_UPLOADER" + secure_regular_file "$UPLOAD_ENV" + secure_regular_file "$LEGACY_HEALTH_POLICY" + install -m 0755 "$LEGACY_COLLECTOR" "$rollback_dir/bin/${PYTHON_ASSETS[0]}" + install -m 0755 "$LEGACY_UPLOADER" "$rollback_dir/bin/${PYTHON_ASSETS[1]}" + install -m 0640 "$UPLOAD_ENV" "$rollback_dir/config/polymarket-market-tape-upload.env" + install -m 0644 "$LEGACY_HEALTH_POLICY" \ + "$rollback_dir/control/polymarket-legacy-health-policy.jq" + + jq -n '{}' >"$state_json" + for asset in "$COLLECTOR_UNIT" "$REFERENCE_UPLOAD_TIMER" "$MARKET_UPLOAD_TIMER"; do + enabled=false + active=false + unit_enabled "$asset" && enabled=true + unit_active "$asset" && active=true + jq --arg unit "$asset" --argjson enabled "$enabled" --argjson active "$active" \ + '.units[$unit] = {enabled:$enabled,active:$active}' "$state_json" \ + >"$state_json.tmp" + mv "$state_json.tmp" "$state_json" + done + ( + cd "$rollback_dir" + sha256sum state.json systemd/* bin/* config/* control/* >manifest.sha256 + ) +} + +restore_legacy() ( + set -e + local evidence_dir=$1 + local rollback_dir=$evidence_dir/rollback active_target actual_manifest_sha asset + local expected_manifest_sha started_epoch rollback_pid current_pid restarts + local rollback_health_policy=$rollback_dir/control/polymarket-legacy-health-policy.jq + direct_directory "$evidence_dir" || die 'rollback evidence directory is indirect' + direct_directory "$rollback_dir" || die 'rollback payload directory is indirect' + secure_regular_file "$rollback_dir/manifest.sha256" + ( + cd "$rollback_dir" + sha256sum --check --strict manifest.sha256 >/dev/null + ) || die 'rollback snapshot checksum failed' + secure_regular_file "$rollback_health_policy" + if [[ -e $evidence_dir/cutover.json || -L $evidence_dir/cutover.json ]]; then + secure_regular_file "$evidence_dir/cutover.json" + expected_manifest_sha=$(jq -er '.rollback_manifest_sha256' "$evidence_dir/cutover.json") \ + || die 'cutover evidence is missing the rollback manifest identity' + actual_manifest_sha=$(sha256sum "$rollback_dir/manifest.sha256" | awk '{print $1}') + [[ $actual_manifest_sha == "$expected_manifest_sha" ]] \ + || die 'rollback manifest differs from the completed cutover evidence' + fi + + systemctl stop "$REFERENCE_UPLOAD_TIMER" "$MARKET_UPLOAD_TIMER" + systemctl stop "$REFERENCE_UPLOAD_UNIT" "$MARKET_UPLOAD_UNIT" + systemctl stop "$COLLECTOR_UNIT" + clear_health_before_restart "$evidence_dir" "pre-rollback-$(date -u +%Y%m%dT%H%M%SZ)-$$" + for asset in "${UNIT_ASSETS[@]}"; do + atomic_install 0644 "$rollback_dir/systemd/$asset" "/etc/systemd/system/$asset" + done + atomic_install 0755 "$rollback_dir/bin/${PYTHON_ASSETS[0]}" "$LEGACY_COLLECTOR" + atomic_install 0755 "$rollback_dir/bin/${PYTHON_ASSETS[1]}" "$LEGACY_UPLOADER" + atomic_install 0640 "$rollback_dir/config/polymarket-market-tape-upload.env" "$UPLOAD_ENV" + if [[ -e $ACTIVE_BINARY || -L $ACTIVE_BINARY ]]; then + [[ -L $ACTIVE_BINARY ]] || die 'refusing to remove a non-symlink active Rust path' + active_target=$(readlink -f -- "$ACTIVE_BINARY") + [[ $active_target == "$RELEASE_ROOT"/*/polymarket-raw-ops ]] \ + || die 'active Rust symlink points outside the immutable release root' + rm -f "$ACTIVE_BINARY" + fi + systemctl daemon-reload + started_epoch=$(date -u +%s) + systemctl restart "$COLLECTOR_UNIT" + rollback_pid= + for _ in $(seq 1 36); do + restarts=$(systemctl show --property=NRestarts --value "$COLLECTOR_UNIT") + [[ $restarts == 0 ]] || die 'Python collector restarted while rollback was being verified' + current_pid=$(systemctl show --property=MainPID --value "$COLLECTOR_UNIT") + if [[ $current_pid =~ ^[1-9][0-9]*$ ]]; then + if [[ -z $rollback_pid ]]; then + rollback_pid=$current_pid + else + [[ $current_pid == "$rollback_pid" ]] \ + || die 'Python collector PID changed while rollback was being verified' + fi + verify_fresh_legacy_runtime "$started_epoch" "$rollback_pid" \ + "$rollback_health_policy" && break + fi + sleep 5 + done + [[ -n $rollback_pid ]] || die 'Python collector never produced a verifiable MainPID' + verify_fresh_legacy_runtime "$started_epoch" "$rollback_pid" "$rollback_health_policy" \ + || die 'Python collector identity or health did not recover during rollback' + + for asset in "$COLLECTOR_UNIT" "$REFERENCE_UPLOAD_TIMER" "$MARKET_UPLOAD_TIMER"; do + if jq -e --arg unit "$asset" '.units[$unit].enabled == true' \ + "$rollback_dir/state.json" >/dev/null; then + systemctl enable "$asset" + else + systemctl disable "$asset" + fi + if jq -e --arg unit "$asset" '.units[$unit].active == true' \ + "$rollback_dir/state.json" >/dev/null; then + systemctl start "$asset" + fi + done + verify_fresh_legacy_runtime "$started_epoch" "$rollback_pid" "$rollback_health_policy" \ + || die 'rollback did not preserve Python runtime identity and health' + verify_saved_unit_state "$rollback_dir/state.json" \ + || die 'rollback did not restore the saved collector/timer state' + printf '%s\n' "$evidence_dir" +) + +[[ ${EUID} -eq 0 ]] || die 'must run as root' +for command in awk date dirname flock grep install journalctl jq ln mkdir mountpoint \ + mv readlink rm sed seq sha256sum sleep stat sync systemctl tr wc; do + command -v "$command" >/dev/null 2>&1 || die "missing required command: $command" +done +mode=${1:-} +case "$mode" in + rollback) + [[ $# -eq 2 ]] || { + usage >&2 + exit 2 + } + ;; + cutover) + [[ $# -eq 3 ]] || { + usage >&2 + exit 2 + } + ;; + *) + usage >&2 + exit 2 + ;; +esac +mountpoint -q /data || die '/data must be a mount point' +for path in /opt/monday /opt/monday/bin /opt/monday/releases "$RELEASE_ROOT" \ + /data/monday /data/monday/spool /data/monday/spool/polymarket-reference \ + /data/monday/evidence "$EVIDENCE_ROOT" "$GATE_EVIDENCE_ROOT"; do + direct_directory_or_absent "$path" || die "fixed path is indirect or a symlink: $path" +done +install -d -m 0755 "$(dirname "$LOCK_FILE")" +exec 9>"$LOCK_FILE" +flock -n 9 || die 'another Polymarket release operation is running' + +if [[ $mode == rollback ]]; then + [[ -d $2 && ! -L $2 ]] || die 'rollback evidence must be a direct directory' + rollback_evidence=$(readlink -f -- "$2") + [[ $rollback_evidence == "$EVIDENCE_ROOT"/* ]] \ + || die 'rollback evidence is outside the fixed cutover evidence root' + restore_legacy "$rollback_evidence" + exit 0 +fi + +# Cutover depends on the current gate bundle and live uploader configuration. +# Manual rollback intentionally branches before these checks and uses only the +# checksum-protected snapshot embedded in the cutover evidence directory. +secure_regular_file "$POLICY" +for asset in "${BUNDLE_ASSETS[@]}"; do + secure_regular_file "$SCRIPT_DIR/$asset" +done +secure_regular_file "$UPLOAD_ENV" +deployment_bundle_sha=$(bundle_sha256) +current_oss_config_sha=$(oss_config_sha256) + +candidate_sha=$(printf '%s' "$2" | tr '[:upper:]' '[:lower:]') +[[ -f $3 && ! -L $3 ]] || die 'gate evidence must be a direct regular file' +gate_json=$(readlink -f -- "$3") +[[ $candidate_sha =~ ^[a-f0-9]{64}$ ]] || die 'candidate SHA-256 is invalid' +[[ $gate_json == "$GATE_EVIDENCE_ROOT"/*/gate.json ]] \ + || die 'gate evidence is outside the fixed shadow evidence root' +secure_regular_file "$gate_json" +gate_dir=$(dirname "$gate_json") +direct_directory "$gate_dir" || die 'gate evidence directory is indirect' +secure_regular_file "$gate_dir/PASSED.sha256" +verify_gate_marker "$gate_dir" \ + || die 'shadow gate marker is not the exact single gate.json checksum' +jq -e -f "$POLICY" "$gate_json" >/dev/null || die 'shadow gate is not production eligible' +[[ $(jq -r '.candidate_sha256' "$gate_json") == "$candidate_sha" ]] \ + || die 'gate belongs to a different candidate' +[[ $(jq -r '.deployment_bundle_sha256' "$gate_json") == "$deployment_bundle_sha" ]] \ + || die 'control-plane bundle changed after the shadow gate' +gate_oss_config_sha=$(jq -er '.oss_config_sha256 | select(type == "string")' "$gate_json") \ + || die 'shadow gate is missing the OSS configuration identity' +[[ $gate_oss_config_sha == "$current_oss_config_sha" ]] \ + || die 'OSS configuration changed after the shadow gate' +gate_completed_at=$(jq -er '.completed_at | select(type == "string")' "$gate_json") \ + || die 'shadow gate completion timestamp is missing' +gate_completed_epoch=$(date -u -d "$gate_completed_at" +%s) \ + || die 'shadow gate completion timestamp is invalid' +now_epoch=$(date -u +%s) +gate_age=$((now_epoch - gate_completed_epoch)) +((gate_age >= 0 && gate_age <= MAX_GATE_AGE_SECONDS)) \ + || die 'shadow gate evidence is stale or from the future' + +candidate_release_dir="$RELEASE_ROOT/$candidate_sha" +secure_release_directory "$candidate_release_dir" \ + || die 'candidate release directory is not root-owned mode 0755' +candidate_binary="$candidate_release_dir/polymarket-raw-ops" +secure_regular_file "$candidate_binary" +[[ -x $candidate_binary ]] || die 'candidate release is not executable' +printf '%s %s\n' "$candidate_sha" "$candidate_binary" \ + | sha256sum --check --strict >/dev/null || die 'candidate release checksum mismatch' +pinned_upload_env="$RELEASE_ROOT/$candidate_sha/polymarket-upload-env-$gate_oss_config_sha.env" +secure_regular_file "$pinned_upload_env" +[[ $(oss_config_sha256 "$pinned_upload_env") == "$gate_oss_config_sha" ]] \ + || die 'pinned uploader environment differs from the shadow gate' +for asset in "$COLLECTOR_UNIT" "$REFERENCE_UPLOAD_UNIT" "$REFERENCE_UPLOAD_TIMER" \ + "$MARKET_UPLOAD_UNIT" "$MARKET_UPLOAD_TIMER"; do + secure_regular_file "$SCRIPT_DIR/$asset" +done +legacy_pid=$(systemctl show --property=MainPID --value "$COLLECTOR_UNIT") +[[ $legacy_pid =~ ^[1-9][0-9]*$ ]] \ + || die 'cutover requires a verifiable active Python reference collector PID' +gate_legacy_pid=$(jq -er '.legacy_runtime.main_pid | select(type == "number" and floor == . and . > 0)' \ + "$gate_json") || die 'shadow gate has no valid legacy MainPID' +[[ $legacy_pid == "$gate_legacy_pid" ]] \ + || die 'legacy collector MainPID changed after the shadow gate' +verify_legacy_runtime "$legacy_pid" \ + || die 'cutover requires the exact restart-free Python reference collector' +legacy_health_not_before=$(($(date -u +%s) - MAX_HEALTH_SILENCE_SECONDS)) +verify_legacy_health "$legacy_health_not_before" \ + || die 'cutover requires current fail-closed legacy health' + +install -d -m 0755 /data/monday /data/monday/evidence "$EVIDENCE_ROOT" +run_id="$(date -u +%Y%m%dT%H%M%SZ)-${candidate_sha:0:12}-$$" +evidence_dir="$EVIDENCE_ROOT/$run_id" +mkdir -m 0750 "$evidence_dir" || die 'cutover evidence directory already exists' +rollback_dir="$evidence_dir/rollback" +snapshot_legacy "$rollback_dir" + +transition_started=false +cutover_succeeded=false +on_exit() { + local status=$? restore_status=0 + if [[ $cutover_succeeded == false && $transition_started == true ]]; then + printf 'cutover failed; restoring snapshotted Python runtime\n' >&2 + trap - EXIT + set +e + restore_legacy "$evidence_dir" >/dev/null + restore_status=$? + set -e + if ((restore_status != 0)); then + printf 'automatic rollback failed; collector and upload timers require operator recovery\n' >&2 + status=1 + fi + fi + exit "$status" +} +trap on_exit EXIT + +# Drain with the still-installed Python uploader before changing any unit. +[[ $(oss_config_sha256) == "$gate_oss_config_sha" ]] \ + || die 'OSS configuration changed before the cutover transition' +transition_started=true +systemctl stop "$REFERENCE_UPLOAD_TIMER" "$MARKET_UPLOAD_TIMER" +systemctl stop "$REFERENCE_UPLOAD_UNIT" "$MARKET_UPLOAD_UNIT" +render_upload_unit "/etc/systemd/system/$REFERENCE_UPLOAD_UNIT" \ + "/etc/systemd/system/$REFERENCE_UPLOAD_UNIT" "$pinned_upload_env" +systemctl daemon-reload +verify_effective_unit "$REFERENCE_UPLOAD_UNIT" \ + "/etc/systemd/system/$REFERENCE_UPLOAD_UNIT" "$LEGACY_REFERENCE_UPLOAD_EXEC" \ + || die 'legacy reference uploader effective unit identity is not exact' +grep -Fxq "EnvironmentFile=$pinned_upload_env" \ + "/etc/systemd/system/$REFERENCE_UPLOAD_UNIT" \ + || die 'legacy reference uploader is not pinned to the gated OSS configuration' +systemctl start "$REFERENCE_UPLOAD_UNIT" +verify_oneshot_success "$REFERENCE_UPLOAD_UNIT" \ + || die 'legacy reference uploader drain did not complete successfully' + +systemctl stop "$COLLECTOR_UNIT" +clear_health_before_restart "$evidence_dir" pre-cutover +install -d -m 0755 /opt/monday/bin +temporary_link="${ACTIVE_BINARY}.new.$$" +rm -f "$temporary_link" +ln -s "$candidate_binary" "$temporary_link" +mv -Tf "$temporary_link" "$ACTIVE_BINARY" +for asset in "${UNIT_ASSETS[@]}"; do + case "$asset" in + "$REFERENCE_UPLOAD_UNIT"|"$MARKET_UPLOAD_UNIT") + render_upload_unit "$SCRIPT_DIR/$asset" "/etc/systemd/system/$asset" \ + "$pinned_upload_env" + ;; + *) atomic_install 0644 "$SCRIPT_DIR/$asset" "/etc/systemd/system/$asset" ;; + esac +done +systemctl daemon-reload +verify_upload_units "$pinned_upload_env" \ + || die 'Rust upload unit or timer identity differs from the gated configuration' + +started_epoch=$(date -u +%s) +systemctl restart "$COLLECTOR_UNIT" +rust_pid= +first_health_updated_at= +health_advanced=false +for _ in $(seq 1 36); do + rust_restarts=$(systemctl show --property=NRestarts --value "$COLLECTOR_UNIT") + [[ $rust_restarts == 0 ]] || die 'Rust collector restarted during cutover verification' + current_pid=$(systemctl show --property=MainPID --value "$COLLECTOR_UNIT") + if [[ $current_pid =~ ^[1-9][0-9]*$ ]]; then + if [[ -z $rust_pid ]]; then + rust_pid=$current_pid + else + [[ $current_pid == "$rust_pid" ]] \ + || die 'Rust collector PID changed during cutover verification' + fi + if verify_rust_runtime "$candidate_binary" "$started_epoch" "$rust_pid"; then + current_health_updated_at=$(jq -er '.updated_at' "$HEALTH") + if [[ -z $first_health_updated_at ]]; then + first_health_updated_at=$current_health_updated_at + elif [[ $current_health_updated_at != "$first_health_updated_at" ]]; then + health_advanced=true + break + fi + fi + fi + sleep 5 +done +[[ -n $rust_pid ]] || die 'Rust collector never produced a verifiable MainPID' +[[ $health_advanced == true ]] \ + || die 'Rust collector health did not advance across two clean polls' +verify_rust_runtime "$candidate_binary" "$started_epoch" "$rust_pid" \ + || die 'Rust collector failed post-restart identity or health checks' + +# Both one-shot services must execute successfully before their timers are re-enabled. +[[ $(oss_config_sha256 "$pinned_upload_env") == "$gate_oss_config_sha" ]] \ + || die 'pinned OSS configuration changed before Rust uploader verification' +verify_upload_units "$pinned_upload_env" \ + || die 'Rust upload unit or timer identity changed before execution' +systemctl start "$REFERENCE_UPLOAD_UNIT" +[[ $(oss_config_sha256 "$pinned_upload_env") == "$gate_oss_config_sha" ]] \ + || die 'pinned OSS configuration changed during reference upload' +systemctl start "$MARKET_UPLOAD_UNIT" +[[ $(oss_config_sha256 "$pinned_upload_env") == "$gate_oss_config_sha" ]] \ + || die 'pinned OSS configuration changed during market upload' +verify_oneshot_success "$REFERENCE_UPLOAD_UNIT" \ + || die 'Rust reference uploader did not complete successfully' +verify_oneshot_success "$MARKET_UPLOAD_UNIT" \ + || die 'Rust market uploader did not complete successfully' +systemctl enable "$COLLECTOR_UNIT" "$REFERENCE_UPLOAD_TIMER" "$MARKET_UPLOAD_TIMER" +systemctl start "$REFERENCE_UPLOAD_TIMER" "$MARKET_UPLOAD_TIMER" +verify_rust_runtime "$candidate_binary" "$started_epoch" "$rust_pid" \ + || die 'Rust collector identity changed while enabling upload timers' +verify_upload_units "$pinned_upload_env" \ + || die 'Rust upload unit or timer identity changed while enabling timers' +[[ $(oss_config_sha256 "$pinned_upload_env") == "$gate_oss_config_sha" ]] \ + || die 'pinned OSS configuration changed while enabling upload timers' + +journal_file="$evidence_dir/post-start.journal" +journalctl --unit "$COLLECTOR_UNIT" --since "@$started_epoch" --no-pager \ + >"$journal_file" +[[ -s $journal_file ]] || die 'post-start journal evidence is empty' +if grep -Eiq 'panic|fatal|segmentation fault|core dumped' "$journal_file"; then + die 'post-start journal contains a fatal runtime signal' +fi + +main_pid=$(systemctl show --property=MainPID --value "$COLLECTOR_UNIT") +[[ $main_pid == "$rust_pid" ]] || die 'Rust collector PID changed before evidence publication' +verify_rust_runtime "$candidate_binary" "$started_epoch" "$rust_pid" \ + || die 'Rust collector identity or health changed before evidence publication' +health_file="$evidence_dir/post-start-health.json" +install -m 0640 "$HEALTH" "$health_file" +sync "$health_file" +verify_rust_health_file "$health_file" "$started_epoch" \ + || die 'snapshotted Rust health is not current and fail-closed clean' +health_sha=$(sha256sum "$health_file" | awk '{print $1}') +journal_sha=$(sha256sum "$journal_file" | awk '{print $1}') +rollback_sha=$(sha256sum "$rollback_dir/manifest.sha256" | awk '{print $1}') +jq -n \ + --arg schema monday.polymarket_cutover.v1 \ + --arg candidate_sha256 "$candidate_sha" \ + --arg deployment_bundle_sha256 "$deployment_bundle_sha" \ + --arg oss_config_sha256 "$gate_oss_config_sha" \ + --arg gate_json_sha256 "$(sha256sum "$gate_json" | awk '{print $1}')" \ + --arg completed_at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + --arg health_sha256 "$health_sha" \ + --arg journal_sha256 "$journal_sha" \ + --arg rollback_manifest_sha256 "$rollback_sha" \ + --argjson main_pid "$main_pid" \ + '{schema:$schema,candidate_sha256:$candidate_sha256, + deployment_bundle_sha256:$deployment_bundle_sha256, + oss_config_sha256:$oss_config_sha256, + gate_json_sha256:$gate_json_sha256,completed_at:$completed_at, + collector:{main_pid:$main_pid,restarts:0,health_sha256:$health_sha256, + journal_sha256:$journal_sha256}, + rollback_manifest_sha256:$rollback_manifest_sha256, + explicit_restart:true,post_start_identity_verified:true, + upload_services_verified:true,rollback_ready:true}' \ + >"$evidence_dir/cutover.json.tmp" +verify_rust_runtime "$candidate_binary" "$started_epoch" "$rust_pid" \ + || die 'Rust collector identity changed while cutover evidence was being prepared' +verify_upload_units "$pinned_upload_env" \ + || die 'Rust upload unit identity changed while cutover evidence was being prepared' +[[ $(oss_config_sha256 "$pinned_upload_env") == "$gate_oss_config_sha" ]] \ + || die 'pinned OSS configuration changed while cutover evidence was being prepared' +mv "$evidence_dir/cutover.json.tmp" "$evidence_dir/cutover.json" +sync "$evidence_dir/cutover.json" + +cutover_succeeded=true +trap - EXIT +printf '%s\n' "$evidence_dir/cutover.json" diff --git a/deployment/aliyun/polymarket-raw-ops-shadow-gate.sh b/deployment/aliyun/polymarket-raw-ops-shadow-gate.sh new file mode 100755 index 000000000..c7220eab6 --- /dev/null +++ b/deployment/aliyun/polymarket-raw-ops-shadow-gate.sh @@ -0,0 +1,640 @@ +#!/usr/bin/env bash +set -euo pipefail + +umask 027 +export LC_ALL=C + +readonly REQUIRED_DURATION_SECONDS=3600 +readonly PARITY_TAIL_SECONDS=300 +readonly MINIMUM_GATE_SECONDS=$((REQUIRED_DURATION_SECONDS + PARITY_TAIL_SECONDS)) +readonly HEALTH_SETTLE_SECONDS=180 +readonly MAX_HEALTH_SILENCE_SECONDS=90 +readonly SAMPLE_SECONDS=30 +readonly PARITY_CUTOFF_LAG_SECONDS=60 +readonly LEGACY_UNIT=polymarket-reference-collector.service +readonly LEGACY_EXEC='/usr/bin/python3 /opt/monday/bin/polymarket_reference_collector.py' +readonly LEGACY_FRAGMENT=/etc/systemd/system/polymarket-reference-collector.service +readonly SHADOW_FRAGMENT=/etc/systemd/system/polymarket-reference-collector-shadow@.service +readonly LEGACY_SPOOL=/data/monday/spool/polymarket-reference +readonly UPLOAD_ENV=/etc/monday/polymarket-market-tape-upload.env +readonly RELEASE_ROOT=/opt/monday/releases/polymarket-raw-ops +readonly SHADOW_ROOT=/data/monday/spool/polymarket-reference-rust-shadow +readonly EVIDENCE_ROOT=/data/monday/evidence/polymarket-shadow-gates +readonly LOCK_FILE=/run/lock/monday-polymarket-raw-ops.lock +SCRIPT_DIR=$(cd -- "$(dirname -- "$0")" && pwd) +readonly SCRIPT_DIR +readonly SERVICE_TEMPLATE="$SCRIPT_DIR/polymarket-reference-collector-shadow@.service" +readonly GATE_POLICY="$SCRIPT_DIR/polymarket-shadow-gate-policy.jq" +readonly LEGACY_HEALTH_POLICY="$SCRIPT_DIR/polymarket-legacy-health-policy.jq" +readonly RUST_HEALTH_POLICY="$SCRIPT_DIR/polymarket-rust-health-policy.jq" +readonly -a BUNDLE_ASSETS=( + polymarket-raw-ops-shadow-gate.sh + polymarket-raw-ops-cutover.sh + polymarket-shadow-gate-policy.jq + polymarket-legacy-health-policy.jq + polymarket-rust-health-policy.jq + polymarket-reference-collector-shadow@.service + polymarket-reference-collector.service + polymarket-reference-upload.service + polymarket-reference-upload.timer + polymarket-market-tape-upload.service + polymarket-market-tape-upload.timer +) + +die() { + printf 'Polymarket shadow gate failed: %s\n' "$*" >&2 + exit 1 +} + +usage() { + printf '%s\n' \ + 'Usage: polymarket-raw-ops-shadow-gate.sh ' \ + '' \ + 'A production-eligible gate observes for 3600 seconds plus a 300-second current-hour parity tail.' +} + +bundle_sha256() { + ( + cd "$SCRIPT_DIR" + sha256sum "${BUNDLE_ASSETS[@]}" | sha256sum | awk '{print $1}' + ) +} + +direct_directory() { + local path=$1 + [[ -d $path && ! -L $path && $(readlink -f -- "$path") == "$path" ]] +} + +direct_directory_or_absent() { + local path=$1 + [[ ! -e $path && ! -L $path ]] || direct_directory "$path" +} + +secure_release_directory() { + local path=$1 owner mode + direct_directory "$path" || return 1 + owner=$(stat -c %u -- "$path") || return 1 + mode=$(stat -c %a -- "$path") || return 1 + [[ $owner == 0 && $mode == 755 ]] +} + +secure_control_file() { + local path=$1 mode owner + [[ -f $path && ! -L $path ]] || die "missing direct control-plane file: $path" + owner=$(stat -c %u -- "$path") + mode=$(stat -c %a -- "$path") + [[ $owner == 0 ]] || die "control-plane file is not root-owned: $path" + (( (8#$mode & 022) == 0 )) || die "control-plane file is group/world writable: $path" +} + +effective_exec_argv() { + local unit=$1 raw argv + raw=$(systemctl show --property=ExecStart --value "$unit") || return 1 + argv=$(sed -nE 's/^.*argv\[\]=([^;]+);.*$/\1/p' <<<"$raw" \ + | sed -E 's/[[:space:]]+$//') + [[ -n $argv ]] || return 1 + printf '%s\n' "$argv" +} + +proc_cmdline() { + local pid=$1 + [[ $pid =~ ^[1-9][0-9]*$ && -r /proc/$pid/cmdline ]] || return 1 + tr '\0' ' ' <"/proc/$pid/cmdline" +} + +verify_legacy_identity() { + local expected_pid=$1 pid restarts fragment drop_ins exec_argv cmdline + systemctl is-active --quiet "$LEGACY_UNIT" || return 1 + fragment=$(systemctl show --property=FragmentPath --value "$LEGACY_UNIT") || return 1 + [[ $fragment == "$LEGACY_FRAGMENT" ]] || return 1 + drop_ins=$(systemctl show --property=DropInPaths --value "$LEGACY_UNIT") || return 1 + [[ -z $drop_ins ]] || return 1 + exec_argv=$(effective_exec_argv "$LEGACY_UNIT") || return 1 + [[ $exec_argv == "$LEGACY_EXEC" ]] || return 1 + pid=$(systemctl show --property=MainPID --value "$LEGACY_UNIT") || return 1 + [[ $pid == "$expected_pid" ]] || return 1 + restarts=$(systemctl show --property=NRestarts --value "$LEGACY_UNIT") || return 1 + [[ $restarts == 0 ]] || return 1 + cmdline=$(proc_cmdline "$pid") || return 1 + [[ $cmdline == "$LEGACY_EXEC " ]] +} + +env_value() { + local key=$1 file=${2:-$UPLOAD_ENV} count value + count=$(grep -c "^${key}=" "$file" || true) + [[ $count -eq 1 ]] || die "$file must contain exactly one $key" + value=$(sed -n "s/^${key}=//p" "$file") + [[ -n $value ]] || die "$file has an empty $key" + printf '%s\n' "$value" +} + +oss_config_sha256() { + local file=${1:-$UPLOAD_ENV} key + for key in OSS_BUCKET OSS_ENDPOINT OSS_REGION ALIYUN_PROFILE \ + ZSTD_TIMEOUT_SECONDS OSS_COPY_TIMEOUT_SECONDS; do + printf '%s=%s\n' "$key" "$(env_value "$key" "$file")" + done | sha256sum | awk '{print $1}' +} + +load_oss_config_snapshot() { + oss_bucket=$(env_value OSS_BUCKET) + oss_endpoint=$(env_value OSS_ENDPOINT) + oss_region=$(env_value OSS_REGION) + aliyun_profile=$(env_value ALIYUN_PROFILE) + zstd_timeout_seconds=$(env_value ZSTD_TIMEOUT_SECONDS) + oss_copy_timeout_seconds=$(env_value OSS_COPY_TIMEOUT_SECONDS) + oss_config_sha=$(printf '%s\n' \ + "OSS_BUCKET=$oss_bucket" \ + "OSS_ENDPOINT=$oss_endpoint" \ + "OSS_REGION=$oss_region" \ + "ALIYUN_PROFILE=$aliyun_profile" \ + "ZSTD_TIMEOUT_SECONDS=$zstd_timeout_seconds" \ + "OSS_COPY_TIMEOUT_SECONDS=$oss_copy_timeout_seconds" \ + | sha256sum | awk '{print $1}') + [[ $(oss_config_sha256) == "$oss_config_sha" ]] \ + || die 'OSS configuration changed while it was being snapshotted' +} + +verify_current_oss_config() { + [[ $(oss_config_sha256) == "$oss_config_sha" ]] \ + || die 'OSS configuration changed during the shadow gate' +} + +install_pinned_upload_env() { + local destination=$1 temporary + if [[ -e $destination || -L $destination ]]; then + secure_control_file "$destination" + [[ $(oss_config_sha256 "$destination") == "$oss_config_sha" ]] \ + || die 'existing pinned OSS environment differs from the gate configuration' + return 0 + fi + temporary="${destination}.new.$$" + ( + umask 077 + printf '%s\n' \ + "OSS_BUCKET=$oss_bucket" \ + "OSS_ENDPOINT=$oss_endpoint" \ + "OSS_REGION=$oss_region" \ + "ALIYUN_PROFILE=$aliyun_profile" \ + "ZSTD_TIMEOUT_SECONDS=$zstd_timeout_seconds" \ + "OSS_COPY_TIMEOUT_SECONDS=$oss_copy_timeout_seconds" >"$temporary" + ) + chmod 0640 "$temporary" + chown root:root "$temporary" + mv -Tf "$temporary" "$destination" + sync "$destination" + secure_control_file "$destination" + [[ $(oss_config_sha256 "$destination") == "$oss_config_sha" ]] \ + || die 'pinned OSS environment identity mismatch' +} + +verify_shadow_identity() { + local expected_pid=$1 pid restarts fragment drop_ins exec_argv cmdline + local expected_exec_raw expected_exec_expanded + systemctl is-active --quiet "$shadow_unit" || return 1 + fragment=$(systemctl show --property=FragmentPath --value "$shadow_unit") || return 1 + [[ $fragment == "$SHADOW_FRAGMENT" ]] || return 1 + drop_ins=$(systemctl show --property=DropInPaths --value "$shadow_unit") || return 1 + [[ -z $drop_ins ]] || return 1 + expected_exec_raw="$release_binary collect-reference --spool-dir \${MONDAY_POLYMARKET_SHADOW_SPOOL}" + expected_exec_expanded="$release_binary collect-reference --spool-dir $shadow_spool" + exec_argv=$(effective_exec_argv "$shadow_unit") || return 1 + [[ $exec_argv == "$expected_exec_raw" || $exec_argv == "$expected_exec_expanded" ]] \ + || return 1 + pid=$(systemctl show --property=MainPID --value "$shadow_unit") || return 1 + [[ $pid == "$expected_pid" ]] || return 1 + restarts=$(systemctl show --property=NRestarts --value "$shadow_unit") || return 1 + [[ $restarts == 0 ]] || return 1 + [[ $(readlink -f "/proc/$pid/exe") == "$release_binary" ]] || return 1 + cmdline=$(proc_cmdline "$pid") || return 1 + [[ $cmdline == "$release_binary collect-reference --spool-dir $shadow_spool " ]] +} + +[[ ${EUID} -eq 0 ]] || die 'must run as root' +[[ $# -eq 3 ]] || { + usage >&2 + exit 2 +} + +for command in awk chown chmod date flock grep install jq mkdir mktemp mountpoint mv \ + readlink rm runuser sed sha256sum sleep stat sync systemctl tr; do + command -v "$command" >/dev/null 2>&1 || die "missing required command: $command" +done + +candidate_source=$1 +candidate_sha=$(printf '%s' "$2" | tr '[:upper:]' '[:lower:]') +source_revision=$(printf '%s' "$3" | tr '[:upper:]' '[:lower:]') +[[ $candidate_sha =~ ^[a-f0-9]{64}$ ]] || die 'candidate SHA-256 is invalid' +[[ $source_revision =~ ^[a-f0-9]{40,64}$ ]] || die 'source revision is invalid' +[[ -f $candidate_source && ! -L $candidate_source && -x $candidate_source ]] \ + || die 'candidate must be a direct executable regular file' +printf '%s %s\n' "$candidate_sha" "$candidate_source" \ + | sha256sum --check --strict >/dev/null || die 'candidate checksum mismatch' + +for asset in "${BUNDLE_ASSETS[@]}"; do + secure_control_file "$SCRIPT_DIR/$asset" +done +secure_control_file "$UPLOAD_ENV" +deployment_bundle_sha=$(bundle_sha256) +load_oss_config_snapshot +mountpoint -q /data || die '/data must be a mount point' + +for path in /opt/monday /opt/monday/releases "$RELEASE_ROOT" \ + /data/monday /data/monday/spool "$SHADOW_ROOT" \ + /data/monday/evidence "$EVIDENCE_ROOT"; do + direct_directory_or_absent "$path" || die "fixed path is indirect or a symlink: $path" +done + +install -d -m 0755 "$(dirname "$LOCK_FILE")" +exec 9>"$LOCK_FILE" +flock -n 9 || die 'another Polymarket release operation is running' + +legacy_pid=$(systemctl show --property=MainPID --value "$LEGACY_UNIT") +[[ $legacy_pid =~ ^[1-9][0-9]*$ ]] || die 'active Python collector has no verifiable MainPID' +verify_legacy_identity "$legacy_pid" \ + || die 'active reference collector identity is not exact and restart-free' + +gate_seconds=${MONDAY_POLYMARKET_GATE_SECONDS:-$MINIMUM_GATE_SECONDS} +[[ $gate_seconds =~ ^[1-9][0-9]*$ ]] || die 'gate duration must be a positive integer' +test_only=false +if ((gate_seconds < MINIMUM_GATE_SECONDS)); then + [[ ${MONDAY_ALLOW_SHORT_GATE_FOR_TESTS:-0} == 1 ]] \ + || die 'short gates require MONDAY_ALLOW_SHORT_GATE_FOR_TESTS=1' + test_only=true +fi + +release_dir="$RELEASE_ROOT/$candidate_sha" +release_binary="$release_dir/polymarket-raw-ops" +cleanup() { + local status=$? + trap - EXIT + systemctl stop "${shadow_unit:-}" >/dev/null 2>&1 || true + rm -rf "${staging:-}" + rm -f "${shadow_env_file:-}" "${shadow_env_tmp:-}" + exit "$status" +} +trap cleanup EXIT +if [[ -e $release_dir || -L $release_dir ]]; then + secure_release_directory "$release_dir" \ + || die 'existing candidate release directory is not root-owned mode 0755' + secure_control_file "$release_binary" + [[ -x $release_binary ]] || die 'existing release is not executable' + printf '%s %s\n' "$candidate_sha" "$release_binary" \ + | sha256sum --check --strict >/dev/null || die 'existing release identity mismatch' +else + install -d -m 0755 "$RELEASE_ROOT" + staging=$(mktemp -d "$RELEASE_ROOT/.${candidate_sha}.new.XXXXXX") + install -m 0755 "$candidate_source" "$staging/polymarket-raw-ops" + printf '%s %s\n' "$candidate_sha" "$staging/polymarket-raw-ops" \ + | sha256sum --check --strict >/dev/null + chown root:root "$staging" + chmod 0755 "$staging" + secure_release_directory "$staging" \ + || die 'staged release directory is not root-owned mode 0755' + mv "$staging" "$release_dir" + staging= +fi +secure_release_directory "$release_dir" \ + || die 'candidate release directory is not root-owned mode 0755' +secure_control_file "$release_binary" +[[ -x $release_binary ]] || die 'candidate release is not executable' +pinned_upload_env="$release_dir/polymarket-upload-env-$oss_config_sha.env" +install_pinned_upload_env "$pinned_upload_env" + +run_id="$(date -u +%Y%m%dT%H%M%SZ)-$$" +shadow_parent="$SHADOW_ROOT/$candidate_sha" +shadow_spool="$shadow_parent/$run_id" +market_shadow_spool="$shadow_parent/${run_id}-market-upload" +shadow_unit="polymarket-reference-collector-shadow@${candidate_sha}.service" +[[ ! -e $shadow_spool && ! -L $shadow_spool ]] \ + || die 'refusing to reuse a shadow spool run' +[[ ! -e $market_shadow_spool && ! -L $market_shadow_spool ]] \ + || die 'refusing to reuse a market upload shadow spool run' +install -d -m 0755 /data/monday /data/monday/spool "$SHADOW_ROOT" "$shadow_parent" +for path in /data/monday /data/monday/spool "$SHADOW_ROOT" "$shadow_parent"; do + direct_directory "$path" || die "created shadow path is indirect: $path" +done +install -d -m 0750 -o hftcollector -g hftcollector "$shadow_spool" +install -d -m 0750 -o hftcollector -g hftcollector "$market_shadow_spool" + +install -d -m 0755 /run/monday +shadow_env_file="/run/monday/polymarket-reference-shadow-${candidate_sha}.env" +# A killed gate can leave its isolated unit/env behind. The global release lock +# proves there is no live gate owner, so stop only that shadow instance and +# replace its root-owned environment with this run's unique spool. +systemctl stop "$shadow_unit" >/dev/null 2>&1 || true +if [[ -e $shadow_env_file || -L $shadow_env_file ]]; then + secure_control_file "$shadow_env_file" + rm -f "$shadow_env_file" +fi +shadow_env_tmp="${shadow_env_file}.new.$$" +printf 'MONDAY_POLYMARKET_SHADOW_SPOOL=%s\n' "$shadow_spool" >"$shadow_env_tmp" +chmod 0644 "$shadow_env_tmp" +mv "$shadow_env_tmp" "$shadow_env_file" +shadow_env_tmp= + +install -m 0644 "$SERVICE_TEMPLATE" \ + /etc/systemd/system/polymarket-reference-collector-shadow@.service +systemctl daemon-reload + +started_at_unix=$(date -u +%s) +started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) +start_uptime=$SECONDS +systemctl start "$shadow_unit" + +last_health= +last_health_change=$start_uptime +last_legacy_health= +last_legacy_health_change=$start_uptime +shadow_pid= +initial_shadow_pid= +common_cutoff= +parity_window_started_at= +while :; do + now_uptime=$SECONDS + elapsed=$((now_uptime - start_uptime)) + verify_legacy_identity "$legacy_pid" \ + || die 'legacy collector PID, restart count, or effective unit identity changed during gate' + shadow_pid=$(systemctl show --property=MainPID --value "$shadow_unit") + [[ $shadow_pid =~ ^[1-9][0-9]*$ ]] || die 'Rust shadow has no MainPID' + if [[ -z $initial_shadow_pid ]]; then + initial_shadow_pid=$shadow_pid + else + [[ $shadow_pid == "$initial_shadow_pid" ]] || die 'Rust shadow MainPID changed during gate' + fi + verify_shadow_identity "$initial_shadow_pid" \ + || die 'Rust shadow systemd identity, PID, or command line changed during gate' + if ((elapsed >= HEALTH_SETTLE_SECONDS)) || [[ $test_only == true ]]; then + health="$shadow_spool/health.json" + [[ -f $health && ! -L $health ]] || die 'Rust shadow health is missing' + jq -e -f "$RUST_HEALTH_POLICY" "$health" >/dev/null \ + || die 'Rust shadow health is not fail-closed clean' + current_health=$(jq -r '.updated_at' "$health") + if [[ $current_health != "$last_health" ]]; then + last_health=$current_health + last_health_change=$now_uptime + fi + ((now_uptime - last_health_change <= MAX_HEALTH_SILENCE_SECONDS)) \ + || die 'Rust shadow health stopped advancing' + + legacy_health="$LEGACY_SPOOL/health.json" + [[ -f $legacy_health && ! -L $legacy_health ]] || die 'Python health is missing' + jq -e -f "$LEGACY_HEALTH_POLICY" "$legacy_health" >/dev/null \ + || die 'Python health is not fail-closed clean during shadow' + current_legacy_health=$(jq -r '.updated_at' "$legacy_health") + if [[ $current_legacy_health != "$last_legacy_health" ]]; then + last_legacy_health=$current_legacy_health + last_legacy_health_change=$now_uptime + fi + ((now_uptime - last_legacy_health_change <= MAX_HEALTH_SILENCE_SECONDS)) \ + || die 'Python health stopped advancing during shadow' + + rust_success_at=$(jq -er '.last_success_at | select(type == "string" and length > 0)' \ + "$health") || die 'Rust health has no last_success_at' + legacy_success_at=$(jq -er '.last_success_at | select(type == "string" and length > 0)' \ + "$legacy_health") || die 'Python health has no last_success_at' + rust_success_epoch=$(date -u -d "$rust_success_at" +%s) \ + || die 'Rust last_success_at is invalid' + legacy_success_epoch=$(date -u -d "$legacy_success_at" +%s) \ + || die 'Python last_success_at is invalid' + now_epoch=$(date -u +%s) + ((rust_success_epoch <= now_epoch && now_epoch - rust_success_epoch <= MAX_HEALTH_SILENCE_SECONDS)) \ + || die 'Rust last_success_at is stale or from the future' + ((legacy_success_epoch <= now_epoch && now_epoch - legacy_success_epoch <= MAX_HEALTH_SILENCE_SECONDS)) \ + || die 'Python last_success_at is stale or from the future' + common_cutoff=$rust_success_epoch + ((legacy_success_epoch < common_cutoff)) && common_cutoff=$legacy_success_epoch + if [[ $test_only == false ]]; then + common_cutoff=$((common_cutoff - PARITY_CUTOFF_LAG_SECONDS)) + fi + parity_window_started_at=$((common_cutoff - common_cutoff % 3600)) + ((parity_window_started_at >= started_at_unix)) \ + || parity_window_started_at=$started_at_unix + fi + + if ((elapsed >= gate_seconds)) && [[ -n $common_cutoff ]]; then + if [[ $test_only == true ]] \ + || ((common_cutoff - parity_window_started_at >= PARITY_TAIL_SECONDS)); then + break + fi + fi + + sleep_for=$SAMPLE_SECONDS + if ((elapsed < gate_seconds)); then + remaining=$((gate_seconds - elapsed)) + ((remaining < sleep_for)) && sleep_for=$remaining + fi + sleep "$sleep_for" +done + +observed_duration_seconds=$elapsed +[[ -n $common_cutoff && -n $parity_window_started_at ]] \ + || die 'no common successful collection cutoff was observed' +if [[ $test_only == false ]]; then + ((observed_duration_seconds >= MINIMUM_GATE_SECONDS)) \ + || die 'production shadow duration is shorter than required' + ((common_cutoff - parity_window_started_at >= PARITY_TAIL_SECONDS)) \ + || die 'production parity window has less than five minutes in one UTC hour' +fi + +shadow_pid=$(systemctl show --property=MainPID --value "$shadow_unit") +[[ $shadow_pid =~ ^[1-9][0-9]*$ ]] || die 'Rust shadow has no final MainPID' +shadow_restarts=$(systemctl show --property=NRestarts --value "$shadow_unit") +[[ $shadow_restarts == 0 && $shadow_pid == "$initial_shadow_pid" ]] \ + || die 'Rust shadow did not remain a single continuous process' +verify_shadow_identity "$initial_shadow_pid" \ + || die 'final Rust shadow systemd identity differs from the gated candidate' +verify_legacy_identity "$legacy_pid" \ + || die 'legacy collector identity changed before parity evidence was captured' +shadow_exec_argv=$(effective_exec_argv "$shadow_unit") \ + || die 'could not capture the effective Rust shadow ExecStart' +shadow_cmdline=$(proc_cmdline "$initial_shadow_pid") \ + || die 'could not capture the exact Rust shadow command line' +shadow_cmdline_argv=${shadow_cmdline% } +shadow_fragment_path=$(systemctl show --property=FragmentPath --value "$shadow_unit") +shadow_drop_ins=$(systemctl show --property=DropInPaths --value "$shadow_unit") +shadow_drop_ins_json=$(jq -cn --arg value "$shadow_drop_ins" \ + '$value | split(" ") | map(select(length > 0))') +systemctl stop "$shadow_unit" + +evidence_parent="$EVIDENCE_ROOT/$candidate_sha" +install -d -m 0755 /data/monday/evidence "$EVIDENCE_ROOT" "$evidence_parent" +for path in /data/monday/evidence "$EVIDENCE_ROOT" "$evidence_parent"; do + direct_directory "$path" || die "evidence path is indirect: $path" +done +evidence_dir="$evidence_parent/$run_id" +mkdir -m 0750 "$evidence_dir" || die 'evidence run already exists' +parity_json="$evidence_dir/parity.json" +"$release_binary" verify-shadow-parity \ + --legacy-spool "$LEGACY_SPOOL" \ + --rust-spool "$shadow_spool" \ + --started-at-unix "$parity_window_started_at" \ + --ended-at-unix "$common_cutoff" \ + --output "$parity_json" || die 'byte/field/dedupe/settlement/rotation parity failed' + +verify_current_oss_config +upload_json=$(runuser -u hftcollector -- env HOME=/var/lib/hft-collector \ + "$release_binary" upload \ + --spool-dir "$shadow_spool" \ + --dataset crypto_expiry_reference_rust_shadow \ + --quote-depth-levels 0 \ + --quote-sample-ms 0 \ + --bucket "$oss_bucket" \ + --endpoint "$oss_endpoint" \ + --region "$oss_region" \ + --profile "$aliyun_profile" \ + --zstd-timeout "$zstd_timeout_seconds" \ + --oss-timeout "$oss_copy_timeout_seconds") \ + || die 'shadow OSS upload/readback failed' +verify_current_oss_config +uploaded_segments=$(jq -er '.uploaded_segments | select(type == "number" and floor == . and . > 0)' \ + <<<"$upload_json") || die 'shadow uploader did not verify a closed segment' +canonical_uploaded_segments=$(jq -er \ + '.canonical_uploaded_segments | select(type == "number" and floor == . and . > 0)' \ + <<<"$upload_json") || die 'shadow uploader did not verify a canonical closed segment' + +# Exercise the market-tape validation/upload/readback path with a deterministic, +# closed fixture. A content-addressed rerun must verify the same remote triplet. +market_fixture="$market_shadow_spool/market-updates.20000101T000003.ndjson" +jq -cn '{sequence:0,recorded_at:"2000-01-01T00:00:00Z",update:{ + kind:"event_discovered",event_id:"shadow-market-upload",symbol:"BTCUSDT", + up_token:"shadow-up",down_token:"shadow-down",end_time:"2000-01-01T00:05:00Z", + window_secs:300,price_to_beat:"100",resolved_up_won:null}}' >"$market_fixture" +jq -cn '{sequence:1,recorded_at:"2000-01-01T00:00:01Z",update:{ + kind:"quote",token_id:"shadow-up",bid:"0.49",ask:"0.51", + bid_size:"10",ask_size:"11",bid_levels:[{price:"0.49",size:"10"}], + ask_levels:[{price:"0.51",size:"11"}],ts:"2000-01-01T00:00:01Z"}}' \ + >>"$market_fixture" +jq -cn '{sequence:2,recorded_at:"2000-01-01T00:00:02Z",update:{ + kind:"reference_price",symbol:"BTCUSDT",source:"binance",asset_class:"crypto", + price:"100",full_accuracy_value:null,is_carried_forward:false, + ts:"2000-01-01T00:00:02Z"}}' >>"$market_fixture" +chown hftcollector:hftcollector "$market_fixture" +chmod 0640 "$market_fixture" +sync "$market_fixture" + +verify_current_oss_config +market_upload_json=$(runuser -u hftcollector -- env HOME=/var/lib/hft-collector \ + "$release_binary" upload \ + --spool-dir "$market_shadow_spool" \ + --dataset crypto_expiry_market_rust_shadow \ + --quote-depth-levels 0 \ + --quote-sample-ms 1000 \ + --bucket "$oss_bucket" \ + --endpoint "$oss_endpoint" \ + --region "$oss_region" \ + --profile "$aliyun_profile" \ + --zstd-timeout "$zstd_timeout_seconds" \ + --oss-timeout "$oss_copy_timeout_seconds") \ + || die 'market-tape shadow OSS upload/readback failed' +verify_current_oss_config +market_uploaded_segments=$(jq -er \ + '.uploaded_segments | select(type == "number" and floor == . and . > 0)' \ + <<<"$market_upload_json") \ + || die 'market-tape shadow uploader did not verify a closed segment' +market_canonical_uploaded_segments=$(jq -er \ + '.canonical_uploaded_segments | select(type == "number" and floor == . and . > 0)' \ + <<<"$market_upload_json") \ + || die 'market-tape shadow uploader did not verify a canonical closed segment' + +verify_legacy_identity "$legacy_pid" \ + || die 'legacy collector identity changed while parity or OSS readback was running' +verify_current_oss_config +legacy_exec_argv=$(effective_exec_argv "$LEGACY_UNIT") \ + || die 'could not capture the effective legacy ExecStart' +legacy_cmdline=$(proc_cmdline "$legacy_pid") \ + || die 'could not capture the exact legacy command line' +legacy_cmdline_argv=${legacy_cmdline% } +legacy_cmdline_sha=$(printf '%s' "$legacy_cmdline_argv" | sha256sum | awk '{print $1}') +legacy_fragment_path=$(systemctl show --property=FragmentPath --value "$LEGACY_UNIT") +legacy_drop_ins=$(systemctl show --property=DropInPaths --value "$LEGACY_UNIT") +legacy_drop_ins_json=$(jq -cn --arg value "$legacy_drop_ins" \ + '$value | split(" ") | map(select(length > 0))') +legacy_restarts=$(systemctl show --property=NRestarts --value "$LEGACY_UNIT") + +completed_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) +production_eligible=true +[[ $test_only == false ]] || production_eligible=false +gate_tmp="$evidence_dir/.gate.json.tmp" +gate_json="$evidence_dir/gate.json" +jq \ + --arg candidate_sha256 "$candidate_sha" \ + --arg deployment_bundle_sha256 "$deployment_bundle_sha" \ + --arg deployment_source_revision "$source_revision" \ + --arg oss_config_sha256 "$oss_config_sha" \ + --arg started_at "$started_at" \ + --arg completed_at "$completed_at" \ + --arg legacy_exec "$legacy_exec_argv" \ + --arg legacy_cmdline "$legacy_cmdline_argv" \ + --arg legacy_cmdline_sha256 "$legacy_cmdline_sha" \ + --arg legacy_fragment_path "$legacy_fragment_path" \ + --argjson legacy_drop_in_paths "$legacy_drop_ins_json" \ + --argjson legacy_pid "$legacy_pid" \ + --argjson legacy_restarts "$legacy_restarts" \ + --arg shadow_exec "$shadow_exec_argv" \ + --arg shadow_cmdline "$shadow_cmdline_argv" \ + --arg shadow_fragment_path "$shadow_fragment_path" \ + --argjson shadow_drop_in_paths "$shadow_drop_ins_json" \ + --argjson shadow_pid "$shadow_pid" \ + --argjson shadow_restarts "$shadow_restarts" \ + --arg shadow_run_id "$run_id" \ + --argjson duration_seconds "$observed_duration_seconds" \ + --argjson parity_window_started_at_unix "$parity_window_started_at" \ + --argjson parity_window_ended_at_unix "$common_cutoff" \ + --argjson production_eligible "$production_eligible" \ + --argjson uploaded_segments "$uploaded_segments" \ + --argjson canonical_uploaded_segments "$canonical_uploaded_segments" \ + --argjson market_uploaded_segments "$market_uploaded_segments" \ + --argjson market_canonical_uploaded_segments "$market_canonical_uploaded_segments" \ + '. + { + schema:"monday.polymarket_shadow_gate.v1", + candidate_sha256:$candidate_sha256, + deployment_bundle_sha256:$deployment_bundle_sha256, + deployment_source_revision:$deployment_source_revision, + oss_config_sha256:$oss_config_sha256, + started_at:$started_at, + completed_at:$completed_at, + shadow_run_id:$shadow_run_id, + duration_seconds:$duration_seconds, + parity_window_started_at_unix:$parity_window_started_at_unix, + parity_window_ended_at_unix:$parity_window_ended_at_unix, + production_eligible:$production_eligible, + legacy_runtime:{exec_start:$legacy_exec,cmdline:$legacy_cmdline, + cmdline_sha256:$legacy_cmdline_sha256, + fragment_path:$legacy_fragment_path,drop_in_paths:$legacy_drop_in_paths, + main_pid:$legacy_pid,restarts:$legacy_restarts}, + shadow_runtime:{exec_start:$shadow_exec,cmdline:$shadow_cmdline, + fragment_path:$shadow_fragment_path,drop_in_paths:$shadow_drop_in_paths, + main_pid:$shadow_pid,restarts:$shadow_restarts}, + checks:(.checks + { + health_freshness:true, + candidate_identity:true, + oss_readback_parity:true, + market_oss_readback_parity:true + }), + metrics:(.metrics + { + oss_uploaded_segments:$uploaded_segments, + oss_canonical_uploaded_segments:$canonical_uploaded_segments, + market_oss_uploaded_segments:$market_uploaded_segments, + market_oss_canonical_uploaded_segments:$market_canonical_uploaded_segments + }) + } | .passed = (.passed and ([.checks[]] | all))' \ + "$parity_json" >"$gate_tmp" +mv "$gate_tmp" "$gate_json" +sync "$gate_json" + +if [[ $production_eligible == true ]]; then + verify_legacy_identity "$legacy_pid" \ + || die 'legacy collector identity changed before the gate marker was published' + verify_current_oss_config + jq -e -f "$GATE_POLICY" "$gate_json" >/dev/null \ + || die 'combined gate evidence failed the production policy' + marker="$evidence_dir/PASSED.sha256" + ( + cd "$evidence_dir" + sha256sum gate.json >".${marker##*/}.tmp" + mv ".${marker##*/}.tmp" "${marker##*/}" + ) +fi + +printf '%s\n' "$gate_json" diff --git a/deployment/aliyun/polymarket-reference-collector-shadow@.service b/deployment/aliyun/polymarket-reference-collector-shadow@.service new file mode 100644 index 000000000..051a060ed --- /dev/null +++ b/deployment/aliyun/polymarket-reference-collector-shadow@.service @@ -0,0 +1,32 @@ +[Unit] +Description=Monday Polymarket Rust reference shadow for artifact %i +After=network-online.target +Wants=network-online.target +RequiresMountsFor=/data +ConditionPathIsMountPoint=/data + +[Service] +Type=simple +User=hftcollector +Group=hftcollector +EnvironmentFile=/run/monday/polymarket-reference-shadow-%i.env +ExecStart=/opt/monday/releases/polymarket-raw-ops/%i/polymarket-raw-ops collect-reference --spool-dir ${MONDAY_POLYMARKET_SHADOW_SPOOL} +Restart=on-failure +RestartSec=5 +RuntimeMaxSec=7500 +TimeoutStopSec=30 +NoNewPrivileges=true +PrivateTmp=true +ProtectSystem=strict +ProtectHome=true +ProtectKernelTunables=true +ProtectKernelModules=true +ProtectControlGroups=true +LockPersonality=true +RestrictSUIDSGID=true +ReadWritePaths=/data/monday/spool/polymarket-reference-rust-shadow/%i +RestrictAddressFamilies=AF_INET AF_INET6 +UMask=0027 +CPUQuota=20% +MemoryHigh=256M +MemoryMax=384M diff --git a/deployment/aliyun/polymarket-reference-collector.service b/deployment/aliyun/polymarket-reference-collector.service index cc0b8bf2c..3711886e7 100644 --- a/deployment/aliyun/polymarket-reference-collector.service +++ b/deployment/aliyun/polymarket-reference-collector.service @@ -9,7 +9,7 @@ ConditionPathIsMountPoint=/data Type=simple User=hftcollector Group=hftcollector -ExecStart=/usr/bin/python3 /opt/monday/bin/polymarket_reference_collector.py +ExecStart=/opt/monday/bin/polymarket-raw-ops collect-reference Restart=always RestartSec=5 RuntimeMaxSec=21600 diff --git a/deployment/aliyun/polymarket-reference-upload.service b/deployment/aliyun/polymarket-reference-upload.service index d939ebb96..e2948e4eb 100644 --- a/deployment/aliyun/polymarket-reference-upload.service +++ b/deployment/aliyun/polymarket-reference-upload.service @@ -11,7 +11,8 @@ User=hftcollector Group=hftcollector Environment=HOME=/var/lib/hft-collector EnvironmentFile=/etc/monday/polymarket-market-tape-upload.env -ExecStart=/usr/bin/python3 /opt/monday/bin/polymarket_market_tape_upload.py --spool-dir /data/monday/spool/polymarket-reference --dataset crypto_expiry_reference --quote-depth-levels 0 --quote-sample-ms 0 +ExecStart=/opt/monday/bin/polymarket-raw-ops upload --spool-dir /data/monday/spool/polymarket-reference --dataset crypto_expiry_reference --quote-depth-levels 0 --quote-sample-ms 0 +TimeoutStartSec=0 NoNewPrivileges=true PrivateTmp=true ProtectSystem=strict diff --git a/deployment/aliyun/polymarket-rust-health-policy.jq b/deployment/aliyun/polymarket-rust-health-policy.jq new file mode 100644 index 000000000..6dd7053ff --- /dev/null +++ b/deployment/aliyun/polymarket-rust-health-policy.jq @@ -0,0 +1,13 @@ +(.updated_at | type == "string" and length > 0) +and (.last_success_at | type == "string" and length > 0) +and (.target_markets | type == "number" and floor == . and . > 0) +and .missing_target_symbols == [] +and .api_errors == [] +and .malformed_trade_rows == 0 +and .truncated_trade_markets == [] +and .non_object_trade_markets == [] +and .invalid_settlement_markets == [] +and .invalid_end_time_markets == [] +and .stale_trade_markets == [] +and .stale_settlement_markets == [] +and .overdue_unresolved_markets == [] diff --git a/deployment/aliyun/polymarket-shadow-gate-policy.jq b/deployment/aliyun/polymarket-shadow-gate-policy.jq new file mode 100644 index 000000000..2850e0c57 --- /dev/null +++ b/deployment/aliyun/polymarket-shadow-gate-policy.jq @@ -0,0 +1,64 @@ +def sha256: type == "string" and test("^[a-f0-9]{64}$"); +def positive_integer: type == "number" and floor == . and . > 0; + +.schema == "monday.polymarket_shadow_gate.v1" +and (.candidate_sha256 | sha256) +and (.deployment_bundle_sha256 | sha256) +and (.deployment_source_revision | type == "string" and test("^[a-f0-9]{40,64}$")) +and (.oss_config_sha256 | sha256) +and (.duration_seconds | positive_integer and . >= 3900) +and (.parity_window_started_at_unix | positive_integer) +and (.parity_window_ended_at_unix | positive_integer) +and (.parity_window_ended_at_unix - .parity_window_started_at_unix >= 300) +and (.completed_at | type == "string" and (fromdateiso8601? | type == "number")) +and .production_eligible == true +and .passed == true +and .legacy_runtime.exec_start == "/usr/bin/python3 /opt/monday/bin/polymarket_reference_collector.py" +and .legacy_runtime.cmdline == "/usr/bin/python3 /opt/monday/bin/polymarket_reference_collector.py" +and .legacy_runtime.cmdline_sha256 == "dffeb118d105e9312898460249f514eb982c20433cd20840ffb2107c64bbca4a" +and .legacy_runtime.fragment_path == "/etc/systemd/system/polymarket-reference-collector.service" +and .legacy_runtime.drop_in_paths == [] +and (.legacy_runtime.main_pid | positive_integer) +and .legacy_runtime.restarts == 0 +and ( + .shadow_runtime.exec_start == ( + "/opt/monday/releases/polymarket-raw-ops/" + .candidate_sha256 + + "/polymarket-raw-ops collect-reference --spool-dir ${MONDAY_POLYMARKET_SHADOW_SPOOL}" + ) + or .shadow_runtime.exec_start == .shadow_runtime.cmdline +) +and .shadow_runtime.cmdline == ( + "/opt/monday/releases/polymarket-raw-ops/" + .candidate_sha256 + + "/polymarket-raw-ops collect-reference --spool-dir " + + "/data/monday/spool/polymarket-reference-rust-shadow/" + + .candidate_sha256 + "/" + .shadow_run_id +) +and .shadow_runtime.fragment_path == "/etc/systemd/system/polymarket-reference-collector-shadow@.service" +and .shadow_runtime.drop_in_paths == [] +and (.shadow_runtime.main_pid | positive_integer) +and .shadow_runtime.restarts == 0 +and .checks.byte_parity == true +and .checks.metadata_parity == true +and .checks.field_parity == true +and .checks.dedupe_parity == true +and .checks.settlement_parity == true +and .checks.rotation_parity == true +and .checks.asset_parity == true +and .checks.health_freshness == true +and .checks.candidate_identity == true +and .checks.oss_readback_parity == true +and .checks.market_oss_readback_parity == true +and (.metrics.oss_uploaded_segments | positive_integer) +and (.metrics.oss_canonical_uploaded_segments | positive_integer) +and (.metrics.market_oss_uploaded_segments | positive_integer) +and (.metrics.market_oss_canonical_uploaded_segments | positive_integer) +and (.metrics.rust_closed_tape_count | positive_integer) +and (.metrics.legacy_trade_count | positive_integer) +and (.metrics.rust_trade_count | positive_integer) +and (.metrics.legacy_metadata_count | positive_integer) +and (.metrics.rust_metadata_count | positive_integer) +and (.metrics.legacy_only_metadata_ids | type == "array" and length == 0) +and (.metrics.rust_only_metadata_ids | type == "array" and length == 0) +and (.metrics.legacy_settlement_count | positive_integer) +and (.metrics.rust_settlement_count | positive_integer) +and (.metrics.rust_duplicate_trade_ids | type == "array" and length == 0) diff --git a/deployment/aliyun/polymarket_market_tape_upload.py b/deployment/aliyun/polymarket_market_tape_upload.py deleted file mode 100644 index 817fd515c..000000000 --- a/deployment/aliyun/polymarket_market_tape_upload.py +++ /dev/null @@ -1,671 +0,0 @@ -#!/usr/bin/env python3 -"""Validate, compress, and upload closed Polymarket tape sessions to OSS.""" - -from __future__ import annotations - -import argparse -import hashlib -import json -import os -import re -import subprocess -import tempfile -from collections import Counter, defaultdict -from dataclasses import dataclass -from datetime import datetime, timezone -from decimal import Decimal, InvalidOperation -from pathlib import Path -from typing import Any - - -ACTIVE_TAPE = "market-updates.ndjson" -ROTATED_TAPE_GLOB = "market-updates.*.ndjson" -ALLOWED_KINDS = { - "quote", - "event_discovered", - "event_expired", - "reference_price", - "market_metadata", - "polymarket_trade", - "market_settlement", -} - - -@dataclass(frozen=True) -class Artifacts: - source: Path - data: Path - manifest: Path - success: Path - object_prefix: str - - -def utc_now() -> str: - return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") - - -def parse_timestamp(value: Any, field: str, line_number: int) -> datetime: - if not isinstance(value, str): - raise ValueError(f"line {line_number}: {field} must be a string") - try: - parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) - except ValueError as error: - raise ValueError(f"line {line_number}: invalid {field}: {value}") from error - if parsed.tzinfo is None: - raise ValueError(f"line {line_number}: {field} must include a timezone") - return parsed.astimezone(timezone.utc) - - -def decimal_or_none(value: Any, field: str, line_number: int) -> Decimal | None: - if value is None: - return None - try: - return Decimal(str(value)) - except (InvalidOperation, ValueError) as error: - raise ValueError(f"line {line_number}: {field} must be numeric") from error - - -def sha256_file(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as handle: - for chunk in iter(lambda: handle.read(1024 * 1024), b""): - digest.update(chunk) - return digest.hexdigest() - - -def atomic_json(path: Path, payload: dict[str, Any]) -> None: - temporary = path.with_suffix(path.suffix + ".tmp") - with temporary.open("w", encoding="utf-8") as handle: - json.dump(payload, handle, sort_keys=True, separators=(",", ":")) - handle.write("\n") - handle.flush() - os.fsync(handle.fileno()) - os.replace(temporary, path) - - -def discover_rotated_tapes(spool_dir: Path) -> list[Path]: - return sorted(path for path in spool_dir.glob(ROTATED_TAPE_GLOB) if path.is_file()) - - -def scan_tape(path: Path, dataset: str, quote_depth_levels: int, quote_sample_ms: int) -> dict[str, Any]: - before = path.stat() - event_types: Counter[str] = Counter() - present_fields: dict[str, Counter[str]] = defaultdict(Counter) - non_null_fields: dict[str, Counter[str]] = defaultdict(Counter) - symbols: set[str] = set() - token_ids: set[str] = set() - known_event_tokens: set[str] = set() - contextless_quote_tokens: set[str] = set() - first_recorded_at: str | None = None - last_recorded_at: str | None = None - previous_recorded_at: datetime | None = None - first_sequence: int | None = None - last_sequence: int | None = None - expected_sequence: int | None = None - crossed_quotes = 0 - one_sided_quotes = 0 - empty_quotes = 0 - out_of_range_prices = 0 - negative_sizes = 0 - max_bid_levels = 0 - max_ask_levels = 0 - contextless_quotes = 0 - market_ids: set[str] = set() - condition_ids: set[str] = set() - record_ids: set[str] = set() - record_id_versions: set[str] = set() - duplicate_record_ids = 0 - source_field_presence: dict[str, Counter[str]] = defaultdict(Counter) - source_field_non_null: dict[str, Counter[str]] = defaultdict(Counter) - - with path.open("rb") as handle: - for line_number, raw_line in enumerate(handle, start=1): - if not raw_line.endswith(b"\n"): - raise ValueError(f"line {line_number}: tape ends with an incomplete record") - try: - record = json.loads(raw_line) - except json.JSONDecodeError as error: - raise ValueError(f"line {line_number}: invalid JSON") from error - if not isinstance(record, dict): - raise ValueError(f"line {line_number}: record must be an object") - - sequence = record.get("sequence") - if not isinstance(sequence, int) or isinstance(sequence, bool) or sequence < 0: - raise ValueError(f"line {line_number}: sequence must be a non-negative integer") - if expected_sequence is None: - expected_sequence = sequence - if sequence != expected_sequence: - raise ValueError( - f"line {line_number}: sequence gap expected={expected_sequence} actual={sequence}" - ) - recorded_at = parse_timestamp(record.get("recorded_at"), "recorded_at", line_number) - if previous_recorded_at is not None and recorded_at < previous_recorded_at: - raise ValueError(f"line {line_number}: recorded_at moved backwards") - - update = record.get("update") - if not isinstance(update, dict): - raise ValueError(f"line {line_number}: update must be an object") - kind = update.get("kind") - if kind not in ALLOWED_KINDS: - raise ValueError(f"line {line_number}: unsupported update kind {kind!r}") - - event_types[kind] += 1 - for field, value in update.items(): - present_fields[kind][field] += 1 - if value is not None: - non_null_fields[kind][field] += 1 - - symbol = update.get("symbol") - if isinstance(symbol, str) and symbol: - symbols.add(symbol) - token_id = update.get("token_id") - if isinstance(token_id, str) and token_id: - token_ids.add(token_id) - if kind == "event_discovered": - for token_field in ("up_token", "down_token"): - event_token = update.get(token_field) - if isinstance(event_token, str) and event_token: - known_event_tokens.add(event_token) - - market_id = update.get("market_id") - if isinstance(market_id, str) and market_id: - market_ids.add(market_id) - condition_id = update.get("condition_id") - if isinstance(condition_id, str) and condition_id: - condition_ids.add(condition_id) - record_id = update.get("record_id") - if isinstance(record_id, str) and record_id: - if record_id in record_ids: - duplicate_record_ids += 1 - record_ids.add(record_id) - if kind == "polymarket_trade": - version = update.get("record_id_version") - record_id_versions.add( - version if isinstance(version, str) and version else "v1_legacy" - ) - - raw_field = { - "market_metadata": "market", - "polymarket_trade": "trade", - "market_settlement": "market", - }.get(kind) - if raw_field is not None: - raw_payload = update.get(raw_field) - if not isinstance(raw_payload, dict): - raise ValueError(f"line {line_number}: {kind}.{raw_field} must be an object") - for field, value in raw_payload.items(): - source_field_presence[kind][field] += 1 - if value is not None: - source_field_non_null[kind][field] += 1 - - if kind == "market_metadata": - for required in ("market_id", "condition_id", "symbol", "retrieved_at"): - if not update.get(required): - raise ValueError(f"line {line_number}: market_metadata requires {required}") - elif kind == "polymarket_trade": - for required in ( - "record_id", - "condition_id", - "token_id", - "symbol", - "side", - "trade_ts", - "transaction_hash", - ): - if update.get(required) in (None, ""): - raise ValueError(f"line {line_number}: polymarket_trade requires {required}") - if update.get("side") not in {"BUY", "SELL"}: - raise ValueError(f"line {line_number}: polymarket_trade side must be BUY or SELL") - decimal_or_none(update.get("size"), "size", line_number) - price = decimal_or_none(update.get("price"), "price", line_number) - if price is None or not Decimal("0") <= price <= Decimal("1"): - raise ValueError(f"line {line_number}: polymarket_trade price must be within [0, 1]") - parse_timestamp(update.get("trade_ts"), "trade_ts", line_number) - elif kind == "market_settlement": - for required in ( - "market_id", - "condition_id", - "symbol", - "winning_token_id", - "winning_outcome", - "resolution_source", - "retrieved_at", - ): - if update.get(required) in (None, ""): - raise ValueError(f"line {line_number}: market_settlement requires {required}") - - if kind == "quote": - if not isinstance(token_id, str) or token_id not in known_event_tokens: - contextless_quotes += 1 - if isinstance(token_id, str): - contextless_quote_tokens.add(token_id) - bid = decimal_or_none(update.get("bid"), "bid", line_number) - ask = decimal_or_none(update.get("ask"), "ask", line_number) - bid_size = decimal_or_none(update.get("bid_size"), "bid_size", line_number) - ask_size = decimal_or_none(update.get("ask_size"), "ask_size", line_number) - bid_levels = update.get("bid_levels") or [] - ask_levels = update.get("ask_levels") or [] - if not isinstance(bid_levels, list) or not isinstance(ask_levels, list): - raise ValueError(f"line {line_number}: quote levels must be arrays") - if quote_depth_levels and ( - len(bid_levels) > quote_depth_levels or len(ask_levels) > quote_depth_levels - ): - raise ValueError(f"line {line_number}: quote exceeds configured depth") - max_bid_levels = max(max_bid_levels, len(bid_levels)) - max_ask_levels = max(max_ask_levels, len(ask_levels)) - for side, levels in (("bid_levels", bid_levels), ("ask_levels", ask_levels)): - for level_index, level in enumerate(levels): - if not isinstance(level, dict): - raise ValueError( - f"line {line_number}: {side}[{level_index}] must be an object" - ) - level_price = decimal_or_none( - level.get("price"), f"{side}[{level_index}].price", line_number - ) - level_size = decimal_or_none( - level.get("size"), f"{side}[{level_index}].size", line_number - ) - if level_price is None or level_size is None: - raise ValueError( - f"line {line_number}: {side}[{level_index}] requires price and size" - ) - if not Decimal("0") <= level_price <= Decimal("1"): - out_of_range_prices += 1 - if level_size < 0: - negative_sizes += 1 - if bid is None and ask is None: - empty_quotes += 1 - elif bid is None or ask is None: - one_sided_quotes += 1 - elif bid > ask: - crossed_quotes += 1 - for price in (bid, ask): - if price is not None and not Decimal("0") <= price <= Decimal("1"): - out_of_range_prices += 1 - for size in (bid_size, ask_size): - if size is not None and size < 0: - negative_sizes += 1 - - recorded_at_text = record["recorded_at"] - first_recorded_at = first_recorded_at or recorded_at_text - last_recorded_at = recorded_at_text - first_sequence = sequence if first_sequence is None else first_sequence - last_sequence = sequence - expected_sequence = sequence + 1 - previous_recorded_at = recorded_at - - after = path.stat() - if (before.st_size, before.st_mtime_ns) != (after.st_size, after.st_mtime_ns): - raise ValueError("tape changed while being validated; refusing to archive an active file") - if first_sequence is None or first_recorded_at is None or last_recorded_at is None: - raise ValueError("tape is empty") - - partition = parse_timestamp(first_recorded_at, "recorded_at", 1) - event_context_complete = contextless_quotes == 0 - has_quotes = event_types.get("quote", 0) > 0 - has_reference_records = any( - event_types.get(kind, 0) > 0 - for kind in ("market_metadata", "polymarket_trade", "market_settlement") - ) - depth_complete = has_quotes and quote_depth_levels == 0 - temporal_updates_complete = has_quotes and quote_sample_ms == 0 - replay_scope = ( - "complete_reference_hour_segment" - if has_reference_records and not has_quotes - else ( - ( - ( - "complete_full_depth_normalized_hour_segment" - if temporal_updates_complete - else "complete_full_depth_sampled_normalized_hour_segment" - ) - if depth_complete - else "complete_sampled_normalized_hour_segment" - ) - if event_context_complete - else "sampled_normalized_hour_segment_requires_prior_event_context" - ) - ) - return { - "schema": "monday.polymarket.raw.v1", - "canonical": True, - "venue": "polymarket", - "dataset": dataset, - "format": "ndjson.zst", - "replay_scope": replay_scope, - "venue_depth_complete": depth_complete, - "temporal_updates_complete": temporal_updates_complete, - "segment_complete": True, - "source_session_closed": True, - "event_context_complete": event_context_complete, - "contextless_quote_tokens": sorted(contextless_quote_tokens), - "events": last_sequence - first_sequence + 1, - "event_types": dict(sorted(event_types.items())), - "start_sequence": first_sequence, - "end_sequence": last_sequence, - "sequence_gaps": 0, - "start_recorded_at": first_recorded_at, - "end_recorded_at": last_recorded_at, - "date": partition.strftime("%Y-%m-%d"), - "hour": partition.strftime("%H"), - "symbols": sorted(symbols), - "token_count": len(token_ids), - "market_count": len(market_ids), - "condition_count": len(condition_ids), - "record_id_versions": sorted(record_id_versions), - "recording_policy": { - "quote_sample_ms": quote_sample_ms, - "quote_depth_levels": quote_depth_levels, - "event_scoped_quotes": True, - }, - "field_presence": { - kind: dict(sorted(fields.items())) for kind, fields in sorted(present_fields.items()) - }, - "field_non_null": { - kind: dict(sorted(fields.items())) for kind, fields in sorted(non_null_fields.items()) - }, - "source_field_presence": { - kind: dict(sorted(fields.items())) - for kind, fields in sorted(source_field_presence.items()) - }, - "source_field_non_null": { - kind: dict(sorted(fields.items())) - for kind, fields in sorted(source_field_non_null.items()) - }, - "quality": { - "crossed_quotes": crossed_quotes, - "one_sided_quotes": one_sided_quotes, - "empty_quotes": empty_quotes, - "out_of_range_prices": out_of_range_prices, - "negative_sizes": negative_sizes, - "max_bid_levels": max_bid_levels, - "max_ask_levels": max_ask_levels, - "contextless_quotes": contextless_quotes, - "duplicate_record_ids": duplicate_record_ids, - }, - "source_file": path.name, - "source_bytes": before.st_size, - } - - -def split_tape_by_utc_hour(source: Path) -> list[Path]: - """Copy a validated closed tape into deterministic UTC-hour chunks.""" - staging_dir = source.parent / ".upload-staging" / source.name - staging_dir.mkdir(parents=True, exist_ok=True) - for temporary in staging_dir.glob("*.tmp"): - temporary.unlink() - - chunks: list[Path] = [] - output = None - output_path: Path | None = None - current_hour: str | None = None - try: - with source.open("rb") as handle: - for line_number, raw_line in enumerate(handle, start=1): - record = json.loads(raw_line) - recorded_at = parse_timestamp(record.get("recorded_at"), "recorded_at", line_number) - hour = recorded_at.strftime("%Y%m%dT%H") - if hour != current_hour: - if output is not None and output_path is not None: - output.flush() - os.fsync(output.fileno()) - output.close() - final_path = output_path.with_suffix("") - os.replace(output_path, final_path) - chunks.append(final_path) - output_path = staging_dir / f"{source.stem}.{hour}.ndjson.tmp" - output = output_path.open("wb") - current_hour = hour - output.write(raw_line) - if output is not None and output_path is not None: - output.flush() - os.fsync(output.fileno()) - output.close() - output = None - final_path = output_path.with_suffix("") - os.replace(output_path, final_path) - chunks.append(final_path) - finally: - if output is not None: - output.close() - return chunks - - -def prepare_artifacts( - source: Path, - dataset: str, - quote_depth_levels: int, - quote_sample_ms: int, - zstd_timeout: int, -) -> tuple[Artifacts, dict[str, Any]]: - metadata = scan_tape(source, dataset, quote_depth_levels, quote_sample_ms) - data = source.with_suffix(source.suffix + ".zst") - temporary_data = data.with_suffix(data.suffix + ".tmp") - subprocess.run( - ["zstd", "-q", "-f", "-T1", "-3", str(source), "-o", str(temporary_data)], - check=True, - timeout=zstd_timeout, - ) - with temporary_data.open("rb") as handle: - os.fsync(handle.fileno()) - os.replace(temporary_data, data) - - digest = sha256_file(data) - metadata.update({"file": data.name, "bytes": data.stat().st_size, "sha256": digest}) - manifest = data.with_name(data.name + ".manifest.json") - success = data.with_name(data.name + "._SUCCESS") - atomic_json(manifest, metadata) - success.write_text(digest + "\n", encoding="utf-8") - with success.open("rb") as handle: - os.fsync(handle.fileno()) - object_prefix = ( - f"lake/raw/venue=polymarket/dataset={dataset}/" - f"date={metadata['date']}/hour={metadata['hour']}" - ) - return Artifacts(source, data, manifest, success, object_prefix), metadata - - -def upload_artifacts( - artifacts: Artifacts, - bucket: str, - endpoint: str, - region: str, - profile: str, - timeout: int, -) -> str: - for source in (artifacts.data, artifacts.manifest, artifacts.success): - destination = f"oss://{bucket}/{artifacts.object_prefix}/{source.name}" - subprocess.run( - [ - "aliyun", - "ossutil", - "cp", - str(source), - destination, - "--profile", - profile, - "--endpoint", - endpoint, - "--region", - region, - "--force", - ], - check=True, - timeout=timeout, - ) - verify_remote_artifacts(artifacts, bucket, endpoint, region, profile, timeout) - for path in (artifacts.source, artifacts.data, artifacts.manifest, artifacts.success): - path.unlink() - return f"oss://{bucket}/{artifacts.object_prefix}/{artifacts.data.name}" - - -def verify_remote_artifacts( - artifacts: Artifacts, - bucket: str, - endpoint: str, - region: str, - profile: str, - timeout: int, -) -> None: - """Read all three objects back before deleting the local closed tape.""" - expected_manifest = json.loads(artifacts.manifest.read_text(encoding="utf-8")) - with tempfile.TemporaryDirectory(prefix=".oss-verify-", dir=artifacts.source.parent) as directory: - verify_dir = Path(directory) - downloaded: dict[str, Path] = {} - for source in (artifacts.data, artifacts.manifest, artifacts.success): - destination = verify_dir / source.name - remote = f"oss://{bucket}/{artifacts.object_prefix}/{source.name}" - subprocess.run( - [ - "aliyun", - "ossutil", - "cp", - remote, - str(destination), - "--profile", - profile, - "--endpoint", - endpoint, - "--region", - region, - "--force", - ], - check=True, - timeout=timeout, - ) - downloaded[source.name] = destination - - remote_data = downloaded[artifacts.data.name] - if remote_data.stat().st_size != expected_manifest["bytes"]: - raise ValueError("remote data size does not match manifest") - if sha256_file(remote_data) != expected_manifest["sha256"]: - raise ValueError("remote data sha256 does not match manifest") - if downloaded[artifacts.manifest.name].read_bytes() != artifacts.manifest.read_bytes(): - raise ValueError("remote manifest does not match local manifest") - if downloaded[artifacts.success.name].read_text(encoding="utf-8").strip() != expected_manifest[ - "sha256" - ]: - raise ValueError("remote _SUCCESS does not match manifest") - - -def archive_source(source: Path, args: argparse.Namespace) -> list[str]: - """Validate one closed session, upload UTC-hour chunks, then delete it.""" - source_manifest = scan_tape( - source, args.dataset, args.quote_depth_levels, args.quote_sample_ms - ) - if source_manifest["start_sequence"] != 0: - raise ValueError( - f"closed source tape must start at sequence 0; actual={source_manifest['start_sequence']}" - ) - uploaded: list[str] = [] - chunks = split_tape_by_utc_hour(source) - for chunk in chunks: - artifacts, _ = prepare_artifacts( - chunk, - args.dataset, - args.quote_depth_levels, - args.quote_sample_ms, - args.zstd_timeout, - ) - uploaded.append( - upload_artifacts( - artifacts, - args.bucket, - args.endpoint, - args.region, - args.profile, - args.oss_timeout, - ) - ) - source.unlink() - staging_dir = source.parent / ".upload-staging" / source.name - try: - staging_dir.rmdir() - staging_dir.parent.rmdir() - except OSError: - pass - return uploaded - - -def read_status(path: Path) -> dict[str, Any]: - try: - value = json.loads(path.read_text(encoding="utf-8")) - return value if isinstance(value, dict) else {} - except (FileNotFoundError, json.JSONDecodeError): - return {} - - -def run(args: argparse.Namespace) -> int: - spool_dir = args.spool_dir - spool_dir.mkdir(parents=True, exist_ok=True) - status_path = spool_dir / "upload-status.json" - status = read_status(status_path) - failures: list[dict[str, str]] = [] - for source in discover_rotated_tapes(spool_dir): - try: - uploaded = archive_source(source, args) - status.update( - { - "last_success_at": utc_now(), - "last_uploaded_object": uploaded[-1], - } - ) - except Exception as error: - failures.append({"source": source.name, "error": str(error)}) - print(f"Polymarket tape upload failed for {source.name}: {error}", flush=True) - - status.update( - { - "updated_at": utc_now(), - "pending_segments": len(discover_rotated_tapes(spool_dir)), - "failed_segments": failures, - "last_error_at": utc_now() if failures else None, - "last_error": failures[-1]["error"] if failures else None, - } - ) - atomic_json(status_path, status) - return 1 if failures else 0 - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser() - parser.add_argument( - "--spool-dir", - type=Path, - default=Path("/data/monday/spool/polymarket"), - ) - parser.add_argument("--dataset", default="crypto_expiry") - parser.add_argument( - "--quote-depth-levels", - type=int, - default=0, - help="maximum persisted levels per side; 0 means unbounded", - ) - parser.add_argument("--quote-sample-ms", type=int, default=0) - parser.add_argument("--bucket", default=os.getenv("OSS_BUCKET", "monday-lob-apne1-1045353359")) - parser.add_argument( - "--endpoint", - default=os.getenv("OSS_ENDPOINT", "oss-ap-northeast-1-internal.aliyuncs.com"), - ) - parser.add_argument("--region", default=os.getenv("OSS_REGION", "ap-northeast-1")) - parser.add_argument("--profile", default=os.getenv("ALIYUN_PROFILE", "ecs-role")) - parser.add_argument( - "--zstd-timeout", - type=int, - default=int(os.getenv("ZSTD_TIMEOUT_SECONDS", "300")), - ) - parser.add_argument( - "--oss-timeout", - type=int, - default=int(os.getenv("OSS_COPY_TIMEOUT_SECONDS", "300")), - ) - args = parser.parse_args() - if not re.fullmatch(r"[a-z0-9_-]+", args.dataset): - parser.error("dataset must match [a-z0-9_-]+") - if args.quote_depth_levels < 0 or args.quote_sample_ms < 0: - parser.error("recording policy values must be non-negative") - return args - - -if __name__ == "__main__": - raise SystemExit(run(parse_args())) diff --git a/deployment/aliyun/polymarket_reference_canonicalize.py b/deployment/aliyun/polymarket_reference_canonicalize.py deleted file mode 100644 index f176c3e73..000000000 --- a/deployment/aliyun/polymarket_reference_canonicalize.py +++ /dev/null @@ -1,86 +0,0 @@ -#!/usr/bin/env python3 -"""Build one canonical v2 reference tape from overlapping historical tapes.""" - -from __future__ import annotations - -import argparse -import json -import os -from pathlib import Path -from typing import Any - -from polymarket_reference_collector import TRADE_ID_VERSION, stable_trade_id - - -def canonicalize(inputs: list[Path], output: Path) -> dict[str, Any]: - rows: list[tuple[str, int, int, dict[str, Any]]] = [] - seen_trades: set[str] = set() - input_rows = 0 - duplicate_trades = 0 - trade_rows = 0 - for source_index, source in enumerate(inputs): - with source.open("rb") as handle: - for line_number, raw_line in enumerate(handle, start=1): - input_rows += 1 - if not raw_line.endswith(b"\n"): - raise ValueError(f"{source}:{line_number}: incomplete record") - row = json.loads(raw_line) - if not isinstance(row, dict) or not isinstance(row.get("update"), dict): - raise ValueError(f"{source}:{line_number}: invalid record") - recorded_at = row.get("recorded_at") - if not isinstance(recorded_at, str) or not recorded_at: - raise ValueError(f"{source}:{line_number}: missing recorded_at") - update = dict(row["update"]) - if update.get("kind") == "polymarket_trade": - trade = update.get("trade") - if not isinstance(trade, dict): - raise ValueError(f"{source}:{line_number}: trade payload missing") - record_id = stable_trade_id(trade) - if record_id in seen_trades: - duplicate_trades += 1 - continue - seen_trades.add(record_id) - update["record_id"] = record_id - update["record_id_version"] = TRADE_ID_VERSION - trade_rows += 1 - rows.append((recorded_at, source_index, line_number, update)) - - rows.sort(key=lambda item: (item[0], item[1], item[2])) - temporary = output.with_suffix(output.suffix + ".tmp") - output.parent.mkdir(parents=True, exist_ok=True) - with temporary.open("wb", buffering=0) as handle: - for sequence, (recorded_at, _source, _line, update) in enumerate(rows): - encoded = json.dumps( - {"sequence": sequence, "recorded_at": recorded_at, "update": update}, - sort_keys=True, - separators=(",", ":"), - ).encode() + b"\n" - view = memoryview(encoded) - written = 0 - while written < len(view): - count = handle.write(view[written:]) - if not isinstance(count, int) or count <= 0: - raise OSError("canonical tape write made no progress") - written += count - handle.flush() - os.fsync(handle.fileno()) - os.replace(temporary, output) - return { - "input_rows": input_rows, - "output_rows": len(rows), - "canonical_v2_trades": trade_rows, - "duplicate_trades_removed": duplicate_trades, - "output": str(output), - } - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser() - parser.add_argument("--output", type=Path, required=True) - parser.add_argument("inputs", nargs="+", type=Path) - return parser.parse_args() - - -if __name__ == "__main__": - args = parse_args() - print(json.dumps(canonicalize(args.inputs, args.output), sort_keys=True)) diff --git a/deployment/aliyun/polymarket_reference_collector.py b/deployment/aliyun/polymarket_reference_collector.py deleted file mode 100644 index c48507572..000000000 --- a/deployment/aliyun/polymarket_reference_collector.py +++ /dev/null @@ -1,813 +0,0 @@ -#!/usr/bin/env python3 -"""Collect Polymarket market metadata, public trades, and settlements to NDJSON.""" - -from __future__ import annotations - -import argparse -import copy -import hashlib -import json -import os -import re -import shutil -import time -import urllib.parse -import urllib.request -from datetime import datetime, timedelta, timezone -from decimal import Decimal, InvalidOperation -from pathlib import Path -from typing import Any - - -GAMMA_MARKETS_URL = "https://gamma-api.polymarket.com/markets/keyset" -GAMMA_MARKET_URL = "https://gamma-api.polymarket.com/markets/{market_id}" -DATA_TRADES_URL = "https://data-api.polymarket.com/trades" -ACTIVE_TAPE = "market-updates.ndjson" -USER_AGENT = "monday-polymarket-reference-collector/1.0" -SETTLEMENT_PRICE = Decimal("0.999") -TRADE_ID_VERSION = "v2" - -SYMBOL_ALIASES = ( - ("BTCUSDT", ("BITCOIN", "BTC")), - ("ETHUSDT", ("ETHEREUM", "ETH")), - ("SOLUSDT", ("SOLANA", "SOL ")), - ("XRPUSDT", ("XRP",)), - ("DOGEUSDT", ("DOGECOIN", "DOGE")), - ("HYPEUSDT", ("HYPERLIQUID", "HYPE")), - ("BNBUSDT", ("BINANCE COIN", "BNB")), -) - - -class DataCompletenessError(RuntimeError): - """Raised when a persisted health failure requires a systemd restart.""" - - -def utc_now() -> datetime: - return datetime.now(timezone.utc) - - -def iso_z(value: datetime) -> str: - return value.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z") - - -def parse_datetime(value: Any) -> datetime | None: - if not isinstance(value, str) or not value: - return None - try: - parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) - except ValueError: - return None - if parsed.tzinfo is None: - parsed = parsed.replace(tzinfo=timezone.utc) - return parsed.astimezone(timezone.utc) - - -def parse_json_array(value: Any) -> list[Any]: - if isinstance(value, list): - return value - if isinstance(value, str): - try: - parsed = json.loads(value) - except json.JSONDecodeError: - return [] - return parsed if isinstance(parsed, list) else [] - return [] - - -def infer_symbol(question: Any) -> str | None: - if not isinstance(question, str): - return None - upper = question.upper() - for symbol, aliases in SYMBOL_ALIASES: - if any(alias in upper for alias in aliases): - return symbol - return None - - -def market_start_time(market: dict[str, Any]) -> datetime | None: - candidates = [market.get("eventStartTime"), market.get("startDate")] - events = market.get("events") - if isinstance(events, list) and events and isinstance(events[0], dict): - candidates.extend((events[0].get("startTime"), events[0].get("startDate"))) - return next((parsed for value in candidates if (parsed := parse_datetime(value)) is not None), None) - - -def infer_window_seconds(market: dict[str, Any]) -> int | None: - end = parse_datetime(market.get("endDate")) - start = market_start_time(market) - if start is not None and end is not None: - duration = round((end - start).total_seconds()) - if duration in (300, 900): - return duration - - text = f"{market.get('slug', '')} {market.get('question', '')}".lower() - if re.search(r"(?:^|[-_ ])15m(?:[-_ ]|$)|15 minutes?", text): - return 900 - if re.search(r"(?:^|[-_ ])5m(?:[-_ ]|$)|5 minutes?", text): - return 300 - return None - - -def is_target_market(market: dict[str, Any], symbols: set[str]) -> tuple[str, int] | None: - symbol = infer_symbol(market.get("question")) - window_seconds = infer_window_seconds(market) - condition_id = market.get("conditionId") - token_ids = parse_json_array(market.get("clobTokenIds")) - if ( - symbol not in symbols - or window_seconds not in (300, 900) - or not isinstance(condition_id, str) - or not condition_id - or len(token_ids) != 2 - ): - return None - return symbol, window_seconds - - -def stable_trade_id(trade: dict[str, Any]) -> str: - parts = ( - str(trade.get("transactionHash", "")), - str(trade.get("conditionId", "")), - str(trade.get("asset", "")), - str(trade.get("side", "")), - str(trade.get("timestamp", "")), - str(trade.get("proxyWallet", "")), - canonical_decimal(trade.get("size")), - canonical_decimal(trade.get("price")), - str(trade.get("outcomeIndex", "")), - ) - return hashlib.sha256("|".join(parts).encode()).hexdigest() - - -def canonical_decimal(value: Any) -> str: - try: - normalized = Decimal(str(value)).normalize() - except (InvalidOperation, ValueError): - return str(value) - return format(normalized, "f") - - -def stable_payload_hash(payload: dict[str, Any]) -> str: - encoded = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode() - return hashlib.sha256(encoded).hexdigest() - - -def advance_trade_finalization( - tracked: dict[str, Any], - now: datetime, - retrieved_at: str, - new_trade_count: int, - truncated: bool, - was_settled: bool, - lag_seconds: int, - stable_polls_required: int, -) -> bool: - """Require both an indexing delay and stable polls before ending trade overlap.""" - tracked.setdefault("settlement_seen_at", retrieved_at) - if new_trade_count: - tracked["last_trade_change_at"] = retrieved_at - tracked["trade_finalization_stable_polls"] = 0 - anchors = [ - value - for value in ( - parse_datetime(tracked.get("settlement_seen_at")), - parse_datetime(tracked.get("last_trade_change_at")), - ) - if value is not None - ] - lag_elapsed = bool(anchors) and (now - max(anchors)).total_seconds() >= lag_seconds - if not lag_elapsed or truncated or new_trade_count or not was_settled: - tracked["trade_finalization_stable_polls"] = 0 - return False - tracked["trade_finalization_stable_polls"] = int( - tracked.get("trade_finalization_stable_polls", 0) - ) + 1 - return int(tracked["trade_finalization_stable_polls"]) >= stable_polls_required - - -def settlement_from_market( - market: dict[str, Any], symbol: str, window_seconds: int, retrieved_at: str -) -> dict[str, Any] | None: - if market.get("closed") is not True: - return None - outcomes = [str(value) for value in parse_json_array(market.get("outcomes"))] - token_ids = [str(value) for value in parse_json_array(market.get("clobTokenIds"))] - raw_prices = parse_json_array(market.get("outcomePrices")) - if len(outcomes) != 2 or len(token_ids) != 2 or len(raw_prices) != 2: - return None - try: - prices = [Decimal(str(value)) for value in raw_prices] - except (InvalidOperation, ValueError): - return None - winners = [index for index, price in enumerate(prices) if price >= SETTLEMENT_PRICE] - if len(winners) != 1: - return None - winner = winners[0] - winning_outcome = outcomes[winner] - lowered = winning_outcome.lower() - resolved_up_won = lowered in {"up", "yes"} - if lowered not in {"up", "down", "yes", "no"}: - resolved_up_won = None - return { - "kind": "market_settlement", - "market_id": str(market.get("id", "")), - "condition_id": str(market.get("conditionId", "")), - "symbol": symbol, - "market_window_secs": window_seconds, - "winning_token_id": token_ids[winner], - "winning_outcome": winning_outcome, - "resolved_up_won": resolved_up_won, - "resolution_source": "gamma_api_closed_market", - "retrieved_at": retrieved_at, - "market": market, - } - - -def atomic_json(path: Path, payload: dict[str, Any]) -> None: - temporary = path.with_suffix(path.suffix + ".tmp") - with temporary.open("w", encoding="utf-8") as handle: - json.dump(payload, handle, sort_keys=True, separators=(",", ":")) - handle.write("\n") - handle.flush() - os.fsync(handle.fileno()) - os.replace(temporary, path) - - -def load_json(path: Path) -> dict[str, Any]: - try: - value = json.loads(path.read_text(encoding="utf-8")) - return value if isinstance(value, dict) else {} - except (FileNotFoundError, json.JSONDecodeError): - return {} - - -class TapeWriter: - def __init__(self, spool_dir: Path) -> None: - self.spool_dir = spool_dir - self.spool_dir.mkdir(parents=True, exist_ok=True) - self.active = spool_dir / ACTIVE_TAPE - self.hour: str | None = None - self.sequence = 0 - self.handle = None - self._recover_active() - - def _recover_active(self) -> None: - if not self.active.exists() or self.active.stat().st_size == 0: - self.handle = self.active.open("ab", buffering=0) - return - expected = 0 - last_complete_offset = 0 - first_recorded: datetime | None = None - with self.active.open("rb") as handle: - for raw_line in handle: - if not raw_line.endswith(b"\n"): - break - row = json.loads(raw_line) - if row.get("sequence") != expected: - raise ValueError(f"active tape sequence gap expected={expected}") - if first_recorded is None: - first_recorded = parse_datetime(row.get("recorded_at")) - expected += 1 - last_complete_offset = handle.tell() - if last_complete_offset != self.active.stat().st_size: - with self.active.open("r+b") as handle: - handle.truncate(last_complete_offset) - handle.flush() - os.fsync(handle.fileno()) - self.sequence = expected - self.hour = first_recorded.strftime("%Y%m%dT%H") if first_recorded else None - self.handle = self.active.open("ab", buffering=0) - - def _rotate(self, now: datetime) -> None: - if self.handle is not None: - self.handle.flush() - os.fsync(self.handle.fileno()) - self.handle.close() - if self.active.exists() and self.active.stat().st_size > 0: - stamp = now.strftime("%Y%m%dT%H%M%S%f") - os.replace(self.active, self.spool_dir / f"market-updates.{stamp}.ndjson") - self.sequence = 0 - self.hour = None - self.handle = self.active.open("ab", buffering=0) - - def write_updates(self, updates: list[dict[str, Any]], now: datetime) -> None: - if not updates: - return - target_hour = now.strftime("%Y%m%dT%H") - if self.hour is not None and self.hour != target_hour: - self._rotate(now) - self.hour = target_hour - recorded_at = iso_z(now) - start_offset = os.fstat(self.handle.fileno()).st_size - start_sequence = self.sequence - try: - for update in updates: - row = {"sequence": self.sequence, "recorded_at": recorded_at, "update": update} - encoded = ( - json.dumps(row, sort_keys=True, separators=(",", ":")).encode() + b"\n" - ) - self._write_all(self.handle, encoded) - self.sequence += 1 - self.handle.flush() - os.fsync(self.handle.fileno()) - except BaseException: - self.sequence = start_sequence - os.ftruncate(self.handle.fileno(), start_offset) - os.fsync(self.handle.fileno()) - raise - - @staticmethod - def _write_all(handle: Any, encoded: bytes) -> None: - view = memoryview(encoded) - written = 0 - while written < len(view): - count = handle.write(view[written:]) - if not isinstance(count, int) or count <= 0: - raise OSError("tape write made no progress") - written += count - - def close(self) -> None: - if self.handle is not None: - self.handle.flush() - os.fsync(self.handle.fileno()) - self.handle.close() - self.handle = None - - -class ReferenceCollector: - def __init__(self, args: argparse.Namespace) -> None: - self.args = args - self.symbols = set(args.symbols) - self.spool_dir = args.spool_dir - self.state_path = self.spool_dir / "collector-state.json" - self.health_path = self.spool_dir / "health.json" - self.state = load_json(self.state_path) - self.state.setdefault("markets", {}) - self.state.setdefault("trade_seen", {}) - if self.state.get("trade_id_version") != TRADE_ID_VERSION: - active = self.spool_dir / ACTIVE_TAPE - if active.exists() and active.stat().st_size: - quarantine = self.spool_dir / f"superseded-v1-{ACTIVE_TAPE}.{time.time_ns()}" - os.replace(active, quarantine) - self.state["trade_seen"] = {} - for tracked in self.state["markets"].values(): - if isinstance(tracked, dict): - tracked["trade_complete"] = False - self.state["trade_id_version"] = TRADE_ID_VERSION - self.writer = TapeWriter(self.spool_dir) - self.recover_state_from_active_tape() - self.last_success_monotonic = time.monotonic() - - def recover_state_from_active_tape(self) -> None: - """Merge durable tape records missing from a pre-crash state checkpoint.""" - if not self.writer.active.exists(): - return - with self.writer.active.open("rb") as handle: - for raw_line in handle: - if not raw_line.endswith(b"\n"): - continue - row = json.loads(raw_line) - update = row.get("update", {}) - kind = update.get("kind") - market_id = update.get("market_id") - condition_id = update.get("condition_id") - if kind == "polymarket_trade": - record_id = update.get("record_id") - timestamp = update.get("trade_ts_unix") - if ( - update.get("record_id_version") == TRADE_ID_VERSION - and isinstance(condition_id, str) - and isinstance(record_id, str) - and isinstance(timestamp, int) - ): - self.state["trade_seen"].setdefault(condition_id, {})[ - record_id - ] = timestamp - elif kind in {"market_metadata", "market_settlement"} and isinstance( - market_id, str - ): - tracked = self.state["markets"].setdefault(market_id, {}) - for source, target in ( - ("condition_id", "condition_id"), - ("symbol", "symbol"), - ("market_window_secs", "market_window_secs"), - ): - if update.get(source) is not None: - tracked[target] = update[source] - market = update.get("market") - if isinstance(market, dict): - tracked["end_time"] = market.get("endDate") - tracked["last_metadata_hash"] = stable_payload_hash(market) - if kind == "market_settlement": - tracked["settled"] = True - - def get_json(self, url: str, params: dict[str, Any] | None = None) -> Any: - if params: - url = f"{url}?{urllib.parse.urlencode(params)}" - request = urllib.request.Request( - url, headers={"User-Agent": USER_AGENT, "Accept": "application/json"} - ) - with urllib.request.urlopen(request, timeout=self.args.http_timeout) as response: - return json.load(response) - - def discover_markets(self, now: datetime) -> list[dict[str, Any]]: - params: dict[str, Any] = { - "end_date_min": iso_z(now - timedelta(seconds=self.args.market_lookback_secs)), - "end_date_max": iso_z(now + timedelta(minutes=30)), - "closed": "false", - "limit": 100, - } - cursor = None - markets: list[dict[str, Any]] = [] - while len(markets) < self.args.max_markets: - call_params = dict(params) - if cursor: - call_params["after_cursor"] = cursor - payload = self.get_json(GAMMA_MARKETS_URL, call_params) - if not isinstance(payload, dict) or not isinstance(payload.get("markets"), list): - raise ValueError("Gamma keyset response is missing markets") - page = [item for item in payload["markets"] if isinstance(item, dict)] - markets.extend(page) - cursor = payload.get("next_cursor") - if len(page) < 100 or not isinstance(cursor, str) or not cursor: - break - return markets[: self.args.max_markets] - - def fetch_trades(self, condition_id: str) -> tuple[list[dict[str, Any]], bool]: - trades: list[dict[str, Any]] = [] - truncated = False - for offset in (0, 10_000): - payload = self.get_json( - DATA_TRADES_URL, - { - "market": condition_id, - "limit": 10_000, - "offset": offset, - "takerOnly": "false", - }, - ) - if not isinstance(payload, list): - raise ValueError("Data API trades response is not an array") - page = [item for item in payload if isinstance(item, dict)] - trades.extend(page) - if len(page) < 10_000: - break - if offset == 10_000: - truncated = True - return trades, truncated - - def trade_updates( - self, - market_id: str, - condition_id: str, - symbol: str, - window_seconds: int, - trades: list[dict[str, Any]], - now: datetime, - state: dict[str, Any] | None = None, - ) -> tuple[list[dict[str, Any]], dict[str, int]]: - state = self.state if state is None else state - seen = state["trade_seen"].setdefault(condition_id, {}) - cutoff = int(now.timestamp()) - self.args.market_lookback_secs - updates = [] - parsed_trades: list[tuple[int, dict[str, Any]]] = [] - malformed_reasons: dict[str, int] = {} - - def reject(reason: str) -> None: - malformed_reasons[reason] = malformed_reasons.get(reason, 0) + 1 - - for trade in trades: - try: - timestamp = int(trade.get("timestamp")) - except (TypeError, ValueError): - reject("invalid_timestamp") - continue - if trade.get("conditionId") != condition_id: - reject("condition_mismatch") - continue - if not isinstance(trade.get("transactionHash"), str) or not trade["transactionHash"]: - reject("missing_transaction_hash") - continue - if not isinstance(trade.get("asset"), str) or not trade["asset"]: - reject("missing_asset") - continue - if trade.get("side") not in {"BUY", "SELL"}: - reject("invalid_side") - continue - if not isinstance(trade.get("proxyWallet"), str) or not trade["proxyWallet"]: - reject("missing_proxy_wallet") - continue - try: - size = Decimal(str(trade.get("size"))) - except (InvalidOperation, ValueError): - size = Decimal("NaN") - if not size.is_finite() or size <= 0: - reject("invalid_size") - continue - try: - price = Decimal(str(trade.get("price"))) - except (InvalidOperation, ValueError): - price = Decimal("NaN") - if not price.is_finite() or not Decimal("0") <= price <= Decimal("1"): - reject("invalid_price") - continue - if trade.get("outcomeIndex") not in {0, 1}: - reject("invalid_outcome_index") - continue - if not isinstance(trade.get("outcome"), str) or not trade["outcome"]: - reject("missing_outcome") - continue - parsed_trades.append((timestamp, trade)) - for timestamp, trade in sorted(parsed_trades, key=lambda value: value[0]): - record_id = stable_trade_id(trade) - if timestamp < cutoff or record_id in seen: - continue - seen[record_id] = timestamp - updates.append( - { - "kind": "polymarket_trade", - "record_id": record_id, - "record_id_version": TRADE_ID_VERSION, - "market_id": market_id, - "condition_id": condition_id, - "token_id": str(trade.get("asset", "")), - "symbol": symbol, - "market_window_secs": window_seconds, - "side": trade.get("side"), - "size": trade.get("size"), - "price": trade.get("price"), - "trade_ts": iso_z(datetime.fromtimestamp(timestamp, tz=timezone.utc)), - "trade_ts_unix": timestamp, - "transaction_hash": trade.get("transactionHash"), - "proxy_wallet": trade.get("proxyWallet"), - "outcome": trade.get("outcome"), - "outcome_index": trade.get("outcomeIndex"), - "source": "polymarket_data_api", - "received_at": iso_z(now), - "trade": trade, - } - ) - state["trade_seen"][condition_id] = { - key: timestamp for key, timestamp in seen.items() if int(timestamp) >= cutoff - } - return updates, malformed_reasons - - def collect_once(self) -> dict[str, Any]: - now = utc_now() - retrieved_at = iso_z(now) - updates: list[dict[str, Any]] = [] - errors: list[str] = [] - truncated_markets: list[str] = [] - trade_polls = 0 - successful_trade_polls = 0 - malformed_trade_reasons: dict[str, int] = {} - overdue_unresolved_markets: list[str] = [] - next_state = copy.deepcopy(self.state) - discovered = self.discover_markets(now) - targets: dict[str, tuple[dict[str, Any], str, int]] = {} - - for market in discovered: - target = is_target_market(market, self.symbols) - if target is None: - continue - symbol, window_seconds = target - market_id = str(market.get("id", "")) - condition_id = str(market.get("conditionId", "")) - targets[market_id] = (market, symbol, window_seconds) - previous = next_state["markets"].get(market_id, {}) - tracked = dict(previous) - tracked.update({ - "condition_id": condition_id, - "symbol": symbol, - "market_window_secs": window_seconds, - "end_time": market.get("endDate"), - "settled": bool(previous.get("settled", False)), - "trade_complete": bool(previous.get("trade_complete", False)), - "last_metadata_hash": previous.get("last_metadata_hash"), - }) - next_state["markets"][market_id] = tracked - - for market_id, tracked in list(next_state["markets"].items()): - if not isinstance(tracked, dict): - continue - end_time = parse_datetime(tracked.get("end_time")) - if end_time is not None and end_time < now - timedelta(seconds=self.args.market_lookback_secs): - if tracked.get("settled") and tracked.get("trade_complete"): - next_state["markets"].pop(market_id, None) - condition_id = tracked.get("condition_id") - if isinstance(condition_id, str): - next_state["trade_seen"].pop(condition_id, None) - continue - if end_time < now - timedelta(seconds=self.args.settlement_lookback_secs): - overdue_unresolved_markets.append(market_id) - if ( - market_id not in targets - and end_time is not None - and end_time <= now - and not (tracked.get("settled") and tracked.get("trade_complete")) - ): - try: - market = self.get_json(GAMMA_MARKET_URL.format(market_id=market_id)) - if isinstance(market, dict): - targets[market_id] = ( - market, - str(tracked.get("symbol", "")), - int(tracked.get("market_window_secs", 0)), - ) - tracked.pop("settlement_failure_since", None) - tracked.pop("settlement_last_error", None) - except Exception as error: - errors.append(f"settlement {market_id}: {error}") - tracked.setdefault("settlement_failure_since", retrieved_at) - tracked["settlement_last_error"] = str(error) - - for market_id, (market, symbol, window_seconds) in targets.items(): - condition_id = str(market.get("conditionId", "")) - tracked = next_state["markets"].setdefault(market_id, {}) - metadata_hash = stable_payload_hash(market) - if tracked.get("last_metadata_hash") != metadata_hash: - updates.append( - { - "kind": "market_metadata", - "market_id": market_id, - "condition_id": condition_id, - "symbol": symbol, - "market_window_secs": window_seconds, - "source": "gamma_api", - "retrieved_at": retrieved_at, - "market": market, - } - ) - tracked["last_metadata_hash"] = metadata_hash - - settlement = settlement_from_market(market, symbol, window_seconds, retrieved_at) - was_settled = bool(tracked.get("settled")) - if condition_id and not tracked.get("trade_complete"): - trade_polls += 1 - try: - trades, truncated = self.fetch_trades(condition_id) - successful_trade_polls += 1 - trade_updates, malformed_reasons = self.trade_updates( - market_id, - condition_id, - symbol, - window_seconds, - trades, - now, - next_state, - ) - updates.extend(trade_updates) - for reason, count in malformed_reasons.items(): - malformed_trade_reasons[reason] = ( - malformed_trade_reasons.get(reason, 0) + count - ) - tracked["last_trade_success_at"] = retrieved_at - if malformed_reasons: - detail = f"malformed trade rows: {dict(sorted(malformed_reasons.items()))}" - errors.append(f"trades {condition_id}: {detail}") - tracked.setdefault("trade_failure_since", retrieved_at) - tracked["trade_last_error"] = detail - else: - tracked.pop("trade_failure_since", None) - tracked.pop("trade_last_error", None) - if truncated: - truncated_markets.append(condition_id) - if settlement is not None: - if advance_trade_finalization( - tracked, - now, - retrieved_at, - len(trade_updates), - truncated or bool(malformed_reasons), - was_settled, - self.args.trade_finalization_lag_secs, - self.args.trade_finalization_stable_polls, - ): - tracked["trade_complete"] = True - except Exception as error: - errors.append(f"trades {condition_id}: {error}") - tracked.setdefault("trade_failure_since", retrieved_at) - tracked["trade_last_error"] = str(error) - if settlement is not None and not tracked.get("settled"): - updates.append(settlement) - tracked["settled"] = True - if self.args.per_market_delay_ms: - time.sleep(self.args.per_market_delay_ms / 1000) - - stale_trade_markets = sorted( - str(tracked.get("condition_id")) - for tracked in next_state["markets"].values() - if isinstance(tracked, dict) - and (failed_at := parse_datetime(tracked.get("trade_failure_since"))) is not None - and (now - failed_at).total_seconds() > self.args.stale_after_secs - ) - stale_settlement_markets = sorted( - str(market_id) - for market_id, tracked in next_state["markets"].items() - if isinstance(tracked, dict) - and (failed_at := parse_datetime(tracked.get("settlement_failure_since"))) is not None - and (now - failed_at).total_seconds() > self.args.stale_after_secs - ) - - self.writer.write_updates(updates, now) - self.state = next_state - atomic_json(self.state_path, self.state) - self.last_success_monotonic = time.monotonic() - - usage = shutil.disk_usage(self.spool_dir) - health = { - "updated_at": retrieved_at, - "last_success_at": retrieved_at, - "target_markets": len(targets), - "tracked_markets": len(self.state["markets"]), - "records_written": len(updates), - "record_types": { - kind: sum(1 for item in updates if item.get("kind") == kind) - for kind in ("market_metadata", "polymarket_trade", "market_settlement") - }, - "api_errors": errors, - "trade_polls": trade_polls, - "successful_trade_polls": successful_trade_polls, - "malformed_trade_rows": sum(malformed_trade_reasons.values()), - "malformed_trade_reasons": dict(sorted(malformed_trade_reasons.items())), - "truncated_trade_markets": truncated_markets, - "stale_trade_markets": stale_trade_markets, - "stale_settlement_markets": stale_settlement_markets, - "overdue_unresolved_markets": sorted(overdue_unresolved_markets), - "active_tape_bytes": self.writer.active.stat().st_size if self.writer.active.exists() else 0, - "free_disk_bytes": usage.free, - } - atomic_json(self.health_path, health) - if truncated_markets: - raise DataCompletenessError( - f"trade pagination exceeded API offset limit for {truncated_markets}" - ) - if stale_trade_markets: - raise DataCompletenessError(f"stale trade markets: {stale_trade_markets}") - if stale_settlement_markets: - raise DataCompletenessError( - f"stale settlement markets: {stale_settlement_markets}" - ) - return health - - def run(self) -> None: - try: - if self.args.once: - print( - json.dumps(self.collect_once(), sort_keys=True, separators=(",", ":")), - flush=True, - ) - return - while True: - started = time.monotonic() - try: - health = self.collect_once() - print(json.dumps(health, sort_keys=True, separators=(",", ":")), flush=True) - except Exception as error: - print(f"Polymarket reference poll failed: {error}", flush=True) - if isinstance(error, DataCompletenessError): - raise - if time.monotonic() - self.last_success_monotonic > self.args.stale_after_secs: - raise - elapsed = time.monotonic() - started - time.sleep(max(0.0, self.args.poll_interval_secs - elapsed)) - finally: - self.writer.close() - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser() - parser.add_argument( - "--spool-dir", type=Path, default=Path("/data/monday/spool/polymarket-reference") - ) - parser.add_argument( - "--symbols", - type=lambda value: [item.strip().upper() for item in value.split(",") if item.strip()], - default=["BTCUSDT", "ETHUSDT", "SOLUSDT", "XRPUSDT", "DOGEUSDT", "HYPEUSDT", "BNBUSDT"], - ) - parser.add_argument("--poll-interval-secs", type=float, default=30.0) - parser.add_argument("--market-lookback-secs", type=int, default=7200) - parser.add_argument("--settlement-lookback-secs", type=int, default=86400) - parser.add_argument("--max-markets", type=int, default=1200) - parser.add_argument("--http-timeout", type=float, default=20.0) - parser.add_argument("--stale-after-secs", type=float, default=180.0) - parser.add_argument("--trade-finalization-lag-secs", type=int, default=1800) - parser.add_argument("--trade-finalization-stable-polls", type=int, default=3) - parser.add_argument("--per-market-delay-ms", type=int, default=100) - parser.add_argument("--once", action="store_true") - args = parser.parse_args() - if ( - args.poll_interval_secs <= 0 - or args.market_lookback_secs <= 0 - or args.settlement_lookback_secs <= 0 - or args.max_markets <= 0 - or args.trade_finalization_lag_secs <= 0 - or args.trade_finalization_stable_polls <= 0 - or args.per_market_delay_ms < 0 - ): - parser.error("poll interval, market lookback, and max markets must be positive") - return args - - -if __name__ == "__main__": - ReferenceCollector(parse_args()).run() diff --git a/deployment/aliyun/test-polymarket-raw-ops-control-plane.sh b/deployment/aliyun/test-polymarket-raw-ops-control-plane.sh new file mode 100755 index 000000000..040c813b7 --- /dev/null +++ b/deployment/aliyun/test-polymarket-raw-ops-control-plane.sh @@ -0,0 +1,458 @@ +#!/usr/bin/env bash +# Static contract greps intentionally use literal shell expressions. +# shellcheck disable=SC1090,SC2016 +set -euo pipefail + +export LC_ALL=C +SCRIPT_DIR=$(cd -- "$(dirname -- "$0")" && pwd) +readonly SCRIPT_DIR +readonly RUST_MANIFEST="$SCRIPT_DIR/../../rust_hft/Cargo.toml" +readonly VERIFY="$SCRIPT_DIR/../../rust_hft/target/debug/polymarket-raw-ops" +readonly POLICY="$SCRIPT_DIR/polymarket-shadow-gate-policy.jq" +readonly LEGACY_HEALTH_POLICY="$SCRIPT_DIR/polymarket-legacy-health-policy.jq" +readonly RUST_HEALTH_POLICY="$SCRIPT_DIR/polymarket-rust-health-policy.jq" +readonly GATE="$SCRIPT_DIR/polymarket-raw-ops-shadow-gate.sh" +readonly CUTOVER="$SCRIPT_DIR/polymarket-raw-ops-cutover.sh" +readonly WORKFLOW="$SCRIPT_DIR/../../.github/workflows/acr-publish.yml" + +for command in cargo chmod cp grep jq mkdir mktemp mv rm sed sha256sum shellcheck; do + command -v "$command" >/dev/null 2>&1 || { + printf 'missing control-plane test dependency: %s\n' "$command" >&2 + exit 2 + } +done + +shellcheck "$GATE" "$CUTOVER" "$0" +cargo build --quiet --manifest-path "$RUST_MANIFEST" -p hft-collector \ + --bin polymarket-raw-ops --no-default-features --locked +"$VERIFY" verify-shadow-parity --help >/dev/null + +tmp_dir=$(mktemp -d) +tmp_dir=$(cd -- "$tmp_dir" && pwd -P) +trap 'rm -rf "$tmp_dir"' EXIT + +# Exercise the production marker verifier itself. A marker is valid only when +# it contains the one content-addressed gate.json entry; sha256sum otherwise +# accepts an unrelated entry or a valid gate entry followed by extra entries. +marker_verifier="$tmp_dir/verify-gate-marker.sh" +sed -n '/^verify_gate_marker() {$/,/^}$/p' "$CUTOVER" >"$marker_verifier" +# shellcheck source=/dev/null +source "$marker_verifier" +declare -F verify_gate_marker >/dev/null || { + printf 'cutover does not expose its gate-marker verifier to contract tests\n' >&2 + exit 1 +} +marker_dir="$tmp_dir/marker" +mkdir "$marker_dir" +printf 'gate evidence\n' >"$marker_dir/gate.json" +printf 'unrelated evidence\n' >"$marker_dir/other.json" +( + cd "$marker_dir" + sha256sum gate.json >PASSED.sha256 +) +verify_gate_marker "$marker_dir" || { + printf 'gate-marker verifier rejected the exact gate.json marker\n' >&2 + exit 1 +} +( + cd "$marker_dir" + sha256sum other.json >PASSED.sha256 +) +if verify_gate_marker "$marker_dir"; then + printf 'gate-marker verifier accepted a marker for another file\n' >&2 + exit 1 +fi +( + cd "$marker_dir" + sha256sum gate.json other.json >PASSED.sha256 +) +if verify_gate_marker "$marker_dir"; then + printf 'gate-marker verifier accepted a multi-entry marker\n' >&2 + exit 1 +fi + +legacy="$tmp_dir/legacy" +rust="$tmp_dir/rust" +mkdir -p "$legacy" "$rust" +legacy_tape="$legacy/market-updates.ndjson" +rust_closed="$rust/market-updates.19700101T000400000000.ndjson" +: >"$legacy_tape" +: >"$rust_closed" +: >"$rust/market-updates.ndjson" + +sequence=0 +append_row() { + local update=$1 row + row=$(jq -cn --argjson sequence "$sequence" --argjson update "$update" \ + '{sequence:$sequence,recorded_at:"1970-01-01T00:03:20Z",update:$update}') + printf '%s\n' "$row" >>"$legacy_tape" + printf '%s\n' "$row" >>"$rust_closed" + sequence=$((sequence + 1)) +} + +for symbol in BTCUSDT ETHUSDT SOLUSDT XRPUSDT DOGEUSDT HYPEUSDT BNBUSDT; do + append_row "$(jq -cn --arg symbol "$symbol" \ + '{kind:"market_metadata",market_id:("market-" + $symbol), + condition_id:("condition-" + $symbol),symbol:$symbol,market_window_secs:300, + source:"gamma_api",retrieved_at:"1970-01-01T00:03:20Z", + market:{id:("market-" + $symbol),conditionId:("condition-" + $symbol), + question:($symbol + " Up or Down"),slug:("market-" + $symbol), + startDate:"1970-01-01T00:00:00Z",endDate:"1970-01-01T00:05:00Z", + outcomes:["Up","Down"],clobTokenIds:[("up-" + $symbol),("down-" + $symbol)], + orderPriceMinTickSize:0.01,orderMinSize:5,feesEnabled:true}}')" +done + +trade=$(jq -cn \ + '{kind:"polymarket_trade",record_id:"trade-1",record_id_version:"v2", + market_id:"market-BTCUSDT",condition_id:"condition-BTCUSDT",token_id:"token-1", + symbol:"BTCUSDT",market_window_secs:300,side:"BUY",size:"1",price:"0.5", + trade_ts:"1970-01-01T00:03:20Z",trade_ts_unix:200, + transaction_hash:"0x1",proxy_wallet:"0x2",outcome:"Up",outcome_index:0, + source:"polymarket_data_api",received_at:"1970-01-01T00:03:20Z", + trade:{transactionHash:"0x1",conditionId:"condition-BTCUSDT",asset:"token-1", + side:"BUY",timestamp:200,proxyWallet:"0x2",size:"1",price:"0.5", + outcomeIndex:0,outcome:"Up"}}') +append_row "$trade" + +append_row "$(jq -cn \ + '{kind:"market_settlement",market_id:"market-BTCUSDT", + condition_id:"condition-BTCUSDT",symbol:"BTCUSDT",market_window_secs:300, + winning_token_id:"token-1",winning_outcome:"Up",resolved_up_won:true, + resolution_source:"gamma_api_closed_market",retrieved_at:"1970-01-01T00:03:20Z", + market:{id:"market-BTCUSDT",conditionId:"condition-BTCUSDT",closed:true, + outcomes:["Up","Down"],clobTokenIds:["token-1","token-2"], + outcomePrices:["1","0"]}}')" + +# A delayed historical trade recorded only after the common cutoff must not make +# the bounded snapshot flaky, even when its source timestamp is in-window. +late_trade=$(jq -c \ + '.record_id = "trade-after-cutoff" + | .trade_ts_unix = 200 + | .trade_ts = "1970-01-01T00:03:20Z" + | .trade.timestamp = 200 + | .post_cutoff_only = true + | .trade.postCutoffOnly = true' <<<"$trade") +jq -cn --argjson sequence "$sequence" --argjson update "$late_trade" \ + '{sequence:$sequence,recorded_at:"1970-01-01T00:05:01Z",update:$update}' \ + >>"$legacy_tape" + +parity="$tmp_dir/parity.json" +"$VERIFY" verify-shadow-parity \ + --legacy-spool "$legacy" --rust-spool "$rust" --started-at-unix 100 \ + --ended-at-unix 300 \ + --output "$parity" +jq -e '.passed == true and .checks.metadata_parity == true + and ([.checks[]] | all)' "$parity" >/dev/null + +rust_bad="$tmp_dir/rust-bad" +cp -R "$rust" "$rust_bad" +jq -cn --argjson update "$trade" \ + '{sequence:0,recorded_at:"1970-01-01T00:03:21Z",update:$update}' \ + >"$rust_bad/market-updates.ndjson" +if "$VERIFY" verify-shadow-parity \ + --legacy-spool "$legacy" --rust-spool "$rust_bad" --started-at-unix 100 \ + --ended-at-unix 300 \ + --output "$tmp_dir/bad-parity.json" 2>/dev/null; then + printf 'parity verifier accepted a duplicate Rust trade ID\n' >&2 + exit 1 +fi +jq -e '.passed == false and .checks.dedupe_parity == false' \ + "$tmp_dir/bad-parity.json" >/dev/null + +rust_bad_metadata="$tmp_dir/rust-bad-metadata" +cp -R "$rust" "$rust_bad_metadata" +jq -c ' + if .update.kind == "market_metadata" and .update.symbol == "BTCUSDT" + then .update.market.clobTokenIds = ["tampered-up", "tampered-down"] + else . end +' "$rust_bad_metadata/market-updates.19700101T000400000000.ndjson" \ + >"$rust_bad_metadata/market-updates.rewritten" +mv "$rust_bad_metadata/market-updates.rewritten" \ + "$rust_bad_metadata/market-updates.19700101T000400000000.ndjson" +if "$VERIFY" verify-shadow-parity \ + --legacy-spool "$legacy" --rust-spool "$rust_bad_metadata" \ + --started-at-unix 100 --ended-at-unix 300 \ + --output "$tmp_dir/bad-metadata-parity.json" 2>/dev/null; then + printf 'parity verifier accepted contradictory metadata values\n' >&2 + exit 1 +fi +jq -e '.passed == false and .checks.metadata_parity == false' \ + "$tmp_dir/bad-metadata-parity.json" >/dev/null + +candidate=$(printf 'a%.0s' {1..64}) +source_revision=$(printf 'b%.0s' {1..40}) +bundle=$(printf 'c%.0s' {1..64}) +oss_config=$(printf 'd%.0s' {1..64}) +legacy_cmdline=dffeb118d105e9312898460249f514eb982c20433cd20840ffb2107c64bbca4a +jq \ + --arg candidate "$candidate" \ + --arg source "$source_revision" \ + --arg bundle "$bundle" \ + --arg oss_config "$oss_config" \ + --arg legacy_cmdline "$legacy_cmdline" \ + '. + { + schema:"monday.polymarket_shadow_gate.v1", + candidate_sha256:$candidate, + deployment_bundle_sha256:$bundle, + deployment_source_revision:$source, + oss_config_sha256:$oss_config, + duration_seconds:3900, + parity_window_started_at_unix:100, + parity_window_ended_at_unix:400, + completed_at:"2026-07-15T00:00:00Z", + shadow_run_id:"run-1", + production_eligible:true, + legacy_runtime:{ + exec_start:"/usr/bin/python3 /opt/monday/bin/polymarket_reference_collector.py", + cmdline:"/usr/bin/python3 /opt/monday/bin/polymarket_reference_collector.py", + cmdline_sha256:$legacy_cmdline, + fragment_path:"/etc/systemd/system/polymarket-reference-collector.service", + drop_in_paths:[],main_pid:10,restarts:0 + }, + shadow_runtime:{ + exec_start:("/opt/monday/releases/polymarket-raw-ops/" + $candidate + + "/polymarket-raw-ops collect-reference --spool-dir ${MONDAY_POLYMARKET_SHADOW_SPOOL}"), + cmdline:("/opt/monday/releases/polymarket-raw-ops/" + $candidate + + "/polymarket-raw-ops collect-reference --spool-dir " + + "/data/monday/spool/polymarket-reference-rust-shadow/" + $candidate + "/run-1"), + fragment_path:"/etc/systemd/system/polymarket-reference-collector-shadow@.service", + drop_in_paths:[],main_pid:11,restarts:0 + }, + checks:(.checks + {health_freshness:true,candidate_identity:true, + oss_readback_parity:true,market_oss_readback_parity:true}), + metrics:(.metrics + { + oss_uploaded_segments:1,oss_canonical_uploaded_segments:1, + market_oss_uploaded_segments:1,market_oss_canonical_uploaded_segments:1 + }) + } | .passed = true' "$parity" >"$tmp_dir/gate.json" +jq -e -f "$POLICY" "$tmp_dir/gate.json" >/dev/null +jq '.duration_seconds = 3599' "$tmp_dir/gate.json" >"$tmp_dir/short.json" +if jq -e -f "$POLICY" "$tmp_dir/short.json" >/dev/null; then + printf 'gate policy accepted a shadow shorter than one hour\n' >&2 + exit 1 +fi +jq '.production_eligible = false' "$tmp_dir/gate.json" >"$tmp_dir/test-only.json" +if jq -e -f "$POLICY" "$tmp_dir/test-only.json" >/dev/null; then + printf 'gate policy accepted test-only evidence\n' >&2 + exit 1 +fi +jq '.parity_window_ended_at_unix = 399' "$tmp_dir/gate.json" \ + >"$tmp_dir/short-parity-tail.json" +if jq -e -f "$POLICY" "$tmp_dir/short-parity-tail.json" >/dev/null; then + printf 'gate policy accepted a parity tail shorter than five minutes\n' >&2 + exit 1 +fi +jq '.checks.metadata_parity = false | .passed = true' "$tmp_dir/gate.json" \ + >"$tmp_dir/bad-metadata-gate.json" +if jq -e -f "$POLICY" "$tmp_dir/bad-metadata-gate.json" >/dev/null; then + printf 'gate policy ignored failed metadata parity\n' >&2 + exit 1 +fi +jq '.checks.market_oss_readback_parity = false | .passed = true' \ + "$tmp_dir/gate.json" >"$tmp_dir/bad-market-upload-gate.json" +if jq -e -f "$POLICY" "$tmp_dir/bad-market-upload-gate.json" >/dev/null; then + printf 'gate policy ignored failed market-tape uploader readback\n' >&2 + exit 1 +fi +jq '.metrics.oss_canonical_uploaded_segments = 0' "$tmp_dir/gate.json" \ + >"$tmp_dir/noncanonical-reference-upload.json" +if jq -e -f "$POLICY" "$tmp_dir/noncanonical-reference-upload.json" >/dev/null; then + printf 'gate policy accepted a noncanonical reference upload\n' >&2 + exit 1 +fi +jq '.metrics.market_oss_canonical_uploaded_segments = 0' "$tmp_dir/gate.json" \ + >"$tmp_dir/noncanonical-market-upload.json" +if jq -e -f "$POLICY" "$tmp_dir/noncanonical-market-upload.json" >/dev/null; then + printf 'gate policy accepted a noncanonical market-tape upload\n' >&2 + exit 1 +fi +jq 'del(.oss_config_sha256)' "$tmp_dir/gate.json" >"$tmp_dir/unbound-oss-config.json" +if jq -e -f "$POLICY" "$tmp_dir/unbound-oss-config.json" >/dev/null; then + printf 'gate policy accepted evidence without an OSS configuration identity\n' >&2 + exit 1 +fi +jq '.legacy_runtime.restarts = 1' "$tmp_dir/gate.json" >"$tmp_dir/restarted-legacy.json" +if jq -e -f "$POLICY" "$tmp_dir/restarted-legacy.json" >/dev/null; then + printf 'gate policy accepted a restarted legacy collector\n' >&2 + exit 1 +fi +jq '.legacy_runtime.drop_in_paths = ["/etc/systemd/system/polymarket-reference-collector.service.d/override.conf"]' \ + "$tmp_dir/gate.json" >"$tmp_dir/legacy-drop-in.json" +if jq -e -f "$POLICY" "$tmp_dir/legacy-drop-in.json" >/dev/null; then + printf 'gate policy accepted a legacy collector with a systemd drop-in\n' >&2 + exit 1 +fi +jq '.legacy_runtime.cmdline_sha256 = "not-a-digest"' "$tmp_dir/gate.json" \ + >"$tmp_dir/unverified-legacy-cmdline.json" +if jq -e -f "$POLICY" "$tmp_dir/unverified-legacy-cmdline.json" >/dev/null; then + printf 'gate policy accepted an unverified legacy command line\n' >&2 + exit 1 +fi +jq '.legacy_runtime.cmdline_sha256 = ("f" * 64)' "$tmp_dir/gate.json" \ + >"$tmp_dir/mismatched-legacy-cmdline-digest.json" +if jq -e -f "$POLICY" "$tmp_dir/mismatched-legacy-cmdline-digest.json" >/dev/null; then + printf 'gate policy accepted a mismatched legacy command-line digest\n' >&2 + exit 1 +fi +jq '.legacy_runtime.cmdline += " --once"' "$tmp_dir/gate.json" \ + >"$tmp_dir/legacy-once-cmdline.json" +if jq -e -f "$POLICY" "$tmp_dir/legacy-once-cmdline.json" >/dev/null; then + printf 'gate policy accepted a legacy --once command line\n' >&2 + exit 1 +fi +jq '.shadow_runtime.drop_in_paths = ["/etc/systemd/system/polymarket-reference-collector-shadow@.service.d/override.conf"]' \ + "$tmp_dir/gate.json" >"$tmp_dir/shadow-drop-in.json" +if jq -e -f "$POLICY" "$tmp_dir/shadow-drop-in.json" >/dev/null; then + printf 'gate policy accepted a Rust shadow with a systemd drop-in\n' >&2 + exit 1 +fi +jq '.shadow_runtime.cmdline += " --once"' "$tmp_dir/gate.json" \ + >"$tmp_dir/shadow-once-cmdline.json" +if jq -e -f "$POLICY" "$tmp_dir/shadow-once-cmdline.json" >/dev/null; then + printf 'gate policy accepted a Rust shadow --once command line\n' >&2 + exit 1 +fi + +jq -n '{ + updated_at:"2026-07-15T00:00:01Z",last_success_at:"2026-07-15T00:00:01Z", + target_markets:14,api_errors:[],malformed_trade_rows:0, + truncated_trade_markets:[],stale_trade_markets:[],stale_settlement_markets:[], + overdue_unresolved_markets:[] +}' >"$tmp_dir/legacy-health.json" +jq -e -f "$LEGACY_HEALTH_POLICY" "$tmp_dir/legacy-health.json" >/dev/null +for mutation in \ + '.api_errors = ["Gamma unavailable"]' \ + '.malformed_trade_rows = 1' \ + '.truncated_trade_markets = ["condition-1"]' \ + '.stale_trade_markets = ["condition-1"]' \ + '.stale_settlement_markets = ["market-1"]' \ + '.overdue_unresolved_markets = ["market-1"]'; do + jq "$mutation" "$tmp_dir/legacy-health.json" >"$tmp_dir/bad-legacy-health.json" + if jq -e -f "$LEGACY_HEALTH_POLICY" "$tmp_dir/bad-legacy-health.json" >/dev/null; then + printf 'legacy health policy accepted failure mutation: %s\n' "$mutation" >&2 + exit 1 + fi +done + +jq -n '{ + updated_at:"2026-07-15T00:00:01Z",last_success_at:"2026-07-15T00:00:01Z", + target_markets:14,missing_target_symbols:[],api_errors:[],malformed_trade_rows:0, + truncated_trade_markets:[],non_object_trade_markets:[],invalid_settlement_markets:[], + invalid_end_time_markets:[],stale_trade_markets:[],stale_settlement_markets:[], + overdue_unresolved_markets:[] +}' >"$tmp_dir/rust-health.json" +jq -e -f "$RUST_HEALTH_POLICY" "$tmp_dir/rust-health.json" >/dev/null +for mutation in \ + 'del(.last_success_at)' \ + '.missing_target_symbols = ["BTCUSDT"]' \ + '.api_errors = ["Gamma unavailable"]' \ + '.non_object_trade_markets = ["condition-1"]' \ + '.invalid_settlement_markets = ["market-1"]' \ + '.invalid_end_time_markets = ["market-1"]'; do + jq "$mutation" "$tmp_dir/rust-health.json" >"$tmp_dir/bad-rust-health.json" + if jq -e -f "$RUST_HEALTH_POLICY" "$tmp_dir/bad-rust-health.json" >/dev/null; then + printf 'Rust health policy accepted failure mutation: %s\n' "$mutation" >&2 + exit 1 + fi +done + +grep -Fq 'readonly REQUIRED_DURATION_SECONDS=3600' "$GATE" +grep -Fq 'readonly PARITY_TAIL_SECONDS=300' "$GATE" +grep -Fq 'verify-shadow-parity' "$GATE" +[[ ! -e "$SCRIPT_DIR/verify-polymarket-shadow-parity.py" ]] +if grep -Fq 'python3 "$PARITY_VERIFIER"' "$GATE"; then + printf 'shadow gate still invokes the retired Python parity verifier\n' >&2 + exit 1 +fi +grep -Fq 'parity_window_ended_at_unix' "$GATE" +grep -Fq 'common_cutoff' "$GATE" +grep -Fq 'shadow_spool="$shadow_parent/$run_id"' "$GATE" +grep -Fq 'MONDAY_POLYMARKET_SHADOW_SPOOL' \ + "$SCRIPT_DIR/polymarket-reference-collector-shadow@.service" +grep -Fq 'crypto_expiry_reference_rust_shadow' "$GATE" +grep -Fq 'crypto_expiry_market_rust_shadow' "$GATE" +grep -Fq '.canonical_uploaded_segments' "$GATE" +grep -Fq 'oss_config_sha256' "$GATE" +grep -Fq 'oss_config_sha256' "$CUTOVER" +grep -Fq 'polymarket-legacy-health-policy.jq' "$GATE" +grep -Fq 'polymarket-legacy-health-policy.jq' "$CUTOVER" +grep -Fq 'polymarket-rust-health-policy.jq' "$GATE" +grep -Fq 'polymarket-rust-health-policy.jq' "$CUTOVER" +grep -Fq 'oss_readback_parity:true' "$GATE" +grep -Fq 'market_oss_readback_parity:true' "$GATE" +grep -Fq '.missing_target_symbols == []' "$RUST_HEALTH_POLICY" +grep -Fq 'systemctl restart "$COLLECTOR_UNIT"' "$CUTOVER" +grep -Fq 'clear_health_before_restart "$evidence_dir" pre-cutover' "$CUTOVER" +grep -Fq 'readlink -f "/proc/$pid/exe"' "$CUTOVER" +grep -Fq 'FragmentPath' "$CUTOVER" +grep -Fq 'DropInPaths' "$CUTOVER" +grep -Fq 'NRestarts' "$CUTOVER" +grep -Fq 'verify_shadow_identity' "$GATE" +grep -Fq 'health_advanced=true' "$CUTOVER" +grep -Fq 'updated_epoch >= started_epoch' "$CUTOVER" +grep -Fq 'gate_legacy_pid' "$CUTOVER" +grep -Fq 'pinned_upload_env' "$GATE" +grep -Fq 'pinned_upload_env' "$CUTOVER" +grep -Fq 'EnvironmentFile=$pinned_upload_env' "$CUTOVER" +grep -Fq 'journalctl --unit "$COLLECTOR_UNIT"' "$CUTOVER" +grep -Fq 'restore_legacy "$evidence_dir"' "$CUTOVER" +grep -Fq 'restore_status=$?' "$CUTOVER" +grep -Fq 'control/polymarket-legacy-health-policy.jq' "$CUTOVER" +if grep -Fq 'if ! restore_legacy' "$CUTOVER"; then + printf 'automatic rollback still invokes restore_legacy from a negated conditional\n' >&2 + exit 1 +fi +grep -Fq 'verify_oneshot_success "$MARKET_UPLOAD_UNIT"' "$CUTOVER" +grep -Fq '/opt/monday/bin/polymarket_reference_collector.py' "$CUTOVER" +grep -Fq 'control-plane bundle changed after the shadow gate' "$CUTOVER" +grep -Fq 'shadow gate evidence is stale or from the future' "$CUTOVER" +grep -Fq 'secure_release_directory "$release_dir"' "$GATE" +grep -Fq 'secure_release_directory "$candidate_release_dir"' "$CUTOVER" +grep -Fxq 'TimeoutStartSec=0' "$SCRIPT_DIR/polymarket-reference-upload.service" +grep -Fxq 'TimeoutStartSec=0' "$SCRIPT_DIR/polymarket-market-tape-upload.service" + +release_chmod_line=$(grep -n '^ chmod 0755 "$staging"$' "$GATE" \ + | cut -d: -f1 || true) +release_move_line=$(grep -n '^ mv "$staging" "$release_dir"$' "$GATE" \ + | cut -d: -f1 || true) +[[ $release_chmod_line =~ ^[1-9][0-9]*$ \ + && $release_move_line =~ ^[1-9][0-9]*$ \ + && $release_chmod_line -lt $release_move_line ]] || { + printf 'shadow gate does not make the release directory traversable before publish\n' >&2 + exit 1 +} + +cutover_stop_line=$(grep -n '^systemctl stop "$COLLECTOR_UNIT"$' "$CUTOVER" | cut -d: -f1) +cutover_clear_line=$(grep -n '^clear_health_before_restart "$evidence_dir" pre-cutover$' \ + "$CUTOVER" | cut -d: -f1) +cutover_restart_line=$(grep -n '^systemctl restart "$COLLECTOR_UNIT"$' "$CUTOVER" \ + | cut -d: -f1) +((cutover_stop_line < cutover_clear_line && cutover_clear_line < cutover_restart_line)) || { + printf 'cutover no longer follows stop -> clear stale health -> explicit restart\n' >&2 + exit 1 +} +rollback_branch_line=$(grep -n '^if \[\[ \$mode == rollback \]\]; then$' "$CUTOVER" \ + | cut -d: -f1) +cutover_policy_line=$(grep -n '^secure_regular_file "$POLICY"$' "$CUTOVER" \ + | tail -1 | cut -d: -f1) +((rollback_branch_line < cutover_policy_line)) || { + printf 'manual rollback still depends on the current cutover bundle preflight\n' >&2 + exit 1 +} +grep -Fq 'shadow_restarts=$(systemctl show --property=NRestarts' "$GATE" + +market_upload_line=$(grep -n '^market_upload_json=' "$GATE" | cut -d: -f1) +legacy_final_line=$(grep -n 'verify_legacy_identity "$legacy_pid"' "$GATE" \ + | tail -1 | cut -d: -f1) +oss_final_line=$(grep -n 'verify_current_oss_config' "$GATE" | tail -1 | cut -d: -f1) +((legacy_final_line > market_upload_line && oss_final_line > market_upload_line)) || { + printf 'gate does not revalidate legacy identity and OSS config after both uploads\n' >&2 + exit 1 +} +grep -Fq 'cd artifact' "$WORKFLOW" +if grep -Fq 'sha256sum artifact/polymarket-raw-ops' "$WORKFLOW"; then + printf 'workflow checksum still embeds the stripped artifact directory\n' >&2 + exit 1 +fi + +printf 'Polymarket raw-ops control-plane tests passed\n' diff --git a/deployment/aliyun/test_polymarket_market_tape_upload.py b/deployment/aliyun/test_polymarket_market_tape_upload.py deleted file mode 100644 index 55c83de6a..000000000 --- a/deployment/aliyun/test_polymarket_market_tape_upload.py +++ /dev/null @@ -1,319 +0,0 @@ -import importlib.util -import json -import shutil -import sys -import tempfile -import unittest -from pathlib import Path -from types import SimpleNamespace -from unittest.mock import patch - - -MODULE_PATH = Path(__file__).with_name("polymarket_market_tape_upload.py") -SPEC = importlib.util.spec_from_file_location("polymarket_market_tape_upload", MODULE_PATH) -UPLOADER = importlib.util.module_from_spec(SPEC) -assert SPEC.loader is not None -sys.modules[SPEC.name] = UPLOADER -SPEC.loader.exec_module(UPLOADER) - - -def record(sequence, recorded_at, update): - return {"sequence": sequence, "recorded_at": recorded_at, "update": update} - - -class TapeValidationTests(unittest.TestCase): - def setUp(self): - self.temporary = tempfile.TemporaryDirectory() - self.spool = Path(self.temporary.name) - - def tearDown(self): - self.temporary.cleanup() - - def write_tape(self, rows, name="market-updates.20260715T010000.ndjson"): - path = self.spool / name - path.write_text("".join(json.dumps(row) + "\n" for row in rows), encoding="utf-8") - return path - - def sample_rows(self): - return [ - record( - 0, - "2026-07-15T01:00:00.000000000Z", - { - "kind": "event_discovered", - "event_id": "event-1", - "symbol": "BTCUSDT", - "up_token": "up-1", - "down_token": "down-1", - "end_time": "2026-07-15T01:05:00Z", - "window_secs": 300, - "price_to_beat": "100", - "resolved_up_won": None, - }, - ), - record( - 1, - "2026-07-15T01:00:01.000000000Z", - { - "kind": "quote", - "token_id": "up-1", - "bid": "0.49", - "ask": "0.51", - "bid_size": "10", - "ask_size": "11", - "bid_levels": [{"price": "0.49", "size": "10"}], - "ask_levels": [{"price": "0.51", "size": "11"}], - "ts": "2026-07-15T01:00:01Z", - }, - ), - record( - 2, - "2026-07-15T01:00:02.000000000Z", - { - "kind": "reference_price", - "symbol": "BTCUSDT", - "source": "binance", - "asset_class": "crypto", - "price": "100", - "full_accuracy_value": None, - "is_carried_forward": False, - "ts": "2026-07-15T01:00:02Z", - }, - ), - ] - - def test_scan_tape_builds_replay_and_field_quality_manifest(self): - path = self.write_tape(self.sample_rows()) - - manifest = UPLOADER.scan_tape(path, "crypto_expiry", 1, 1000) - - self.assertEqual(manifest["events"], 3) - self.assertEqual(manifest["sequence_gaps"], 0) - self.assertEqual(manifest["event_types"]["quote"], 1) - self.assertEqual(manifest["symbols"], ["BTCUSDT"]) - self.assertEqual(manifest["token_count"], 1) - self.assertEqual(manifest["quality"]["crossed_quotes"], 0) - self.assertEqual(manifest["quality"]["max_bid_levels"], 1) - self.assertEqual(manifest["field_non_null"]["quote"]["bid"], 1) - - def test_scan_tape_rejects_sequence_gap(self): - rows = self.sample_rows() - rows[1]["sequence"] = 2 - path = self.write_tape(rows) - - with self.assertRaisesRegex(ValueError, "sequence gap"): - UPLOADER.scan_tape(path, "crypto_expiry", 1, 1000) - - def test_scan_tape_rejects_incomplete_final_record(self): - path = self.write_tape(self.sample_rows()) - with path.open("ab") as handle: - handle.write(b'{"sequence":3') - - with self.assertRaisesRegex(ValueError, "incomplete record"): - UPLOADER.scan_tape(path, "crypto_expiry", 1, 1000) - - def test_scan_tape_rejects_malformed_quote_numbers(self): - rows = self.sample_rows() - rows[1]["update"]["bid"] = "not-a-number" - path = self.write_tape(rows) - - with self.assertRaisesRegex(ValueError, "bid must be numeric"): - UPLOADER.scan_tape(path, "crypto_expiry", 1, 1000) - - def test_scan_tape_rejects_malformed_depth_level(self): - rows = self.sample_rows() - rows[1]["update"]["bid_levels"][0]["size"] = None - path = self.write_tape(rows) - - with self.assertRaisesRegex(ValueError, "requires price and size"): - UPLOADER.scan_tape(path, "crypto_expiry", 1, 1000) - - def test_zero_depth_limit_accepts_full_visible_book(self): - rows = self.sample_rows() - rows[1]["update"]["bid_levels"].append({"price": "0.48", "size": "12"}) - rows[1]["update"]["ask_levels"].append({"price": "0.52", "size": "13"}) - path = self.write_tape(rows) - - manifest = UPLOADER.scan_tape(path, "crypto_expiry", 0, 1000) - - self.assertTrue(manifest["venue_depth_complete"]) - self.assertFalse(manifest["temporal_updates_complete"]) - self.assertEqual(manifest["quality"]["max_bid_levels"], 2) - self.assertEqual(manifest["recording_policy"]["quote_depth_levels"], 0) - - def test_reference_tape_validates_raw_fields_and_trade_quality(self): - rows = [ - record( - 0, - "2026-07-15T01:00:00Z", - { - "kind": "market_metadata", - "market_id": "market-1", - "condition_id": "0xcondition", - "symbol": "BTCUSDT", - "market_window_secs": 300, - "retrieved_at": "2026-07-15T01:00:00Z", - "market": { - "volume": 100, - "orderPriceMinTickSize": 0.01, - "orderMinSize": 5, - "makerBaseFee": 1000, - "takerBaseFee": 1000, - }, - }, - ), - record( - 1, - "2026-07-15T01:00:01Z", - { - "kind": "polymarket_trade", - "record_id": "trade-1", - "record_id_version": "v2", - "market_id": "market-1", - "condition_id": "0xcondition", - "token_id": "token-1", - "symbol": "BTCUSDT", - "side": "BUY", - "size": 10, - "price": 0.51, - "trade_ts": "2026-07-15T01:00:00Z", - "transaction_hash": "0xtx", - "trade": {"asset": "token-1", "price": 0.51, "size": 10}, - }, - ), - ] - path = self.write_tape(rows) - - manifest = UPLOADER.scan_tape(path, "crypto_expiry_reference", 0, 0) - - self.assertEqual(manifest["market_count"], 1) - self.assertEqual(manifest["condition_count"], 1) - self.assertTrue(manifest["canonical"]) - self.assertEqual(manifest["record_id_versions"], ["v2"]) - self.assertEqual(manifest["quality"]["duplicate_record_ids"], 0) - self.assertEqual( - manifest["source_field_non_null"]["market_metadata"]["makerBaseFee"], 1 - ) - - def test_scan_tape_marks_quotes_without_event_context_as_non_self_contained(self): - path = self.write_tape([self.sample_rows()[1]]) - row = json.loads(path.read_text()) - row["sequence"] = 0 - path.write_text(json.dumps(row) + "\n", encoding="utf-8") - - manifest = UPLOADER.scan_tape(path, "crypto_expiry", 1, 1000) - - self.assertFalse(manifest["event_context_complete"]) - self.assertEqual(manifest["quality"]["contextless_quotes"], 1) - self.assertIn("requires_prior_event_context", manifest["replay_scope"]) - - def test_discovery_excludes_active_tape(self): - self.write_tape(self.sample_rows(), "market-updates.ndjson") - rotated = self.write_tape(self.sample_rows()) - - self.assertEqual(UPLOADER.discover_rotated_tapes(self.spool), [rotated]) - - def test_split_tape_by_utc_hour_preserves_contiguous_global_sequences(self): - rows = self.sample_rows() - rows.append( - record( - 3, - "2026-07-15T02:00:00.000000000Z", - {"kind": "event_expired", "event_id": "event-1", "end_time": None}, - ) - ) - source = self.write_tape(rows) - - chunks = UPLOADER.split_tape_by_utc_hour(source) - - self.assertEqual(len(chunks), 2) - first = UPLOADER.scan_tape(chunks[0], "crypto_expiry", 1, 1000) - second = UPLOADER.scan_tape(chunks[1], "crypto_expiry", 1, 1000) - self.assertEqual((first["start_sequence"], first["end_sequence"]), (0, 2)) - self.assertEqual((second["start_sequence"], second["end_sequence"]), (3, 3)) - self.assertEqual((first["hour"], second["hour"]), ("01", "02")) - - @unittest.skipUnless(shutil.which("zstd"), "zstd is required for artifact check") - def test_prepare_artifacts_emits_hash_bound_manifest_and_success(self): - source = self.write_tape(self.sample_rows()) - - artifacts, manifest = UPLOADER.prepare_artifacts( - source, "crypto_expiry", 1, 1000, 30 - ) - - self.assertTrue(source.exists()) - self.assertTrue(artifacts.data.exists()) - self.assertEqual(artifacts.success.read_text().strip(), manifest["sha256"]) - self.assertEqual(json.loads(artifacts.manifest.read_text())["sha256"], manifest["sha256"]) - self.assertEqual( - artifacts.object_prefix, - "lake/raw/venue=polymarket/dataset=crypto_expiry/date=2026-07-15/hour=01", - ) - - @unittest.skipUnless(shutil.which("zstd"), "zstd is required for artifact check") - def test_remote_verification_reads_back_hash_bound_triplet(self): - source = self.write_tape(self.sample_rows()) - artifacts, _ = UPLOADER.prepare_artifacts( - source, "crypto_expiry", 1, 1000, 30 - ) - remote = { - path.name: path.read_bytes() - for path in (artifacts.data, artifacts.manifest, artifacts.success) - } - - def download(command, **_kwargs): - Path(command[4]).write_bytes(remote[Path(command[3]).name]) - - with patch.object(UPLOADER.subprocess, "run", side_effect=download): - UPLOADER.verify_remote_artifacts( - artifacts, "bucket", "endpoint", "region", "profile", 30 - ) - - def test_run_continues_after_one_bad_closed_tape(self): - first = self.write_tape(self.sample_rows(), "market-updates.20260715T010000.ndjson") - second = self.write_tape(self.sample_rows(), "market-updates.20260715T020000.ndjson") - args = SimpleNamespace( - spool_dir=self.spool, - dataset="crypto_expiry", - quote_depth_levels=1, - quote_sample_ms=1000, - zstd_timeout=30, - bucket="bucket", - endpoint="endpoint", - region="region", - profile="profile", - oss_timeout=30, - ) - - def archive(source, _args): - if source == first: - raise ValueError("bad tape") - source.unlink() - return ["oss://bucket/second"] - - with patch.object(UPLOADER, "archive_source", side_effect=archive) as mocked: - result = UPLOADER.run(args) - - self.assertEqual(result, 1) - self.assertEqual(mocked.call_count, 2) - status = json.loads((self.spool / "upload-status.json").read_text()) - self.assertEqual(status["pending_segments"], 1) - self.assertEqual(status["failed_segments"][0]["source"], first.name) - - def test_archive_source_rejects_a_truncated_session_prefix(self): - rows = self.sample_rows() - for sequence, row in enumerate(rows, start=5): - row["sequence"] = sequence - source = self.write_tape(rows) - args = SimpleNamespace( - dataset="crypto_expiry", - quote_depth_levels=1, - quote_sample_ms=1000, - ) - - with self.assertRaisesRegex(ValueError, "must start at sequence 0"): - UPLOADER.archive_source(source, args) - - -if __name__ == "__main__": - unittest.main() diff --git a/deployment/aliyun/test_polymarket_reference_canonicalize.py b/deployment/aliyun/test_polymarket_reference_canonicalize.py deleted file mode 100644 index 59a898aa7..000000000 --- a/deployment/aliyun/test_polymarket_reference_canonicalize.py +++ /dev/null @@ -1,59 +0,0 @@ -import importlib.util -import json -import sys -import tempfile -import unittest -from pathlib import Path - - -BASE = Path(__file__).parent -sys.path.insert(0, str(BASE)) -SPEC = importlib.util.spec_from_file_location( - "polymarket_reference_canonicalize", BASE / "polymarket_reference_canonicalize.py" -) -CANONICALIZE = importlib.util.module_from_spec(SPEC) -assert SPEC.loader is not None -SPEC.loader.exec_module(CANONICALIZE) - - -class CanonicalizeTests(unittest.TestCase): - def test_union_rekeys_v1_trades_and_removes_only_v2_duplicates(self): - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - first = root / "first.ndjson" - second = root / "second.ndjson" - output = root / "market-updates.repair.ndjson" - trade = { - "transactionHash": "0xtx", - "conditionId": "0xcondition", - "asset": "token", - "side": "BUY", - "timestamp": 1784084400, - "proxyWallet": "0xwallet", - "size": 10, - "price": 0.5, - "outcome": "Up", - "outcomeIndex": 0, - } - collision = dict(trade, proxyWallet="0xother", size=11) - rows = [ - {"sequence": 0, "recorded_at": "2026-07-15T03:00:00Z", "update": {"kind": "polymarket_trade", "record_id": "v1", "trade": trade}}, - {"sequence": 1, "recorded_at": "2026-07-15T03:00:01Z", "update": {"kind": "polymarket_trade", "record_id": "v1", "trade": collision}}, - ] - first.write_text("".join(json.dumps(row) + "\n" for row in rows), encoding="utf-8") - second.write_text(json.dumps(rows[0]) + "\n", encoding="utf-8") - - summary = CANONICALIZE.canonicalize([first, second], output) - written = [json.loads(line) for line in output.read_text().splitlines()] - - self.assertEqual(summary["canonical_v2_trades"], 2) - self.assertEqual(summary["duplicate_trades_removed"], 1) - self.assertEqual([row["sequence"] for row in written], [0, 1]) - self.assertEqual( - {row["update"]["record_id_version"] for row in written}, {"v2"} - ) - self.assertEqual(len({row["update"]["record_id"] for row in written}), 2) - - -if __name__ == "__main__": - unittest.main() diff --git a/deployment/aliyun/test_polymarket_reference_collector.py b/deployment/aliyun/test_polymarket_reference_collector.py deleted file mode 100644 index af1d21b76..000000000 --- a/deployment/aliyun/test_polymarket_reference_collector.py +++ /dev/null @@ -1,388 +0,0 @@ -import importlib.util -import json -import sys -import tempfile -import unittest -from unittest import mock -from datetime import datetime, timezone -from pathlib import Path -from types import SimpleNamespace - - -MODULE_PATH = Path(__file__).with_name("polymarket_reference_collector.py") -SPEC = importlib.util.spec_from_file_location("polymarket_reference_collector", MODULE_PATH) -COLLECTOR = importlib.util.module_from_spec(SPEC) -assert SPEC.loader is not None -sys.modules[SPEC.name] = COLLECTOR -SPEC.loader.exec_module(COLLECTOR) - - -def sample_market(**overrides): - market = { - "id": "2916403", - "question": "Dogecoin Up or Down - July 14, 11:00PM-11:15PM ET", - "conditionId": "0xcondition", - "slug": "doge-updown-15m-1784084400", - "startDate": "2026-07-15T03:00:00Z", - "endDate": "2026-07-15T03:15:00Z", - "closed": False, - "clobTokenIds": '["up-token","down-token"]', - "outcomes": '["Up","Down"]', - "outcomePrices": '["0.445","0.555"]', - "volume": 100, - "orderPriceMinTickSize": 0.01, - "orderMinSize": 5, - "makerBaseFee": 1000, - "takerBaseFee": 1000, - } - market.update(overrides) - return market - - -class ReferenceCollectorUnitTests(unittest.TestCase): - def test_v2_trade_identity_migration_reopens_completed_markets(self): - with tempfile.TemporaryDirectory() as directory: - spool = Path(directory) - (spool / "collector-state.json").write_text( - json.dumps( - { - "markets": {"market-1": {"trade_complete": True}}, - "trade_seen": {"0xcondition": {"old-id": 1}}, - } - ), - encoding="utf-8", - ) - (spool / COLLECTOR.ACTIVE_TAPE).write_text("legacy\n", encoding="utf-8") - collector = COLLECTOR.ReferenceCollector( - SimpleNamespace(spool_dir=spool, symbols=["BTCUSDT"]) - ) - - self.assertEqual(collector.state["trade_id_version"], "v2") - self.assertEqual(collector.state["trade_seen"], {}) - self.assertEqual(len(list(spool.glob("superseded-v1-market-updates.ndjson.*"))), 1) - self.assertFalse(collector.state["markets"]["market-1"]["trade_complete"]) - collector.writer.close() - - def test_target_market_keeps_all_seven_assets_and_supported_windows(self): - symbols = {symbol for symbol, _aliases in COLLECTOR.SYMBOL_ALIASES} - questions = { - "BTCUSDT": "Bitcoin Up or Down - 5 minutes", - "ETHUSDT": "Ethereum Up or Down - 5 minutes", - "SOLUSDT": "Solana Up or Down - 5 minutes", - "XRPUSDT": "XRP Up or Down - 5 minutes", - "DOGEUSDT": "Dogecoin Up or Down - 5 minutes", - "HYPEUSDT": "Hyperliquid Up or Down - 5 minutes", - "BNBUSDT": "BNB Up or Down - 5 minutes", - } - for expected, question in questions.items(): - market = sample_market( - question=question, - slug=f"{expected.lower()}-updown-5m-1784084400", - startDate="2026-07-15T03:00:00Z", - endDate="2026-07-15T03:05:00Z", - ) - self.assertEqual(COLLECTOR.is_target_market(market, symbols), (expected, 300)) - - def test_settlement_requires_closed_terminal_prices(self): - market = sample_market(closed=True, outcomePrices='["0.0005","0.9995"]') - update = COLLECTOR.settlement_from_market( - market, "DOGEUSDT", 900, "2026-07-15T03:16:00Z" - ) - self.assertIsNotNone(update) - self.assertEqual(update["winning_token_id"], "down-token") - self.assertFalse(update["resolved_up_won"]) - self.assertEqual(update["market"]["makerBaseFee"], 1000) - self.assertIsNone( - COLLECTOR.settlement_from_market( - sample_market(), "DOGEUSDT", 900, "2026-07-15T03:16:00Z" - ) - ) - - def test_trade_updates_are_stable_and_deduplicated_across_polls(self): - collector = COLLECTOR.ReferenceCollector.__new__(COLLECTOR.ReferenceCollector) - collector.state = {"trade_seen": {}} - collector.args = SimpleNamespace(market_lookback_secs=7200) - now = datetime(2026, 7, 15, 3, 10, tzinfo=timezone.utc) - trade = { - "proxyWallet": "0xwallet", - "side": "BUY", - "asset": "up-token", - "conditionId": "0xcondition", - "size": 10, - "price": 0.78, - "timestamp": int(now.timestamp()) - 5, - "outcome": "Up", - "outcomeIndex": 0, - "transactionHash": "0xtx", - } - second_trade = dict(trade, proxyWallet="0xother", size=11, price=0.79) - first, malformed = collector.trade_updates( - "2916403", "0xcondition", "DOGEUSDT", 900, [trade, second_trade], now - ) - second, second_malformed = collector.trade_updates( - "2916403", "0xcondition", "DOGEUSDT", 900, [trade, second_trade], now - ) - self.assertEqual(len(first), 2) - self.assertEqual(second, []) - self.assertEqual((malformed, second_malformed), ({}, {})) - self.assertEqual(first[0]["market_id"], "2916403") - self.assertEqual(first[0]["record_id_version"], "v2") - self.assertEqual(first[0]["record_id"], COLLECTOR.stable_trade_id(trade)) - self.assertNotEqual( - COLLECTOR.stable_trade_id(trade), COLLECTOR.stable_trade_id(second_trade) - ) - - def test_staged_trade_updates_do_not_advance_live_state(self): - collector = COLLECTOR.ReferenceCollector.__new__(COLLECTOR.ReferenceCollector) - collector.state = {"trade_seen": {}} - collector.args = SimpleNamespace(market_lookback_secs=7200) - staged = {"trade_seen": {}} - now = datetime(2026, 7, 15, 3, 10, tzinfo=timezone.utc) - trade = { - "proxyWallet": "0xwallet", - "side": "BUY", - "asset": "up-token", - "conditionId": "0xcondition", - "size": 10, - "price": 0.78, - "timestamp": int(now.timestamp()) - 5, - "outcome": "Up", - "outcomeIndex": 0, - "transactionHash": "0xtx", - } - - collector.trade_updates( - "2916403", "0xcondition", "DOGEUSDT", 900, [trade], now, staged - ) - - self.assertEqual(collector.state, {"trade_seen": {}}) - self.assertIn(COLLECTOR.stable_trade_id(trade), staged["trade_seen"]["0xcondition"]) - - def test_v2_replay_preserves_all_prints_that_collided_under_v1(self): - collector = COLLECTOR.ReferenceCollector.__new__(COLLECTOR.ReferenceCollector) - collector.args = SimpleNamespace(market_lookback_secs=7200) - now = datetime(2026, 7, 15, 3, 10, tzinfo=timezone.utc) - first = { - "transactionHash": "0xtx", - "conditionId": "0xcondition", - "asset": "up-token", - "side": "BUY", - "timestamp": int(now.timestamp()) - 5, - "proxyWallet": "0xfirst", - "size": 10, - "price": 0.78, - "outcome": "Up", - "outcomeIndex": 0, - } - collision = dict(first, proxyWallet="0xsecond", size=11, price=0.79) - collector.state = {"trade_seen": {}} - - updates, malformed = collector.trade_updates( - "2916403", "0xcondition", "DOGEUSDT", 900, [first, collision], now - ) - - self.assertEqual( - [item["record_id"] for item in updates], - [COLLECTOR.stable_trade_id(first), COLLECTOR.stable_trade_id(collision)], - ) - self.assertEqual(malformed, {}) - - def test_malformed_trade_is_counted_without_blocking_valid_rows(self): - collector = COLLECTOR.ReferenceCollector.__new__(COLLECTOR.ReferenceCollector) - collector.args = SimpleNamespace(market_lookback_secs=7200) - collector.state = {"trade_seen": {}} - now = datetime(2026, 7, 15, 3, 10, tzinfo=timezone.utc) - valid = { - "transactionHash": "0xtx", - "conditionId": "0xcondition", - "asset": "up-token", - "side": "BUY", - "timestamp": int(now.timestamp()) - 5, - "proxyWallet": "0xwallet", - "size": 10, - "price": 0.78, - "outcome": "Up", - "outcomeIndex": 0, - } - - updates, malformed = collector.trade_updates( - "2916403", - "0xcondition", - "DOGEUSDT", - 900, - [ - dict(valid, timestamp=None), - dict(valid, side="HOLD"), - dict(valid, size="bad"), - dict(valid, price=2), - dict(valid, transactionHash=""), - valid, - ], - now, - ) - - self.assertEqual( - malformed, - { - "invalid_timestamp": 1, - "invalid_side": 1, - "invalid_size": 1, - "invalid_price": 1, - "missing_transaction_hash": 1, - }, - ) - self.assertEqual(len(updates), 1) - - def test_trade_finalization_requires_lag_and_stable_polls_and_resets_on_late_trade(self): - settled_at = datetime(2026, 7, 15, 3, 0, tzinfo=timezone.utc) - tracked = {} - self.assertFalse( - COLLECTOR.advance_trade_finalization( - tracked, settled_at, COLLECTOR.iso_z(settled_at), 0, False, False, 1800, 2 - ) - ) - after_lag = settled_at.replace(minute=30) - self.assertFalse( - COLLECTOR.advance_trade_finalization( - tracked, after_lag, COLLECTOR.iso_z(after_lag), 0, False, True, 1800, 2 - ) - ) - late_trade_at = settled_at.replace(minute=31) - self.assertFalse( - COLLECTOR.advance_trade_finalization( - tracked, - late_trade_at, - COLLECTOR.iso_z(late_trade_at), - 1, - False, - True, - 1800, - 2, - ) - ) - final_poll_one = settled_at.replace(hour=4, minute=1) - self.assertFalse( - COLLECTOR.advance_trade_finalization( - tracked, - final_poll_one, - COLLECTOR.iso_z(final_poll_one), - 0, - False, - True, - 1800, - 2, - ) - ) - final_poll_two = settled_at.replace(hour=4, minute=2) - self.assertTrue( - COLLECTOR.advance_trade_finalization( - tracked, - final_poll_two, - COLLECTOR.iso_z(final_poll_two), - 0, - False, - True, - 1800, - 2, - ) - ) - - -class TapeWriterTests(unittest.TestCase): - def test_write_all_retries_short_writes(self): - class ShortWriter: - def __init__(self): - self.data = bytearray() - - def write(self, value): - count = min(2, len(value)) - self.data.extend(value[:count]) - return count - - handle = ShortWriter() - COLLECTOR.TapeWriter._write_all(handle, b"complete") - self.assertEqual(bytes(handle.data), b"complete") - - def test_failed_batch_is_truncated_and_sequence_is_restored(self): - with tempfile.TemporaryDirectory() as directory: - spool = Path(directory) - writer = COLLECTOR.TapeWriter(spool) - with mock.patch.object( - COLLECTOR.os, "fsync", side_effect=[OSError("disk full"), None] - ): - with self.assertRaisesRegex(OSError, "disk full"): - writer.write_updates( - [{"kind": "market_metadata"}, {"kind": "polymarket_trade"}], - datetime(2026, 7, 15, 3, 0, tzinfo=timezone.utc), - ) - self.assertEqual(writer.sequence, 0) - self.assertEqual((spool / COLLECTOR.ACTIVE_TAPE).read_bytes(), b"") - writer.close() - - def test_rotation_produces_closed_zero_based_sessions(self): - with tempfile.TemporaryDirectory() as directory: - spool = Path(directory) - writer = COLLECTOR.TapeWriter(spool) - writer.write_updates( - [{"kind": "market_metadata"}], - datetime(2026, 7, 15, 3, 59, tzinfo=timezone.utc), - ) - writer.write_updates( - [{"kind": "market_metadata"}], - datetime(2026, 7, 15, 4, 0, tzinfo=timezone.utc), - ) - writer.close() - closed = list(spool.glob("market-updates.*.ndjson")) - self.assertEqual(len(closed), 1) - self.assertEqual(json.loads(closed[0].read_text())["sequence"], 0) - self.assertEqual(json.loads((spool / COLLECTOR.ACTIVE_TAPE).read_text())["sequence"], 0) - - def test_startup_truncates_only_incomplete_tail_and_resumes_sequence(self): - with tempfile.TemporaryDirectory() as directory: - spool = Path(directory) - active = spool / COLLECTOR.ACTIVE_TAPE - active.write_bytes( - b'{"sequence":0,"recorded_at":"2026-07-15T03:00:00Z","update":{}}\n' - b'{"sequence":1' - ) - writer = COLLECTOR.TapeWriter(spool) - writer.write_updates( - [{"kind": "market_metadata"}], - datetime(2026, 7, 15, 3, 1, tzinfo=timezone.utc), - ) - writer.close() - rows = [json.loads(line) for line in active.read_text().splitlines()] - self.assertEqual([row["sequence"] for row in rows], [0, 1]) - - def test_collector_recovers_trade_dedupe_state_from_durable_tape(self): - with tempfile.TemporaryDirectory() as directory: - spool = Path(directory) - writer = COLLECTOR.TapeWriter(spool) - writer.write_updates( - [ - { - "kind": "polymarket_trade", - "market_id": "market-1", - "condition_id": "0xcondition", - "record_id": "trade-1", - "record_id_version": "v2", - "trade_ts_unix": 1784084400, - } - ], - datetime(2026, 7, 15, 3, 0, tzinfo=timezone.utc), - ) - writer.close() - collector = COLLECTOR.ReferenceCollector.__new__(COLLECTOR.ReferenceCollector) - collector.state = {"markets": {}, "trade_seen": {}} - collector.writer = COLLECTOR.TapeWriter(spool) - - collector.recover_state_from_active_tape() - collector.writer.close() - - self.assertEqual( - collector.state["trade_seen"]["0xcondition"]["trade-1"], 1784084400 - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/products/ploy/tasks/todo.md b/products/ploy/tasks/todo.md index 205e6a5da..53ae588c5 100644 --- a/products/ploy/tasks/todo.md +++ b/products/ploy/tasks/todo.md @@ -29,6 +29,7 @@ Integrate the maintained PLOY product workspace into Monday without transferring - [x] Archive hourly Polymarket tapes to OSS with manifest and readback verification. - [x] Retain full visible CLOB depth in one-second snapshots for all seven configured crypto assets. - [x] Add a collection-only metadata/trade/settlement tape and verify a complete OSS artifact. +- [x] Port the long-lived Polymarket reference collector and raw-tape uploader to Rust, and retire their Python runtime entry points. ## Safety diff --git a/products/ploy/tests/workspace_runtime_retirement.rs b/products/ploy/tests/workspace_runtime_retirement.rs index 03bbb35b6..fb88f574f 100644 --- a/products/ploy/tests/workspace_runtime_retirement.rs +++ b/products/ploy/tests/workspace_runtime_retirement.rs @@ -258,6 +258,14 @@ fn monday_polymarket_data_service_is_read_only_and_fail_closed() { monday_root.join("deployment/aliyun/polymarket-reference-collector.service"), ) .expect("read Polymarket reference service"); + let market_upload_service = fs::read_to_string( + monday_root.join("deployment/aliyun/polymarket-market-tape-upload.service"), + ) + .expect("read Polymarket market tape upload service"); + let reference_upload_service = fs::read_to_string( + monday_root.join("deployment/aliyun/polymarket-reference-upload.service"), + ) + .expect("read Polymarket reference upload service"); for required in [ "mode = \"dryrun\"", @@ -295,7 +303,7 @@ fn monday_polymarket_data_service_is_read_only_and_fail_closed() { } for required in [ "User=hftcollector", - "polymarket_reference_collector.py", + "/opt/monday/bin/polymarket-raw-ops collect-reference", "NoNewPrivileges=true", "ProtectSystem=strict", "ReadWritePaths=/data/monday/spool/polymarket-reference", @@ -305,4 +313,29 @@ fn monday_polymarket_data_service_is_read_only_and_fail_closed() { "reference service missing {required}" ); } + for (name, upload_service) in [ + ("market", &market_upload_service), + ("reference", &reference_upload_service), + ] { + assert!( + upload_service.contains("/opt/monday/bin/polymarket-raw-ops upload"), + "{name} uploader must use the Rust raw-ops binary" + ); + for forbidden in ["python3", ".py", "PRIVATE_KEY", "live-execution"] { + assert!( + !upload_service.contains(forbidden), + "{name} uploader contains forbidden runtime surface {forbidden}" + ); + } + } + for retired in [ + "polymarket_reference_collector.py", + "polymarket_market_tape_upload.py", + "polymarket_reference_canonicalize.py", + ] { + assert!( + !monday_root.join("deployment/aliyun").join(retired).exists(), + "retired Python runtime still exists: {retired}" + ); + } } diff --git a/rust_hft/deployment/docker/Dockerfile.binance-lob-archiver b/rust_hft/deployment/docker/Dockerfile.binance-lob-archiver index cd412c682..b3e0bfe83 100644 --- a/rust_hft/deployment/docker/Dockerfile.binance-lob-archiver +++ b/rust_hft/deployment/docker/Dockerfile.binance-lob-archiver @@ -19,8 +19,11 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \ --no-default-features \ -p hft-collector \ --bin binance-lob-archiver \ + --bin polymarket-raw-ops \ && install -D -m 0755 /work/target/release/binance-lob-archiver \ - /out/bin/binance-lob-archiver + /out/bin/binance-lob-archiver \ + && install -D -m 0755 /work/target/release/polymarket-raw-ops \ + /out/bin/polymarket-raw-ops FROM debian:bookworm-slim AS aliyun-cli @@ -54,6 +57,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ && chown -R hftcollector:hftcollector /data/monday COPY --from=builder /out/bin/binance-lob-archiver /usr/local/bin/binance-lob-archiver +COPY --from=builder /out/bin/polymarket-raw-ops /usr/local/bin/polymarket-raw-ops COPY --from=aliyun-cli /usr/local/bin/aliyun /usr/local/bin/aliyun USER hftcollector diff --git a/rust_hft/tools/collector/Cargo.toml b/rust_hft/tools/collector/Cargo.toml index e35c53bb1..34430b92e 100644 --- a/rust_hft/tools/collector/Cargo.toml +++ b/rust_hft/tools/collector/Cargo.toml @@ -79,6 +79,10 @@ path = "src/main.rs" name = "binance-lob-archiver" path = "src/bin/binance-lob-archiver.rs" +[[bin]] +name = "polymarket-raw-ops" +path = "src/bin/polymarket-raw-ops.rs" + [lib] name = "hft_collector" path = "src/lib.rs" diff --git a/rust_hft/tools/collector/src/bin/polymarket-raw-ops.rs b/rust_hft/tools/collector/src/bin/polymarket-raw-ops.rs new file mode 100644 index 000000000..50da10b96 --- /dev/null +++ b/rust_hft/tools/collector/src/bin/polymarket-raw-ops.rs @@ -0,0 +1,217 @@ +use anyhow::{bail, Context, Result}; +use clap::{Parser, Subcommand}; +use hft_collector::polymarket_parity::{verify_shadow_parity, ShadowParityConfig}; +use hft_collector::polymarket_raw::{run_reference, ReferenceConfig, DEFAULT_MAX_MARKETS_PER_LANE}; +use hft_collector::polymarket_upload::{run_upload, UploadConfig}; +use std::env; +use std::path::PathBuf; +use std::time::Duration; + +#[derive(Debug, Parser)] +#[command( + name = "polymarket-raw-ops", + about = "Fail-closed Polymarket reference collection and raw tape archival" +)] +struct Cli { + #[command(subcommand)] + command: Command, +} + +#[derive(Debug, Subcommand)] +enum Command { + /// Collect public market metadata, trades, and settlement evidence. + CollectReference { + #[arg(long, default_value = "/data/monday/spool/polymarket-reference")] + spool_dir: PathBuf, + #[arg( + long, + value_delimiter = ',', + default_value = "BTCUSDT,ETHUSDT,SOLUSDT,XRPUSDT,DOGEUSDT,HYPEUSDT,BNBUSDT" + )] + symbols: Vec, + #[arg(long, default_value_t = 30.0)] + poll_interval_secs: f64, + #[arg(long, default_value_t = 7_200)] + market_lookback_secs: i64, + #[arg(long, default_value_t = 86_400)] + settlement_lookback_secs: i64, + #[arg(long, default_value_t = DEFAULT_MAX_MARKETS_PER_LANE)] + max_markets: usize, + #[arg(long, default_value_t = 20.0)] + http_timeout: f64, + #[arg(long, default_value_t = 180.0)] + stale_after_secs: f64, + #[arg(long, default_value_t = 1_800)] + trade_finalization_lag_secs: i64, + #[arg(long, default_value_t = 3)] + trade_finalization_stable_polls: u64, + #[arg(long, default_value_t = 100)] + per_market_delay_ms: u64, + #[arg(long)] + once: bool, + }, + /// Validate, compress, upload, and read back all closed tape segments. + Upload { + #[arg(long, default_value = "/data/monday/spool/polymarket")] + spool_dir: PathBuf, + #[arg(long, default_value = "crypto_expiry")] + dataset: String, + #[arg(long, default_value_t = 0)] + quote_depth_levels: usize, + #[arg(long, default_value_t = 0)] + quote_sample_ms: u64, + #[arg(long)] + bucket: Option, + #[arg(long)] + endpoint: Option, + #[arg(long)] + region: Option, + #[arg(long)] + profile: Option, + #[arg(long)] + zstd_timeout: Option, + #[arg(long)] + oss_timeout: Option, + }, + /// Compare a bounded Python reference lane with an isolated Rust shadow. + VerifyShadowParity { + #[arg(long)] + legacy_spool: PathBuf, + #[arg(long)] + rust_spool: PathBuf, + #[arg(long)] + started_at_unix: i64, + #[arg(long)] + ended_at_unix: i64, + #[arg(long)] + output: PathBuf, + }, +} + +fn positive_duration(value: f64, name: &str) -> Result { + if !value.is_finite() || value <= 0.0 { + bail!("{name} must be finite and positive"); + } + Ok(Duration::from_secs_f64(value)) +} + +fn env_or(value: Option, name: &str, fallback: &str) -> String { + value.unwrap_or_else(|| env::var(name).unwrap_or_else(|_| fallback.to_owned())) +} + +fn env_u64(value: Option, name: &str, fallback: u64) -> Result { + match value { + Some(value) => Ok(value), + None => env::var(name) + .ok() + .map(|raw| { + raw.parse::() + .with_context(|| format!("{name} must be an unsigned integer")) + }) + .transpose() + .map(|value| value.unwrap_or(fallback)), + } +} + +#[tokio::main] +async fn main() -> Result<()> { + match Cli::parse().command { + Command::CollectReference { + spool_dir, + symbols, + poll_interval_secs, + market_lookback_secs, + settlement_lookback_secs, + max_markets, + http_timeout, + stale_after_secs, + trade_finalization_lag_secs, + trade_finalization_stable_polls, + per_market_delay_ms, + once, + } => { + let config = ReferenceConfig { + spool_dir, + symbols, + poll_interval: positive_duration(poll_interval_secs, "poll interval")?, + market_lookback_secs, + settlement_lookback_secs, + max_markets, + http_timeout: positive_duration(http_timeout, "HTTP timeout")?, + stale_after: positive_duration(stale_after_secs, "stale interval")?, + trade_finalization_lag_secs, + trade_finalization_stable_polls, + per_market_delay: Duration::from_millis(per_market_delay_ms), + }; + run_reference(config, once).await + } + Command::Upload { + spool_dir, + dataset, + quote_depth_levels, + quote_sample_ms, + bucket, + endpoint, + region, + profile, + zstd_timeout, + oss_timeout, + } => { + let zstd_timeout = env_u64(zstd_timeout, "ZSTD_TIMEOUT_SECONDS", 300)?; + let oss_timeout = env_u64(oss_timeout, "OSS_COPY_TIMEOUT_SECONDS", 300)?; + let config = UploadConfig { + spool_dir, + dataset, + quote_depth_levels, + quote_sample_ms, + bucket: env_or(bucket, "OSS_BUCKET", "monday-lob-apne1-1045353359"), + endpoint: env_or( + endpoint, + "OSS_ENDPOINT", + "oss-ap-northeast-1-internal.aliyuncs.com", + ), + region: env_or(region, "OSS_REGION", "ap-northeast-1"), + profile: env_or(profile, "ALIYUN_PROFILE", "ecs-role"), + zstd_timeout: Duration::from_secs(zstd_timeout), + oss_timeout: Duration::from_secs(oss_timeout), + }; + println!("{}", serde_json::to_string(&run_upload(&config)?)?); + Ok(()) + } + Command::VerifyShadowParity { + legacy_spool, + rust_spool, + started_at_unix, + ended_at_unix, + output, + } => { + let config = ShadowParityConfig { + legacy_spool, + rust_spool, + started_at_unix, + ended_at_unix, + output, + }; + if !verify_shadow_parity(&config)? { + bail!("byte/field/dedupe/settlement/rotation parity failed"); + } + Ok(()) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn collect_reference_cli_uses_the_library_discovery_capacity_default() { + let command = Cli::try_parse_from(["polymarket-raw-ops", "collect-reference"]) + .expect("default collector CLI must parse") + .command; + let Command::CollectReference { max_markets, .. } = command else { + panic!("collect-reference must select the collector command"); + }; + assert_eq!(max_markets, ReferenceConfig::default().max_markets); + } +} diff --git a/rust_hft/tools/collector/src/lib.rs b/rust_hft/tools/collector/src/lib.rs index 438d65683..caed115ce 100644 --- a/rust_hft/tools/collector/src/lib.rs +++ b/rust_hft/tools/collector/src/lib.rs @@ -2,6 +2,9 @@ pub mod feature_matrix; pub mod lob_archiver; +pub mod polymarket_parity; +pub mod polymarket_raw; +pub mod polymarket_upload; pub mod source_catalog; pub use feature_matrix::{ diff --git a/rust_hft/tools/collector/src/lob_archiver.rs b/rust_hft/tools/collector/src/lob_archiver.rs index ab0ea6391..05025b680 100644 --- a/rust_hft/tools/collector/src/lob_archiver.rs +++ b/rust_hft/tools/collector/src/lob_archiver.rs @@ -1,4 +1,5 @@ use engine::binance_md::{parse_fixed_6, BookSync, SequenceDecision, UpdateMeta}; +use rand::random; use rust_decimal::Decimal; use serde::{Deserialize, Serialize}; use serde_json::{json, Value}; @@ -960,20 +961,39 @@ fn atomic_write_json(path: &Path, value: &Value) -> anyhow::Result<()> { } fn atomic_write(path: &Path, bytes: &[u8]) -> anyhow::Result<()> { - let temporary = path.with_extension(format!( - "{}.tmp", - path.extension() - .and_then(|ext| ext.to_str()) - .unwrap_or_default() - )); - let mut output = OpenOptions::new() - .create(true) - .truncate(true) - .write(true) - .open(&temporary)?; - output.write_all(bytes)?; - output.sync_all()?; - fs::rename(temporary, path)?; + let parent = path + .parent() + .ok_or_else(|| anyhow::anyhow!("atomic target has no parent"))?; + let file_name = path + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or("state"); + let (temporary, mut output) = (0..32) + .find_map(|_| { + let temporary = parent.join(format!(".{file_name}.{:016x}.tmp", random::())); + match OpenOptions::new() + .create_new(true) + .write(true) + .open(&temporary) + { + Ok(output) => Some(Ok((temporary, output))), + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => None, + Err(error) => Some(Err(error)), + } + }) + .transpose()? + .ok_or_else(|| anyhow::anyhow!("could not allocate exclusive atomic-write temporary"))?; + let write_result = (|| -> anyhow::Result<()> { + output.write_all(bytes)?; + output.sync_all()?; + drop(output); + fs::rename(&temporary, path)?; + Ok(()) + })(); + if write_result.is_err() { + let _ = fs::remove_file(&temporary); + } + write_result?; sync_parent(path) } @@ -1241,4 +1261,23 @@ mod tests { ); fs::remove_dir_all(root).unwrap(); } + + #[cfg(unix)] + #[test] + fn success_marker_atomic_write_does_not_follow_predictable_temp_symlink() { + use std::os::unix::fs::symlink; + + let root = std::env::temp_dir().join(format!("monday-marker-test-{}", now_ns().unwrap())); + fs::create_dir(&root).unwrap(); + let data = root.join("segment.ndjson.zst"); + fs::write(&data, b"compressed").unwrap(); + let victim = root.join("victim"); + fs::write(&victim, b"do-not-touch\n").unwrap(); + symlink(&victim, root.join("segment.ndjson.zst._SUCCESS.tmp")).unwrap(); + + let marker = write_success_marker(&data, "abcd").unwrap(); + assert_eq!(fs::read_to_string(marker).unwrap(), "abcd\n"); + assert_eq!(fs::read(&victim).unwrap(), b"do-not-touch\n"); + fs::remove_dir_all(root).unwrap(); + } } diff --git a/rust_hft/tools/collector/src/polymarket_parity.rs b/rust_hft/tools/collector/src/polymarket_parity.rs new file mode 100644 index 000000000..a6a6a0fc8 --- /dev/null +++ b/rust_hft/tools/collector/src/polymarket_parity.rs @@ -0,0 +1,924 @@ +use anyhow::{anyhow, bail, Context, Result}; +use chrono::{DateTime, NaiveDateTime}; +use rand::random; +use serde_json::{json, Map, Value}; +use sha2::{Digest, Sha256}; +use std::collections::{BTreeMap, BTreeSet}; +use std::fs::{self, File, OpenOptions}; +use std::io::{Read, Write}; +use std::os::unix::fs::MetadataExt; +use std::path::{Path, PathBuf}; +use std::thread; +use std::time::Duration; + +const ACTIVE_TAPE: &str = "market-updates.ndjson"; +const EXPECTED_SYMBOLS: [&str; 7] = [ + "BTCUSDT", "ETHUSDT", "SOLUSDT", "XRPUSDT", "DOGEUSDT", "HYPEUSDT", "BNBUSDT", +]; +const KINDS: [&str; 3] = ["market_metadata", "polymarket_trade", "market_settlement"]; +const METADATA_CONTRACT_FIELDS: [&str; 23] = [ + "id", + "conditionId", + "question", + "slug", + "startDate", + "startDateIso", + "endDate", + "endDateIso", + "eventStartTime", + "outcomes", + "clobTokenIds", + "orderPriceMinTickSize", + "orderMinSize", + "makerBaseFee", + "takerBaseFee", + "feesEnabled", + "active", + "closed", + "acceptingOrders", + "enableOrderBook", + "negRisk", + "umaEndDate", + "umaEndDateIso", +]; + +#[derive(Debug, Clone)] +pub struct ShadowParityConfig { + pub legacy_spool: PathBuf, + pub rust_spool: PathBuf, + pub started_at_unix: i64, + pub ended_at_unix: i64, + pub output: PathBuf, +} + +#[derive(Debug, Clone)] +struct TapeRow { + recorded_at: i64, + update: Value, +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +struct FileFingerprint { + device: u64, + inode: u64, + bytes: u64, + modified_seconds: i64, + modified_nanoseconds: i64, +} + +impl FileFingerprint { + fn from_metadata(metadata: &fs::Metadata) -> Self { + Self { + device: metadata.dev(), + inode: metadata.ino(), + bytes: metadata.len(), + modified_seconds: metadata.mtime(), + modified_nanoseconds: metadata.mtime_nsec(), + } + } +} + +fn parse_timestamp(value: Option<&Value>) -> Option { + let value = value?.as_str()?; + DateTime::parse_from_rfc3339(value) + .map(|parsed| parsed.timestamp()) + .ok() + .or_else(|| { + NaiveDateTime::parse_from_str(value, "%Y-%m-%dT%H:%M:%S%.f") + .ok() + .map(|parsed| parsed.and_utc().timestamp()) + }) +} + +fn ensure_direct_directory(path: &Path) -> Result<()> { + if !path.is_absolute() { + bail!("spool path must be absolute: {}", path.display()); + } + let metadata = fs::symlink_metadata(path) + .with_context(|| format!("inspect spool directory {}", path.display()))?; + if metadata.file_type().is_symlink() || !metadata.is_dir() { + bail!("spool is not a direct directory: {}", path.display()); + } + if fs::canonicalize(path)? != path { + bail!("spool has an indirect ancestor: {}", path.display()); + } + Ok(()) +} + +fn strict_rotation_name(name: &str) -> bool { + let Some(middle) = name + .strip_prefix("market-updates.") + .and_then(|name| name.strip_suffix(".ndjson")) + else { + return false; + }; + let mut parts = middle.split('.'); + let Some(stamp) = parts.next() else { + return false; + }; + let format = match stamp.len() { + 15 => "%Y%m%dT%H%M%S", + 21 => "%Y%m%dT%H%M%S%6f", + _ => return false, + }; + if NaiveDateTime::parse_from_str(stamp, format).is_err() { + return false; + } + match (parts.next(), parts.next()) { + (None, None) => true, + (Some(uuid), None) => { + uuid.len() == 36 + && uuid.bytes().enumerate().all(|(index, byte)| { + if matches!(index, 8 | 13 | 18 | 23) { + byte == b'-' + } else { + byte.is_ascii_hexdigit() + } + }) + } + _ => false, + } +} + +fn tape_paths(spool: &Path) -> Result> { + ensure_direct_directory(spool)?; + let mut paths = Vec::new(); + for entry in fs::read_dir(spool)? { + let entry = entry?; + let name = entry.file_name(); + let Some(name) = name.to_str() else { + continue; + }; + if name == ACTIVE_TAPE || strict_rotation_name(name) { + let metadata = fs::symlink_metadata(entry.path())?; + if metadata.file_type().is_symlink() || !metadata.is_file() { + bail!( + "tape is not a direct regular file: {}", + entry.path().display() + ); + } + paths.push(entry.path()); + } else if name.starts_with("market-updates.") && name.ends_with(".ndjson") { + bail!("invalid rotated tape name: {name}"); + } + } + paths.sort(); + if paths.is_empty() { + bail!("no reference tapes found in {}", spool.display()); + } + Ok(paths) +} + +fn stable_file_bytes(path: &Path) -> Result> { + let mut last_reason = "file changed while being read".to_owned(); + for _ in 0..5 { + let before = fs::symlink_metadata(path)?; + if before.file_type().is_symlink() || !before.is_file() { + bail!("tape is not a direct regular file: {}", path.display()); + } + let expected = FileFingerprint::from_metadata(&before); + let mut file = File::open(path)?; + if FileFingerprint::from_metadata(&file.metadata()?) != expected { + last_reason = "tape identity changed while being opened".to_owned(); + thread::sleep(Duration::from_millis(20)); + continue; + } + let mut bytes = Vec::new(); + file.read_to_end(&mut bytes)?; + let after = FileFingerprint::from_metadata(&file.metadata()?); + if after != expected { + last_reason = "tape changed while being read".to_owned(); + thread::sleep(Duration::from_millis(20)); + continue; + } + if !bytes.is_empty() && !bytes.ends_with(b"\n") { + last_reason = "tape ends with an incomplete record".to_owned(); + thread::sleep(Duration::from_millis(20)); + continue; + } + return Ok(bytes); + } + bail!("{}: {last_reason}", path.display()) +} + +fn load_rows(spool: &Path) -> Result<(Vec, usize, bool)> { + for _ in 0..5 { + let paths = tape_paths(spool)?; + let mut rows = Vec::new(); + let mut closed_count = 0_usize; + let mut active_present = false; + for path in &paths { + let active = path.file_name().and_then(|name| name.to_str()) == Some(ACTIVE_TAPE); + active_present |= active; + closed_count += usize::from(!active); + let bytes = stable_file_bytes(path)?; + let mut expected_sequence = 0_u64; + let lines = bytes.split(|byte| *byte == b'\n').collect::>(); + for (line_index, raw) in lines.iter().enumerate() { + if raw.is_empty() { + if line_index + 1 == lines.len() { + continue; + } + bail!("blank row in {}:{}", path.display(), line_index + 1); + } + let row: Value = serde_json::from_slice(raw).with_context(|| { + format!("invalid JSON in {}:{}", path.display(), line_index + 1) + })?; + let object = row.as_object().ok_or_else(|| { + anyhow!("non-object row in {}:{}", path.display(), line_index + 1) + })?; + let sequence = object + .get("sequence") + .and_then(Value::as_u64) + .ok_or_else(|| { + anyhow!("invalid sequence in {}:{}", path.display(), line_index + 1) + })?; + if sequence != expected_sequence { + bail!( + "sequence gap in {}:{} expected={expected_sequence} actual={sequence}", + path.display(), + line_index + 1 + ); + } + expected_sequence = expected_sequence + .checked_add(1) + .context("tape sequence overflow")?; + let recorded_at = parse_timestamp(object.get("recorded_at")).ok_or_else(|| { + anyhow!( + "invalid recorded_at in {}:{}", + path.display(), + line_index + 1 + ) + })?; + let update = object + .get("update") + .filter(|value| value.is_object()) + .cloned() + .ok_or_else(|| { + anyhow!("invalid update in {}:{}", path.display(), line_index + 1) + })?; + let kind = update.get("kind").and_then(Value::as_str); + if !kind.is_some_and(|kind| KINDS.contains(&kind)) { + bail!( + "invalid update kind in {}:{}", + path.display(), + line_index + 1 + ); + } + rows.push(TapeRow { + recorded_at, + update, + }); + } + } + if tape_paths(spool)? == paths { + return Ok((rows, closed_count, active_present)); + } + thread::sleep(Duration::from_millis(20)); + } + bail!( + "spool changed repeatedly while enumerating tapes: {}", + spool.display() + ) +} + +fn field_paths(value: &Value, prefix: &str, paths: &mut BTreeSet) { + let Some(object) = value.as_object() else { + if !prefix.is_empty() { + paths.insert(prefix.to_owned()); + } + return; + }; + for (key, child) in object { + let path = if prefix.is_empty() { + key.clone() + } else { + format!("{prefix}.{key}") + }; + paths.insert(path.clone()); + if child.is_object() { + field_paths(child, &path, paths); + } + } +} + +fn observed_fields(rows: &[TapeRow], ended_at: i64) -> BTreeMap> { + let mut result = BTreeMap::>::new(); + for row in rows.iter().filter(|row| row.recorded_at <= ended_at) { + let kind = row.update["kind"].as_str().expect("kind was validated"); + field_paths(&row.update, "", result.entry(kind.to_owned()).or_default()); + } + result +} + +fn normalized_array(value: Option<&Value>, field: &str) -> Result> { + match value { + Some(Value::Array(values)) => Ok(values.clone()), + Some(Value::String(value)) => serde_json::from_str::>(value) + .with_context(|| format!("metadata {field} is not a JSON array")), + _ => bail!("metadata {field} is not an array"), + } +} + +fn required_text<'a>(object: &'a Map, field: &str, label: &str) -> Result<&'a str> { + object + .get(field) + .and_then(Value::as_str) + .filter(|value| !value.is_empty()) + .ok_or_else(|| anyhow!("{label} {field} is empty")) +} + +fn metadata_contract(value: &Value) -> Result { + let update = value + .as_object() + .context("metadata update is not an object")?; + let market = update + .get("market") + .and_then(Value::as_object) + .context("metadata market is not an object")?; + let market_id = required_text(update, "market_id", "metadata")?; + let condition_id = required_text(update, "condition_id", "metadata")?; + let symbol = required_text(update, "symbol", "metadata")?; + let window = update + .get("market_window_secs") + .and_then(Value::as_u64) + .context("metadata market_window_secs is invalid")?; + if !EXPECTED_SYMBOLS.contains(&symbol) || !matches!(window, 300 | 900) { + bail!("metadata symbol/window is outside the configured contract"); + } + if market.get("id").and_then(Value::as_str) != Some(market_id) + || market.get("conditionId").and_then(Value::as_str) != Some(condition_id) + { + bail!("metadata wrapper IDs contradict the embedded market"); + } + let outcomes = normalized_array(market.get("outcomes"), "outcomes")?; + let token_ids = normalized_array(market.get("clobTokenIds"), "clobTokenIds")?; + for (label, values) in [("outcomes", &outcomes), ("token IDs", &token_ids)] { + let strings = values + .iter() + .map(Value::as_str) + .collect::>>() + .ok_or_else(|| anyhow!("metadata {label} must be strings"))?; + if strings.len() != 2 + || strings.iter().any(|value| value.is_empty()) + || strings[0] == strings[1] + { + bail!("metadata requires two unique non-empty {label}"); + } + } + + let mut raw = Map::new(); + for field in METADATA_CONTRACT_FIELDS { + let Some(field_value) = market.get(field) else { + continue; + }; + let normalized = match field { + "outcomes" | "clobTokenIds" => { + Value::Array(normalized_array(Some(field_value), field)?) + } + _ => field_value.clone(), + }; + raw.insert(field.to_owned(), normalized); + } + Ok(json!({ + "market_id": market_id, + "condition_id": condition_id, + "symbol": symbol, + "market_window_secs": window, + "source": update.get("source").cloned().unwrap_or(Value::Null), + "market": raw, + })) +} + +fn metadata_map( + rows: &[TapeRow], + started_at: i64, + ended_at: i64, +) -> Result> { + let mut result = BTreeMap::new(); + for row in rows.iter().filter(|row| row.recorded_at <= ended_at) { + if row.update["kind"] != "market_metadata" { + continue; + } + let contract = metadata_contract(&row.update)?; + let market = row.update["market"] + .as_object() + .expect("metadata contract requires a market object"); + let end_epoch = parse_timestamp(market.get("endDate")) + .or_else(|| parse_timestamp(market.get("endDateIso"))) + .ok_or_else(|| anyhow!("metadata {} has no valid end time", contract["market_id"]))?; + if !(started_at.saturating_sub(900)..=ended_at.saturating_add(900)).contains(&end_epoch) { + continue; + } + let identity = contract["market_id"] + .as_str() + .expect("metadata contract has a market_id") + .to_owned(); + result.insert(identity, contract); + } + Ok(result) +} + +fn canonical_bytes(value: &Value) -> Result> { + let mut normalized = value.clone(); + if let Some(object) = normalized.as_object_mut() { + object.remove("retrieved_at"); + object.remove("received_at"); + } + Ok(serde_json::to_vec(&normalized)?) +} + +fn digest_values<'a>(values: impl Iterator) -> Result { + let mut digest = Sha256::new(); + let mut first = true; + for value in values { + if !first { + digest.update(b"\n"); + } + digest.update(canonical_bytes(value)?); + first = false; + } + Ok(hex::encode(digest.finalize())) +} + +fn trade_map( + rows: &[TapeRow], + started_at: i64, + ended_at: i64, +) -> Result<(BTreeMap, Vec)> { + let mut result = BTreeMap::new(); + let mut counts = BTreeMap::::new(); + for row in rows.iter().filter(|row| row.recorded_at <= ended_at) { + if row.update["kind"] != "polymarket_trade" { + continue; + } + let Some(timestamp) = row.update.get("trade_ts_unix").and_then(Value::as_i64) else { + bail!("trade has an invalid trade_ts_unix"); + }; + if !(started_at..=ended_at).contains(×tamp) { + continue; + } + let record_id = row + .update + .get("record_id") + .and_then(Value::as_str) + .filter(|value| !value.is_empty()) + .context("trade has an empty record_id")? + .to_owned(); + *counts.entry(record_id.clone()).or_default() += 1; + result.insert(record_id, row.update.clone()); + } + let duplicates = counts + .into_iter() + .filter_map(|(identity, count)| (count > 1).then_some(identity)) + .collect(); + Ok((result, duplicates)) +} + +fn settlement_map( + rows: &[TapeRow], + started_at: i64, + ended_at: i64, +) -> Result> { + let mut result = BTreeMap::new(); + for row in rows + .iter() + .filter(|row| (started_at..=ended_at).contains(&row.recorded_at)) + { + if row.update["kind"] != "market_settlement" { + continue; + } + let market_id = row + .update + .get("market_id") + .and_then(Value::as_str) + .filter(|value| !value.is_empty()) + .context("settlement has an empty market_id")? + .to_owned(); + result.insert(market_id, row.update.clone()); + } + Ok(result) +} + +fn map_ids(values: &BTreeMap) -> BTreeSet { + values.keys().cloned().collect() +} + +fn shared_values_match( + left: &BTreeMap, + right: &BTreeMap, +) -> Result { + for identity in map_ids(left).intersection(&map_ids(right)) { + if canonical_bytes(&left[identity])? != canonical_bytes(&right[identity])? { + return Ok(false); + } + } + Ok(true) +} + +fn required_fields_present(fields: &BTreeMap>) -> bool { + let required = [ + ( + "market_metadata", + [ + "kind", + "market_id", + "condition_id", + "symbol", + "market_window_secs", + "source", + "retrieved_at", + "market", + ] + .as_slice(), + ), + ( + "polymarket_trade", + [ + "kind", + "record_id", + "record_id_version", + "market_id", + "condition_id", + "symbol", + "trade_ts_unix", + "received_at", + "trade", + ] + .as_slice(), + ), + ( + "market_settlement", + [ + "kind", + "market_id", + "condition_id", + "symbol", + "winning_token_id", + "winning_outcome", + "resolution_source", + "retrieved_at", + "market", + ] + .as_slice(), + ), + ]; + required.into_iter().all(|(kind, required)| { + fields + .get(kind) + .is_some_and(|present| required.iter().all(|field| present.contains(*field))) + }) +} + +fn legacy_fields_preserved( + legacy: &BTreeMap>, + rust: &BTreeMap>, +) -> bool { + legacy.iter().all(|(kind, fields)| { + rust.get(kind) + .is_some_and(|present| fields.is_subset(present)) + }) +} + +fn compare(config: &ShadowParityConfig) -> Result { + if config.ended_at_unix <= config.started_at_unix { + bail!("parity window end must be after its start"); + } + let (legacy_rows, _, legacy_active) = load_rows(&config.legacy_spool)?; + let (rust_rows, rust_closed, rust_active) = load_rows(&config.rust_spool)?; + + let legacy_fields = observed_fields(&legacy_rows, config.ended_at_unix); + let rust_fields = observed_fields(&rust_rows, config.ended_at_unix); + let field_parity = required_fields_present(&rust_fields) + && legacy_fields_preserved(&legacy_fields, &rust_fields); + + let legacy_metadata = metadata_map(&legacy_rows, config.started_at_unix, config.ended_at_unix)?; + let rust_metadata = metadata_map(&rust_rows, config.started_at_unix, config.ended_at_unix)?; + let legacy_metadata_ids = map_ids(&legacy_metadata); + let rust_metadata_ids = map_ids(&rust_metadata); + let metadata_parity = !legacy_metadata_ids.is_empty() + && legacy_metadata_ids == rust_metadata_ids + && shared_values_match(&legacy_metadata, &rust_metadata)?; + + let (legacy_trades, legacy_duplicates) = + trade_map(&legacy_rows, config.started_at_unix, config.ended_at_unix)?; + let (rust_trades, rust_duplicates) = + trade_map(&rust_rows, config.started_at_unix, config.ended_at_unix)?; + let legacy_trade_ids = map_ids(&legacy_trades); + let rust_trade_ids = map_ids(&rust_trades); + let trade_bytes_match = shared_values_match(&legacy_trades, &rust_trades)?; + let dedupe_parity = legacy_duplicates.is_empty() + && rust_duplicates.is_empty() + && !legacy_trade_ids.is_empty() + && legacy_trade_ids == rust_trade_ids; + + let legacy_settlements = + settlement_map(&legacy_rows, config.started_at_unix, config.ended_at_unix)?; + let rust_settlements = + settlement_map(&rust_rows, config.started_at_unix, config.ended_at_unix)?; + let legacy_settlement_ids = map_ids(&legacy_settlements); + let rust_settlement_ids = map_ids(&rust_settlements); + let settlement_parity = !legacy_settlement_ids.is_empty() + && legacy_settlement_ids.is_subset(&rust_settlement_ids) + && shared_values_match(&legacy_settlements, &rust_settlements)?; + + let rust_symbols = rust_metadata + .values() + .filter_map(|value| value.get("symbol").and_then(Value::as_str)) + .map(str::to_owned) + .collect::>(); + let asset_parity = EXPECTED_SYMBOLS + .iter() + .all(|symbol| rust_symbols.contains(*symbol)); + let rotation_parity = legacy_active && rust_active && rust_closed >= 1; + let byte_parity = metadata_parity && dedupe_parity && trade_bytes_match && settlement_parity; + + let checks = json!({ + "byte_parity": byte_parity, + "metadata_parity": metadata_parity, + "field_parity": field_parity, + "dedupe_parity": dedupe_parity, + "settlement_parity": settlement_parity, + "rotation_parity": rotation_parity, + "asset_parity": asset_parity, + }); + let passed = checks + .as_object() + .expect("checks is an object") + .values() + .all(|value| value == &Value::Bool(true)); + Ok(json!({ + "schema": "monday.polymarket_shadow_parity.v1", + "passed": passed, + "checks": checks, + "metrics": { + "legacy_trade_count": legacy_trade_ids.len(), + "rust_trade_count": rust_trade_ids.len(), + "legacy_metadata_count": legacy_metadata_ids.len(), + "rust_metadata_count": rust_metadata_ids.len(), + "legacy_only_metadata_ids": legacy_metadata_ids.difference(&rust_metadata_ids).cloned().collect::>(), + "rust_only_metadata_ids": rust_metadata_ids.difference(&legacy_metadata_ids).cloned().collect::>(), + "legacy_duplicate_trade_ids": legacy_duplicates, + "rust_duplicate_trade_ids": rust_duplicates, + "legacy_settlement_count": legacy_settlement_ids.len(), + "rust_settlement_count": rust_settlement_ids.len(), + "rust_closed_tape_count": rust_closed, + "rust_symbols": rust_symbols, + "normalized_trade_sha256": digest_values(rust_trades.values())?, + "normalized_metadata_sha256": digest_values(rust_metadata.values())?, + }, + })) +} + +fn atomic_write_json(path: &Path, value: &Value) -> Result<()> { + let parent = path + .parent() + .context("parity output has no parent directory")?; + ensure_direct_directory(parent)?; + let name = path + .file_name() + .and_then(|value| value.to_str()) + .context("parity output name is not UTF-8")?; + let (temporary, mut output) = (0..32) + .find_map(|_| { + let candidate = parent.join(format!(".{name}.{:016x}.tmp", random::())); + match OpenOptions::new() + .create_new(true) + .write(true) + .open(&candidate) + { + Ok(output) => Some(Ok((candidate, output))), + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => None, + Err(error) => Some(Err(error)), + } + }) + .transpose()? + .context("could not allocate parity output temporary")?; + let write_result = (|| -> Result<()> { + serde_json::to_writer(&mut output, value)?; + output.write_all(b"\n")?; + output.sync_all()?; + drop(output); + fs::rename(&temporary, path)?; + File::open(parent)?.sync_all()?; + Ok(()) + })(); + if write_result.is_err() { + let _ = fs::remove_file(&temporary); + } + write_result +} + +/// Compare a bounded active Python lane with an isolated Rust shadow, write +/// fail-closed evidence, and return whether every parity check passed. +pub fn verify_shadow_parity(config: &ShadowParityConfig) -> Result { + let evidence = compare(config)?; + let passed = evidence["passed"] == Value::Bool(true); + atomic_write_json(&config.output, &evidence)?; + Ok(passed) +} + +#[cfg(test)] +mod tests { + use super::*; + + struct TestDir(PathBuf); + + impl TestDir { + fn new() -> Self { + let path = std::env::temp_dir().join(format!( + "monday-polymarket-parity-{}-{:016x}", + std::process::id(), + random::() + )); + fs::create_dir(&path).unwrap(); + Self(fs::canonicalize(path).unwrap()) + } + + fn path(&self) -> &Path { + &self.0 + } + } + + impl Drop for TestDir { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } + } + + fn metadata(symbol: &str) -> Value { + json!({ + "kind":"market_metadata", + "market_id":format!("market-{symbol}"), + "condition_id":format!("condition-{symbol}"), + "symbol":symbol, + "market_window_secs":300, + "source":"gamma_api", + "retrieved_at":"1970-01-01T00:03:20Z", + "market":{ + "id":format!("market-{symbol}"), + "conditionId":format!("condition-{symbol}"), + "question":format!("{symbol} Up or Down"), + "slug":format!("market-{symbol}"), + "startDate":"1970-01-01T00:00:00Z", + "endDate":"1970-01-01T00:05:00Z", + "outcomes":["Up","Down"], + "clobTokenIds":[format!("up-{symbol}"),format!("down-{symbol}")], + "orderPriceMinTickSize":0.01, + "orderMinSize":5, + "feesEnabled":true + } + }) + } + + fn trade(record_id: &str) -> Value { + json!({ + "kind":"polymarket_trade","record_id":record_id,"record_id_version":"v2", + "market_id":"market-BTCUSDT","condition_id":"condition-BTCUSDT", + "token_id":"up-BTCUSDT","symbol":"BTCUSDT","market_window_secs":300, + "side":"BUY","size":"1","price":"0.5", + "trade_ts":"1970-01-01T00:03:20Z","trade_ts_unix":200, + "transaction_hash":"0x1","proxy_wallet":"0x2","outcome":"Up", + "outcome_index":0,"source":"polymarket_data_api", + "received_at":"1970-01-01T00:03:20Z", + "trade":{"transactionHash":"0x1","conditionId":"condition-BTCUSDT", + "asset":"up-BTCUSDT","side":"BUY","timestamp":200, + "proxyWallet":"0x2","size":"1","price":"0.5", + "outcomeIndex":0,"outcome":"Up"} + }) + } + + fn settlement() -> Value { + json!({ + "kind":"market_settlement","market_id":"market-BTCUSDT", + "condition_id":"condition-BTCUSDT","symbol":"BTCUSDT", + "market_window_secs":300,"winning_token_id":"up-BTCUSDT", + "winning_outcome":"Up","resolved_up_won":true, + "resolution_source":"gamma_api_closed_market", + "retrieved_at":"1970-01-01T00:03:20Z", + "market":{"id":"market-BTCUSDT","conditionId":"condition-BTCUSDT", + "closed":true,"outcomes":["Up","Down"], + "clobTokenIds":["up-BTCUSDT","down-BTCUSDT"], + "outcomePrices":["1","0"]} + }) + } + + fn fixture_rows() -> Vec { + let mut rows = EXPECTED_SYMBOLS + .iter() + .map(|symbol| metadata(symbol)) + .collect::>(); + rows.push(trade("trade-1")); + rows.push(settlement()); + rows + } + + fn write_tape(path: &Path, updates: &[Value], recorded_at: &str) { + let mut output = File::create(path).unwrap(); + for (sequence, update) in updates.iter().enumerate() { + serde_json::to_writer( + &mut output, + &json!({"sequence":sequence,"recorded_at":recorded_at,"update":update}), + ) + .unwrap(); + output.write_all(b"\n").unwrap(); + } + output.sync_all().unwrap(); + } + + fn fixture() -> (TestDir, ShadowParityConfig) { + let root = TestDir::new(); + let legacy = root.path().join("legacy"); + let rust = root.path().join("rust"); + fs::create_dir(&legacy).unwrap(); + fs::create_dir(&rust).unwrap(); + let mut legacy_rows = fixture_rows(); + let mut delayed = trade("trade-after-cutoff"); + delayed["post_cutoff_only"] = Value::Bool(true); + delayed["trade"]["postCutoffOnly"] = Value::Bool(true); + legacy_rows.push(delayed); + write_tape( + &legacy.join(ACTIVE_TAPE), + &legacy_rows[..legacy_rows.len() - 1], + "1970-01-01T00:03:20Z", + ); + let mut legacy_active = OpenOptions::new() + .append(true) + .open(legacy.join(ACTIVE_TAPE)) + .unwrap(); + serde_json::to_writer( + &mut legacy_active, + &json!({"sequence":legacy_rows.len() - 1,"recorded_at":"1970-01-01T00:05:01Z","update":legacy_rows.last().unwrap()}), + ) + .unwrap(); + legacy_active.write_all(b"\n").unwrap(); + legacy_active.sync_all().unwrap(); + + write_tape( + &rust.join("market-updates.19700101T000400000000.ndjson"), + &fixture_rows(), + "1970-01-01T00:03:20Z", + ); + File::create(rust.join(ACTIVE_TAPE)).unwrap(); + let config = ShadowParityConfig { + legacy_spool: legacy, + rust_spool: rust, + started_at_unix: 100, + ended_at_unix: 300, + output: root.path().join("parity.json"), + }; + (root, config) + } + + #[test] + fn bounded_parity_ignores_delayed_trade_recorded_after_cutoff() { + let (_root, config) = fixture(); + let evidence = compare(&config).unwrap(); + assert_eq!(evidence["passed"], true); + assert_eq!(evidence["checks"]["metadata_parity"], true); + assert_eq!(evidence["metrics"]["legacy_trade_count"], 1); + } + + #[test] + fn duplicate_rust_trade_fails_dedupe_parity() { + let (_root, config) = fixture(); + write_tape( + &config.rust_spool.join(ACTIVE_TAPE), + &[trade("trade-1")], + "1970-01-01T00:03:21Z", + ); + let evidence = compare(&config).unwrap(); + assert_eq!(evidence["passed"], false); + assert_eq!(evidence["checks"]["dedupe_parity"], false); + } + + #[test] + fn contradictory_metadata_values_fail_parity() { + let (_root, config) = fixture(); + let mut rows = fixture_rows(); + rows[0]["market"]["clobTokenIds"] = json!(["tampered-up", "tampered-down"]); + write_tape( + &config + .rust_spool + .join("market-updates.19700101T000400000000.ndjson"), + &rows, + "1970-01-01T00:03:20Z", + ); + let evidence = compare(&config).unwrap(); + assert_eq!(evidence["passed"], false); + assert_eq!(evidence["checks"]["metadata_parity"], false); + } + + #[test] + fn evidence_is_written_even_when_semantic_parity_fails() { + let (_root, config) = fixture(); + write_tape( + &config.rust_spool.join(ACTIVE_TAPE), + &[trade("trade-1")], + "1970-01-01T00:03:21Z", + ); + assert!(!verify_shadow_parity(&config).unwrap()); + let evidence: Value = serde_json::from_slice(&fs::read(&config.output).unwrap()).unwrap(); + assert_eq!(evidence["passed"], false); + } +} diff --git a/rust_hft/tools/collector/src/polymarket_raw.rs b/rust_hft/tools/collector/src/polymarket_raw.rs new file mode 100644 index 000000000..76a10d1a3 --- /dev/null +++ b/rust_hft/tools/collector/src/polymarket_raw.rs @@ -0,0 +1,2384 @@ +//! Fail-closed Polymarket reference collection and raw tape archival. + +use crate::polymarket_upload::ensure_canonical_directory; +use anyhow::{anyhow, bail, Context, Result}; +use chrono::{DateTime, NaiveDateTime, TimeDelta, Utc}; +use rand::random; +use rust_decimal::Decimal; +use serde::{de::DeserializeOwned, Deserialize, Serialize}; +use serde_json::{json, Value}; +use sha2::{Digest, Sha256}; +use std::collections::{BTreeMap, BTreeSet}; +use std::fs::{self, File, OpenOptions}; +use std::io::{BufRead, BufReader, Seek, SeekFrom, Write}; +use std::os::unix::fs::OpenOptionsExt; +use std::path::{Path, PathBuf}; +use std::str::FromStr; +use std::time::{Duration, Instant}; + +pub const ACTIVE_TAPE: &str = "market-updates.ndjson"; +const GAMMA_MARKETS_URL: &str = "https://gamma-api.polymarket.com/markets/keyset"; +const GAMMA_MARKET_URL: &str = "https://gamma-api.polymarket.com/markets"; +const DATA_TRADES_URL: &str = "https://data-api.polymarket.com/trades"; +const USER_AGENT: &str = "monday-polymarket-reference-collector/2.0"; +const TRADE_ID_VERSION: &str = "v2"; +const CRYPTO_TAG_ID: u64 = 21; +const MIN_SETTLEMENT_LOOKBACK_SECS: i64 = 86_400; +pub const DEFAULT_MAX_MARKETS_PER_LANE: usize = 10_000; +const TARGET_MARKET_WINDOWS_SECS: [usize; 2] = [300, 900]; +const SETTLEMENT_PRICE: Decimal = Decimal::from_parts(999, 0, 0, false, 3); +const SETTLEMENT_LOSER_PRICE: Decimal = Decimal::from_parts(1, 0, 0, false, 3); +const SETTLEMENT_SUM_TOLERANCE: Decimal = Decimal::from_parts(1, 0, 0, false, 6); + +const SYMBOL_ALIASES: [(&str, &[&str]); 7] = [ + ("BTCUSDT", &["BITCOIN", "BTC"]), + ("ETHUSDT", &["ETHEREUM", "ETH"]), + ("SOLUSDT", &["SOLANA", "SOL "]), + ("XRPUSDT", &["XRP"]), + ("DOGEUSDT", &["DOGECOIN", "DOGE"]), + ("HYPEUSDT", &["HYPERLIQUID", "HYPE"]), + ("BNBUSDT", &["BINANCE COIN", "BNB"]), +]; + +#[derive(Debug)] +struct DataCompletenessError(String); + +impl std::fmt::Display for DataCompletenessError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str(&self.0) + } +} + +impl std::error::Error for DataCompletenessError {} + +fn completeness_error(message: impl Into) -> anyhow::Error { + DataCompletenessError(message.into()).into() +} + +#[derive(Debug, Clone)] +pub struct ReferenceConfig { + pub spool_dir: PathBuf, + pub symbols: Vec, + pub poll_interval: Duration, + pub market_lookback_secs: i64, + pub settlement_lookback_secs: i64, + pub max_markets: usize, + pub http_timeout: Duration, + pub stale_after: Duration, + pub trade_finalization_lag_secs: i64, + pub trade_finalization_stable_polls: u64, + pub per_market_delay: Duration, +} + +impl Default for ReferenceConfig { + fn default() -> Self { + Self { + spool_dir: PathBuf::from("/data/monday/spool/polymarket-reference"), + symbols: SYMBOL_ALIASES + .iter() + .map(|(symbol, _)| (*symbol).to_owned()) + .collect(), + poll_interval: Duration::from_secs(30), + market_lookback_secs: 7_200, + settlement_lookback_secs: 86_400, + max_markets: DEFAULT_MAX_MARKETS_PER_LANE, + http_timeout: Duration::from_secs(20), + stale_after: Duration::from_secs(180), + trade_finalization_lag_secs: 1_800, + trade_finalization_stable_polls: 3, + per_market_delay: Duration::from_millis(100), + } + } +} + +impl ReferenceConfig { + pub fn validate(&self) -> Result<()> { + if self.symbols.is_empty() + || self.poll_interval.is_zero() + || self.market_lookback_secs <= 0 + || self.settlement_lookback_secs <= 0 + || self.max_markets == 0 + || self.http_timeout.is_zero() + || self.stale_after.is_zero() + || self.trade_finalization_lag_secs <= 0 + || self.trade_finalization_stable_polls == 0 + { + bail!("reference collector limits must be positive"); + } + if self.settlement_lookback_secs < MIN_SETTLEMENT_LOOKBACK_SECS { + bail!("settlement lookback must cover at least {MIN_SETTLEMENT_LOOKBACK_SECS} seconds"); + } + let symbol_count = self + .symbols + .iter() + .map(|symbol| symbol.trim().to_ascii_uppercase()) + .filter(|symbol| !symbol.is_empty()) + .collect::>() + .len(); + let settlement_lookback = usize::try_from(self.settlement_lookback_secs) + .context("settlement lookback exceeds this platform's capacity")?; + let markets_per_symbol = TARGET_MARKET_WINDOWS_SECS + .iter() + .try_fold(0_usize, |total, window| { + total.checked_add(settlement_lookback.div_ceil(*window)) + }) + .context("settlement discovery capacity overflow")?; + let required_markets = symbol_count + .checked_mul(markets_per_symbol) + .context("settlement discovery capacity overflow")?; + if self.max_markets < required_markets { + bail!( + "max_markets={} cannot cover the settlement lane minimum of {required_markets} markets for {symbol_count} symbols", + self.max_markets + ); + } + Ok(()) + } +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +struct CollectorState { + #[serde(default)] + trade_id_version: Option, + #[serde(default)] + context_seed_hour: Option, + #[serde(default)] + markets: BTreeMap, + #[serde(default)] + trade_seen: BTreeMap>, + #[serde(flatten)] + extra: BTreeMap, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +struct TrackedMarket { + #[serde(default)] + condition_id: Option, + #[serde(default)] + symbol: Option, + #[serde(default)] + market_window_secs: u64, + #[serde(default)] + end_time: Option, + #[serde(default)] + settled: bool, + #[serde(default)] + trade_complete: bool, + #[serde(default)] + last_metadata_hash: Option, + #[serde(default)] + settlement_failure_since: Option, + #[serde(default)] + settlement_last_error: Option, + #[serde(default)] + trade_failure_since: Option, + #[serde(default)] + trade_last_error: Option, + #[serde(default)] + last_trade_success_at: Option, + #[serde(default)] + settlement_seen_at: Option, + #[serde(default)] + last_trade_change_at: Option, + #[serde(default)] + trade_finalization_stable_polls: u64, + #[serde(flatten)] + extra: BTreeMap, +} + +#[derive(Debug, Clone)] +struct TargetMarket { + symbol: String, + window_secs: u64, +} + +fn utc_now() -> DateTime { + Utc::now() +} + +fn iso_z(value: DateTime) -> String { + value.format("%Y-%m-%dT%H:%M:%S%.6fZ").to_string() +} + +fn hour_key(value: DateTime) -> String { + value.format("%Y%m%dT%H").to_string() +} + +fn parse_datetime(value: &Value) -> Option> { + let value = value.as_str()?; + DateTime::parse_from_rfc3339(value) + .map(|parsed| parsed.with_timezone(&Utc)) + .ok() + .or_else(|| { + NaiveDateTime::parse_from_str(value, "%Y-%m-%dT%H:%M:%S%.f") + .ok() + .map(|parsed| parsed.and_utc()) + }) +} + +fn parse_optional_datetime(value: Option<&str>) -> Option> { + value.and_then(|value| parse_datetime(&Value::String(value.to_owned()))) +} + +fn parse_json_array(value: Option<&Value>) -> Vec { + match value { + Some(Value::Array(values)) => values.clone(), + Some(Value::String(value)) => serde_json::from_str::>(value).unwrap_or_default(), + _ => Vec::new(), + } +} + +fn value_text(value: Option<&Value>) -> String { + match value { + Some(Value::String(value)) => value.clone(), + Some(Value::Null) | None => "None".to_owned(), + Some(Value::Bool(true)) => "True".to_owned(), + Some(Value::Bool(false)) => "False".to_owned(), + Some(value) => value.to_string(), + } +} + +fn canonical_decimal(value: Option<&Value>) -> String { + let raw = value_text(value); + Decimal::from_str(&raw) + .map(|value| value.normalize().to_string()) + .unwrap_or(raw) +} + +fn stable_trade_id(trade: &Value) -> String { + let parts = [ + value_text(trade.get("transactionHash")), + value_text(trade.get("conditionId")), + value_text(trade.get("asset")), + value_text(trade.get("side")), + value_text(trade.get("timestamp")), + value_text(trade.get("proxyWallet")), + canonical_decimal(trade.get("size")), + canonical_decimal(trade.get("price")), + value_text(trade.get("outcomeIndex")), + ]; + hex::encode(Sha256::digest(parts.join("|").as_bytes())) +} + +fn stable_payload_hash(payload: &Value) -> Result { + Ok(hex::encode(Sha256::digest(serde_json::to_vec(payload)?))) +} + +fn infer_symbol(question: Option<&Value>) -> Option<&'static str> { + let upper = question?.as_str()?.to_ascii_uppercase(); + SYMBOL_ALIASES.iter().find_map(|(symbol, aliases)| { + aliases + .iter() + .any(|alias| upper.contains(alias)) + .then_some(*symbol) + }) +} + +fn market_start_time(market: &Value) -> Option> { + let event = market + .get("events") + .and_then(Value::as_array) + .and_then(|events| events.first()); + [ + market.get("eventStartTime"), + market.get("startDate"), + event.and_then(|event| event.get("startTime")), + event.and_then(|event| event.get("startDate")), + ] + .into_iter() + .flatten() + .find_map(parse_datetime) +} + +fn infer_window_seconds(market: &Value) -> Option { + if let (Some(start), Some(end)) = ( + market_start_time(market), + market.get("endDate").and_then(parse_datetime), + ) { + let seconds = (end - start).num_seconds(); + if matches!(seconds, 300 | 900) { + return u64::try_from(seconds).ok(); + } + } + let text = format!( + "{} {}", + market + .get("slug") + .and_then(Value::as_str) + .unwrap_or_default(), + market + .get("question") + .and_then(Value::as_str) + .unwrap_or_default() + ) + .to_ascii_lowercase(); + if contains_window_token(&text, "15m") || text.contains("15 minute") { + Some(900) + } else if contains_window_token(&text, "5m") || text.contains("5 minute") { + Some(300) + } else { + None + } +} + +fn contains_window_token(text: &str, token: &str) -> bool { + text.match_indices(token).any(|(start, _)| { + let before = text[..start].chars().next_back(); + let after = text[start + token.len()..].chars().next(); + before.is_none_or(|value| matches!(value, '-' | '_' | ' ')) + && after.is_none_or(|value| matches!(value, '-' | '_' | ' ')) + }) +} + +fn target_market(market: &Value, symbols: &BTreeSet) -> Option { + let symbol = infer_symbol(market.get("question"))?; + let window_secs = infer_window_seconds(market)?; + let condition = market.get("conditionId")?.as_str()?; + let token_ids = parse_json_array(market.get("clobTokenIds")); + if !symbols.contains(symbol) + || !matches!(window_secs, 300 | 900) + || condition.is_empty() + || token_ids.len() != 2 + { + return None; + } + Some(TargetMarket { + symbol: symbol.to_owned(), + window_secs, + }) +} + +fn atomic_write_json(path: &Path, value: &T) -> Result<()> { + let mut bytes = serde_json::to_vec(value)?; + bytes.push(b'\n'); + atomic_write(path, &bytes) +} + +fn atomic_write(path: &Path, bytes: &[u8]) -> Result<()> { + let parent = path.parent().context("atomic target has no parent")?; + fs::create_dir_all(parent)?; + let name = path + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or("state"); + let (temporary, mut output) = (0..32) + .find_map(|_| { + let temporary = parent.join(format!(".{name}.{:016x}.tmp", random::())); + match OpenOptions::new() + .create_new(true) + .write(true) + .open(&temporary) + { + Ok(output) => Some(Ok((temporary, output))), + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => None, + Err(error) => Some(Err(error)), + } + }) + .transpose()? + .context("could not allocate exclusive atomic-write temporary")?; + let result = (|| -> Result<()> { + output.write_all(bytes)?; + output.sync_all()?; + drop(output); + fs::rename(&temporary, path)?; + File::open(parent)?.sync_all()?; + Ok(()) + })(); + if result.is_err() { + let _ = fs::remove_file(&temporary); + } + result +} + +fn read_optional_json(path: &Path) -> Result { + match fs::symlink_metadata(path) { + Ok(_) => serde_json::from_reader(open_read_regular(path)?) + .with_context(|| format!("invalid JSON in {}", path.display())), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(T::default()), + Err(error) => Err(error.into()), + } +} + +fn open_read_regular(path: &Path) -> Result { + let file = OpenOptions::new() + .read(true) + .custom_flags(libc::O_CLOEXEC | libc::O_NOFOLLOW) + .open(path) + .with_context(|| format!("failed to open regular file {}", path.display()))?; + if !file.metadata()?.is_file() { + bail!("refusing non-regular file {}", path.display()); + } + Ok(file) +} + +fn strict_existing_regular(path: &Path) -> Result<()> { + let metadata = fs::symlink_metadata(path)?; + if metadata.file_type().is_symlink() || !metadata.is_file() { + bail!("refusing non-regular file {}", path.display()); + } + Ok(()) +} + +fn open_append(path: &Path) -> Result { + let file = OpenOptions::new() + .create(true) + .read(true) + .append(true) + .custom_flags(libc::O_CLOEXEC | libc::O_NOFOLLOW) + .open(path) + .with_context(|| format!("failed to open append-only tape {}", path.display()))?; + if !file.metadata()?.is_file() { + bail!("refusing non-regular tape {}", path.display()); + } + Ok(file) +} + +struct TapeWriter { + spool_dir: PathBuf, + active: PathBuf, + hour: Option, + sequence: u64, + file: Option, +} + +impl TapeWriter { + fn new(spool_dir: &Path) -> Result { + fs::create_dir_all(spool_dir)?; + let active = spool_dir.join(ACTIVE_TAPE); + let mut writer = Self { + spool_dir: spool_dir.to_path_buf(), + active, + hour: None, + sequence: 0, + file: None, + }; + writer.recover_active()?; + Ok(writer) + } + + fn recover_active(&mut self) -> Result<()> { + if !self.active.exists() { + self.file = Some(open_append(&self.active)?); + return Ok(()); + } + let mut reader = BufReader::new(open_read_regular(&self.active)?); + let mut expected = 0_u64; + let mut valid_bytes = 0_u64; + let mut first_recorded = None; + loop { + let mut line = Vec::new(); + let bytes = reader.read_until(b'\n', &mut line)?; + if bytes == 0 { + break; + } + if !line.ends_with(b"\n") { + break; + } + let row: Value = serde_json::from_slice(&line)?; + if row.get("sequence").and_then(Value::as_u64) != Some(expected) { + bail!("active tape sequence gap expected={expected}"); + } + if first_recorded.is_none() { + first_recorded = row.get("recorded_at").and_then(parse_datetime); + } + expected += 1; + valid_bytes += u64::try_from(bytes)?; + } + if fs::metadata(&self.active)?.len() != valid_bytes { + let file = OpenOptions::new() + .write(true) + .custom_flags(libc::O_CLOEXEC | libc::O_NOFOLLOW) + .open(&self.active)?; + file.set_len(valid_bytes)?; + file.sync_all()?; + } + self.sequence = expected; + self.hour = first_recorded.map(|value| value.format("%Y%m%dT%H").to_string()); + self.file = Some(open_append(&self.active)?); + Ok(()) + } + + fn rotate(&mut self, now: DateTime) -> Result<()> { + let current = self.file.take().context("active tape is closed")?; + current.sync_all()?; + drop(current); + let staged = self + .spool_dir + .join(format!(".{ACTIVE_TAPE}.{}.rotate", random::())); + let staged_file = OpenOptions::new() + .create_new(true) + .write(true) + .open(&staged)?; + staged_file.sync_all()?; + drop(staged_file); + let rotated = self.spool_dir.join(format!( + "market-updates.{}.ndjson", + now.format("%Y%m%dT%H%M%S%6f") + )); + fs::rename(&self.active, &rotated)?; + if let Err(error) = fs::rename(&staged, &self.active) { + let rollback = fs::rename(&rotated, &self.active); + return Err(anyhow!( + "failed to publish next tape: {error}; rollback={rollback:?}" + )); + } + File::open(&self.spool_dir)?.sync_all()?; + self.sequence = 0; + self.hour = None; + self.file = Some(open_append(&self.active)?); + Ok(()) + } + + fn write_updates(&mut self, updates: &[Value], now: DateTime) -> Result<()> { + self.write_updates_with_sync(updates, now, |file| { + file.sync_all()?; + Ok(()) + }) + } + + fn write_updates_with_sync( + &mut self, + updates: &[Value], + now: DateTime, + sync: F, + ) -> Result<()> + where + F: FnOnce(&mut File) -> Result<()>, + { + if updates.is_empty() { + return Ok(()); + } + let target_hour = hour_key(now); + if self.hour.as_ref().is_some_and(|hour| hour != &target_hour) { + self.rotate(now)?; + } + let start_hour = self.hour.clone(); + let start_sequence = self.sequence; + let start_offset = self + .file + .as_ref() + .context("active tape is closed")? + .metadata()? + .len(); + self.hour = Some(target_hour); + let recorded_at = iso_z(now); + let result = (|| -> Result<()> { + let file = self.file.as_mut().context("active tape is closed")?; + for update in updates { + let mut encoded = serde_json::to_vec(&json!({ + "sequence": self.sequence, + "recorded_at": recorded_at, + "update": update, + }))?; + encoded.push(b'\n'); + file.write_all(&encoded)?; + self.sequence += 1; + } + sync(file)?; + Ok(()) + })(); + if let Err(error) = result { + self.hour = start_hour; + self.sequence = start_sequence; + let file = self.file.as_mut().context("active tape is closed")?; + file.set_len(start_offset)?; + file.seek(SeekFrom::End(0))?; + file.sync_all()?; + return Err(error); + } + Ok(()) + } + + fn needs_hour_context(&self, now: DateTime) -> bool { + self.hour.as_deref() != Some(hour_key(now).as_str()) + } + + fn close(&mut self) -> Result<()> { + if let Some(file) = self.file.take() { + file.sync_all()?; + } + Ok(()) + } +} + +impl Drop for TapeWriter { + fn drop(&mut self) { + let _ = self.close(); + } +} + +fn context_seed_required(state: &CollectorState, writer: &TapeWriter, now: DateTime) -> bool { + state.context_seed_hour.as_deref() != Some(hour_key(now).as_str()) + || writer.needs_hour_context(now) +} + +fn missing_symbols(configured: &BTreeSet, discovered: &BTreeSet) -> Vec { + configured.difference(discovered).cloned().collect() +} + +fn settlement_from_market( + market: &Value, + symbol: &str, + window_secs: u64, + retrieved_at: &str, +) -> Result> { + if market.get("closed").and_then(Value::as_bool) != Some(true) { + return Ok(None); + } + let outcomes = parse_json_array(market.get("outcomes")); + let token_ids = parse_json_array(market.get("clobTokenIds")); + let raw_prices = parse_json_array(market.get("outcomePrices")); + if outcomes.len() != 2 || token_ids.len() != 2 || raw_prices.len() != 2 { + return Err(completeness_error( + "closed market is missing its two-outcome settlement vectors", + )); + } + let market_id = market + .get("id") + .and_then(Value::as_str) + .filter(|value| !value.is_empty()) + .ok_or_else(|| completeness_error("closed market id is missing"))?; + let condition_id = market + .get("conditionId") + .and_then(Value::as_str) + .filter(|value| !value.is_empty()) + .ok_or_else(|| completeness_error("closed market condition id is missing"))?; + let outcome_names = outcomes + .iter() + .map(|value| value.as_str().filter(|value| !value.is_empty())) + .collect::>>() + .ok_or_else(|| completeness_error("closed market outcomes are missing"))?; + let outcome_set = outcome_names + .iter() + .map(|value| value.to_ascii_lowercase()) + .collect::>(); + if outcome_set != BTreeSet::from(["down".to_owned(), "up".to_owned()]) + && outcome_set != BTreeSet::from(["no".to_owned(), "yes".to_owned()]) + { + return Err(completeness_error( + "closed market outcomes are not a binary up/down or yes/no pair", + )); + } + let settlement_token_ids = token_ids + .iter() + .map(|value| value.as_str().filter(|value| !value.is_empty())) + .collect::>>() + .ok_or_else(|| completeness_error("closed market token ids are missing"))?; + if settlement_token_ids[0] == settlement_token_ids[1] { + return Err(completeness_error( + "closed market settlement token ids are not unique", + )); + } + let prices = raw_prices + .iter() + .map(|value| Decimal::from_str(&value_text(Some(value))).ok()) + .collect::>>() + .ok_or_else(|| completeness_error("closed market has invalid settlement prices"))?; + if prices + .iter() + .any(|price| *price < Decimal::ZERO || *price > Decimal::ONE) + { + return Err(completeness_error( + "closed market settlement prices are outside [0, 1]", + )); + } + let winners = prices + .iter() + .enumerate() + .filter_map(|(index, price)| (*price >= SETTLEMENT_PRICE).then_some(index)) + .collect::>(); + if winners.len() != 1 { + return Err(completeness_error(format!( + "closed market has {} settlement winners", + winners.len() + ))); + } + let winner = winners[0]; + let loser = 1 - winner; + if prices[loser] > SETTLEMENT_LOSER_PRICE { + return Err(completeness_error( + "closed market losing settlement price must be near zero", + )); + } + let price_sum = prices[0] + prices[1]; + if price_sum < Decimal::ONE - SETTLEMENT_SUM_TOLERANCE + || price_sum > Decimal::ONE + SETTLEMENT_SUM_TOLERANCE + { + return Err(completeness_error( + "closed market settlement prices must sum to one", + )); + } + let winning_token_id = settlement_token_ids[winner]; + let winning_outcome = outcome_names[winner]; + let resolved_up_won = match winning_outcome.to_ascii_lowercase().as_str() { + "up" | "yes" => true, + "down" | "no" => false, + _ => { + return Err(completeness_error(format!( + "closed market winner outcome is unsupported: {winning_outcome}" + ))) + } + }; + Ok(Some(json!({ + "kind": "market_settlement", + "market_id": market_id, + "condition_id": condition_id, + "symbol": symbol, + "market_window_secs": window_secs, + "winning_token_id": winning_token_id, + "winning_outcome": winning_outcome, + "resolved_up_won": resolved_up_won, + "resolution_source": "gamma_api_closed_market", + "retrieved_at": retrieved_at, + "market": market, + }))) +} + +fn settlement_is_overdue( + tracked: &TrackedMarket, + now: DateTime, + lookback_secs: i64, +) -> Result { + let end_time = parse_optional_datetime(tracked.end_time.as_deref()) + .ok_or_else(|| completeness_error("tracked market end time is missing or invalid"))?; + Ok(!tracked.settled && end_time < now - TimeDelta::seconds(lookback_secs)) +} + +#[derive(Clone, Copy)] +enum GammaLane { + Open, + Closed, +} + +impl GammaLane { + fn is_closed(self) -> bool { + matches!(self, Self::Closed) + } +} + +#[derive(Default)] +struct GammaDiscovery { + open: Vec, + closed: Vec, +} + +impl GammaDiscovery { + fn append_page( + &mut self, + lane: GammaLane, + payload: &Value, + max_markets: usize, + ) -> Result> { + let markets = match lane { + GammaLane::Open => &mut self.open, + GammaLane::Closed => &mut self.closed, + }; + append_gamma_page(markets, payload, max_markets) + } + + fn into_markets(mut self) -> Vec { + self.open.append(&mut self.closed); + self.open + } +} + +fn append_gamma_page( + markets: &mut Vec, + payload: &Value, + max_markets: usize, +) -> Result> { + let page = payload + .get("markets") + .and_then(Value::as_array) + .ok_or_else(|| completeness_error("Gamma keyset response is missing markets"))?; + if page.iter().any(|value| !value.is_object()) { + return Err(completeness_error( + "Gamma keyset response contains a non-object market", + )); + } + let next_cursor = match payload.get("next_cursor") { + None | Some(Value::Null) => None, + Some(Value::String(value)) if value.is_empty() => None, + Some(Value::String(value)) => Some(value.clone()), + Some(_) => { + return Err(completeness_error( + "Gamma keyset response has an invalid next_cursor", + )) + } + }; + let total = markets + .len() + .checked_add(page.len()) + .ok_or_else(|| completeness_error("Gamma market count overflow"))?; + if total > max_markets || (total == max_markets && next_cursor.is_some()) { + return Err(completeness_error(format!( + "Gamma market discovery reached max_markets={max_markets} before exhausting its cursor" + ))); + } + markets.extend(page.iter().cloned()); + Ok(next_cursor) +} + +fn gamma_discovery_params( + config: &ReferenceConfig, + now: DateTime, + closed: bool, +) -> Vec<(String, String)> { + let lookback_secs = if closed { + config.settlement_lookback_secs + } else { + config.market_lookback_secs + }; + vec![ + ( + "end_date_min".to_owned(), + iso_z(now - TimeDelta::seconds(lookback_secs)), + ), + ( + "end_date_max".to_owned(), + iso_z(now + TimeDelta::minutes(30)), + ), + ("closed".to_owned(), closed.to_string()), + ("tag_id".to_owned(), CRYPTO_TAG_ID.to_string()), + ("related_tags".to_owned(), "false".to_owned()), + ("limit".to_owned(), "100".to_owned()), + ] +} + +fn object_rows(page: &[Value]) -> (Vec, u64) { + let objects = page + .iter() + .filter(|value| value.is_object()) + .cloned() + .collect::>(); + let rejected = u64::try_from(page.len() - objects.len()).unwrap_or(u64::MAX); + (objects, rejected) +} + +fn market_metadata_update( + market_id: &str, + market: &Value, + target: &TargetMarket, + retrieved_at: &str, + force_context: bool, + tracked: &mut TrackedMarket, +) -> Result> { + let metadata_hash = stable_payload_hash(market)?; + if !force_context && tracked.last_metadata_hash.as_deref() == Some(&metadata_hash) { + return Ok(None); + } + tracked.last_metadata_hash = Some(metadata_hash); + Ok(Some(json!({ + "kind": "market_metadata", + "market_id": market_id, + "condition_id": market.get("conditionId").and_then(Value::as_str).unwrap_or_default(), + "symbol": target.symbol, + "market_window_secs": target.window_secs, + "source": "gamma_api", + "retrieved_at": retrieved_at, + "market": market, + }))) +} + +fn trade_timestamp(value: Option<&Value>) -> Option { + match value { + Some(Value::Number(value)) => value.as_i64(), + Some(Value::String(value)) => value.parse().ok(), + _ => None, + } +} + +fn positive_decimal(value: Option<&Value>) -> Option { + Decimal::from_str(&value_text(value)) + .ok() + .filter(|value| *value > Decimal::ZERO) +} + +fn probability_decimal(value: Option<&Value>) -> Option { + Decimal::from_str(&value_text(value)) + .ok() + .filter(|value| *value >= Decimal::ZERO && *value <= Decimal::ONE) +} + +fn reject(counter: &mut BTreeMap, reason: &str) { + *counter.entry(reason.to_owned()).or_default() += 1; +} + +#[allow(clippy::too_many_arguments)] +fn trade_updates( + config: &ReferenceConfig, + state: &mut CollectorState, + market_id: &str, + condition_id: &str, + symbol: &str, + window_secs: u64, + trades: &[Value], + now: DateTime, +) -> (Vec, BTreeMap) { + let cutoff = now.timestamp() - config.market_lookback_secs; + let mut parsed = Vec::new(); + let mut malformed = BTreeMap::new(); + for trade in trades { + let Some(timestamp) = trade_timestamp(trade.get("timestamp")) else { + reject(&mut malformed, "invalid_timestamp"); + continue; + }; + if trade.get("conditionId").and_then(Value::as_str) != Some(condition_id) { + reject(&mut malformed, "condition_mismatch"); + continue; + } + if trade + .get("transactionHash") + .and_then(Value::as_str) + .is_none_or(str::is_empty) + { + reject(&mut malformed, "missing_transaction_hash"); + continue; + } + if trade + .get("asset") + .and_then(Value::as_str) + .is_none_or(str::is_empty) + { + reject(&mut malformed, "missing_asset"); + continue; + } + if !matches!( + trade.get("side").and_then(Value::as_str), + Some("BUY" | "SELL") + ) { + reject(&mut malformed, "invalid_side"); + continue; + } + if trade + .get("proxyWallet") + .and_then(Value::as_str) + .is_none_or(str::is_empty) + { + reject(&mut malformed, "missing_proxy_wallet"); + continue; + } + if positive_decimal(trade.get("size")).is_none() { + reject(&mut malformed, "invalid_size"); + continue; + } + if probability_decimal(trade.get("price")).is_none() { + reject(&mut malformed, "invalid_price"); + continue; + } + if !matches!( + trade.get("outcomeIndex").and_then(Value::as_i64), + Some(0 | 1) + ) { + reject(&mut malformed, "invalid_outcome_index"); + continue; + } + if trade + .get("outcome") + .and_then(Value::as_str) + .is_none_or(str::is_empty) + { + reject(&mut malformed, "missing_outcome"); + continue; + } + parsed.push((timestamp, trade)); + } + parsed.sort_by_key(|(timestamp, _)| *timestamp); + + let seen = state.trade_seen.entry(condition_id.to_owned()).or_default(); + let mut updates = Vec::new(); + for (timestamp, trade) in parsed { + let record_id = stable_trade_id(trade); + if timestamp < cutoff || seen.contains_key(&record_id) { + continue; + } + seen.insert(record_id.clone(), timestamp); + updates.push(json!({ + "kind": "polymarket_trade", + "record_id": record_id, + "record_id_version": TRADE_ID_VERSION, + "market_id": market_id, + "condition_id": condition_id, + "token_id": trade.get("asset").and_then(Value::as_str).unwrap_or_default(), + "symbol": symbol, + "market_window_secs": window_secs, + "side": trade.get("side"), + "size": trade.get("size"), + "price": trade.get("price"), + "trade_ts": iso_z(DateTime::from_timestamp(timestamp, 0).unwrap_or(DateTime::::MIN_UTC)), + "trade_ts_unix": timestamp, + "transaction_hash": trade.get("transactionHash"), + "proxy_wallet": trade.get("proxyWallet"), + "outcome": trade.get("outcome"), + "outcome_index": trade.get("outcomeIndex"), + "source": "polymarket_data_api", + "received_at": iso_z(now), + "trade": trade, + })); + } + seen.retain(|_, timestamp| *timestamp >= cutoff); + (updates, malformed) +} + +#[allow(clippy::too_many_arguments)] +fn advance_trade_finalization( + tracked: &mut TrackedMarket, + now: DateTime, + retrieved_at: &str, + new_trade_count: usize, + truncated: bool, + was_settled: bool, + lag_secs: i64, + stable_polls_required: u64, +) -> bool { + if tracked.settlement_seen_at.is_none() { + tracked.settlement_seen_at = Some(retrieved_at.to_owned()); + } + if new_trade_count > 0 { + tracked.last_trade_change_at = Some(retrieved_at.to_owned()); + tracked.trade_finalization_stable_polls = 0; + } + let latest_anchor = [ + parse_optional_datetime(tracked.settlement_seen_at.as_deref()), + parse_optional_datetime(tracked.last_trade_change_at.as_deref()), + ] + .into_iter() + .flatten() + .max(); + let lag_elapsed = latest_anchor.is_some_and(|anchor| (now - anchor).num_seconds() >= lag_secs); + if !lag_elapsed || truncated || new_trade_count > 0 || !was_settled { + tracked.trade_finalization_stable_polls = 0; + return false; + } + tracked.trade_finalization_stable_polls += 1; + tracked.trade_finalization_stable_polls >= stable_polls_required +} + +struct ReferenceCollector { + config: ReferenceConfig, + symbols: BTreeSet, + state_path: PathBuf, + health_path: PathBuf, + state: CollectorState, + writer: TapeWriter, + http: reqwest::Client, + last_success: Instant, +} + +impl ReferenceCollector { + fn new(config: ReferenceConfig) -> Result { + config.validate()?; + let symbols = config + .symbols + .iter() + .map(|symbol| symbol.trim().to_ascii_uppercase()) + .filter(|symbol| !symbol.is_empty()) + .collect::>(); + let supported = SYMBOL_ALIASES + .iter() + .map(|(symbol, _)| (*symbol).to_owned()) + .collect::>(); + if symbols.is_empty() || !symbols.is_subset(&supported) { + bail!("symbols must be a non-empty subset of the supported Polymarket catalog"); + } + ensure_canonical_directory(&config.spool_dir) + .with_context(|| format!("invalid spool directory {}", config.spool_dir.display()))?; + let state_path = config.spool_dir.join("collector-state.json"); + let health_path = config.spool_dir.join("health.json"); + let mut state: CollectorState = read_optional_json(&state_path)?; + if state.trade_id_version.as_deref() != Some(TRADE_ID_VERSION) { + let active = config.spool_dir.join(ACTIVE_TAPE); + if active.exists() { + strict_existing_regular(&active)?; + if fs::metadata(&active)?.len() > 0 { + let quarantine = config.spool_dir.join(format!( + "superseded-v1-{ACTIVE_TAPE}.{}", + utc_now().timestamp_nanos_opt().unwrap_or_default() + )); + fs::rename(&active, quarantine)?; + File::open(&config.spool_dir)?.sync_all()?; + } + } + state.trade_seen.clear(); + for tracked in state.markets.values_mut() { + tracked.trade_complete = false; + } + state.trade_id_version = Some(TRADE_ID_VERSION.to_owned()); + atomic_write_json(&state_path, &state)?; + } + let writer = TapeWriter::new(&config.spool_dir)?; + let http = reqwest::Client::builder() + .timeout(config.http_timeout) + .user_agent(USER_AGENT) + .build()?; + let mut collector = Self { + config, + symbols, + state_path, + health_path, + state, + writer, + http, + last_success: Instant::now(), + }; + collector.recover_state_from_active_tape()?; + Ok(collector) + } + + fn recover_state_from_active_tape(&mut self) -> Result<()> { + if !self.writer.active.exists() { + return Ok(()); + } + let reader = BufReader::new(open_read_regular(&self.writer.active)?); + for line in reader.split(b'\n') { + let line = line?; + if line.is_empty() { + continue; + } + let row: Value = serde_json::from_slice(&line)?; + let update = row + .get("update") + .and_then(Value::as_object) + .cloned() + .unwrap_or_default(); + let kind = update + .get("kind") + .and_then(Value::as_str) + .unwrap_or_default(); + let market_id = update.get("market_id").and_then(Value::as_str); + let condition_id = update.get("condition_id").and_then(Value::as_str); + if kind == "polymarket_trade" { + if update.get("record_id_version").and_then(Value::as_str) == Some(TRADE_ID_VERSION) + { + if let (Some(condition_id), Some(record_id), Some(timestamp)) = ( + condition_id, + update.get("record_id").and_then(Value::as_str), + update.get("trade_ts_unix").and_then(Value::as_i64), + ) { + self.state + .trade_seen + .entry(condition_id.to_owned()) + .or_default() + .insert(record_id.to_owned(), timestamp); + } + } + } else if matches!(kind, "market_metadata" | "market_settlement") { + if let Some(market_id) = market_id { + let tracked = self.state.markets.entry(market_id.to_owned()).or_default(); + tracked.condition_id = condition_id.map(str::to_owned); + tracked.symbol = update + .get("symbol") + .and_then(Value::as_str) + .map(str::to_owned); + if let Some(window) = update.get("market_window_secs").and_then(Value::as_u64) { + tracked.market_window_secs = window; + } + if let Some(market) = update.get("market") { + tracked.end_time = market + .get("endDate") + .and_then(Value::as_str) + .map(str::to_owned); + tracked.last_metadata_hash = Some(stable_payload_hash(market)?); + } + if kind == "market_settlement" { + tracked.settled = true; + } + } + } + } + Ok(()) + } + + async fn get_json(&self, url: &str, params: &[(String, String)]) -> Result { + Ok(self + .http + .get(url) + .query(params) + .send() + .await? + .error_for_status()? + .json() + .await?) + } + + async fn discover_markets(&self, now: DateTime) -> Result> { + let mut discovery = GammaDiscovery::default(); + for lane in [GammaLane::Open, GammaLane::Closed] { + let base = gamma_discovery_params(&self.config, now, lane.is_closed()); + let mut cursor: Option = None; + let mut seen_cursors = BTreeSet::new(); + loop { + let mut params = base.clone(); + if let Some(cursor) = cursor.as_ref() { + params.push(("after_cursor".to_owned(), cursor.clone())); + } + let payload = self.get_json(GAMMA_MARKETS_URL, ¶ms).await?; + cursor = discovery.append_page(lane, &payload, self.config.max_markets)?; + let Some(next_cursor) = cursor.as_ref() else { + break; + }; + if !seen_cursors.insert(next_cursor.clone()) { + return Err(completeness_error(format!( + "Gamma keyset cursor repeated: {next_cursor}" + ))); + } + } + } + Ok(discovery.into_markets()) + } + + async fn fetch_trades(&self, condition_id: &str) -> Result<(Vec, bool, u64)> { + let mut trades = Vec::new(); + let mut truncated = false; + let mut non_object_rows = 0_u64; + for offset in [0_u64, 10_000] { + let payload = self + .get_json( + DATA_TRADES_URL, + &[ + ("market".to_owned(), condition_id.to_owned()), + ("limit".to_owned(), "10000".to_owned()), + ("offset".to_owned(), offset.to_string()), + ("takerOnly".to_owned(), "false".to_owned()), + ], + ) + .await?; + let page = payload + .as_array() + .context("Data API trades response is not an array")?; + let (objects, rejected) = object_rows(page); + trades.extend(objects); + non_object_rows = non_object_rows.saturating_add(rejected); + if page.len() < 10_000 { + break; + } + if offset == 10_000 { + truncated = true; + } + } + Ok((trades, truncated, non_object_rows)) + } + + async fn collect_once(&mut self) -> Result { + let now = utc_now(); + let retrieved_at = iso_z(now); + let mut updates = Vec::new(); + let mut errors = Vec::new(); + let mut truncated_markets = Vec::new(); + let mut non_object_trade_markets = Vec::new(); + let mut invalid_settlement_markets = Vec::new(); + let mut trade_polls = 0_u64; + let mut successful_trade_polls = 0_u64; + let mut malformed_trade_reasons = BTreeMap::::new(); + let mut next_state = self.state.clone(); + let discovered = self.discover_markets(now).await?; + let mut targets = BTreeMap::::new(); + let mut discovered_target_symbols = BTreeSet::new(); + let target_hour = hour_key(now); + let force_hour_context = context_seed_required(&self.state, &self.writer, now); + + for market in discovered { + let Some(target) = target_market(&market, &self.symbols) else { + continue; + }; + let Some(market_id) = market + .get("id") + .and_then(Value::as_str) + .filter(|value| !value.is_empty()) + else { + continue; + }; + let condition_id = market + .get("conditionId") + .and_then(Value::as_str) + .unwrap_or_default() + .to_owned(); + let tracked = next_state.markets.entry(market_id.to_owned()).or_default(); + tracked.condition_id = Some(condition_id); + tracked.symbol = Some(target.symbol.clone()); + tracked.market_window_secs = target.window_secs; + tracked.end_time = market + .get("endDate") + .and_then(Value::as_str) + .map(str::to_owned); + discovered_target_symbols.insert(target.symbol.clone()); + targets.insert(market_id.to_owned(), (market, target)); + } + let missing_target_symbols = missing_symbols(&self.symbols, &discovered_target_symbols); + + let tracked_ids = next_state.markets.keys().cloned().collect::>(); + for market_id in tracked_ids { + let Some(mut tracked) = next_state.markets.remove(&market_id) else { + continue; + }; + let end_time = parse_optional_datetime(tracked.end_time.as_deref()); + if end_time + .is_some_and(|end| end < now - TimeDelta::seconds(self.config.market_lookback_secs)) + && tracked.settled + && tracked.trade_complete + { + if let Some(condition_id) = tracked.condition_id.as_ref() { + next_state.trade_seen.remove(condition_id); + } + continue; + } + let needs_detail = !targets.contains_key(&market_id) + && end_time.is_some_and(|end| end <= now) + && !(tracked.settled && tracked.trade_complete); + if needs_detail { + let url = format!("{GAMMA_MARKET_URL}/{market_id}"); + match self.get_json(&url, &[]).await { + Ok(market) if market.is_object() => { + targets.insert( + market_id.clone(), + ( + market, + TargetMarket { + symbol: tracked.symbol.clone().unwrap_or_default(), + window_secs: tracked.market_window_secs, + }, + ), + ); + tracked.settlement_failure_since = None; + tracked.settlement_last_error = None; + } + Ok(_) => { + let error = "Gamma market detail response is not an object".to_owned(); + errors.push(format!("settlement {market_id}: {error}")); + tracked + .settlement_failure_since + .get_or_insert_with(|| retrieved_at.clone()); + tracked.settlement_last_error = Some(error); + } + Err(error) => { + errors.push(format!("settlement {market_id}: {error}")); + tracked + .settlement_failure_since + .get_or_insert_with(|| retrieved_at.clone()); + tracked.settlement_last_error = Some(error.to_string()); + } + } + } + next_state.markets.insert(market_id, tracked); + } + + for (market_id, (market, target)) in &targets { + let condition_id = market + .get("conditionId") + .and_then(Value::as_str) + .unwrap_or_default() + .to_owned(); + let mut tracked = next_state.markets.remove(market_id).unwrap_or_default(); + tracked.condition_id = Some(condition_id.clone()); + tracked.symbol = Some(target.symbol.clone()); + tracked.market_window_secs = target.window_secs; + if let Some(end_time) = market.get("endDate").and_then(Value::as_str) { + tracked.end_time = Some(end_time.to_owned()); + } + + if let Some(metadata) = market_metadata_update( + market_id, + market, + target, + &retrieved_at, + force_hour_context, + &mut tracked, + )? { + updates.push(metadata); + } + + let settlement = match settlement_from_market( + market, + &target.symbol, + target.window_secs, + &retrieved_at, + ) { + Ok(settlement) => settlement, + Err(error) => { + let detail = error.to_string(); + errors.push(format!("settlement {market_id}: {detail}")); + invalid_settlement_markets.push(market_id.clone()); + tracked + .settlement_failure_since + .get_or_insert_with(|| retrieved_at.clone()); + tracked.settlement_last_error = Some(detail); + None + } + }; + if settlement.is_some() { + tracked.settlement_failure_since = None; + tracked.settlement_last_error = None; + } + let was_settled = tracked.settled; + if !condition_id.is_empty() && !tracked.trade_complete { + trade_polls += 1; + match self.fetch_trades(&condition_id).await { + Ok((trades, truncated, non_object_rows)) => { + successful_trade_polls += 1; + let (new_updates, mut malformed) = trade_updates( + &self.config, + &mut next_state, + market_id, + &condition_id, + &target.symbol, + target.window_secs, + &trades, + now, + ); + if non_object_rows > 0 { + *malformed.entry("non_object_trade".to_owned()).or_default() += + non_object_rows; + non_object_trade_markets.push(condition_id.clone()); + } + let new_trade_count = new_updates.len(); + updates.extend(new_updates); + for (reason, count) in &malformed { + *malformed_trade_reasons.entry(reason.clone()).or_default() += count; + } + tracked.last_trade_success_at = Some(retrieved_at.clone()); + if malformed.is_empty() { + tracked.trade_failure_since = None; + tracked.trade_last_error = None; + } else { + let detail = format!("malformed trade rows: {malformed:?}"); + errors.push(format!("trades {condition_id}: {detail}")); + tracked + .trade_failure_since + .get_or_insert_with(|| retrieved_at.clone()); + tracked.trade_last_error = Some(detail); + } + if truncated { + truncated_markets.push(condition_id.clone()); + } + if settlement.is_some() + && advance_trade_finalization( + &mut tracked, + now, + &retrieved_at, + new_trade_count, + truncated || !malformed.is_empty(), + was_settled, + self.config.trade_finalization_lag_secs, + self.config.trade_finalization_stable_polls, + ) + { + tracked.trade_complete = true; + } + } + Err(error) => { + errors.push(format!("trades {condition_id}: {error}")); + tracked + .trade_failure_since + .get_or_insert_with(|| retrieved_at.clone()); + tracked.trade_last_error = Some(error.to_string()); + } + } + } + if let Some(settlement) = settlement { + if !tracked.settled { + updates.push(settlement); + tracked.settled = true; + } + } + next_state.markets.insert(market_id.clone(), tracked); + if !self.config.per_market_delay.is_zero() { + tokio::time::sleep(self.config.per_market_delay).await; + } + } + + let mut overdue_unresolved_markets = Vec::new(); + let mut invalid_end_time_markets = Vec::new(); + for (market_id, tracked) in &next_state.markets { + match settlement_is_overdue(tracked, now, self.config.settlement_lookback_secs) { + Ok(true) => overdue_unresolved_markets.push(market_id.clone()), + Ok(false) => {} + Err(_) => invalid_end_time_markets.push(market_id.clone()), + } + } + let mut stale_trade_markets = next_state + .markets + .values() + .filter_map(|tracked| { + let failed_at = parse_optional_datetime(tracked.trade_failure_since.as_deref())?; + ((now - failed_at).to_std().ok()? > self.config.stale_after) + .then(|| tracked.condition_id.clone().unwrap_or_default()) + }) + .collect::>(); + let mut stale_settlement_markets = next_state + .markets + .iter() + .filter_map(|(market_id, tracked)| { + let failed_at = + parse_optional_datetime(tracked.settlement_failure_since.as_deref())?; + ((now - failed_at).to_std().ok()? > self.config.stale_after) + .then(|| market_id.clone()) + }) + .collect::>(); + + self.writer.write_updates(&updates, now)?; + if missing_target_symbols.is_empty() { + next_state.context_seed_hour = Some(target_hour); + } + self.state = next_state; + atomic_write_json(&self.state_path, &self.state)?; + self.last_success = Instant::now(); + + overdue_unresolved_markets.sort(); + truncated_markets.sort(); + non_object_trade_markets.sort(); + invalid_settlement_markets.sort(); + invalid_end_time_markets.sort(); + stale_trade_markets.sort(); + stale_settlement_markets.sort(); + let record_types = ["market_metadata", "polymarket_trade", "market_settlement"] + .into_iter() + .map(|kind| { + ( + kind, + updates + .iter() + .filter(|update| update.get("kind").and_then(Value::as_str) == Some(kind)) + .count(), + ) + }) + .collect::>(); + let health = json!({ + "updated_at": retrieved_at, + "last_success_at": retrieved_at, + "target_markets": targets.len(), + "missing_target_symbols": missing_target_symbols, + "tracked_markets": self.state.markets.len(), + "records_written": updates.len(), + "record_types": record_types, + "api_errors": errors, + "trade_polls": trade_polls, + "successful_trade_polls": successful_trade_polls, + "malformed_trade_rows": malformed_trade_reasons.values().sum::(), + "malformed_trade_reasons": malformed_trade_reasons, + "truncated_trade_markets": truncated_markets, + "non_object_trade_markets": non_object_trade_markets, + "invalid_settlement_markets": invalid_settlement_markets, + "invalid_end_time_markets": invalid_end_time_markets, + "stale_trade_markets": stale_trade_markets, + "stale_settlement_markets": stale_settlement_markets, + "overdue_unresolved_markets": overdue_unresolved_markets, + "active_tape_bytes": fs::metadata(&self.writer.active).map(|value| value.len()).unwrap_or_default(), + "free_disk_bytes": fs4::available_space(&self.config.spool_dir)?, + }); + atomic_write_json(&self.health_path, &health)?; + + if !missing_target_symbols.is_empty() { + return Err(DataCompletenessError(format!( + "target discovery missed configured symbols: {missing_target_symbols:?}" + )) + .into()); + } + if !truncated_markets.is_empty() { + return Err(DataCompletenessError(format!( + "trade pagination exceeded API offset limit for {truncated_markets:?}" + )) + .into()); + } + if !non_object_trade_markets.is_empty() { + return Err(DataCompletenessError(format!( + "non-object trade rows for {non_object_trade_markets:?}" + )) + .into()); + } + if !invalid_settlement_markets.is_empty() { + return Err(DataCompletenessError(format!( + "invalid settlement payloads for {invalid_settlement_markets:?}" + )) + .into()); + } + if !invalid_end_time_markets.is_empty() { + return Err(DataCompletenessError(format!( + "tracked markets with missing or invalid end time: {invalid_end_time_markets:?}" + )) + .into()); + } + if !overdue_unresolved_markets.is_empty() { + return Err(DataCompletenessError(format!( + "overdue unresolved settlements: {overdue_unresolved_markets:?}" + )) + .into()); + } + if !stale_trade_markets.is_empty() { + return Err(DataCompletenessError(format!( + "stale trade markets: {stale_trade_markets:?}" + )) + .into()); + } + if !stale_settlement_markets.is_empty() { + return Err(DataCompletenessError(format!( + "stale settlement markets: {stale_settlement_markets:?}" + )) + .into()); + } + Ok(health) + } +} + +pub async fn run_reference(config: ReferenceConfig, once: bool) -> Result<()> { + let poll_interval = config.poll_interval; + let stale_after = config.stale_after; + let mut collector = ReferenceCollector::new(config)?; + if once { + println!( + "{}", + serde_json::to_string(&collector.collect_once().await?)? + ); + collector.writer.close()?; + return Ok(()); + } + loop { + let started = Instant::now(); + match collector.collect_once().await { + Ok(health) => println!("{}", serde_json::to_string(&health)?), + Err(error) => { + eprintln!("Polymarket reference poll failed: {error:#}"); + if error.downcast_ref::().is_some() + || collector.last_success.elapsed() > stale_after + { + collector.writer.close()?; + return Err(error); + } + } + } + tokio::time::sleep(poll_interval.saturating_sub(started.elapsed())).await; + } +} + +#[cfg(test)] +mod tests { + use super::*; + + struct TestDir(PathBuf); + + impl TestDir { + fn new() -> Self { + let path = std::env::temp_dir().join(format!( + "monday-polymarket-reference-test-{:016x}", + random::() + )); + fs::create_dir(&path).unwrap(); + Self(fs::canonicalize(path).unwrap()) + } + + fn path(&self) -> &Path { + &self.0 + } + } + + impl Drop for TestDir { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } + } + + fn fixed_time(value: &str) -> DateTime { + DateTime::parse_from_rfc3339(value) + .unwrap() + .with_timezone(&Utc) + } + + fn market(question: &str, start: &str, end: &str) -> Value { + json!({ + "id": "market-1", + "conditionId": "condition-1", + "question": question, + "startDate": start, + "endDate": end, + "clobTokenIds": ["up-token", "down-token"], + }) + } + + fn valid_trade(timestamp: i64) -> Value { + json!({ + "transactionHash": "0xtx", + "conditionId": "condition-1", + "asset": "up-token", + "side": "BUY", + "timestamp": timestamp, + "proxyWallet": "0xwallet", + "size": "10.0", + "price": "0.780", + "outcome": "Up", + "outcomeIndex": 0, + "sourceOnlyField": {"preserved": true}, + }) + } + + #[test] + fn targets_all_configured_symbols_and_only_five_or_fifteen_minute_markets() { + let symbols = SYMBOL_ALIASES + .iter() + .map(|(symbol, _)| (*symbol).to_owned()) + .collect::>(); + for (symbol, aliases) in SYMBOL_ALIASES { + let five = market( + &format!("Will {} go up?", aliases[0]), + "2026-07-15T00:00:00Z", + "2026-07-15T00:05:00Z", + ); + assert_eq!(target_market(&five, &symbols).unwrap().symbol, symbol); + let fifteen = market( + &format!("Will {} go up?", aliases[0]), + "2026-07-15T00:00:00Z", + "2026-07-15T00:15:00Z", + ); + assert_eq!(target_market(&fifteen, &symbols).unwrap().window_secs, 900); + } + assert!(target_market( + &market( + "Will Bitcoin go up?", + "2026-07-15T00:00:00Z", + "2026-07-15T00:10:00Z", + ), + &symbols, + ) + .is_none()); + let misleading = json!({ + "id": "market-2", + "conditionId": "condition-2", + "question": "Will Bitcoin go up?", + "slug": "bitcoin-115ms-sample", + "clobTokenIds": ["up", "down"], + }); + assert!(target_market(&misleading, &symbols).is_none()); + } + + #[test] + fn missing_target_symbols_reports_every_configured_gap() { + let configured = ["BTCUSDT", "ETHUSDT", "SOLUSDT"] + .into_iter() + .map(str::to_owned) + .collect::>(); + let discovered = ["ETHUSDT"] + .into_iter() + .map(str::to_owned) + .collect::>(); + assert_eq!( + missing_symbols(&configured, &discovered), + vec!["BTCUSDT", "SOLUSDT"] + ); + assert!(missing_symbols(&configured, &configured).is_empty()); + } + + #[test] + fn settlement_requires_closed_market_and_single_near_certain_winner() { + let mut resolved = json!({ + "id": "market-1", + "conditionId": "condition-1", + "closed": true, + "outcomes": ["Up", "Down"], + "clobTokenIds": ["up-token", "down-token"], + "outcomePrices": ["0.999", "0.001"], + }); + let settlement = settlement_from_market(&resolved, "BTCUSDT", 300, "now") + .unwrap() + .unwrap(); + assert_eq!(settlement["winning_token_id"], "up-token"); + assert_eq!(settlement["resolved_up_won"], true); + + resolved["closed"] = json!(false); + assert!(settlement_from_market(&resolved, "BTCUSDT", 300, "now") + .unwrap() + .is_none()); + resolved["closed"] = json!(true); + resolved["outcomePrices"] = json!(["0.9989", "0.0011"]); + assert!(settlement_from_market(&resolved, "BTCUSDT", 300, "now").is_err()); + resolved["outcomePrices"] = json!(["1", "1"]); + assert!(settlement_from_market(&resolved, "BTCUSDT", 300, "now").is_err()); + resolved["outcomePrices"] = json!(["0.999", "0.5"]); + assert!(settlement_from_market(&resolved, "BTCUSDT", 300, "now").is_err()); + resolved["outcomePrices"] = json!(["1", "0.001"]); + assert!(settlement_from_market(&resolved, "BTCUSDT", 300, "now").is_err()); + resolved["outcomePrices"] = json!(["1", "0"]); + resolved["outcomes"] = json!(["Maybe", "Never"]); + assert!(settlement_from_market(&resolved, "BTCUSDT", 300, "now").is_err()); + } + + #[test] + fn only_unresolved_markets_become_overdue() { + let now = fixed_time("2026-07-15T02:00:00Z"); + let mut tracked = TrackedMarket { + end_time: Some("2026-07-15T00:00:00Z".to_owned()), + ..TrackedMarket::default() + }; + assert!(settlement_is_overdue(&tracked, now, 3_600).unwrap()); + tracked.settled = true; + assert!(!settlement_is_overdue(&tracked, now, 3_600).unwrap()); + } + + #[test] + fn missing_or_invalid_market_end_time_is_a_completeness_error() { + let now = fixed_time("2026-07-15T02:00:00Z"); + for end_time in [None, Some("not-a-time".to_owned())] { + let tracked = TrackedMarket { + end_time, + settled: true, + ..TrackedMarket::default() + }; + let error = settlement_is_overdue(&tracked, now, 3_600).unwrap_err(); + assert!(error.downcast_ref::().is_some()); + } + } + + #[test] + fn gamma_cursor_pages_are_exhausted_or_fail_at_the_configured_cap() { + let mut markets = Vec::new(); + let cursor = append_gamma_page( + &mut markets, + &json!({"markets": [{"id": "one"}], "next_cursor": "page-2"}), + 2, + ) + .unwrap(); + assert_eq!(cursor.as_deref(), Some("page-2")); + assert!(append_gamma_page( + &mut markets, + &json!({"markets": [{"id": "two"}], "next_cursor": ""}), + 2, + ) + .unwrap() + .is_none()); + assert_eq!(markets.len(), 2); + + let mut capped = Vec::new(); + let error = append_gamma_page( + &mut capped, + &json!({"markets": [{"id": "one"}], "next_cursor": "page-2"}), + 1, + ) + .unwrap_err(); + assert!(error.downcast_ref::().is_some()); + } + + #[test] + fn open_and_closed_discovery_lanes_have_independent_caps() { + let mut discovery = GammaDiscovery::default(); + assert!(discovery + .append_page( + GammaLane::Open, + &json!({"markets": [{"id": "open"}], "next_cursor": ""}), + 1, + ) + .unwrap() + .is_none()); + assert!(discovery + .append_page( + GammaLane::Closed, + &json!({"markets": [{"id": "closed"}], "next_cursor": ""}), + 1, + ) + .unwrap() + .is_none()); + + let ids = discovery + .into_markets() + .into_iter() + .map(|market| market["id"].as_str().unwrap().to_owned()) + .collect::>(); + assert_eq!(ids, ["open", "closed"]); + } + + #[test] + fn settlement_discovery_capacity_covers_every_configured_five_and_fifteen_minute_market() { + let default = ReferenceConfig::default(); + assert_eq!(default.max_markets, 10_000); + default.validate().unwrap(); + + let exact_capacity = ReferenceConfig { + max_markets: 2_688, + ..default.clone() + }; + exact_capacity.validate().unwrap(); + + let undersized = ReferenceConfig { + max_markets: 2_687, + ..default.clone() + }; + let error = undersized.validate().unwrap_err(); + assert!(error.to_string().contains("max_markets")); + + let short_settlement_window = ReferenceConfig { + settlement_lookback_secs: 86_399, + ..default + }; + let error = short_settlement_window.validate().unwrap_err(); + assert!(error.to_string().contains("settlement lookback")); + } + + #[test] + fn closed_discovery_uses_the_24_hour_settlement_lookback() { + let now = fixed_time("2026-07-15T02:00:00Z"); + let config = ReferenceConfig { + market_lookback_secs: 7_200, + settlement_lookback_secs: 86_400, + ..ReferenceConfig::default() + }; + let open = gamma_discovery_params(&config, now, false) + .into_iter() + .collect::>(); + let closed = gamma_discovery_params(&config, now, true) + .into_iter() + .collect::>(); + assert_eq!(open["closed"], "false"); + assert_eq!(closed["closed"], "true"); + assert_eq!(open["tag_id"], "21"); + assert_eq!(closed["tag_id"], "21"); + assert_eq!(open["related_tags"], "false"); + assert_eq!(closed["related_tags"], "false"); + assert_eq!(open["end_date_min"], "2026-07-15T00:00:00.000000Z"); + assert_eq!(closed["end_date_min"], "2026-07-14T02:00:00.000000Z"); + } + + #[cfg(unix)] + #[test] + fn collector_rejects_symlinked_or_noncanonical_spool_ancestors() { + use std::os::unix::fs::symlink; + + let root = TestDir::new(); + let actual = root.path().join("actual"); + fs::create_dir(&actual).unwrap(); + let linked = root.path().join("linked"); + symlink(&actual, &linked).unwrap(); + for spool_dir in [ + linked.join("spool"), + actual.join("child").join("..").join("child"), + ] { + let config = ReferenceConfig { + spool_dir, + symbols: vec!["BTCUSDT".to_owned()], + ..ReferenceConfig::default() + }; + assert!(ReferenceCollector::new(config) + .err() + .expect("indirect spool must fail") + .to_string() + .contains("spool")); + } + } + + #[test] + fn gamma_and_trade_pages_reject_schema_loss() { + let mut markets = Vec::new(); + assert!( + append_gamma_page(&mut markets, &json!({"markets": [{"id": "one"}, null]}), 10,) + .is_err() + ); + + let (objects, rejected) = object_rows(&[json!({"id": "trade"}), Value::Null]); + assert_eq!(objects.len(), 1); + assert_eq!(rejected, 1); + } + + #[test] + fn unchanged_metadata_is_reseeded_for_a_new_hour() { + let market = market( + "Will Bitcoin go up?", + "2026-07-15T00:00:00Z", + "2026-07-15T00:05:00Z", + ); + let target = TargetMarket { + symbol: "BTCUSDT".to_owned(), + window_secs: 300, + }; + let mut tracked = TrackedMarket::default(); + assert!( + market_metadata_update("market-1", &market, &target, "first", false, &mut tracked,) + .unwrap() + .is_some() + ); + assert!(market_metadata_update( + "market-1", + &market, + &target, + "same-hour", + false, + &mut tracked, + ) + .unwrap() + .is_none()); + assert!(market_metadata_update( + "market-1", + &market, + &target, + "next-hour", + true, + &mut tracked, + ) + .unwrap() + .is_some()); + } + + #[test] + fn trade_id_matches_v2_golden_vector() { + let mut trade = valid_trade(1_784_084_995); + trade["conditionId"] = json!("0xcondition"); + assert_eq!( + stable_trade_id(&trade), + "6a476c3be58fcb8d789224feba3e079f372bc933e838cd0dbb76435d4fad9cbe" + ); + } + + #[test] + fn trade_rows_are_validated_deduplicated_and_preserve_raw_payload() { + let now = fixed_time("2026-07-15T01:00:00Z"); + let mut state = CollectorState::default(); + let mut malformed = valid_trade(now.timestamp()); + malformed["price"] = json!("1.01"); + let mut formerly_colliding = valid_trade(now.timestamp()); + formerly_colliding["proxyWallet"] = json!("0xother"); + formerly_colliding["size"] = json!("11"); + formerly_colliding["price"] = json!("0.79"); + let (updates, reasons) = trade_updates( + &ReferenceConfig::default(), + &mut state, + "market-1", + "condition-1", + "BTCUSDT", + 300, + &[ + valid_trade(now.timestamp()), + valid_trade(now.timestamp()), + formerly_colliding, + malformed, + ], + now, + ); + assert_eq!(updates.len(), 2); + assert_ne!(updates[0]["record_id"], updates[1]["record_id"]); + assert_eq!(reasons.get("invalid_price"), Some(&1)); + assert_eq!(updates[0]["record_id_version"], TRADE_ID_VERSION); + assert_eq!(updates[0]["trade"]["sourceOnlyField"]["preserved"], true); + + let (again, _) = trade_updates( + &ReferenceConfig::default(), + &mut state, + "market-1", + "condition-1", + "BTCUSDT", + 300, + &[valid_trade(now.timestamp())], + now, + ); + assert!(again.is_empty()); + } + + #[test] + fn finalization_requires_lag_and_consecutive_stable_post_settlement_polls() { + let now = fixed_time("2026-07-15T02:00:00Z"); + let mut tracked = TrackedMarket { + settlement_seen_at: Some("2026-07-15T01:00:00Z".to_owned()), + ..TrackedMarket::default() + }; + assert!(!advance_trade_finalization( + &mut tracked, + now, + &iso_z(now), + 0, + false, + true, + 1_800, + 2, + )); + assert!(advance_trade_finalization( + &mut tracked, + now, + &iso_z(now), + 0, + false, + true, + 1_800, + 2, + )); + assert!(!advance_trade_finalization( + &mut tracked, + now, + &iso_z(now), + 1, + false, + true, + 1_800, + 2, + )); + assert_eq!(tracked.trade_finalization_stable_polls, 0); + } + + #[test] + fn tape_recovery_truncates_incomplete_tail_and_continues_sequence() { + let root = TestDir::new(); + let now = fixed_time("2026-07-15T01:00:00Z"); + { + let mut writer = TapeWriter::new(root.path()).unwrap(); + writer + .write_updates(&[json!({"kind": "first"}), json!({"kind": "second"})], now) + .unwrap(); + } + OpenOptions::new() + .append(true) + .open(root.path().join(ACTIVE_TAPE)) + .unwrap() + .write_all(b"{incomplete") + .unwrap(); + { + let mut writer = TapeWriter::new(root.path()).unwrap(); + assert_eq!(writer.sequence, 2); + writer + .write_updates(&[json!({"kind": "third"})], now) + .unwrap(); + } + let rows = BufReader::new(File::open(root.path().join(ACTIVE_TAPE)).unwrap()) + .lines() + .map(|line| serde_json::from_str::(&line.unwrap()).unwrap()) + .collect::>(); + assert_eq!(rows.len(), 3); + assert_eq!(rows[2]["sequence"], 2); + } + + #[test] + fn rotation_publishes_new_active_tape_starting_at_zero() { + let root = TestDir::new(); + let first = fixed_time("2026-07-15T01:59:59Z"); + let second = fixed_time("2026-07-15T02:00:00Z"); + let mut writer = TapeWriter::new(root.path()).unwrap(); + writer + .write_updates(&[json!({"kind": "first"})], first) + .unwrap(); + assert!(!writer.needs_hour_context(first)); + assert!(writer.needs_hour_context(second)); + writer + .write_updates(&[json!({"kind": "second"})], second) + .unwrap(); + assert!(!writer.needs_hour_context(second)); + writer.close().unwrap(); + let active: Value = + serde_json::from_str(&fs::read_to_string(root.path().join(ACTIVE_TAPE)).unwrap()) + .unwrap(); + assert_eq!(active["sequence"], 0); + let rotated = fs::read_dir(root.path()) + .unwrap() + .filter_map(|entry| entry.ok()) + .filter(|entry| { + let name = entry.file_name(); + let name = name.to_string_lossy(); + name.as_ref() != ACTIVE_TAPE + && name.starts_with("market-updates.") + && name.ends_with(".ndjson") + }) + .count(); + assert_eq!(rotated, 1); + } + + #[test] + fn failed_batch_restores_hour_so_retry_still_requires_context() { + let root = TestDir::new(); + let now = fixed_time("2026-07-15T02:00:00Z"); + let mut writer = TapeWriter::new(root.path()).unwrap(); + let error = writer + .write_updates_with_sync( + &[json!({"kind": "market_metadata", "market_id": "market-1"})], + now, + |_| bail!("injected sync failure"), + ) + .unwrap_err(); + assert!(error.to_string().contains("injected sync failure")); + assert!(writer.needs_hour_context(now)); + assert_eq!(writer.sequence, 0); + assert_eq!( + fs::metadata(root.path().join(ACTIVE_TAPE)).unwrap().len(), + 0 + ); + + writer + .write_updates( + &[json!({"kind": "market_metadata", "market_id": "market-1"})], + now, + ) + .unwrap(); + assert!(!writer.needs_hour_context(now)); + } + + #[test] + fn context_seed_is_required_for_old_state_same_hour_and_next_hour() { + let root = TestDir::new(); + let current = fixed_time("2026-07-15T02:00:00Z"); + let next = fixed_time("2026-07-15T03:00:00Z"); + let mut writer = TapeWriter::new(root.path()).unwrap(); + writer + .write_updates( + &[json!({"kind": "market_metadata", "market_id": "market-1"})], + current, + ) + .unwrap(); + let mut old_state: CollectorState = + serde_json::from_value(json!({"trade_id_version": "v2"})).unwrap(); + + assert!(!writer.needs_hour_context(current)); + assert!(context_seed_required(&old_state, &writer, current)); + old_state.context_seed_hour = Some(hour_key(current)); + let state_path = root.path().join("collector-state.json"); + atomic_write_json(&state_path, &old_state).unwrap(); + let reloaded: CollectorState = read_optional_json(&state_path).unwrap(); + assert!(!context_seed_required(&reloaded, &writer, current)); + assert!(context_seed_required(&reloaded, &writer, next)); + } + + #[test] + fn invalid_state_fails_closed_before_network_access() { + let root = TestDir::new(); + fs::write(root.path().join("collector-state.json"), b"not-json\n").unwrap(); + let config = ReferenceConfig { + spool_dir: root.path().to_path_buf(), + ..ReferenceConfig::default() + }; + assert!(ReferenceCollector::new(config) + .err() + .expect("invalid state must fail") + .to_string() + .contains("invalid JSON")); + } + + #[test] + fn empty_or_unknown_symbol_catalog_fails_closed() { + for symbols in [vec![" ".to_owned()], vec!["NOT_A_MARKET".to_owned()]] { + let root = TestDir::new(); + let config = ReferenceConfig { + spool_dir: root.path().to_path_buf(), + symbols, + ..ReferenceConfig::default() + }; + assert!(ReferenceCollector::new(config) + .err() + .expect("unsupported symbols must fail") + .to_string() + .contains("supported Polymarket catalog")); + } + } + + #[test] + fn v2_migration_quarantines_active_tape_and_reopens_trade_collection() { + let root = TestDir::new(); + atomic_write_json( + &root.path().join("collector-state.json"), + &json!({ + "markets": {"market-1": {"trade_complete": true}}, + "trade_seen": {"condition-1": {"old-id": 1}}, + }), + ) + .unwrap(); + fs::write(root.path().join(ACTIVE_TAPE), b"legacy-v1\n").unwrap(); + let config = ReferenceConfig { + spool_dir: root.path().to_path_buf(), + symbols: vec!["BTCUSDT".to_owned()], + ..ReferenceConfig::default() + }; + + let mut collector = ReferenceCollector::new(config.clone()).unwrap(); + assert_eq!(collector.state.trade_id_version.as_deref(), Some("v2")); + assert!(collector.state.trade_seen.is_empty()); + assert!(!collector.state.markets["market-1"].trade_complete); + let durable_state: CollectorState = + read_optional_json(&root.path().join("collector-state.json")).unwrap(); + assert_eq!(durable_state.trade_id_version.as_deref(), Some("v2")); + let quarantined = fs::read_dir(root.path()) + .unwrap() + .filter_map(|entry| entry.ok()) + .filter(|entry| { + entry + .file_name() + .to_string_lossy() + .starts_with("superseded-v1-market-updates.ndjson.") + }) + .count(); + assert_eq!(quarantined, 1); + collector + .writer + .write_updates( + &[json!({ + "kind": "polymarket_trade", + "condition_id": "condition-1", + "record_id": "durable-v2", + "record_id_version": "v2", + "trade_ts_unix": 1_784_084_400_i64, + })], + fixed_time("2026-07-15T03:00:00Z"), + ) + .unwrap(); + collector.writer.close().unwrap(); + drop(collector); + + let mut restarted = ReferenceCollector::new(config).unwrap(); + assert!(fs::read_to_string(root.path().join(ACTIVE_TAPE)) + .unwrap() + .contains("durable-v2")); + let quarantined_after_restart = fs::read_dir(root.path()) + .unwrap() + .filter_map(|entry| entry.ok()) + .filter(|entry| { + entry + .file_name() + .to_string_lossy() + .starts_with("superseded-v1-market-updates.ndjson.") + }) + .count(); + assert_eq!(quarantined_after_restart, 1); + restarted.writer.close().unwrap(); + } + + #[test] + fn collector_recovers_trade_dedupe_state_from_durable_active_tape() { + let root = TestDir::new(); + atomic_write_json( + &root.path().join("collector-state.json"), + &json!({"trade_id_version": "v2"}), + ) + .unwrap(); + { + let mut writer = TapeWriter::new(root.path()).unwrap(); + writer + .write_updates( + &[json!({ + "kind": "polymarket_trade", + "market_id": "market-1", + "condition_id": "condition-1", + "record_id": "trade-1", + "record_id_version": "v2", + "trade_ts_unix": 1_784_084_400_i64, + })], + fixed_time("2026-07-15T03:00:00Z"), + ) + .unwrap(); + } + let config = ReferenceConfig { + spool_dir: root.path().to_path_buf(), + symbols: vec!["BTCUSDT".to_owned()], + ..ReferenceConfig::default() + }; + + let mut collector = ReferenceCollector::new(config).unwrap(); + assert_eq!( + collector.state.trade_seen["condition-1"]["trade-1"], + 1_784_084_400 + ); + collector.writer.close().unwrap(); + } + + #[cfg(unix)] + #[test] + fn symlink_active_tape_is_refused() { + use std::os::unix::fs::symlink; + + let root = TestDir::new(); + let target = root.path().join("victim"); + fs::write(&target, b"do not touch\n").unwrap(); + symlink(&target, root.path().join(ACTIVE_TAPE)).unwrap(); + let error = TapeWriter::new(root.path()) + .err() + .expect("symlink active tape must fail") + .to_string(); + assert!(error.contains("non-regular") || error.contains("failed to open")); + assert_eq!(fs::read(target).unwrap(), b"do not touch\n"); + } +} diff --git a/rust_hft/tools/collector/src/polymarket_upload.rs b/rust_hft/tools/collector/src/polymarket_upload.rs new file mode 100644 index 000000000..e6f6cb008 --- /dev/null +++ b/rust_hft/tools/collector/src/polymarket_upload.rs @@ -0,0 +1,2673 @@ +//! Validation and fail-closed OSS upload for closed Polymarket raw tapes. + +use crate::lob_archiver::{command_status_with_timeout, sha256_file, write_success_marker}; +use anyhow::{anyhow, bail, Context, Result}; +use chrono::{DateTime, NaiveDateTime, Utc}; +use rand::random; +use rust_decimal::Decimal; +use serde::Serialize; +use serde_json::{json, Map, Value}; +use sha2::{Digest, Sha256}; +use std::collections::{BTreeMap, BTreeSet}; +use std::fs::{self, DirBuilder, File, Metadata, OpenOptions}; +use std::io::{BufRead, BufReader, BufWriter, Write}; +use std::os::unix::fs::{DirBuilderExt, MetadataExt}; +use std::path::{Component, Path, PathBuf}; +use std::process::{Command, ExitStatus, Stdio}; +use std::str::FromStr; +use std::time::Duration; + +const ALLOWED_KINDS: [&str; 7] = [ + "quote", + "event_discovered", + "event_expired", + "reference_price", + "market_metadata", + "polymarket_trade", + "market_settlement", +]; +const SUPPORTED_SYMBOL_ALIASES: [(&str, &[&str]); 7] = [ + ("BTCUSDT", &["BITCOIN", "BTC"]), + ("ETHUSDT", &["ETHEREUM", "ETH"]), + ("SOLUSDT", &["SOLANA", "SOL "]), + ("XRPUSDT", &["XRP"]), + ("DOGEUSDT", &["DOGECOIN", "DOGE"]), + ("HYPEUSDT", &["HYPERLIQUID", "HYPE"]), + ("BNBUSDT", &["BINANCE COIN", "BNB"]), +]; +const SETTLEMENT_PRICE: Decimal = Decimal::from_parts(999, 0, 0, false, 3); +const SETTLEMENT_LOSER_PRICE: Decimal = Decimal::from_parts(1, 0, 0, false, 3); +const SETTLEMENT_SUM_TOLERANCE: Decimal = Decimal::from_parts(1, 0, 0, false, 6); + +#[derive(Debug, Clone)] +pub struct UploadConfig { + pub spool_dir: PathBuf, + pub dataset: String, + pub quote_depth_levels: usize, + pub quote_sample_ms: u64, + pub bucket: String, + pub endpoint: String, + pub region: String, + pub profile: String, + pub zstd_timeout: Duration, + pub oss_timeout: Duration, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +pub struct UploadSummary { + pub uploaded_segments: usize, + pub canonical_uploaded_segments: usize, +} + +impl UploadConfig { + pub fn validate(&self) -> Result<()> { + if self.dataset.is_empty() + || !self.dataset.bytes().all(|byte| { + byte.is_ascii_lowercase() || byte.is_ascii_digit() || b"_-".contains(&byte) + }) + { + bail!("dataset must match [a-z0-9_-]+"); + } + if self.bucket.trim().is_empty() + || self.endpoint.trim().is_empty() + || self.region.trim().is_empty() + || self.profile.trim().is_empty() + || self.zstd_timeout.is_zero() + || self.oss_timeout.is_zero() + { + bail!("upload destination and timeouts must be non-empty"); + } + Ok(()) + } +} + +#[derive(Debug)] +struct Artifacts { + source: PathBuf, + data: PathBuf, + manifest: PathBuf, + success: PathBuf, + object_prefix: String, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct FileIdentity { + device: u64, + inode: u64, + bytes: u64, + modified_seconds: i64, + modified_nanoseconds: i64, +} + +impl FileIdentity { + fn from_metadata(metadata: &Metadata) -> Self { + Self { + device: metadata.dev(), + inode: metadata.ino(), + bytes: metadata.len(), + modified_seconds: metadata.mtime(), + modified_nanoseconds: metadata.mtime_nsec(), + } + } +} + +#[derive(Debug)] +struct ScanResult { + manifest: Value, + identity: FileIdentity, +} + +#[derive(Debug)] +struct UploadedSegment { + object: String, + canonical_complete: bool, +} + +struct ExclusiveTempDir(PathBuf); + +impl ExclusiveTempDir { + fn create(parent: &Path, prefix: &str) -> Result { + ensure_canonical_directory(parent)?; + for _ in 0..32 { + let path = parent.join(format!("{prefix}.{:016x}", random::())); + match DirBuilder::new().mode(0o700).create(&path) { + Ok(()) => return Ok(Self(path)), + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => continue, + Err(error) => return Err(error.into()), + } + } + bail!("could not allocate an exclusive temporary directory") + } + + fn path(&self) -> &Path { + &self.0 + } +} + +impl Drop for ExclusiveTempDir { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } +} + +fn utc_now() -> String { + Utc::now().format("%Y-%m-%dT%H:%M:%S%.6fZ").to_string() +} + +pub(crate) fn ensure_canonical_directory(path: &Path) -> Result<()> { + if !path.is_absolute() + || path + .components() + .any(|component| matches!(component, Component::CurDir | Component::ParentDir)) + { + bail!( + "directory path must be absolute and canonical: {}", + path.display() + ); + } + let mut existing = path; + while !existing.exists() { + existing = existing + .parent() + .ok_or_else(|| anyhow!("directory has no existing ancestor"))?; + } + let metadata = fs::symlink_metadata(existing)?; + if metadata.file_type().is_symlink() || !metadata.is_dir() { + bail!( + "directory ancestor is a symlink or non-directory: {}", + existing.display() + ); + } + if fs::canonicalize(existing)? != existing { + bail!("directory has an indirect ancestor: {}", path.display()); + } + let missing = path.strip_prefix(existing)?; + let mut current = existing.to_path_buf(); + for component in missing.components() { + current.push(component.as_os_str()); + match fs::create_dir(¤t) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {} + Err(error) => return Err(error.into()), + } + let metadata = fs::symlink_metadata(¤t)?; + if metadata.file_type().is_symlink() || !metadata.is_dir() { + bail!( + "directory path component is a symlink or non-directory: {}", + current.display() + ); + } + } + if fs::canonicalize(path)? != path { + bail!("directory has an indirect ancestor: {}", path.display()); + } + Ok(()) +} + +fn regular_identity(path: &Path) -> Result { + let metadata = fs::symlink_metadata(path) + .with_context(|| format!("inspect closed tape {}", path.display()))?; + if metadata.file_type().is_symlink() || !metadata.file_type().is_file() { + bail!( + "closed tape must be a regular non-symlink file: {}", + path.display() + ); + } + Ok(FileIdentity::from_metadata(&metadata)) +} + +fn ensure_identity(path: &Path, expected: FileIdentity) -> Result<()> { + let actual = regular_identity(path)?; + if actual != expected { + bail!("tape changed while being validated; refusing to archive an active file"); + } + Ok(()) +} + +fn parse_timestamp( + value: Option<&Value>, + field: &str, + line_number: usize, +) -> Result> { + let value = value + .and_then(Value::as_str) + .ok_or_else(|| anyhow!("line {line_number}: {field} must be a string"))?; + DateTime::parse_from_rfc3339(value) + .map(|parsed| parsed.with_timezone(&Utc)) + .map_err(|_| anyhow!("line {line_number}: invalid {field}: {value}")) +} + +fn decimal_or_none( + value: Option<&Value>, + field: &str, + line_number: usize, +) -> Result> { + let Some(value) = value else { + return Ok(None); + }; + if value.is_null() { + return Ok(None); + } + let text = value + .as_str() + .map(str::to_owned) + .unwrap_or_else(|| value.to_string()); + Decimal::from_str(&text) + .map(Some) + .map_err(|_| anyhow!("line {line_number}: {field} must be numeric")) +} + +fn value_text(value: Option<&Value>) -> String { + match value { + Some(Value::String(value)) => value.clone(), + Some(Value::Null) | None => "None".to_owned(), + Some(Value::Bool(true)) => "True".to_owned(), + Some(Value::Bool(false)) => "False".to_owned(), + Some(value) => value.to_string(), + } +} + +fn canonical_decimal(value: Option<&Value>) -> String { + let raw = value_text(value); + Decimal::from_str(&raw) + .map(|value| value.normalize().to_string()) + .unwrap_or(raw) +} + +fn required_object_text<'a>( + object: &'a Map, + field: &str, + object_label: &str, + line_number: usize, +) -> Result<&'a str> { + object + .get(field) + .and_then(Value::as_str) + .filter(|value| !value.is_empty()) + .ok_or_else(|| anyhow!("line {line_number}: {object_label} requires {field}")) +} + +fn raw_trade_timestamp(value: Option<&Value>, line_number: usize) -> Result { + let timestamp = match value { + Some(Value::Number(value)) => value.as_i64(), + Some(Value::String(value)) => value.parse().ok(), + _ => None, + } + .ok_or_else(|| anyhow!("line {line_number}: raw trade timestamp must be integer seconds"))?; + DateTime::from_timestamp(timestamp, 0) + .ok_or_else(|| anyhow!("line {line_number}: raw trade timestamp is out of range"))?; + Ok(timestamp) +} + +fn parse_market_datetime(value: Option<&Value>) -> Option> { + let value = value?.as_str()?; + DateTime::parse_from_rfc3339(value) + .map(|parsed| parsed.with_timezone(&Utc)) + .ok() + .or_else(|| { + NaiveDateTime::parse_from_str(value, "%Y-%m-%dT%H:%M:%S%.f") + .ok() + .map(|parsed| parsed.and_utc()) + }) +} + +fn contains_window_token(text: &str, token: &str) -> bool { + text.match_indices(token).any(|(start, _)| { + let before = text[..start].chars().next_back(); + let after = text[start + token.len()..].chars().next(); + before.is_none_or(|value| matches!(value, '-' | '_' | ' ')) + && after.is_none_or(|value| matches!(value, '-' | '_' | ' ')) + }) +} + +fn raw_market_window(market: &Map) -> Option { + let event = market + .get("events") + .and_then(Value::as_array) + .and_then(|events| events.first()); + let start = [ + market.get("eventStartTime"), + market.get("startDate"), + event.and_then(|event| event.get("startTime")), + event.and_then(|event| event.get("startDate")), + ] + .into_iter() + .flatten() + .find_map(|value| parse_market_datetime(Some(value))); + if let (Some(start), Some(end)) = (start, parse_market_datetime(market.get("endDate"))) { + let seconds = (end - start).num_seconds(); + if matches!(seconds, 300 | 900) { + return u64::try_from(seconds).ok(); + } + } + let text = format!( + "{} {}", + market + .get("slug") + .and_then(Value::as_str) + .unwrap_or_default(), + market + .get("question") + .and_then(Value::as_str) + .unwrap_or_default() + ) + .to_ascii_lowercase(); + if contains_window_token(&text, "15m") || text.contains("15 minute") { + Some(900) + } else if contains_window_token(&text, "5m") || text.contains("5 minute") { + Some(300) + } else { + None + } +} + +fn parse_json_array(value: Option<&Value>) -> Option> { + match value { + Some(Value::Array(values)) => Some(values.clone()), + Some(Value::String(value)) => serde_json::from_str::>(value).ok(), + _ => None, + } +} + +fn unique_string_pair( + market: &Map, + field: &str, + record_kind: &str, + line_number: usize, +) -> Result<[String; 2]> { + let values = parse_json_array(market.get(field)).unwrap_or_default(); + let strings = values + .iter() + .map(|value| value.as_str().filter(|value| !value.is_empty())) + .collect::>>(); + let Some(strings) = strings else { + bail!("line {line_number}: {record_kind} raw market requires two unique {field}"); + }; + if strings.len() != 2 || strings[0] == strings[1] { + bail!("line {line_number}: {record_kind} raw market requires two unique {field}"); + } + Ok([strings[0].to_owned(), strings[1].to_owned()]) +} + +fn derived_trade_record_id(trade: &Map) -> String { + let parts = [ + value_text(trade.get("transactionHash")), + value_text(trade.get("conditionId")), + value_text(trade.get("asset")), + value_text(trade.get("side")), + value_text(trade.get("timestamp")), + value_text(trade.get("proxyWallet")), + canonical_decimal(trade.get("size")), + canonical_decimal(trade.get("price")), + value_text(trade.get("outcomeIndex")), + ]; + hex::encode(Sha256::digest(parts.join("|").as_bytes())) +} + +fn validate_canonical_trade(update: &Map, line_number: usize) -> Result { + if update.get("record_id_version").and_then(Value::as_str) != Some("v2") { + bail!("line {line_number}: polymarket_trade record_id_version must be v2"); + } + for field in [ + "record_id", + "market_id", + "condition_id", + "token_id", + "symbol", + "side", + "trade_ts", + "transaction_hash", + "proxy_wallet", + "outcome", + ] { + required_text(update, field, line_number) + .map_err(|_| anyhow!("line {line_number}: polymarket_trade requires {field}"))?; + } + + let trade = update + .get("trade") + .and_then(Value::as_object) + .ok_or_else(|| anyhow!("line {line_number}: polymarket_trade.trade must be an object"))?; + let transaction_hash = + required_object_text(trade, "transactionHash", "raw trade", line_number)?; + let condition_id = required_object_text(trade, "conditionId", "raw trade", line_number)?; + let asset = required_object_text(trade, "asset", "raw trade", line_number)?; + let side = required_object_text(trade, "side", "raw trade", line_number)?; + if !matches!(side, "BUY" | "SELL") { + bail!("line {line_number}: raw trade side must be BUY or SELL"); + } + let proxy_wallet = required_object_text(trade, "proxyWallet", "raw trade", line_number)?; + let outcome = required_object_text(trade, "outcome", "raw trade", line_number)?; + let timestamp = raw_trade_timestamp(trade.get("timestamp"), line_number)?; + let size = decimal_or_none(trade.get("size"), "raw trade size", line_number)?; + if size.is_none_or(|value| value <= Decimal::ZERO) { + bail!("line {line_number}: raw trade size must be positive"); + } + let price = decimal_or_none(trade.get("price"), "raw trade price", line_number)?; + if !price.is_some_and(|value| (Decimal::ZERO..=Decimal::ONE).contains(&value)) { + bail!("line {line_number}: raw trade price must be within [0, 1]"); + } + let outcome_index = trade.get("outcomeIndex").and_then(Value::as_i64); + if !matches!(outcome_index, Some(0 | 1)) { + bail!("line {line_number}: raw trade outcomeIndex must be 0 or 1"); + } + + for (field, expected) in [ + ("transaction_hash", transaction_hash), + ("condition_id", condition_id), + ("token_id", asset), + ("side", side), + ("proxy_wallet", proxy_wallet), + ("outcome", outcome), + ] { + if update.get(field).and_then(Value::as_str) != Some(expected) { + bail!("line {line_number}: polymarket_trade {field} does not match raw trade"); + } + } + if decimal_or_none(update.get("size"), "size", line_number)? != size { + bail!("line {line_number}: polymarket_trade size does not match raw trade"); + } + if decimal_or_none(update.get("price"), "price", line_number)? != price { + bail!("line {line_number}: polymarket_trade price does not match raw trade"); + } + if update.get("outcome_index").and_then(Value::as_i64) != outcome_index { + bail!("line {line_number}: polymarket_trade outcome_index does not match raw trade"); + } + if update.get("trade_ts_unix").and_then(Value::as_i64) != Some(timestamp) { + bail!("line {line_number}: polymarket_trade trade_ts_unix does not match raw trade"); + } + let trade_ts = parse_timestamp(update.get("trade_ts"), "trade_ts", line_number)?; + if trade_ts != DateTime::from_timestamp(timestamp, 0).expect("timestamp was validated") { + bail!("line {line_number}: polymarket_trade trade_ts does not match raw trade"); + } + + let record_id = required_text(update, "record_id", line_number)?.to_owned(); + if record_id != derived_trade_record_id(trade) { + bail!("line {line_number}: record_id does not match raw trade"); + } + Ok(record_id) +} + +fn validate_market_context( + update: &Map, + record_kind: &str, + line_number: usize, +) -> Result<([String; 2], [String; 2])> { + let market_id = required_text(update, "market_id", line_number) + .map_err(|_| anyhow!("line {line_number}: {record_kind} requires market_id"))?; + let condition_id = required_text(update, "condition_id", line_number) + .map_err(|_| anyhow!("line {line_number}: {record_kind} requires condition_id"))?; + let market = update + .get("market") + .and_then(Value::as_object) + .ok_or_else(|| anyhow!("line {line_number}: {record_kind}.market must be an object"))?; + let raw_market_id = required_object_text( + market, + "id", + &format!("{record_kind} raw market"), + line_number, + )?; + let raw_condition_id = required_object_text( + market, + "conditionId", + &format!("{record_kind} raw market"), + line_number, + )?; + if market_id != raw_market_id { + bail!("line {line_number}: {record_kind} market_id does not match raw market"); + } + if condition_id != raw_condition_id { + bail!("line {line_number}: {record_kind} condition_id does not match raw market"); + } + let symbol = required_text(update, "symbol", line_number) + .map_err(|_| anyhow!("line {line_number}: {record_kind} requires symbol"))?; + let raw_question = market + .get("question") + .and_then(Value::as_str) + .unwrap_or_default() + .to_ascii_uppercase(); + let raw_symbol = SUPPORTED_SYMBOL_ALIASES + .iter() + .find_map(|(candidate, aliases)| { + aliases + .iter() + .any(|alias| raw_question.contains(alias)) + .then_some(*candidate) + }); + if raw_symbol != Some(symbol) { + bail!("line {line_number}: {record_kind} symbol is unsupported or contradicts raw market"); + } + let window = update.get("market_window_secs").and_then(Value::as_u64); + if !matches!(window, Some(300 | 900)) || raw_market_window(market) != window { + bail!("line {line_number}: {record_kind} window is unsupported or contradicts raw market"); + } + let tokens = unique_string_pair(market, "clobTokenIds", record_kind, line_number)?; + let outcomes = unique_string_pair(market, "outcomes", record_kind, line_number)?; + Ok((tokens, outcomes)) +} + +fn validate_market_settlement(update: &Map, line_number: usize) -> Result<()> { + let (tokens, outcomes) = validate_market_context(update, "market_settlement", line_number)?; + let market = update + .get("market") + .and_then(Value::as_object) + .expect("market context validation requires an object"); + if market.get("closed").and_then(Value::as_bool) != Some(true) { + bail!("line {line_number}: market_settlement raw market must be closed"); + } + let raw_prices = parse_json_array(market.get("outcomePrices")).unwrap_or_default(); + if raw_prices.len() != 2 { + bail!("line {line_number}: market_settlement requires exactly two outcomePrices"); + } + let prices = raw_prices + .iter() + .map(|value| Decimal::from_str(&value_text(Some(value))).ok()) + .collect::>>() + .ok_or_else(|| { + anyhow!("line {line_number}: market_settlement outcomePrices must be numeric") + })?; + if prices + .iter() + .any(|price| !(Decimal::ZERO..=Decimal::ONE).contains(price)) + { + bail!("line {line_number}: market_settlement outcomePrices must be within [0, 1]"); + } + let winners = prices + .iter() + .enumerate() + .filter_map(|(index, price)| (*price >= SETTLEMENT_PRICE).then_some(index)) + .collect::>(); + if winners.len() != 1 { + bail!("line {line_number}: market_settlement requires exactly one winning price"); + } + let winner = winners[0]; + let loser = 1 - winner; + if prices[loser] > SETTLEMENT_LOSER_PRICE { + bail!("line {line_number}: market_settlement losing price must be near zero"); + } + let price_sum = prices[0] + prices[1]; + if price_sum < Decimal::ONE - SETTLEMENT_SUM_TOLERANCE + || price_sum > Decimal::ONE + SETTLEMENT_SUM_TOLERANCE + { + bail!("line {line_number}: market_settlement prices must sum to one"); + } + if update.get("winning_token_id").and_then(Value::as_str) != Some(tokens[winner].as_str()) { + bail!("line {line_number}: market_settlement winning_token_id does not match raw market"); + } + if update.get("winning_outcome").and_then(Value::as_str) != Some(outcomes[winner].as_str()) { + bail!("line {line_number}: market_settlement winning_outcome does not match raw market"); + } + let outcome_set = outcomes + .iter() + .map(|outcome| outcome.to_ascii_lowercase()) + .collect::>(); + if outcome_set != BTreeSet::from(["down".to_owned(), "up".to_owned()]) + && outcome_set != BTreeSet::from(["no".to_owned(), "yes".to_owned()]) + { + bail!("line {line_number}: market_settlement outcomes must be Up/Down or Yes/No"); + } + let resolved_up_won = matches!(outcomes[winner].to_ascii_lowercase().as_str(), "up" | "yes"); + if update.get("resolved_up_won").and_then(Value::as_bool) != Some(resolved_up_won) { + bail!("line {line_number}: market_settlement resolved_up_won does not match raw market"); + } + Ok(()) +} + +fn required_text<'a>( + update: &'a Map, + field: &str, + line_number: usize, +) -> Result<&'a str> { + update + .get(field) + .and_then(Value::as_str) + .filter(|value| !value.is_empty()) + .ok_or_else(|| anyhow!("line {line_number}: update requires {field}")) +} + +fn reference_context(update: &Map, line_number: usize) -> Result<(String, String)> { + Ok(( + required_text(update, "market_id", line_number)?.to_owned(), + required_text(update, "condition_id", line_number)?.to_owned(), + )) +} + +fn increment_nested(counts: &mut BTreeMap>, kind: &str, field: &str) { + *counts + .entry(kind.to_owned()) + .or_default() + .entry(field.to_owned()) + .or_default() += 1; +} + +fn strict_rotation_name(name: &str) -> bool { + let Some(middle) = name + .strip_prefix("market-updates.") + .and_then(|name| name.strip_suffix(".ndjson")) + else { + return false; + }; + let mut parts = middle.split('.'); + let Some(stamp) = parts.next() else { + return false; + }; + if !strict_timestamp(stamp) { + return false; + } + match (parts.next(), parts.next()) { + (None, None) => true, + (Some(uuid), None) => strict_uuid(uuid), + _ => false, + } +} + +fn strict_timestamp(value: &str) -> bool { + let format = match value.len() { + 15 => "%Y%m%dT%H%M%S", + 21 => "%Y%m%dT%H%M%S%6f", + _ => return false, + }; + NaiveDateTime::parse_from_str(value, format).is_ok() +} + +fn strict_uuid(value: &str) -> bool { + value.len() == 36 + && value.bytes().enumerate().all(|(index, byte)| { + if matches!(index, 8 | 13 | 18 | 23) { + byte == b'-' + } else { + byte.is_ascii_hexdigit() + } + }) +} + +fn discover_rotated_tapes(spool_dir: &Path) -> Result> { + ensure_canonical_directory(spool_dir)?; + let mut paths = Vec::new(); + for entry in fs::read_dir(spool_dir)? { + let entry = entry?; + let name = entry.file_name(); + let name = name + .to_str() + .ok_or_else(|| anyhow!("spool entry name is not UTF-8"))?; + if strict_rotation_name(name) { + regular_identity(&entry.path())?; + paths.push(entry.path()); + } else if name != "market-updates.ndjson" && name.starts_with("market-updates.") { + bail!("invalid rotated tape name: {name}"); + } + } + paths.sort(); + Ok(paths) +} + +/// Validate a closed tape and return the manifest body used by the uploader. +pub fn scan_tape( + path: &Path, + dataset: &str, + quote_depth_levels: usize, + quote_sample_ms: u64, +) -> Result { + Ok(scan_tape_with_identity(path, dataset, quote_depth_levels, quote_sample_ms)?.manifest) +} + +fn scan_tape_with_identity( + path: &Path, + dataset: &str, + quote_depth_levels: usize, + quote_sample_ms: u64, +) -> Result { + let identity = regular_identity(path)?; + let file = File::open(path).with_context(|| format!("open tape {}", path.display()))?; + if FileIdentity::from_metadata(&file.metadata()?) != identity { + bail!("tape changed while being opened; refusing to archive an active file"); + } + let mut reader = BufReader::new(file); + let mut event_types = BTreeMap::::new(); + let mut present_fields = BTreeMap::>::new(); + let mut non_null_fields = BTreeMap::>::new(); + let mut symbols = BTreeSet::new(); + let mut token_ids = BTreeSet::new(); + let mut known_event_tokens = BTreeSet::new(); + let mut contextless_quote_tokens = BTreeSet::new(); + let mut first_recorded_at: Option = None; + let mut last_recorded_at: Option = None; + let mut previous_recorded_at: Option> = None; + let mut first_sequence: Option = None; + let mut last_sequence: Option = None; + let mut expected_sequence: Option = None; + let mut crossed_quotes = 0_u64; + let mut one_sided_quotes = 0_u64; + let mut empty_quotes = 0_u64; + let mut out_of_range_prices = 0_u64; + let mut negative_sizes = 0_u64; + let mut max_bid_levels = 0_usize; + let mut max_ask_levels = 0_usize; + let mut contextless_quotes = 0_u64; + let mut market_ids = BTreeSet::new(); + let mut condition_ids = BTreeSet::new(); + let mut record_ids = BTreeSet::new(); + let mut record_id_versions = BTreeSet::new(); + let mut metadata_contexts = BTreeSet::new(); + let mut dependent_reference_contexts = BTreeSet::new(); + let mut source_field_presence = BTreeMap::>::new(); + let mut source_field_non_null = BTreeMap::>::new(); + let mut raw_line = Vec::new(); + let mut line_number = 0_usize; + + loop { + raw_line.clear(); + let read = reader.read_until(b'\n', &mut raw_line)?; + if read == 0 { + break; + } + line_number += 1; + if !raw_line.ends_with(b"\n") { + bail!("line {line_number}: tape ends with an incomplete record"); + } + let record: Value = serde_json::from_slice(&raw_line) + .map_err(|_| anyhow!("line {line_number}: invalid JSON"))?; + let record = record + .as_object() + .ok_or_else(|| anyhow!("line {line_number}: record must be an object"))?; + let sequence = record + .get("sequence") + .and_then(Value::as_u64) + .ok_or_else(|| { + anyhow!("line {line_number}: sequence must be a non-negative integer") + })?; + let expected = expected_sequence.get_or_insert(sequence); + if sequence != *expected { + bail!( + "line {line_number}: sequence gap expected={} actual={sequence}", + *expected + ); + } + let recorded_at = parse_timestamp(record.get("recorded_at"), "recorded_at", line_number)?; + if previous_recorded_at.is_some_and(|previous| recorded_at < previous) { + bail!("line {line_number}: recorded_at moved backwards"); + } + let update = record + .get("update") + .and_then(Value::as_object) + .ok_or_else(|| anyhow!("line {line_number}: update must be an object"))?; + let kind = update + .get("kind") + .and_then(Value::as_str) + .ok_or_else(|| anyhow!("line {line_number}: unsupported update kind None"))?; + if !ALLOWED_KINDS.contains(&kind) { + bail!("line {line_number}: unsupported update kind {kind:?}"); + } + + *event_types.entry(kind.to_owned()).or_default() += 1; + for (field, value) in update { + increment_nested(&mut present_fields, kind, field); + if !value.is_null() { + increment_nested(&mut non_null_fields, kind, field); + } + } + if let Some(symbol) = update + .get("symbol") + .and_then(Value::as_str) + .filter(|v| !v.is_empty()) + { + symbols.insert(symbol.to_owned()); + } + let token_id = update.get("token_id").and_then(Value::as_str); + if let Some(token_id) = token_id.filter(|value| !value.is_empty()) { + token_ids.insert(token_id.to_owned()); + } + if kind == "event_discovered" { + for field in ["up_token", "down_token"] { + if let Some(token) = update + .get(field) + .and_then(Value::as_str) + .filter(|v| !v.is_empty()) + { + known_event_tokens.insert(token.to_owned()); + } + } + } + if let Some(market_id) = update + .get("market_id") + .and_then(Value::as_str) + .filter(|v| !v.is_empty()) + { + market_ids.insert(market_id.to_owned()); + } + if let Some(condition_id) = update + .get("condition_id") + .and_then(Value::as_str) + .filter(|v| !v.is_empty()) + { + condition_ids.insert(condition_id.to_owned()); + } + let raw_field = match kind { + "market_metadata" | "market_settlement" => Some("market"), + "polymarket_trade" => Some("trade"), + _ => None, + }; + if let Some(raw_field) = raw_field { + let raw_payload = update + .get(raw_field) + .and_then(Value::as_object) + .ok_or_else(|| { + anyhow!("line {line_number}: {kind}.{raw_field} must be an object") + })?; + for (field, value) in raw_payload { + increment_nested(&mut source_field_presence, kind, field); + if !value.is_null() { + increment_nested(&mut source_field_non_null, kind, field); + } + } + } + + match kind { + "market_metadata" => { + for field in ["market_id", "condition_id", "symbol", "retrieved_at"] { + required_text(update, field, line_number).map_err(|_| { + anyhow!("line {line_number}: market_metadata requires {field}") + })?; + } + validate_market_context(update, "market_metadata", line_number)?; + metadata_contexts.insert(reference_context(update, line_number)?); + } + "polymarket_trade" => { + let record_id = validate_canonical_trade(update, line_number)?; + if !record_ids.insert(record_id) { + bail!("line {line_number}: duplicate polymarket_trade record_id"); + } + record_id_versions.insert("v2".to_owned()); + dependent_reference_contexts.insert(reference_context(update, line_number)?); + } + "market_settlement" => { + for field in [ + "market_id", + "condition_id", + "symbol", + "winning_token_id", + "winning_outcome", + "resolution_source", + "retrieved_at", + ] { + required_text(update, field, line_number).map_err(|_| { + anyhow!("line {line_number}: market_settlement requires {field}") + })?; + } + validate_market_settlement(update, line_number)?; + dependent_reference_contexts.insert(reference_context(update, line_number)?); + } + _ => {} + } + + if kind == "quote" { + if !token_id.is_some_and(|token| known_event_tokens.contains(token)) { + contextless_quotes += 1; + if let Some(token) = token_id { + contextless_quote_tokens.insert(token.to_owned()); + } + } + let bid = decimal_or_none(update.get("bid"), "bid", line_number)?; + let ask = decimal_or_none(update.get("ask"), "ask", line_number)?; + let bid_size = decimal_or_none(update.get("bid_size"), "bid_size", line_number)?; + let ask_size = decimal_or_none(update.get("ask_size"), "ask_size", line_number)?; + let bid_levels = quote_levels(update.get("bid_levels"), line_number)?; + let ask_levels = quote_levels(update.get("ask_levels"), line_number)?; + if quote_depth_levels > 0 + && (bid_levels.len() > quote_depth_levels || ask_levels.len() > quote_depth_levels) + { + bail!("line {line_number}: quote exceeds configured depth"); + } + max_bid_levels = max_bid_levels.max(bid_levels.len()); + max_ask_levels = max_ask_levels.max(ask_levels.len()); + for (side, levels) in [("bid_levels", bid_levels), ("ask_levels", ask_levels)] { + for (level_index, level) in levels.iter().enumerate() { + let level = level.as_object().ok_or_else(|| { + anyhow!("line {line_number}: {side}[{level_index}] must be an object") + })?; + let level_price = decimal_or_none( + level.get("price"), + &format!("{side}[{level_index}].price"), + line_number, + )?; + let level_size = decimal_or_none( + level.get("size"), + &format!("{side}[{level_index}].size"), + line_number, + )?; + let (Some(level_price), Some(level_size)) = (level_price, level_size) else { + bail!("line {line_number}: {side}[{level_index}] requires price and size"); + }; + if !(Decimal::ZERO..=Decimal::ONE).contains(&level_price) { + out_of_range_prices += 1; + } + if level_size < Decimal::ZERO { + negative_sizes += 1; + } + } + } + match (bid, ask) { + (None, None) => empty_quotes += 1, + (None, Some(_)) | (Some(_), None) => one_sided_quotes += 1, + (Some(bid), Some(ask)) if bid > ask => crossed_quotes += 1, + _ => {} + } + for price in [bid, ask].into_iter().flatten() { + if !(Decimal::ZERO..=Decimal::ONE).contains(&price) { + out_of_range_prices += 1; + } + } + for size in [bid_size, ask_size].into_iter().flatten() { + if size < Decimal::ZERO { + negative_sizes += 1; + } + } + } + + let recorded_at_text = record + .get("recorded_at") + .and_then(Value::as_str) + .expect("recorded_at was validated") + .to_owned(); + first_recorded_at.get_or_insert_with(|| recorded_at_text.clone()); + last_recorded_at = Some(recorded_at_text); + first_sequence.get_or_insert(sequence); + last_sequence = Some(sequence); + expected_sequence = sequence.checked_add(1); + if expected_sequence.is_none() { + bail!("line {line_number}: sequence overflow"); + } + previous_recorded_at = Some(recorded_at); + } + + let after = FileIdentity::from_metadata(&reader.get_ref().metadata()?); + if after != identity { + bail!("tape changed while being validated; refusing to archive an active file"); + } + ensure_identity(path, identity)?; + let first_sequence = first_sequence.ok_or_else(|| anyhow!("tape is empty"))?; + let last_sequence = last_sequence.expect("non-empty tape has a final sequence"); + let first_recorded_at = first_recorded_at.expect("non-empty tape has a first timestamp"); + let last_recorded_at = last_recorded_at.expect("non-empty tape has a final timestamp"); + let partition = DateTime::parse_from_rfc3339(&first_recorded_at) + .expect("recorded_at was validated") + .with_timezone(&Utc); + let event_context_complete = contextless_quotes == 0; + let has_quotes = event_types.get("quote").copied().unwrap_or_default() > 0; + let has_reference_records = ["market_metadata", "polymarket_trade", "market_settlement"] + .iter() + .any(|kind| event_types.get(*kind).copied().unwrap_or_default() > 0); + let reference_context_complete = dependent_reference_contexts.is_subset(&metadata_contexts); + let depth_complete = has_quotes && quote_depth_levels == 0; + let temporal_updates_complete = has_quotes && quote_sample_ms == 0; + let replay_scope = if !reference_context_complete { + "reference_hour_segment_requires_market_metadata_context" + } else if has_reference_records && !has_quotes { + "complete_reference_hour_segment" + } else if event_context_complete { + if depth_complete { + if temporal_updates_complete { + "complete_full_depth_normalized_hour_segment" + } else { + "complete_full_depth_sampled_normalized_hour_segment" + } + } else { + "complete_sampled_normalized_hour_segment" + } + } else { + "sampled_normalized_hour_segment_requires_prior_event_context" + }; + let source_file = path + .file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| anyhow!("source file name is not UTF-8"))?; + let mut manifest = json!({ + "schema": "monday.polymarket.raw.v1", + "canonical": reference_context_complete, + "venue": "polymarket", + "dataset": dataset, + "format": "ndjson.zst", + "replay_scope": replay_scope, + "venue_depth_complete": depth_complete, + "temporal_updates_complete": temporal_updates_complete, + "segment_complete": reference_context_complete, + "source_session_closed": true, + "event_context_complete": event_context_complete, + "contextless_quote_tokens": contextless_quote_tokens, + "events": last_sequence - first_sequence + 1, + "event_types": event_types, + "start_sequence": first_sequence, + "end_sequence": last_sequence, + "sequence_gaps": 0, + "start_recorded_at": first_recorded_at, + "end_recorded_at": last_recorded_at, + "date": partition.format("%Y-%m-%d").to_string(), + "hour": partition.format("%H").to_string(), + "symbols": symbols, + "token_count": token_ids.len(), + "market_count": market_ids.len(), + "condition_count": condition_ids.len(), + "record_id_versions": record_id_versions, + "recording_policy": { + "quote_sample_ms": quote_sample_ms, + "quote_depth_levels": quote_depth_levels, + "event_scoped_quotes": true, + }, + "field_presence": present_fields, + "field_non_null": non_null_fields, + "source_field_presence": source_field_presence, + "source_field_non_null": source_field_non_null, + "quality": { + "crossed_quotes": crossed_quotes, + "one_sided_quotes": one_sided_quotes, + "empty_quotes": empty_quotes, + "out_of_range_prices": out_of_range_prices, + "negative_sizes": negative_sizes, + "max_bid_levels": max_bid_levels, + "max_ask_levels": max_ask_levels, + "contextless_quotes": contextless_quotes, + "duplicate_record_ids": 0, + }, + "source_file": source_file, + "source_bytes": identity.bytes, + }); + manifest + .as_object_mut() + .expect("manifest is an object") + .insert( + "reference_context_complete".to_owned(), + json!(reference_context_complete), + ); + Ok(ScanResult { manifest, identity }) +} + +fn quote_levels(value: Option<&Value>, line_number: usize) -> Result<&[Value]> { + match value { + None | Some(Value::Null) => Ok(&[]), + Some(Value::Array(levels)) => Ok(levels), + _ => bail!("line {line_number}: quote levels must be arrays"), + } +} + +fn split_tape_by_utc_hour(source: &Path, staging_dir: &Path) -> Result> { + let file = File::open(source)?; + let mut reader = BufReader::new(file); + let stem = source + .file_stem() + .and_then(|stem| stem.to_str()) + .ok_or_else(|| anyhow!("source file name is not UTF-8"))?; + let mut chunks = Vec::new(); + let mut current_hour: Option = None; + let mut output: Option<(PathBuf, BufWriter)> = None; + let mut raw_line = Vec::new(); + let mut line_number = 0_usize; + loop { + raw_line.clear(); + let read = reader.read_until(b'\n', &mut raw_line)?; + if read == 0 { + break; + } + line_number += 1; + let record: Value = serde_json::from_slice(&raw_line)?; + let hour = parse_timestamp(record.get("recorded_at"), "recorded_at", line_number)? + .format("%Y%m%dT%H") + .to_string(); + if current_hour.as_deref() != Some(&hour) { + if let Some((path, mut writer)) = output.take() { + writer.flush()?; + writer.get_ref().sync_all()?; + chunks.push(path); + } + let path = staging_dir.join(format!("{stem}.{hour}.ndjson")); + let file = OpenOptions::new() + .create_new(true) + .write(true) + .open(&path)?; + output = Some((path, BufWriter::new(file))); + current_hour = Some(hour); + } + output + .as_mut() + .expect("a chunk is opened before writing") + .1 + .write_all(&raw_line)?; + } + if let Some((path, mut writer)) = output { + writer.flush()?; + writer.get_ref().sync_all()?; + chunks.push(path); + } + File::open(staging_dir)?.sync_all()?; + Ok(chunks) +} + +fn append_name(path: &Path, suffix: &str) -> Result { + let name = path + .file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| anyhow!("file name is not UTF-8"))?; + Ok(path.with_file_name(format!("{name}{suffix}"))) +} + +fn exclusive_sibling(path: &Path, suffix: &str) -> Result<(PathBuf, File)> { + let parent = path.parent().ok_or_else(|| anyhow!("file has no parent"))?; + let name = path + .file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| anyhow!("file name is not UTF-8"))?; + for _ in 0..32 { + let temporary = parent.join(format!(".{name}.{:016x}{suffix}", random::())); + match OpenOptions::new() + .create_new(true) + .write(true) + .open(&temporary) + { + Ok(file) => return Ok((temporary, file)), + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => continue, + Err(error) => return Err(error.into()), + } + } + bail!("could not allocate an exclusive temporary file") +} + +fn atomic_json(path: &Path, payload: &Value) -> Result<()> { + let (temporary, mut file) = exclusive_sibling(path, ".tmp")?; + let result = (|| -> Result<()> { + serde_json::to_writer(&mut file, payload)?; + file.write_all(b"\n")?; + file.sync_all()?; + fs::rename(&temporary, path)?; + if let Some(parent) = path.parent() { + File::open(parent)?.sync_all()?; + } + Ok(()) + })(); + if result.is_err() { + let _ = fs::remove_file(temporary); + } + result +} + +fn prepare_artifacts(source: &Path, config: &UploadConfig) -> Result<(Artifacts, Value)> { + let scan = scan_tape_with_identity( + source, + &config.dataset, + config.quote_depth_levels, + config.quote_sample_ms, + )?; + let data = append_name(source, ".zst")?; + let (temporary_data, temporary_file) = exclusive_sibling(&data, ".tmp")?; + let output = temporary_file.try_clone()?; + let mut command = Command::new("zstd"); + command + .args(["-q", "-T1", "-3", "-c"]) + .arg(source) + .stdout(Stdio::from(output)); + match command_status_with_timeout(&mut command, config.zstd_timeout) { + Ok(status) if status.success() => {} + Ok(status) => { + let _ = fs::remove_file(&temporary_data); + bail!("zstd exited with {status}"); + } + Err(error) => { + let _ = fs::remove_file(&temporary_data); + return Err(error); + } + } + temporary_file.sync_all()?; + if let Err(error) = ensure_identity(source, scan.identity) { + let _ = fs::remove_file(&temporary_data); + return Err(error); + } + fs::rename(&temporary_data, &data)?; + let digest = sha256_file(&data)?; + let mut metadata = scan + .manifest + .as_object() + .cloned() + .expect("scan manifest is an object"); + metadata.insert( + "file".to_owned(), + json!(data.file_name().and_then(|name| name.to_str())), + ); + metadata.insert("bytes".to_owned(), json!(fs::metadata(&data)?.len())); + metadata.insert("sha256".to_owned(), json!(digest.clone())); + let manifest_value = Value::Object(metadata); + let manifest = append_name(&data, ".manifest.json")?; + atomic_json(&manifest, &manifest_value)?; + let success = write_success_marker(&data, &digest)?; + let date = manifest_value["date"] + .as_str() + .expect("scan manifest has a date"); + let hour = manifest_value["hour"] + .as_str() + .expect("scan manifest has an hour"); + let object_prefix = format!( + "lake/raw/venue=polymarket/dataset={}/date={date}/hour={hour}/sha256={digest}", + config.dataset, + ); + Ok(( + Artifacts { + source: source.to_path_buf(), + data, + manifest, + success, + object_prefix, + }, + manifest_value, + )) +} + +fn oss_copy_command(source: &str, destination: &str, config: &UploadConfig) -> Command { + let mut command = Command::new("aliyun"); + command.args([ + "ossutil", + "cp", + source, + destination, + "--profile", + &config.profile, + "--endpoint", + &config.endpoint, + "--region", + &config.region, + ]); + command +} + +fn oss_upload_command(source: &str, destination: &str, config: &UploadConfig) -> Command { + let mut command = oss_copy_command(source, destination, config); + command.arg("--ignore-existing"); + command +} + +fn run_checked(command: &mut Command, timeout: Duration) -> Result { + let status = command_status_with_timeout(command, timeout)?; + if !status.success() { + bail!("child process exited with {status}"); + } + Ok(status) +} + +fn download_remote_artifacts_with( + artifacts: &Artifacts, + config: &UploadConfig, + runner: &mut F, +) -> Result<(ExclusiveTempDir, BTreeMap)> +where + F: FnMut(&mut Command, Duration) -> Result, +{ + let parent = artifacts + .source + .parent() + .ok_or_else(|| anyhow!("artifact has no parent"))?; + let verify_dir = ExclusiveTempDir::create(parent, ".oss-verify")?; + let mut downloaded = BTreeMap::new(); + for source in [&artifacts.data, &artifacts.manifest, &artifacts.success] { + let name = source + .file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| anyhow!("artifact file name is not UTF-8"))?; + let destination = verify_dir.path().join(name); + let remote = format!("oss://{}/{}/{name}", config.bucket, artifacts.object_prefix); + let mut command = oss_copy_command( + &remote, + destination + .to_str() + .ok_or_else(|| anyhow!("verification path is not UTF-8"))?, + config, + ); + runner(&mut command, config.oss_timeout)?; + regular_identity(&destination)?; + downloaded.insert(name.to_owned(), destination); + } + Ok((verify_dir, downloaded)) +} + +fn verify_downloaded_artifacts( + artifacts: &Artifacts, + downloaded: &BTreeMap, +) -> Result<()> { + let expected_manifest: Value = serde_json::from_slice(&fs::read(&artifacts.manifest)?)?; + let data_name = artifacts + .data + .file_name() + .and_then(|name| name.to_str()) + .unwrap(); + let manifest_name = artifacts + .manifest + .file_name() + .and_then(|name| name.to_str()) + .unwrap(); + let success_name = artifacts + .success + .file_name() + .and_then(|name| name.to_str()) + .unwrap(); + let remote_data = &downloaded[data_name]; + let expected_bytes = expected_manifest["bytes"] + .as_u64() + .ok_or_else(|| anyhow!("manifest bytes must be an integer"))?; + let expected_sha = expected_manifest["sha256"] + .as_str() + .ok_or_else(|| anyhow!("manifest sha256 must be a string"))?; + if fs::metadata(remote_data)?.len() != expected_bytes { + bail!("remote data size does not match manifest"); + } + if sha256_file(remote_data)? != expected_sha { + bail!("remote data sha256 does not match manifest"); + } + if fs::read(&downloaded[manifest_name])? != fs::read(&artifacts.manifest)? { + bail!("remote manifest does not match local manifest"); + } + if fs::read_to_string(&downloaded[success_name])?.trim() != expected_sha { + bail!("remote _SUCCESS does not match manifest"); + } + Ok(()) +} + +fn verify_remote_artifacts_with( + artifacts: &Artifacts, + config: &UploadConfig, + runner: &mut F, +) -> Result<()> +where + F: FnMut(&mut Command, Duration) -> Result, +{ + let (_verify_dir, downloaded) = download_remote_artifacts_with(artifacts, config, runner)?; + verify_downloaded_artifacts(artifacts, &downloaded) +} + +fn remote_artifacts_exist_and_match_with( + artifacts: &Artifacts, + config: &UploadConfig, + runner: &mut F, +) -> Result +where + F: FnMut(&mut Command, Duration) -> Result, +{ + let Ok((_verify_dir, downloaded)) = download_remote_artifacts_with(artifacts, config, runner) + else { + return Ok(false); + }; + verify_downloaded_artifacts(artifacts, &downloaded)?; + Ok(true) +} + +fn remove_regular(path: &Path) -> Result<()> { + regular_identity(path)?; + fs::remove_file(path)?; + Ok(()) +} + +fn remove_artifacts(artifacts: &Artifacts) -> Result<()> { + let paths = [ + artifacts.source.as_path(), + artifacts.data.as_path(), + artifacts.manifest.as_path(), + artifacts.success.as_path(), + ]; + let identities = paths + .iter() + .map(|path| regular_identity(path)) + .collect::>>()?; + for (path, identity) in paths.iter().zip(identities) { + ensure_identity(path, identity)?; + } + for path in paths { + remove_regular(path)?; + } + Ok(()) +} + +fn upload_artifacts(artifacts: &Artifacts, config: &UploadConfig) -> Result { + upload_artifacts_with(artifacts, config, &mut run_checked) +} + +fn upload_artifacts_with( + artifacts: &Artifacts, + config: &UploadConfig, + runner: &mut F, +) -> Result +where + F: FnMut(&mut Command, Duration) -> Result, +{ + if !remote_artifacts_exist_and_match_with(artifacts, config, runner)? { + for source in [&artifacts.data, &artifacts.manifest, &artifacts.success] { + let name = source + .file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| anyhow!("artifact file name is not UTF-8"))?; + let destination = format!("oss://{}/{}/{name}", config.bucket, artifacts.object_prefix); + let mut command = oss_upload_command( + source + .to_str() + .ok_or_else(|| anyhow!("artifact path is not UTF-8"))?, + &destination, + config, + ); + runner(&mut command, config.oss_timeout)?; + } + verify_remote_artifacts_with(artifacts, config, runner)?; + } + remove_artifacts(artifacts)?; + let data_name = artifacts + .data + .file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| anyhow!("artifact file name is not UTF-8"))?; + Ok(format!( + "oss://{}/{}/{data_name}", + config.bucket, artifacts.object_prefix + )) +} + +fn canonical_complete_manifest(manifest: &Value) -> bool { + manifest.get("canonical").and_then(Value::as_bool) == Some(true) + && manifest.get("segment_complete").and_then(Value::as_bool) == Some(true) +} + +fn archive_source(source: &Path, config: &UploadConfig) -> Result> { + let source_scan = scan_tape_with_identity( + source, + &config.dataset, + config.quote_depth_levels, + config.quote_sample_ms, + )?; + if source_scan.manifest["start_sequence"].as_u64() != Some(0) { + bail!( + "closed source tape must start at sequence 0; actual={}", + source_scan.manifest["start_sequence"] + ); + } + let spool_dir = source + .parent() + .ok_or_else(|| anyhow!("source has no parent"))?; + let staging_root = spool_dir.join(".upload-staging"); + match fs::symlink_metadata(&staging_root) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + fs::create_dir(&staging_root)?; + } + Ok(metadata) if metadata.file_type().is_dir() && !metadata.file_type().is_symlink() => {} + Ok(_) => bail!("upload staging root must be a plain directory"), + Err(error) => return Err(error.into()), + } + let staging = ExclusiveTempDir::create(&staging_root, "session")?; + let chunks = split_tape_by_utc_hour(source, staging.path())?; + ensure_identity(source, source_scan.identity)?; + let mut uploaded = Vec::new(); + for chunk in chunks { + let (artifacts, manifest) = prepare_artifacts(&chunk, config)?; + uploaded.push(UploadedSegment { + object: upload_artifacts(&artifacts, config)?, + canonical_complete: canonical_complete_manifest(&manifest), + }); + } + ensure_identity(source, source_scan.identity)?; + fs::remove_file(source)?; + File::open(spool_dir)?.sync_all()?; + Ok(uploaded) +} + +fn read_status(path: &Path) -> Result> { + match fs::symlink_metadata(path) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(Map::new()), + Ok(metadata) if metadata.file_type().is_file() && !metadata.file_type().is_symlink() => { + Ok(serde_json::from_slice::(&fs::read(path)?) + .ok() + .and_then(|value| value.as_object().cloned()) + .unwrap_or_default()) + } + Ok(_) => bail!("upload status must be a regular non-symlink file"), + Err(error) => Err(error.into()), + } +} + +/// Upload all closed tapes, continuing past bad segments while returning failure. +pub fn upload_pending(config: &UploadConfig) -> Result { + upload_pending_with(config, archive_source) +} + +/// Binary-friendly alias for [`upload_pending`]. +pub fn run_upload(config: &UploadConfig) -> Result { + upload_pending(config) +} + +fn upload_pending_with(config: &UploadConfig, mut archive: F) -> Result +where + F: FnMut(&Path, &UploadConfig) -> Result>, +{ + config.validate()?; + ensure_canonical_directory(&config.spool_dir)?; + let status_path = config.spool_dir.join("upload-status.json"); + let mut status = read_status(&status_path)?; + let mut failures = Vec::new(); + let mut uploaded_segments = 0_usize; + let mut canonical_uploaded_segments = 0_usize; + for source in discover_rotated_tapes(&config.spool_dir)? { + match archive(&source, config) { + Ok(uploaded) if !uploaded.is_empty() => { + uploaded_segments += uploaded.len(); + canonical_uploaded_segments += uploaded + .iter() + .filter(|segment| segment.canonical_complete) + .count(); + status.insert("last_success_at".to_owned(), json!(utc_now())); + status.insert( + "last_uploaded_object".to_owned(), + json!(&uploaded.last().expect("non-empty upload result").object), + ); + } + Ok(_) => failures.push(json!({ + "source": source.file_name().and_then(|name| name.to_str()), + "error": "closed tape produced no upload artifacts", + })), + Err(error) => { + eprintln!( + "Polymarket tape upload failed for {}: {error}", + source.display() + ); + failures.push(json!({ + "source": source.file_name().and_then(|name| name.to_str()), + "error": error.to_string(), + })); + } + } + } + let now = utc_now(); + let pending = discover_rotated_tapes(&config.spool_dir)?.len(); + status.insert("updated_at".to_owned(), json!(now)); + status.insert("uploaded_segments".to_owned(), json!(uploaded_segments)); + status.insert( + "canonical_uploaded_segments".to_owned(), + json!(canonical_uploaded_segments), + ); + status.insert("pending_segments".to_owned(), json!(pending)); + status.insert("failed_segments".to_owned(), Value::Array(failures.clone())); + status.insert( + "last_error_at".to_owned(), + if failures.is_empty() { + Value::Null + } else { + json!(utc_now()) + }, + ); + status.insert( + "last_error".to_owned(), + failures + .last() + .and_then(|failure| failure.get("error")) + .cloned() + .unwrap_or(Value::Null), + ); + atomic_json(&status_path, &Value::Object(status))?; + if failures.is_empty() { + Ok(UploadSummary { + uploaded_segments, + canonical_uploaded_segments, + }) + } else { + bail!( + "{} closed Polymarket tape segment(s) failed", + failures.len() + ) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + use std::os::unix::fs::symlink; + use std::process::ExitStatus; + + struct TestDir(PathBuf); + + impl TestDir { + fn new() -> Self { + for _ in 0..32 { + let path = std::env::temp_dir().join(format!( + "monday-polymarket-upload-test-{:016x}", + random::() + )); + if fs::create_dir(&path).is_ok() { + return Self(fs::canonicalize(path).unwrap()); + } + } + panic!("could not create test directory") + } + + fn path(&self) -> &Path { + &self.0 + } + } + + impl Drop for TestDir { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } + } + + fn record(sequence: u64, recorded_at: &str, update: Value) -> Value { + json!({"sequence": sequence, "recorded_at": recorded_at, "update": update}) + } + + fn sample_rows() -> Vec { + vec![ + record( + 0, + "2026-07-15T01:00:00.000000000Z", + json!({ + "kind": "event_discovered", "event_id": "event-1", "symbol": "BTCUSDT", + "up_token": "up-1", "down_token": "down-1", + "end_time": "2026-07-15T01:05:00Z", "window_secs": 300, + "price_to_beat": "100", "resolved_up_won": null, + }), + ), + record( + 1, + "2026-07-15T01:00:01.000000000Z", + json!({ + "kind": "quote", "token_id": "up-1", "bid": "0.49", "ask": "0.51", + "bid_size": "10", "ask_size": "11", + "bid_levels": [{"price": "0.49", "size": "10"}], + "ask_levels": [{"price": "0.51", "size": "11"}], + "ts": "2026-07-15T01:00:01Z", + }), + ), + record( + 2, + "2026-07-15T01:00:02.000000000Z", + json!({ + "kind": "reference_price", "symbol": "BTCUSDT", "source": "binance", + "asset_class": "crypto", "price": "100", "full_accuracy_value": null, + "is_carried_forward": false, "ts": "2026-07-15T01:00:02Z", + }), + ), + ] + } + + fn valid_v2_trade_update() -> Value { + json!({ + "kind": "polymarket_trade", + "record_id": "6a476c3be58fcb8d789224feba3e079f372bc933e838cd0dbb76435d4fad9cbe", + "record_id_version": "v2", + "market_id": "market-1", + "condition_id": "0xcondition", + "token_id": "up-token", + "symbol": "BTCUSDT", + "market_window_secs": 300, + "side": "BUY", + "size": "10.0", + "price": "0.780", + "trade_ts": "2026-07-15T03:09:55Z", + "trade_ts_unix": 1_784_084_995, + "transaction_hash": "0xtx", + "proxy_wallet": "0xwallet", + "outcome": "Up", + "outcome_index": 0, + "source": "polymarket_data_api", + "received_at": "2026-07-15T03:10:00Z", + "trade": { + "transactionHash": "0xtx", + "conditionId": "0xcondition", + "asset": "up-token", + "side": "BUY", + "timestamp": 1_784_084_995, + "proxyWallet": "0xwallet", + "size": "10.0", + "price": "0.780", + "outcome": "Up", + "outcomeIndex": 0, + "sourceOnlyField": {"preserved": true}, + }, + }) + } + + fn valid_market_metadata_update() -> Value { + json!({ + "kind": "market_metadata", + "market_id": "market-1", + "condition_id": "0xcondition", + "symbol": "BTCUSDT", + "market_window_secs": 300, + "source": "gamma_api", + "retrieved_at": "2026-07-15T03:00:00Z", + "market": { + "id": "market-1", + "conditionId": "0xcondition", + "question": "Bitcoin Up or Down - 5 minutes", + "slug": "btc-updown-5m-1784084400", + "startDate": "2026-07-15T03:00:00Z", + "endDate": "2026-07-15T03:05:00Z", + "clobTokenIds": "[\"up-token\",\"down-token\"]", + "outcomes": "[\"Up\",\"Down\"]", + "makerBaseFee": 1000, + "takerBaseFee": 1000, + }, + }) + } + + fn valid_market_settlement_update() -> Value { + let mut update = valid_market_metadata_update(); + update["kind"] = json!("market_settlement"); + update["winning_token_id"] = json!("up-token"); + update["winning_outcome"] = json!("Up"); + update["resolved_up_won"] = json!(true); + update["resolution_source"] = json!("gamma_api_closed_market"); + update["market"]["closed"] = json!(true); + update["market"]["outcomePrices"] = json!("[\"0.999\",\"0.001\"]"); + update + } + + fn write_tape(root: &Path, name: &str, rows: &[Value]) -> PathBuf { + let path = root.join(name); + let mut file = File::create(&path).unwrap(); + for row in rows { + serde_json::to_writer(&mut file, row).unwrap(); + file.write_all(b"\n").unwrap(); + } + file.sync_all().unwrap(); + path + } + + fn config(root: &Path) -> UploadConfig { + UploadConfig { + spool_dir: root.to_path_buf(), + dataset: "crypto_expiry".to_owned(), + quote_depth_levels: 1, + quote_sample_ms: 1_000, + bucket: "bucket".to_owned(), + endpoint: "endpoint".to_owned(), + region: "region".to_owned(), + profile: "profile".to_owned(), + zstd_timeout: Duration::from_secs(30), + oss_timeout: Duration::from_secs(30), + } + } + + #[test] + fn validates_manifest_quality_and_reference_fields() { + let root = TestDir::new(); + let tape = write_tape( + root.path(), + "market-updates.20260715T010000.ndjson", + &sample_rows(), + ); + let manifest = scan_tape(&tape, "crypto_expiry", 1, 1_000).unwrap(); + assert_eq!(manifest["events"], 3); + assert_eq!(manifest["event_types"]["quote"], 1); + assert_eq!(manifest["quality"]["max_bid_levels"], 1); + assert_eq!(manifest["field_non_null"]["quote"]["bid"], 1); + assert_eq!(manifest["field_non_null"]["reference_price"]["price"], 1); + } + + #[test] + fn rejects_sequence_gap_incomplete_numeric_and_depth_errors() { + let root = TestDir::new(); + let mut rows = sample_rows(); + rows[1]["sequence"] = json!(2); + let gap = write_tape(root.path(), "market-updates.20260715T010000.ndjson", &rows); + assert!(scan_tape(&gap, "crypto_expiry", 1, 1_000) + .unwrap_err() + .to_string() + .contains("sequence gap")); + + let incomplete = root.path().join("market-updates.20260715T020000.ndjson"); + fs::write(&incomplete, b"{\"sequence\":0").unwrap(); + assert!(scan_tape(&incomplete, "crypto_expiry", 1, 1_000) + .unwrap_err() + .to_string() + .contains("incomplete record")); + + let mut rows = sample_rows(); + rows[1]["update"]["bid"] = json!("not-a-number"); + let numeric = write_tape(root.path(), "market-updates.20260715T030000.ndjson", &rows); + assert!(scan_tape(&numeric, "crypto_expiry", 1, 1_000) + .unwrap_err() + .to_string() + .contains("bid must be numeric")); + + let mut rows = sample_rows(); + rows[1]["update"]["bid_levels"][0]["size"] = Value::Null; + let depth = write_tape(root.path(), "market-updates.20260715T040000.ndjson", &rows); + assert!(scan_tape(&depth, "crypto_expiry", 1, 1_000) + .unwrap_err() + .to_string() + .contains("requires price and size")); + } + + #[test] + fn full_book_context_and_reference_records_match_python_contract() { + let root = TestDir::new(); + let mut rows = sample_rows(); + rows[1]["update"]["bid_levels"] + .as_array_mut() + .unwrap() + .push(json!({"price": "0.48", "size": "12"})); + let full = write_tape(root.path(), "market-updates.20260715T010000.ndjson", &rows); + let manifest = scan_tape(&full, "crypto_expiry", 0, 1_000).unwrap(); + assert_eq!(manifest["venue_depth_complete"], true); + assert_eq!(manifest["quality"]["max_bid_levels"], 2); + + let mut quote = sample_rows().remove(1); + quote["sequence"] = json!(0); + let contextless = write_tape( + root.path(), + "market-updates.20260715T020000.ndjson", + &[quote], + ); + let manifest = scan_tape(&contextless, "crypto_expiry", 1, 1_000).unwrap(); + assert_eq!(manifest["event_context_complete"], false); + assert!(manifest["replay_scope"] + .as_str() + .unwrap() + .contains("requires_prior_event_context")); + + let reference = vec![ + record(0, "2026-07-15T03:00:00Z", valid_market_metadata_update()), + record(1, "2026-07-15T03:10:00Z", valid_v2_trade_update()), + ]; + let tape = write_tape( + root.path(), + "market-updates.20260715T030000.ndjson", + &reference, + ); + let manifest = scan_tape(&tape, "crypto_expiry_reference", 0, 0).unwrap(); + assert_eq!(manifest["replay_scope"], "complete_reference_hour_segment"); + assert_eq!(manifest["record_id_versions"], json!(["v2"])); + assert_eq!( + manifest["source_field_non_null"]["market_metadata"]["makerBaseFee"], + 1 + ); + + for (stamp, invalid_size) in [("040000", Value::Null), ("050000", json!(-1))] { + let mut invalid = reference.clone(); + invalid[1]["update"]["size"] = invalid_size.clone(); + invalid[1]["update"]["trade"]["size"] = invalid_size; + let tape = write_tape( + root.path(), + &format!("market-updates.20260715T{stamp}.ndjson"), + &invalid, + ); + assert!(scan_tape(&tape, "crypto_expiry_reference", 0, 0) + .unwrap_err() + .to_string() + .contains("size must be positive")); + } + } + + #[test] + fn rejects_market_metadata_wrapper_id_contradiction() { + let root = TestDir::new(); + let mut update = valid_market_metadata_update(); + update["market_id"] = json!("different-market"); + let tape = write_tape( + root.path(), + "market-updates.20260715T030000.ndjson", + &[record(0, "2026-07-15T03:00:00Z", update)], + ); + + assert!(scan_tape(&tape, "crypto_expiry_reference", 0, 0) + .unwrap_err() + .to_string() + .contains("market_metadata market_id does not match raw market")); + } + + #[test] + fn rejects_unsupported_market_metadata_symbol() { + let root = TestDir::new(); + let mut update = valid_market_metadata_update(); + update["symbol"] = json!("ADAUSDT"); + let tape = write_tape( + root.path(), + "market-updates.20260715T030000.ndjson", + &[record(0, "2026-07-15T03:00:00Z", update)], + ); + + assert!(scan_tape(&tape, "crypto_expiry_reference", 0, 0) + .unwrap_err() + .to_string() + .contains("market_metadata symbol is unsupported or contradicts raw market")); + } + + #[test] + fn rejects_market_metadata_window_contradiction() { + let root = TestDir::new(); + let mut update = valid_market_metadata_update(); + update["market_window_secs"] = json!(900); + let tape = write_tape( + root.path(), + "market-updates.20260715T030000.ndjson", + &[record(0, "2026-07-15T03:00:00Z", update)], + ); + + assert!(scan_tape(&tape, "crypto_expiry_reference", 0, 0) + .unwrap_err() + .to_string() + .contains("market_metadata window is unsupported or contradicts raw market")); + } + + #[test] + fn rejects_market_metadata_without_two_unique_tokens() { + let root = TestDir::new(); + let mut update = valid_market_metadata_update(); + update["market"]["clobTokenIds"] = json!(["same-token", "same-token"]); + let tape = write_tape( + root.path(), + "market-updates.20260715T030000.ndjson", + &[record(0, "2026-07-15T03:00:00Z", update)], + ); + + assert!(scan_tape(&tape, "crypto_expiry_reference", 0, 0) + .unwrap_err() + .to_string() + .contains("market_metadata raw market requires two unique clobTokenIds")); + } + + #[test] + fn rejects_market_metadata_without_two_unique_outcomes() { + let root = TestDir::new(); + let mut update = valid_market_metadata_update(); + update["market"]["outcomes"] = json!(["Up", "Up"]); + let tape = write_tape( + root.path(), + "market-updates.20260715T030000.ndjson", + &[record(0, "2026-07-15T03:00:00Z", update)], + ); + + assert!(scan_tape(&tape, "crypto_expiry_reference", 0, 0) + .unwrap_err() + .to_string() + .contains("market_metadata raw market requires two unique outcomes")); + } + + #[test] + fn rejects_non_object_market_metadata_payload() { + let root = TestDir::new(); + let mut update = valid_market_metadata_update(); + update["market"] = json!("not-an-object"); + let tape = write_tape( + root.path(), + "market-updates.20260715T030000.ndjson", + &[record(0, "2026-07-15T03:00:00Z", update)], + ); + + assert!(scan_tape(&tape, "crypto_expiry_reference", 0, 0) + .unwrap_err() + .to_string() + .contains("market_metadata.market must be an object")); + } + + #[test] + fn rejects_settlement_when_raw_market_is_not_closed() { + let root = TestDir::new(); + let mut update = valid_market_settlement_update(); + update["market"]["closed"] = json!(false); + let tape = write_tape( + root.path(), + "market-updates.20260715T030000.ndjson", + &[record(0, "2026-07-15T03:06:00Z", update)], + ); + + assert!(scan_tape(&tape, "crypto_expiry_reference", 0, 0) + .unwrap_err() + .to_string() + .contains("market_settlement raw market must be closed")); + } + + #[test] + fn rejects_settlement_wrapper_winner_contradiction() { + let root = TestDir::new(); + let mut update = valid_market_settlement_update(); + update["winning_token_id"] = json!("down-token"); + let tape = write_tape( + root.path(), + "market-updates.20260715T030000.ndjson", + &[record(0, "2026-07-15T03:06:00Z", update)], + ); + + assert!(scan_tape(&tape, "crypto_expiry_reference", 0, 0) + .unwrap_err() + .to_string() + .contains("market_settlement winning_token_id does not match raw market")); + } + + #[test] + fn rejects_invalid_or_ambiguous_settlement_prices() { + for (index, prices, expected) in [ + (0, json!(["1", "1"]), "exactly one winning price"), + (1, json!(["1.1", "0"]), "within [0, 1]"), + (2, json!(["invalid", "0"]), "must be numeric"), + (3, json!(["0.999", "0.5"]), "losing price must be near zero"), + (4, json!(["1", "0.001"]), "prices must sum to one"), + ] { + let root = TestDir::new(); + let mut update = valid_market_settlement_update(); + update["market"]["outcomePrices"] = prices; + let tape = write_tape( + root.path(), + &format!("market-updates.20260715T03000{index}.ndjson"), + &[record(0, "2026-07-15T03:06:00Z", update)], + ); + assert!(scan_tape(&tape, "crypto_expiry_reference", 0, 0) + .unwrap_err() + .to_string() + .contains(expected)); + } + } + + #[test] + fn rejects_settlement_outcome_and_resolution_contradictions() { + for (index, field, value, expected) in [ + (0, "winning_outcome", json!("Down"), "winning_outcome"), + (1, "resolved_up_won", json!(false), "resolved_up_won"), + ] { + let root = TestDir::new(); + let mut update = valid_market_settlement_update(); + update[field] = value; + let tape = write_tape( + root.path(), + &format!("market-updates.20260715T03001{index}.ndjson"), + &[record(0, "2026-07-15T03:06:00Z", update)], + ); + assert!(scan_tape(&tape, "crypto_expiry_reference", 0, 0) + .unwrap_err() + .to_string() + .contains(expected)); + } + } + + #[test] + fn trade_only_reference_segment_is_not_complete_or_canonical() { + let root = TestDir::new(); + let tape = write_tape( + root.path(), + "market-updates.20260715T030000.ndjson", + &[record(0, "2026-07-15T03:10:00Z", valid_v2_trade_update())], + ); + + let manifest = scan_tape(&tape, "crypto_expiry_reference", 0, 0).unwrap(); + assert_eq!(manifest["canonical"], false); + assert_eq!(manifest["segment_complete"], false); + assert_eq!(manifest["reference_context_complete"], false); + assert_eq!( + manifest["replay_scope"], + "reference_hour_segment_requires_market_metadata_context" + ); + } + + #[test] + fn settlement_only_reference_segment_is_not_complete_or_canonical() { + let root = TestDir::new(); + let tape = write_tape( + root.path(), + "market-updates.20260715T030000.ndjson", + &[record( + 0, + "2026-07-15T03:06:00Z", + valid_market_settlement_update(), + )], + ); + + let manifest = scan_tape(&tape, "crypto_expiry_reference", 0, 0).unwrap(); + assert_eq!(manifest["canonical"], false); + assert_eq!(manifest["segment_complete"], false); + assert_eq!(manifest["reference_context_complete"], false); + } + + #[test] + fn metadata_context_makes_settlement_segment_complete() { + let root = TestDir::new(); + let tape = write_tape( + root.path(), + "market-updates.20260715T030000.ndjson", + &[ + record(0, "2026-07-15T03:00:00Z", valid_market_metadata_update()), + record(1, "2026-07-15T03:06:00Z", valid_market_settlement_update()), + ], + ); + + let manifest = scan_tape(&tape, "crypto_expiry_reference", 0, 0).unwrap(); + assert_eq!(manifest["canonical"], true); + assert_eq!(manifest["segment_complete"], true); + assert_eq!(manifest["reference_context_complete"], true); + assert_eq!(manifest["replay_scope"], "complete_reference_hour_segment"); + } + + #[test] + fn canonical_upload_count_requires_both_manifest_flags() { + assert!(canonical_complete_manifest( + &json!({"canonical": true, "segment_complete": true}) + )); + for manifest in [ + json!({"canonical": false, "segment_complete": true}), + json!({"canonical": true, "segment_complete": false}), + json!({"canonical": true}), + ] { + assert!(!canonical_complete_manifest(&manifest)); + } + } + + #[test] + fn rejects_trade_record_id_not_derived_from_raw_payload() { + let root = TestDir::new(); + let mut update = valid_v2_trade_update(); + update["record_id"] = json!("0".repeat(64)); + let tape = write_tape( + root.path(), + "market-updates.20260715T030000.ndjson", + &[record(0, "2026-07-15T03:10:00Z", update)], + ); + + assert!(scan_tape(&tape, "crypto_expiry_reference", 0, 0) + .unwrap_err() + .to_string() + .contains("record_id does not match raw trade")); + } + + #[test] + fn rejects_legacy_trade_id_version() { + let root = TestDir::new(); + let mut update = valid_v2_trade_update(); + update["record_id_version"] = json!("v1_legacy"); + let tape = write_tape( + root.path(), + "market-updates.20260715T030000.ndjson", + &[record(0, "2026-07-15T03:10:00Z", update)], + ); + + assert!(scan_tape(&tape, "crypto_expiry_reference", 0, 0) + .unwrap_err() + .to_string() + .contains("record_id_version must be v2")); + } + + #[test] + fn rejects_duplicate_trade_record_ids() { + let root = TestDir::new(); + let update = valid_v2_trade_update(); + let tape = write_tape( + root.path(), + "market-updates.20260715T030000.ndjson", + &[ + record(0, "2026-07-15T03:10:00Z", update.clone()), + record(1, "2026-07-15T03:10:01Z", update), + ], + ); + + assert!(scan_tape(&tape, "crypto_expiry_reference", 0, 0) + .unwrap_err() + .to_string() + .contains("duplicate polymarket_trade record_id")); + } + + #[test] + fn rejects_non_object_raw_trade() { + let root = TestDir::new(); + let mut update = valid_v2_trade_update(); + update["trade"] = json!([]); + let tape = write_tape( + root.path(), + "market-updates.20260715T030000.ndjson", + &[record(0, "2026-07-15T03:10:00Z", update)], + ); + + assert!(scan_tape(&tape, "crypto_expiry_reference", 0, 0) + .unwrap_err() + .to_string() + .contains("polymarket_trade.trade must be an object")); + } + + #[test] + fn rejects_invalid_raw_trade_fields() { + let root = TestDir::new(); + let mut update = valid_v2_trade_update(); + update["trade"]["proxyWallet"] = Value::Null; + let tape = write_tape( + root.path(), + "market-updates.20260715T030000.ndjson", + &[record(0, "2026-07-15T03:10:00Z", update)], + ); + + assert!(scan_tape(&tape, "crypto_expiry_reference", 0, 0) + .unwrap_err() + .to_string() + .contains("raw trade requires proxyWallet")); + } + + #[test] + fn splits_utc_hours_without_rebasing_sequences() { + let root = TestDir::new(); + let mut rows = sample_rows(); + rows.push(record( + 3, + "2026-07-15T02:00:00Z", + json!({"kind": "event_expired", "event_id": "event-1", "end_time": null}), + )); + let tape = write_tape(root.path(), "market-updates.20260715T010000.ndjson", &rows); + let staging = ExclusiveTempDir::create(root.path(), ".split").unwrap(); + let chunks = split_tape_by_utc_hour(&tape, staging.path()).unwrap(); + assert_eq!(chunks.len(), 2); + let first = scan_tape(&chunks[0], "crypto_expiry", 1, 1_000).unwrap(); + let second = scan_tape(&chunks[1], "crypto_expiry", 1, 1_000).unwrap(); + assert_eq!( + ( + first["start_sequence"].as_u64(), + first["end_sequence"].as_u64() + ), + (Some(0), Some(2)) + ); + assert_eq!( + ( + second["start_sequence"].as_u64(), + second["end_sequence"].as_u64() + ), + (Some(3), Some(3)) + ); + assert_eq!( + (first["hour"].as_str(), second["hour"].as_str()), + (Some("01"), Some("02")) + ); + } + + #[test] + fn hash_triplet_and_remote_tamper_are_fail_closed() { + if Command::new("zstd").arg("--version").output().is_err() { + return; + } + let root = TestDir::new(); + let upload_config = config(root.path()); + let source = write_tape( + root.path(), + "market-updates.20260715T010000.ndjson", + &sample_rows(), + ); + let (artifacts, manifest) = prepare_artifacts(&source, &upload_config).unwrap(); + assert_eq!( + fs::read_to_string(&artifacts.success).unwrap().trim(), + manifest["sha256"].as_str().unwrap() + ); + assert_eq!( + artifacts.object_prefix, + format!( + "lake/raw/venue=polymarket/dataset=crypto_expiry/date=2026-07-15/hour=01/sha256={}", + manifest["sha256"].as_str().unwrap() + ) + ); + let remote = BTreeMap::from([ + ( + artifacts + .data + .file_name() + .unwrap() + .to_str() + .unwrap() + .to_owned(), + fs::read(&artifacts.data).unwrap(), + ), + ( + artifacts + .manifest + .file_name() + .unwrap() + .to_str() + .unwrap() + .to_owned(), + fs::read(&artifacts.manifest).unwrap(), + ), + ( + artifacts + .success + .file_name() + .unwrap() + .to_str() + .unwrap() + .to_owned(), + fs::read(&artifacts.success).unwrap(), + ), + ]); + let mut good_runner = |command: &mut Command, _: Duration| -> Result { + let args = command + .get_args() + .map(|arg| arg.to_string_lossy().into_owned()) + .collect::>(); + let remote_name = Path::new(&args[2]).file_name().unwrap().to_str().unwrap(); + fs::write(&args[3], &remote[remote_name])?; + Ok(success_status()) + }; + verify_remote_artifacts_with(&artifacts, &upload_config, &mut good_runner).unwrap(); + + for tampered_suffix in [".zst", ".manifest.json", "._SUCCESS"] { + let root = TestDir::new(); + let config = config(root.path()); + let source = write_tape( + root.path(), + "market-updates.20260715T010000.ndjson", + &sample_rows(), + ); + let (artifacts, _) = prepare_artifacts(&source, &config).unwrap(); + let mut uploaded_remote = BTreeMap::>::new(); + let mut bad_runner = |command: &mut Command, _: Duration| -> Result { + let args = command + .get_args() + .map(|arg| arg.to_string_lossy().into_owned()) + .collect::>(); + if args[2].starts_with("oss://") { + let remote_name = Path::new(&args[2]).file_name().unwrap().to_str().unwrap(); + let Some(remote_bytes) = uploaded_remote.get(remote_name) else { + bail!("remote object not found"); + }; + let bytes = if remote_name.ends_with(tampered_suffix) { + b"tampered".to_vec() + } else { + remote_bytes.clone() + }; + fs::write(&args[3], bytes)?; + } else { + assert!(args.iter().any(|arg| arg == "--ignore-existing")); + assert!(!args.iter().any(|arg| arg == "--force")); + let remote_name = Path::new(&args[3]) + .file_name() + .unwrap() + .to_str() + .unwrap() + .to_owned(); + uploaded_remote.insert(remote_name, fs::read(&args[2])?); + } + Ok(success_status()) + }; + assert!(upload_artifacts_with(&artifacts, &config, &mut bad_runner).is_err()); + for local in [ + &source, + &artifacts.data, + &artifacts.manifest, + &artifacts.success, + ] { + assert!( + local.exists(), + "readback failure deleted {}", + local.display() + ); + } + } + } + + #[test] + fn existing_remote_triplet_is_verified_without_overwrite() { + if Command::new("zstd").arg("--version").output().is_err() { + return; + } + let root = TestDir::new(); + let config = config(root.path()); + let source = write_tape( + root.path(), + "market-updates.20260715T010000.ndjson", + &sample_rows(), + ); + let (artifacts, _) = prepare_artifacts(&source, &config).unwrap(); + let remote = [&artifacts.data, &artifacts.manifest, &artifacts.success] + .into_iter() + .map(|path| { + ( + path.file_name().unwrap().to_str().unwrap().to_owned(), + fs::read(path).unwrap(), + ) + }) + .collect::>(); + let mut uploads = 0; + let mut runner = |command: &mut Command, _: Duration| -> Result { + let args = command + .get_args() + .map(|arg| arg.to_string_lossy().into_owned()) + .collect::>(); + if args[2].starts_with("oss://") { + let name = Path::new(&args[2]).file_name().unwrap().to_str().unwrap(); + fs::write(&args[3], &remote[name])?; + } else { + uploads += 1; + } + Ok(success_status()) + }; + + upload_artifacts_with(&artifacts, &config, &mut runner).unwrap(); + + assert_eq!(uploads, 0); + } + + #[test] + fn new_remote_triplet_is_no_clobber_uploaded_then_read_back() { + if Command::new("zstd").arg("--version").output().is_err() { + return; + } + let root = TestDir::new(); + let config = config(root.path()); + let source = write_tape( + root.path(), + "market-updates.20260715T010000.ndjson", + &sample_rows(), + ); + let (artifacts, _) = prepare_artifacts(&source, &config).unwrap(); + let mut remote = BTreeMap::>::new(); + let mut uploads = 0; + let mut downloads = 0; + let mut runner = |command: &mut Command, _: Duration| -> Result { + let args = command + .get_args() + .map(|arg| arg.to_string_lossy().into_owned()) + .collect::>(); + if args[2].starts_with("oss://") { + downloads += 1; + let name = Path::new(&args[2]).file_name().unwrap().to_str().unwrap(); + let bytes = remote + .get(name) + .ok_or_else(|| anyhow!("remote object not found"))?; + fs::write(&args[3], bytes)?; + } else { + assert!(args.iter().any(|arg| arg == "--ignore-existing")); + let name = Path::new(&args[3]) + .file_name() + .unwrap() + .to_str() + .unwrap() + .to_owned(); + remote.entry(name).or_insert(fs::read(&args[2])?); + uploads += 1; + } + Ok(success_status()) + }; + + upload_artifacts_with(&artifacts, &config, &mut runner).unwrap(); + + assert_eq!((uploads, downloads), (3, 4)); + assert!(!source.exists()); + } + + #[test] + fn mismatched_existing_remote_is_refused_before_upload() { + if Command::new("zstd").arg("--version").output().is_err() { + return; + } + let root = TestDir::new(); + let config = config(root.path()); + let source = write_tape( + root.path(), + "market-updates.20260715T010000.ndjson", + &sample_rows(), + ); + let (artifacts, _) = prepare_artifacts(&source, &config).unwrap(); + let mut remote = [&artifacts.data, &artifacts.manifest, &artifacts.success] + .into_iter() + .map(|path| { + ( + path.file_name().unwrap().to_str().unwrap().to_owned(), + fs::read(path).unwrap(), + ) + }) + .collect::>(); + let manifest_name = artifacts.manifest.file_name().unwrap().to_str().unwrap(); + remote.insert(manifest_name.to_owned(), b"tampered".to_vec()); + let mut uploads = 0; + let mut runner = |command: &mut Command, _: Duration| -> Result { + let args = command + .get_args() + .map(|arg| arg.to_string_lossy().into_owned()) + .collect::>(); + if args[2].starts_with("oss://") { + let name = Path::new(&args[2]).file_name().unwrap().to_str().unwrap(); + fs::write(&args[3], &remote[name])?; + } else { + uploads += 1; + } + Ok(success_status()) + }; + + assert!(upload_artifacts_with(&artifacts, &config, &mut runner).is_err()); + assert_eq!(uploads, 0); + for local in [ + &source, + &artifacts.data, + &artifacts.manifest, + &artifacts.success, + ] { + assert!(local.exists(), "mismatch deleted {}", local.display()); + } + } + + #[test] + fn strict_discovery_rejects_malformed_names_and_symlinks() { + assert!(strict_rotation_name( + "market-updates.20260715T010000123456.ndjson" + )); + assert!(strict_rotation_name( + "market-updates.20260715T010000123456.123e4567-e89b-12d3-a456-426614174000.ndjson" + )); + assert!(!strict_rotation_name( + "market-updates.20269999T999999.ndjson" + )); + let root = TestDir::new(); + write_tape(root.path(), "market-updates.ndjson", &sample_rows()); + write_tape(root.path(), "market-updates.backup.ndjson", &sample_rows()); + assert!(discover_rotated_tapes(root.path()) + .unwrap_err() + .to_string() + .contains("invalid rotated tape name")); + fs::remove_file(root.path().join("market-updates.backup.ndjson")).unwrap(); + let valid = write_tape( + root.path(), + "market-updates.20260715T010000.ndjson", + &sample_rows(), + ); + let target = root.path().join("target"); + fs::write(&target, b"victim").unwrap(); + let linked = root.path().join("market-updates.20260715T020000.ndjson"); + symlink(&target, &linked).unwrap(); + let error = discover_rotated_tapes(root.path()).unwrap_err().to_string(); + assert!(error.contains("regular non-symlink")); + fs::remove_file(&linked).unwrap(); + assert_eq!(discover_rotated_tapes(root.path()).unwrap(), vec![valid]); + assert_eq!(fs::read(target).unwrap(), b"victim"); + } + + #[cfg(unix)] + #[test] + fn upload_rejects_symlinked_or_noncanonical_spool_ancestors() { + let root = TestDir::new(); + let actual = root.path().join("actual"); + fs::create_dir(&actual).unwrap(); + let linked = root.path().join("linked"); + symlink(&actual, &linked).unwrap(); + + for spool_dir in [ + linked.join("spool"), + actual.join("child").join("..").join("child"), + ] { + let upload_config = config(&spool_dir); + assert!(upload_pending(&upload_config) + .unwrap_err() + .to_string() + .contains("directory")); + } + } + + #[test] + fn continues_after_bad_segment_and_persists_failure() { + let root = TestDir::new(); + let first = write_tape( + root.path(), + "market-updates.20260715T010000.ndjson", + &sample_rows(), + ); + let second = write_tape( + root.path(), + "market-updates.20260715T020000.ndjson", + &sample_rows(), + ); + let config = config(root.path()); + let mut visited = Vec::new(); + let result = upload_pending_with(&config, |source, _| { + visited.push(source.to_path_buf()); + if source == first { + bail!("bad tape"); + } + fs::remove_file(source)?; + Ok(vec![UploadedSegment { + object: "oss://bucket/second".to_owned(), + canonical_complete: true, + }]) + }); + assert!(result.is_err()); + assert_eq!(visited, vec![first.clone(), second]); + let status: Value = + serde_json::from_slice(&fs::read(root.path().join("upload-status.json")).unwrap()) + .unwrap(); + assert_eq!(status["pending_segments"], 1); + assert_eq!( + status["failed_segments"][0]["source"], + first.file_name().unwrap().to_str().unwrap() + ); + } + + #[test] + fn reports_uploaded_hour_segments_not_source_files() { + let root = TestDir::new(); + let source = write_tape( + root.path(), + "market-updates.20260715T010000.ndjson", + &sample_rows(), + ); + let config = config(root.path()); + let uploaded = upload_pending_with(&config, |source, _| { + fs::remove_file(source)?; + Ok(vec![ + UploadedSegment { + object: "oss://bucket/hour=01/data".to_owned(), + canonical_complete: true, + }, + UploadedSegment { + object: "oss://bucket/hour=02/data".to_owned(), + canonical_complete: false, + }, + ]) + }) + .unwrap(); + assert_eq!( + uploaded, + UploadSummary { + uploaded_segments: 2, + canonical_uploaded_segments: 1, + } + ); + assert!(!source.exists()); + let status: Value = + serde_json::from_slice(&fs::read(root.path().join("upload-status.json")).unwrap()) + .unwrap(); + assert_eq!(status["canonical_uploaded_segments"], 1); + } + + #[cfg(unix)] + fn success_status() -> ExitStatus { + use std::os::unix::process::ExitStatusExt; + ExitStatus::from_raw(0) + } +} From 1a8d2f6df3bbd90a9f2ecc4edc980d73f7da90a0 Mon Sep 17 00:00:00 2001 From: proerror Date: Wed, 15 Jul 2026 18:53:00 +0800 Subject: [PATCH 2/2] test(collector): isolate temporary directories --- rust_hft/tools/collector/src/lob_archiver.rs | 8 +++-- .../tools/collector/src/polymarket_parity.rs | 26 +++++++---------- .../tools/collector/src/polymarket_raw.rs | 25 +++++++--------- .../tools/collector/src/polymarket_upload.rs | 29 +++++++------------ 4 files changed, 38 insertions(+), 50 deletions(-) diff --git a/rust_hft/tools/collector/src/lob_archiver.rs b/rust_hft/tools/collector/src/lob_archiver.rs index 05025b680..545a753ff 100644 --- a/rust_hft/tools/collector/src/lob_archiver.rs +++ b/rust_hft/tools/collector/src/lob_archiver.rs @@ -1267,8 +1267,11 @@ mod tests { fn success_marker_atomic_write_does_not_follow_predictable_temp_symlink() { use std::os::unix::fs::symlink; - let root = std::env::temp_dir().join(format!("monday-marker-test-{}", now_ns().unwrap())); - fs::create_dir(&root).unwrap(); + let root = tempfile::Builder::new() + .prefix("monday-marker-test-") + .tempdir() + .unwrap(); + let root = root.path(); let data = root.join("segment.ndjson.zst"); fs::write(&data, b"compressed").unwrap(); let victim = root.join("victim"); @@ -1278,6 +1281,5 @@ mod tests { let marker = write_success_marker(&data, "abcd").unwrap(); assert_eq!(fs::read_to_string(marker).unwrap(), "abcd\n"); assert_eq!(fs::read(&victim).unwrap(), b"do-not-touch\n"); - fs::remove_dir_all(root).unwrap(); } } diff --git a/rust_hft/tools/collector/src/polymarket_parity.rs b/rust_hft/tools/collector/src/polymarket_parity.rs index a6a6a0fc8..e9df429c2 100644 --- a/rust_hft/tools/collector/src/polymarket_parity.rs +++ b/rust_hft/tools/collector/src/polymarket_parity.rs @@ -722,27 +722,23 @@ pub fn verify_shadow_parity(config: &ShadowParityConfig) -> Result { mod tests { use super::*; - struct TestDir(PathBuf); + struct TestDir { + _temp: tempfile::TempDir, + path: PathBuf, + } impl TestDir { fn new() -> Self { - let path = std::env::temp_dir().join(format!( - "monday-polymarket-parity-{}-{:016x}", - std::process::id(), - random::() - )); - fs::create_dir(&path).unwrap(); - Self(fs::canonicalize(path).unwrap()) + let temp = tempfile::Builder::new() + .prefix("monday-polymarket-parity-") + .tempdir() + .unwrap(); + let path = fs::canonicalize(temp.path()).unwrap(); + Self { _temp: temp, path } } fn path(&self) -> &Path { - &self.0 - } - } - - impl Drop for TestDir { - fn drop(&mut self) { - let _ = fs::remove_dir_all(&self.0); + &self.path } } diff --git a/rust_hft/tools/collector/src/polymarket_raw.rs b/rust_hft/tools/collector/src/polymarket_raw.rs index 76a10d1a3..335c25fc3 100644 --- a/rust_hft/tools/collector/src/polymarket_raw.rs +++ b/rust_hft/tools/collector/src/polymarket_raw.rs @@ -1653,26 +1653,23 @@ pub async fn run_reference(config: ReferenceConfig, once: bool) -> Result<()> { mod tests { use super::*; - struct TestDir(PathBuf); + struct TestDir { + _temp: tempfile::TempDir, + path: PathBuf, + } impl TestDir { fn new() -> Self { - let path = std::env::temp_dir().join(format!( - "monday-polymarket-reference-test-{:016x}", - random::() - )); - fs::create_dir(&path).unwrap(); - Self(fs::canonicalize(path).unwrap()) + let temp = tempfile::Builder::new() + .prefix("monday-polymarket-reference-test-") + .tempdir() + .unwrap(); + let path = fs::canonicalize(temp.path()).unwrap(); + Self { _temp: temp, path } } fn path(&self) -> &Path { - &self.0 - } - } - - impl Drop for TestDir { - fn drop(&mut self) { - let _ = fs::remove_dir_all(&self.0); + &self.path } } diff --git a/rust_hft/tools/collector/src/polymarket_upload.rs b/rust_hft/tools/collector/src/polymarket_upload.rs index e6f6cb008..93da27680 100644 --- a/rust_hft/tools/collector/src/polymarket_upload.rs +++ b/rust_hft/tools/collector/src/polymarket_upload.rs @@ -1621,30 +1621,23 @@ mod tests { use std::os::unix::fs::symlink; use std::process::ExitStatus; - struct TestDir(PathBuf); + struct TestDir { + _temp: tempfile::TempDir, + path: PathBuf, + } impl TestDir { fn new() -> Self { - for _ in 0..32 { - let path = std::env::temp_dir().join(format!( - "monday-polymarket-upload-test-{:016x}", - random::() - )); - if fs::create_dir(&path).is_ok() { - return Self(fs::canonicalize(path).unwrap()); - } - } - panic!("could not create test directory") + let temp = tempfile::Builder::new() + .prefix("monday-polymarket-upload-test-") + .tempdir() + .unwrap(); + let path = fs::canonicalize(temp.path()).unwrap(); + Self { _temp: temp, path } } fn path(&self) -> &Path { - &self.0 - } - } - - impl Drop for TestDir { - fn drop(&mut self) { - let _ = fs::remove_dir_all(&self.0); + &self.path } }