diff --git a/.github/workflows/acr-publish.yml b/.github/workflows/acr-publish.yml index a9efd685f..b107cffb7 100644 --- a/.github/workflows/acr-publish.yml +++ b/.github/workflows/acr-publish.yml @@ -316,6 +316,7 @@ jobs: ) reference_control_assets=( binance-usdm-reference-collector-shadow@.service + binance-usdm-reference-shadow-gate.sh binance-usdm-reference-shadow-gate-policy.jq ) ( diff --git a/deployment/aliyun/binance-usdm-reference-shadow-gate.sh b/deployment/aliyun/binance-usdm-reference-shadow-gate.sh new file mode 100755 index 000000000..8c114080b --- /dev/null +++ b/deployment/aliyun/binance-usdm-reference-shadow-gate.sh @@ -0,0 +1,420 @@ +#!/usr/bin/env bash +set -euo pipefail + +umask 027 +export LC_ALL=C + +readonly REQUIRED_DURATION_SECONDS=3600 +readonly OBSERVATION_GRACE_SECONDS=90 +readonly MAX_ARTIFACT_GAP_NS=90000000000 +readonly RELEASE_SCHEMA=monday.binance_usdm_reference_release.v1 +readonly GATE_SCHEMA=monday.binance_usdm_reference_shadow_gate.v1 +readonly HEALTH_SCHEMA=binance.usdm_reference_health.v1 +readonly VERIFICATION_SCHEMA=monday.binance_usdm_reference_artifact_verification.v1 +readonly SERVICE_TEMPLATE=binance-usdm-reference-collector-shadow@.service +readonly GATE_POLICY=binance-usdm-reference-shadow-gate-policy.jq +readonly RUNNER=binance-usdm-reference-shadow-gate.sh + +die() { + printf 'Binance USD-M reference shadow gate failed: %s\n' "$*" >&2 + exit 1 +} + +usage() { + printf '%s\n' \ + 'Usage: binance-usdm-reference-shadow-gate.sh ' \ + '' \ + 'The gate only observes an already-running isolated shadow service.' \ + 'A production-eligible gate requires at least 3600 seconds of artifacts.' +} + +[[ $# -eq 1 ]] || { + usage >&2 + exit 2 +} + +test_mode=false +root_prefix= +if [[ ${MONDAY_ALLOW_REFERENCE_GATE_TEST_MODE:-0} == 1 ]]; then + [[ -n ${MONDAY_REFERENCE_GATE_TEST_ROOT:-} ]] \ + || die 'test mode requires MONDAY_REFERENCE_GATE_TEST_ROOT' + root_prefix=$(cd -- "$MONDAY_REFERENCE_GATE_TEST_ROOT" && pwd -P) + [[ $root_prefix == /* && $root_prefix != / ]] || die 'invalid test root' + test_mode=true +else + [[ -z ${MONDAY_REFERENCE_GATE_TEST_ROOT+x} \ + && -z ${MONDAY_REFERENCE_GATE_TEST_SECONDS+x} \ + && -z ${MONDAY_REFERENCE_GATE_TEST_GRACE_SECONDS+x} \ + && -z ${MONDAY_REFERENCE_GATE_TEST_NOW_NS+x} ]] \ + || die 'test overrides require explicit test mode' + [[ ${EUID} -eq 0 ]] || die 'must run as root' +fi + +for command in awk cmp date dirname find jq mkdir mktemp readlink rm sha256sum \ + sleep sort stat systemctl tr; do + command -v "$command" >/dev/null 2>&1 || die "missing required command: $command" +done + +prefix_path() { + printf '%s%s\n' "$root_prefix" "$1" +} + +candidate_sha=$(printf '%s' "$1" | tr '[:upper:]' '[:lower:]') +[[ $candidate_sha =~ ^[a-f0-9]{64}$ ]] \ + || die 'candidate SHA-256 must be 64 hexadecimal characters' +release_root=$(prefix_path /opt/monday/releases/binance-usdm-reference-collector) +release="$release_root/$candidate_sha" +deployment="$release/deployment" +collector="$release/binance-usdm-reference-collector" +verifier="$release/binance-usdm-reference-artifact-verifier" +release_json="$release/release.json" +control_manifest="$release/binance-usdm-reference-control-assets.sha256" +control_archive="$release/binance-usdm-reference-control.tar.gz" +spool=$(prefix_path "/data/monday/spool/binance-usdm-reference-shadow/$candidate_sha") +evidence_root=$(prefix_path /data/monday/evidence/binance-usdm-reference-shadow-gates) +installed_service=$(prefix_path "/etc/systemd/system/$SERVICE_TEMPLATE") +proc_root=$(prefix_path /proc) +lock_file=$(prefix_path /run/monday/binance-usdm-reference-release.lock) +unit="binance-usdm-reference-collector-shadow@$candidate_sha.service" +script_dir=$(cd -- "$(dirname -- "$0")" && pwd -P) +[[ $script_dir == "$deployment" ]] || die 'gate runner is outside the candidate deployment bundle' + +direct_directory() { + local path=$1 + [[ -d $path && ! -L $path && $(cd -- "$path" && pwd -P) == "$path" ]] +} + +secure_root_directory() { + local path=$1 mode owner + direct_directory "$path" || return 1 + if [[ $test_mode == false ]]; then + owner=$(stat -c %u -- "$path") + mode=$(stat -c %a -- "$path") + [[ $owner == 0 ]] || return 1 + (( (8#$mode & 022) == 0 )) || return 1 + fi +} + +secure_spool_directory() { + local path=$1 mode owner + direct_directory "$path" || return 1 + if [[ $test_mode == false ]]; then + owner=$(stat -c %U -- "$path") + mode=$(stat -c %a -- "$path") + [[ $owner == hftcollector ]] || return 1 + (( (8#$mode & 022) == 0 )) || return 1 + fi +} + +direct_file() { + local path=$1 + [[ -f $path && ! -L $path && $(readlink -f -- "$path") == "$path" ]] +} + +secure_control_file() { + local path=$1 mode owner + direct_file "$path" || die "missing direct control file: $path" + if [[ $test_mode == false ]]; then + owner=$(stat -c %u -- "$path") + mode=$(stat -c %a -- "$path") + [[ $owner == 0 ]] || die "control file is not root-owned: $path" + (( (8#$mode & 022) == 0 )) || die "control file is group/world writable: $path" + fi +} + +secure_data_file() { + local path=$1 mode owner + direct_file "$path" || die "missing direct collector file: $path" + if [[ $test_mode == false ]]; then + owner=$(stat -c %U -- "$path") + mode=$(stat -c %a -- "$path") + [[ $owner == hftcollector ]] || die "collector file has the wrong owner: $path" + (( (8#$mode & 022) == 0 )) \ + || die "collector file is group/world writable: $path" + fi +} + +check_sha() { + local expected=$1 path=$2 + if [[ $test_mode == true ]]; then + [[ $(sha256sum "$path" | awk '{print $1}') == "$expected" ]] + else + printf '%s %s\n' "$expected" "$path" | sha256sum --check --strict >/dev/null + fi +} + +for directory in "$release_root" "$release" "$deployment"; do + secure_root_directory "$directory" \ + || die "release directory is indirect or insecure: $directory" +done +secure_spool_directory "$spool" || die 'shadow spool directory is indirect or insecure' +for file in "$collector" "$verifier" "$release_json" "$control_manifest" \ + "$control_archive" "$deployment/$RUNNER" "$deployment/$GATE_POLICY" \ + "$deployment/$SERVICE_TEMPLATE" "$installed_service"; do + secure_control_file "$file" +done +[[ -x $collector && -x $verifier && -x "$deployment/$RUNNER" ]] \ + || die 'candidate, verifier, and gate runner must be executable' +cmp -s "$deployment/$SERVICE_TEMPLATE" "$installed_service" \ + || die 'installed shadow service differs from the release bundle' + +manifest_sha=$(sha256sum "$control_manifest" | awk '{print $1}') +archive_sha=$(sha256sum "$control_archive" | awk '{print $1}') +verifier_sha=$(sha256sum "$verifier" | awk '{print $1}') +jq -e --arg candidate "$candidate_sha" --arg verifier "$verifier_sha" \ + --arg manifest "$manifest_sha" --arg archive "$archive_sha" \ + --arg schema "$RELEASE_SCHEMA" ' + .schema == $schema + and (keys | sort) == (["candidate","control_archive","control_manifest", + "schema","source_revision","verifier"] | sort) + and (.source_revision | type == "string" and test("^[a-f0-9]{40,64}$")) + and .candidate == {file:"binance-usdm-reference-collector",sha256:$candidate} + and .verifier == {file:"binance-usdm-reference-artifact-verifier",sha256:$verifier} + and .control_manifest == { + file:"binance-usdm-reference-control-assets.sha256",sha256:$manifest} + and .control_archive == { + file:"binance-usdm-reference-control.tar.gz",sha256:$archive} +' "$release_json" >/dev/null || die 'release identity or asset binding is invalid' +source_revision=$(jq -er .source_revision "$release_json") +check_sha "$candidate_sha" "$collector" || die 'candidate SHA-256 drifted' + +expected_assets=$(printf '%s\n' "$GATE_POLICY" "$RUNNER" "$SERVICE_TEMPLATE" | sort) +actual_assets=$(awk 'NF == 2 && $1 ~ /^[a-f0-9]{64}$/ {print $2}' "$control_manifest" | sort) +[[ $actual_assets == "$expected_assets" \ + && $(awk 'NF == 2 && $1 ~ /^[a-f0-9]{64}$/ {count++} END {print count+0}' \ + "$control_manifest") == 3 ]] || die 'control manifest has an unexpected asset set' +( + cd "$deployment" + if [[ $test_mode == true ]]; then + sha256sum --check "$control_manifest" >/dev/null + else + sha256sum --check --strict "$control_manifest" >/dev/null + fi +) || die 'control bundle asset digest mismatch' + +mkdir -p -- "$(dirname -- "$lock_file")" +secure_root_directory "$(dirname -- "$lock_file")" \ + || die 'release lock directory is indirect or insecure' +if [[ $test_mode == true ]]; then + mkdir -- "$lock_file.test-lock" || die 'another test USD-M reference gate is running' +else + command -v flock >/dev/null 2>&1 || die 'missing required command: flock' + command -v mountpoint >/dev/null 2>&1 || die 'missing required command: mountpoint' + mountpoint -q /data || die '/data must be a mount point' + [[ ! -e $lock_file || -f $lock_file && ! -L $lock_file ]] \ + || die 'release lock is not a direct regular file' + exec 9>>"$lock_file" + secure_control_file "$lock_file" + flock -n 9 || die 'another USD-M reference gate is running' +fi + +gate_seconds=${MONDAY_REFERENCE_GATE_TEST_SECONDS:-$REQUIRED_DURATION_SECONDS} +grace_seconds=${MONDAY_REFERENCE_GATE_TEST_GRACE_SECONDS:-$OBSERVATION_GRACE_SECONDS} +[[ $gate_seconds =~ ^[1-9][0-9]*$ && $grace_seconds =~ ^[0-9]+$ ]] \ + || die 'gate and grace durations must be integers' +if [[ $test_mode == false ]]; then + ((gate_seconds >= REQUIRED_DURATION_SECONDS)) \ + || die 'production gate duration cannot be shorter than 3600 seconds' + [[ $gate_seconds == "$REQUIRED_DURATION_SECONDS" \ + && $grace_seconds == "$OBSERVATION_GRACE_SECONDS" ]] \ + || die 'production duration overrides are not allowed' +fi + +systemctl_value() { + systemctl show "$unit" --property="$1" --value +} + +start_pid='' +start_restarts='' +start_invocation='' +end_pid='' +end_restarts='' +end_invocation='' + +verify_process() { + local pid=$1 expected actual + [[ $pid =~ ^[1-9][0-9]*$ ]] || return 1 + [[ -L $proc_root/$pid/exe && $(readlink -f -- "$proc_root/$pid/exe") == "$collector" ]] \ + || return 1 + [[ -f $proc_root/$pid/cmdline && ! -L $proc_root/$pid/cmdline ]] || return 1 + expected=$(printf '%s\n' "$collector" --output-root "$spool" --interval-seconds 30 \ + --request-timeout-seconds 10 --oi-concurrency 8 --max-staleness-ms 30000) + actual=$(tr '\0' '\n' <"$proc_root/$pid/cmdline") + [[ $actual == "$expected" ]] +} + +capture_identity() { + local prefix=$1 fragment drop_ins pid restarts invocation + systemctl is-active --quiet "$unit" || die 'shadow service is not active' + fragment=$(systemctl_value FragmentPath) + drop_ins=$(systemctl_value DropInPaths) + pid=$(systemctl_value MainPID) + restarts=$(systemctl_value NRestarts) + invocation=$(systemctl_value InvocationID) + [[ $fragment == "$installed_service" && -z $drop_ins ]] \ + || die 'shadow service fragment or drop-ins do not match the release' + [[ $restarts == 0 && $invocation =~ ^[a-f0-9]{32}$ ]] \ + || die 'shadow service restarted or has no stable invocation identity' + verify_process "$pid" || die 'shadow process identity or argv is invalid' + printf -v "${prefix}_pid" '%s' "$pid" + printf -v "${prefix}_restarts" '%s' "$restarts" + printf -v "${prefix}_invocation" '%s' "$invocation" +} + +capture_identity start +if [[ $test_mode == true ]]; then + start_ns=${MONDAY_REFERENCE_GATE_TEST_NOW_NS:-} + [[ $start_ns =~ ^[1-9][0-9]*$ ]] || die 'test mode requires a positive start time' + end_ns=$((start_ns + (gate_seconds + grace_seconds) * 1000000000)) +else + [[ -r /proc/uptime ]] || die '/proc/uptime is required for monotonic timing' + start_uptime_seconds=$(awk '{print int($1)}' /proc/uptime) + start_ns=$(date -u +%s%N) + [[ $start_ns =~ ^[1-9][0-9]{18}$ ]] || die 'could not read nanosecond wall clock' + sleep "$((gate_seconds + grace_seconds))" + end_ns=$(date -u +%s%N) + end_uptime_seconds=$(awk '{print int($1)}' /proc/uptime) + [[ $end_ns =~ ^[1-9][0-9]{18}$ && $end_ns -ge $start_ns ]] \ + || die 'wall clock regressed during the gate' + ((end_uptime_seconds - start_uptime_seconds >= gate_seconds + grace_seconds)) \ + || die 'monotonic observation duration is too short' +fi + +temp_dir=$(mktemp -d) +cleanup() { + rm -rf -- "$temp_dir" +} +trap cleanup EXIT + +symlinked=$(find "$spool" -type l \( -name reference.ndjson \ + -o -name reference.ndjson.manifest.json -o -name reference.ndjson._SUCCESS \) \ + -print -quit) +[[ -z $symlinked ]] || die "reference artifact contains a symlink: $symlinked" + +artifact_count=0 +while IFS= read -r manifest; do + secure_data_file "$manifest" + observed=$(jq -er '.observed_at_ns | select(type == "number" and . == floor)' "$manifest") + [[ $observed =~ ^[1-9][0-9]*$ ]] || die "invalid artifact observation time: $manifest" + ((observed >= start_ns && observed <= end_ns)) || continue + data=${manifest%.manifest.json} + success="$data._SUCCESS" + secure_data_file "$data" + secure_data_file "$success" + data_sha=$(jq -er '.sha256 | select(type == "string" and test("^[a-f0-9]{64}$"))' \ + "$manifest") + current_manifest_sha=$(sha256sum "$manifest" | awk '{print $1}') + verification=$("$verifier" --data-path "$data" --data-sha256 "$data_sha" \ + --manifest-sha256 "$current_manifest_sha") \ + || die "canonical verifier rejected artifact: $manifest" + jq -e --arg schema "$VERIFICATION_SCHEMA" --arg path "$data" \ + --arg data "$data_sha" --arg manifest "$current_manifest_sha" \ + --slurpfile source "$manifest" ' + .schema == $schema and .data_path == $path and .data_sha256 == $data + and .manifest_sha256 == $manifest and .content_rows_verified == true + and .metadata_observations == $source[0].coverage.metadata_observations + and .mark_index_funding_observations == + $source[0].coverage.mark_index_funding_observations + and .open_interest_observations == $source[0].coverage.open_interest_observations + ' <<<"$verification" >/dev/null || die "verifier output is not bound to artifact: $manifest" + [[ ! -e $temp_dir/$observed.json ]] || die 'duplicate artifact observation time' + jq -c --arg manifest_sha "$current_manifest_sha" ' + {canonical_readback:true,dataset:.dataset,venue:.venue, + manifest_schema:.schema,data_schema:.data_schema,source_origin:.source_origin, + source_endpoints:.source_endpoints,max_staleness_ms:.max_staleness_ms, + data_sha256:.sha256,manifest_sha256:$manifest_sha,success_sha256:.sha256, + content_rows_verified:true,observed_at_ns:.observed_at_ns, + time_bounds:.time_bounds,coverage:.coverage} + ' "$manifest" >"$temp_dir/$observed.json" + check_sha "$data_sha" "$data" || die "artifact changed after readback: $data" + check_sha "$current_manifest_sha" "$manifest" \ + || die "manifest changed after readback: $manifest" + artifact_count=$((artifact_count + 1)) +done < <(find "$spool" -type f -name reference.ndjson.manifest.json | sort) +((artifact_count >= 3)) || die 'fewer than three new canonical artifacts were observed' +artifacts=$(jq -s 'sort_by(.observed_at_ns)' "$temp_dir"/*.json) +span_ns=$(jq -er '.[-1].observed_at_ns - .[0].observed_at_ns' <<<"$artifacts") +max_gap_ns=$(jq -er '[range(1; length) as $i | + .[$i].observed_at_ns - .[$i - 1].observed_at_ns] as $gaps + | select(all($gaps[]; . > 0 and . <= 90000000000)) | ($gaps | max)' \ + <<<"$artifacts") || die 'artifact observations are discontinuous' +((span_ns >= gate_seconds * 1000000000)) \ + || die 'artifact observation span is shorter than the gate duration' +((max_gap_ns <= MAX_ARTIFACT_GAP_NS)) || die 'artifact gap exceeds 90 seconds' + +health="$spool/health.json" +secure_data_file "$health" +latest_data_sha=$(jq -er '.[-1].data_sha256' <<<"$artifacts") +latest_manifest_sha=$(jq -er '.[-1].manifest_sha256' <<<"$artifacts") +latest_observed=$(jq -er '.[-1].observed_at_ns' <<<"$artifacts") +latest_data=$(find "$spool" -type f -name reference.ndjson.manifest.json \ + -exec jq -er --argjson observed "$latest_observed" \ + 'select(.observed_at_ns == $observed) | input_filename' {} \; | head -n 1) +latest_data=${latest_data%.manifest.json} +jq -e --arg schema "$HEALTH_SCHEMA" --arg data "$latest_data_sha" \ + --arg manifest "$latest_manifest_sha" --arg path "$latest_data" \ + --argjson observed "$latest_observed" ' + .schema == $schema and .status == "healthy" + and .source_origin == "https://fapi.binance.com" + and .api_error_count == 0 and .total_api_errors == 0 + and .artifact_error_count == 0 and .total_artifact_errors == 0 + and .last_error == null and .data_path == $path + and .data_sha256 == $data and .manifest_sha256 == $manifest + and (.last_success_at_ns | type == "number" and . == floor) + and .last_success_at_ns >= $observed + and .last_success_at_ns - $observed <= 90000000000 +' "$health" >/dev/null || die 'health is stale, erroneous, or not bound to the latest artifact' +health_evidence=$(jq '{schema,status,source_origin,api_error_count,total_api_errors, + artifact_error_count,total_artifact_errors,last_success_at_ns,data_sha256, + manifest_sha256}' "$health") + +capture_identity end +[[ $end_pid == "$start_pid" && $end_restarts == "$start_restarts" \ + && $end_invocation == "$start_invocation" ]] \ + || die 'shadow service identity changed during the gate' +check_sha "$candidate_sha" "$collector" || die 'candidate changed during the gate' +check_sha "$verifier_sha" "$verifier" || die 'verifier changed during the gate' + +for directory in "$(prefix_path /data/monday)" "$(prefix_path /data/monday/evidence)" \ + "$evidence_root"; do + mkdir -p -- "$directory" + secure_root_directory "$directory" || die "evidence directory is insecure: $directory" +done +bundle_dir="$evidence_root/$candidate_sha/$manifest_sha/runs" +mkdir -p -- "$bundle_dir" +secure_root_directory "$bundle_dir" || die 'gate runs directory is indirect or insecure' +run_id="$(date -u +%Y%m%dT%H%M%SZ)-${start_ns}-$$" +evidence_dir="$bundle_dir/$run_id" +mkdir -- "$evidence_dir" || die 'gate evidence run already exists' +gate_json="$evidence_dir/gate.json" +production_eligible=true +[[ $test_mode == false ]] || production_eligible=false +jq -S -n --arg schema "$GATE_SCHEMA" --arg candidate "$candidate_sha" \ + --arg bundle "$manifest_sha" --arg source "$source_revision" \ + --arg unit "$unit" --arg invocation "$start_invocation" \ + --argjson duration "$gate_seconds" --argjson eligible "$production_eligible" \ + --argjson health "$health_evidence" --argjson artifacts "$artifacts" \ + --argjson max_gap "$max_gap_ns" ' + {schema:$schema,candidate_sha256:$candidate, + deployment_bundle_sha256:$bundle,deployment_source_revision:$source, + passed:$eligible,production_eligible:$eligible,duration_seconds:$duration, + service:{unit:$unit,active:true,restart_count:0,binary_sha256:$candidate, + invocation_id_start:$invocation,invocation_id_end:$invocation}, + health:$health,artifact_count:($artifacts|length),max_artifact_gap_ns:$max_gap, + artifacts:$artifacts} +' >"$gate_json" +chmod 0640 "$gate_json" + +if [[ $test_mode == false ]]; then + jq -e --arg candidate_sha256 "$candidate_sha" \ + --arg deployment_bundle_sha256 "$manifest_sha" \ + --arg deployment_source_revision "$source_revision" \ + -f "$deployment/$GATE_POLICY" "$gate_json" >/dev/null \ + || die 'compiled production evidence failed the release gate policy' + gate_sha=$(sha256sum "$gate_json" | awk '{print $1}') + (set -C; printf '%s gate.json\n' "$gate_sha" >"$evidence_dir/PASSED.sha256") \ + || die 'could not publish immutable PASSED marker' + chmod 0640 "$evidence_dir/PASSED.sha256" +fi + +printf '%s\n' "$gate_json" diff --git a/deployment/aliyun/test-binance-usdm-reference-host-gate.sh b/deployment/aliyun/test-binance-usdm-reference-host-gate.sh new file mode 100755 index 000000000..53e7ef042 --- /dev/null +++ b/deployment/aliyun/test-binance-usdm-reference-host-gate.sh @@ -0,0 +1,245 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir=$(cd -- "$(dirname -- "$0")" && pwd) +runner="$script_dir/binance-usdm-reference-shadow-gate.sh" +policy="$script_dir/binance-usdm-reference-shadow-gate-policy.jq" +service="$script_dir/binance-usdm-reference-collector-shadow@.service" +start_ns=2000000000000000000 +tmp_dir=$(mktemp -d) +tmp_dir=$(cd -- "$tmp_dir" && pwd -P) +trap 'rm -rf "$tmp_dir"' EXIT +collector_fixture="$tmp_dir/binance-usdm-reference-collector" +printf '#!/bin/sh\nexit 0\n' >"$collector_fixture" +chmod 0755 "$collector_fixture" +candidate=$(sha256sum "$collector_fixture" | awk '{print $1}') + +[[ -x $runner ]] || { + printf '%s\n' 'missing executable Binance USD-M reference host gate' >&2 + exit 1 +} + +write_artifact() { + local root=$1 observed_ns=$2 label=$3 + local batch data data_sha manifest + batch="$root/data/monday/spool/binance-usdm-reference-shadow/$candidate/lake/raw/venue=binance_usdm/dataset=reference/date=2033-05-18/hour=03/batch=$observed_ns" + mkdir -p "$batch" + data="$batch/reference.ndjson" + printf '{"fixture":"%s"}\n' "$label" >"$data" + data_sha=$(sha256sum "$data" | awk '{print $1}') + manifest="$data.manifest.json" + jq -S -n --arg data_sha "$data_sha" --argjson observed "$observed_ns" ' + {schema:"binance.usdm_reference_manifest.v1",venue:"binance_usdm", + dataset:"reference",data_schema:"binance.usdm_reference.v2",format:"ndjson", + source_origin:"https://fapi.binance.com", + source_endpoints:["https://fapi.binance.com/fapi/v1/time", + "https://fapi.binance.com/fapi/v1/exchangeInfo", + "https://fapi.binance.com/fapi/v1/premiumIndex", + "https://fapi.binance.com/fapi/v1/openInterest"], + file:"reference.ndjson",bytes:21,sha256:$data_sha,rows:1500, + observed_at_ns:$observed,max_staleness_ms:30000, + coverage:{active_contracts:500,metadata_observations:500, + mark_index_funding_observations:500,open_interest_observations:500, + stale_metadata:0,stale_mark_index_funding:0,stale_open_interest:0, + api_error_count:0}, + time_bounds:{min_source_time_ms:1999999999000, + max_source_time_ms:2000000000000, + min_received_at_ns:1999999999500000000, + max_received_at_ns:1999999999900000000}}' >"$manifest" + printf '%s\n' "$data_sha" >"$data._SUCCESS" +} + +setup_fixture() { + local name=$1 + root="$tmp_dir/$name" + release="$root/opt/monday/releases/binance-usdm-reference-collector/$candidate" + deployment="$release/deployment" + spool="$root/data/monday/spool/binance-usdm-reference-shadow/$candidate" + evidence="$root/data/monday/evidence/binance-usdm-reference-shadow-gates" + fake_bin="$root/fake-bin" + state="$root/systemctl-state.json" + mkdir -p "$deployment" "$spool" "$fake_bin" "$root/etc/systemd/system" \ + "$root/proc/4242" "$root/run/lock" "$evidence" + + cp "$collector_fixture" "$release/binance-usdm-reference-collector" + chmod 0755 "$release/binance-usdm-reference-collector" + cat >"$release/binance-usdm-reference-artifact-verifier" <<'VERIFIER' +#!/usr/bin/env bash +set -euo pipefail +while (($#)); do + case "$1" in + --data-path) data=$2; shift 2 ;; + --data-sha256) data_sha=$2; shift 2 ;; + --manifest-sha256) manifest_sha=$2; shift 2 ;; + *) exit 2 ;; + esac +done +manifest="$data.manifest.json" +success="$data._SUCCESS" +if [[ ! -f $data || -L $data || ! -f $manifest || -L $manifest \ + || ! -f $success || -L $success \ + || $(sha256sum "$data" | awk '{print $1}') != "$data_sha" \ + || $(sha256sum "$manifest" | awk '{print $1}') != "$manifest_sha" \ + || $(cat "$success") != "$data_sha" ]]; then + exit 1 +fi +jq -c --arg path "$data" --arg data "$data_sha" --arg manifest "$manifest_sha" ' + {schema:"monday.binance_usdm_reference_artifact_verification.v1", + data_path:$path,data_sha256:$data,manifest_sha256:$manifest, + metadata_observations:.coverage.metadata_observations, + mark_index_funding_observations:.coverage.mark_index_funding_observations, + open_interest_observations:.coverage.open_interest_observations, + content_rows_verified:true}' "$manifest" +VERIFIER + chmod 0755 "$release/binance-usdm-reference-artifact-verifier" + cp "$runner" "$policy" "$service" "$deployment/" + cp "$service" "$root/etc/systemd/system/" + ( + cd "$deployment" + sha256sum binance-usdm-reference-shadow-gate.sh \ + binance-usdm-reference-shadow-gate-policy.jq \ + binance-usdm-reference-collector-shadow@.service \ + >"$release/binance-usdm-reference-control-assets.sha256" + ) + printf 'fixture archive\n' >"$release/binance-usdm-reference-control.tar.gz" + local collector_sha verifier_sha manifest_sha archive_sha + collector_sha=$(sha256sum "$release/binance-usdm-reference-collector" | awk '{print $1}') + [[ $collector_sha == "$candidate" ]] + verifier_sha=$(sha256sum "$release/binance-usdm-reference-artifact-verifier" | awk '{print $1}') + manifest_sha=$(sha256sum "$release/binance-usdm-reference-control-assets.sha256" | awk '{print $1}') + archive_sha=$(sha256sum "$release/binance-usdm-reference-control.tar.gz" | awk '{print $1}') + jq -S -n --arg candidate "$candidate" --arg verifier "$verifier_sha" \ + --arg manifest "$manifest_sha" --arg archive "$archive_sha" \ + '{schema:"monday.binance_usdm_reference_release.v1",source_revision:("c"*40), + candidate:{file:"binance-usdm-reference-collector",sha256:$candidate}, + verifier:{file:"binance-usdm-reference-artifact-verifier",sha256:$verifier}, + control_manifest:{file:"binance-usdm-reference-control-assets.sha256",sha256:$manifest}, + control_archive:{file:"binance-usdm-reference-control.tar.gz",sha256:$archive}}' \ + >"$release/release.json" + + write_artifact "$root" "$start_ns" first + write_artifact "$root" "$((start_ns + 1000000000))" second + write_artifact "$root" "$((start_ns + 2000000000))" third + latest_manifest=$(find "$spool" -name reference.ndjson.manifest.json -type f \ + | sort | tail -n 1) + latest_data=${latest_manifest%.manifest.json} + latest_data_sha=$(jq -r .sha256 "$latest_manifest") + latest_manifest_sha=$(sha256sum "$latest_manifest" | awk '{print $1}') + jq -S -n --arg path "$latest_data" --arg data "$latest_data_sha" \ + --arg manifest "$latest_manifest_sha" --argjson success "$((start_ns + 2500000000))" ' + {schema:"binance.usdm_reference_health.v1",status:"healthy", + source_origin:"https://fapi.binance.com",last_attempt_at_ns:$success, + last_success_at_ns:$success,api_error_count:0,total_api_errors:0, + artifact_error_count:0,total_artifact_errors:0,last_error:null, + data_path:$path,data_sha256:$data,manifest_sha256:$manifest}' >"$spool/health.json" + + jq -n --arg fragment "$root/etc/systemd/system/binance-usdm-reference-collector-shadow@.service" \ + '{fragment:$fragment,pid:"4242",restarts:"0",invocation:("f"*32), + replacement:("0"*32),replace_after_first:false}' >"$state" + cat >"$fake_bin/systemctl" <<'SYSTEMCTL' +#!/usr/bin/env bash +set -euo pipefail +case "$1" in + is-active) exit 0 ;; + show) + property= + for arg in "$@"; do + case "$arg" in --property=*) property=${arg#--property=} ;; esac + done + case "$property" in + FragmentPath) jq -r .fragment "$FAKE_SYSTEMCTL_STATE" ;; + DropInPaths) printf '\n' ;; + MainPID) jq -r .pid "$FAKE_SYSTEMCTL_STATE" ;; + NRestarts) jq -r .restarts "$FAKE_SYSTEMCTL_STATE" ;; + InvocationID) + count_file="$FAKE_SYSTEMCTL_STATE.count" + count=0; [[ ! -f $count_file ]] || count=$(cat "$count_file") + if [[ $(jq -r .replace_after_first "$FAKE_SYSTEMCTL_STATE") == true && $count -gt 0 ]]; then + jq -r .replacement "$FAKE_SYSTEMCTL_STATE" + else + jq -r .invocation "$FAKE_SYSTEMCTL_STATE" + fi + printf '%s\n' "$((count + 1))" >"$count_file" + ;; + *) exit 2 ;; + esac + ;; + *) exit 2 ;; +esac +SYSTEMCTL + chmod 0755 "$fake_bin/systemctl" + ln -s "$release/binance-usdm-reference-collector" "$root/proc/4242/exe" + printf '%s\0' "$release/binance-usdm-reference-collector" --output-root "$spool" \ + --interval-seconds 30 --request-timeout-seconds 10 --oi-concurrency 8 \ + --max-staleness-ms 30000 >"$root/proc/4242/cmdline" +} + +run_gate() { + MONDAY_ALLOW_REFERENCE_GATE_TEST_MODE=1 \ + MONDAY_REFERENCE_GATE_TEST_ROOT="$root" \ + MONDAY_REFERENCE_GATE_TEST_SECONDS=2 \ + MONDAY_REFERENCE_GATE_TEST_GRACE_SECONDS="${test_grace_seconds:-1}" \ + MONDAY_REFERENCE_GATE_TEST_NOW_NS="$start_ns" \ + FAKE_SYSTEMCTL_STATE="$state" \ + PATH="$fake_bin:$PATH" \ + "$deployment/binance-usdm-reference-shadow-gate.sh" "$candidate" +} + +expect_failure() { + local label=$1 + if run_gate >"$root/$label.stdout" 2>"$root/$label.stderr"; then + printf 'host gate accepted invalid fixture: %s\n' "$label" >&2 + exit 1 + fi +} + +setup_fixture positive +gate_json=$(run_gate) +jq -e ' + .schema == "monday.binance_usdm_reference_shadow_gate.v1" + and .passed == false and .production_eligible == false + and .duration_seconds == 2 and .artifact_count == 3 + and .service.active == true and .service.restart_count == 0 + and all(.artifacts[]; .canonical_readback and .content_rows_verified) +' "$gate_json" >/dev/null +[[ ! -e ${gate_json%/gate.json}/PASSED.sha256 ]] + +if grep -Eq 'systemctl[[:space:]]+(start|stop|restart|enable|disable|daemon-reload)' "$runner"; then + printf '%s\n' 'host gate must not mutate systemd service state' >&2 + exit 1 +fi + +setup_fixture tampered +printf 'tampered\n' >>"$latest_data" +expect_failure tampered-data + +setup_fixture symlink +mv "$latest_manifest" "$latest_manifest.real" +ln -s "$latest_manifest.real" "$latest_manifest" +expect_failure symlinked-manifest + +setup_fixture invocation +jq '.replace_after_first=true' "$state" >"$state.tmp" && mv "$state.tmp" "$state" +expect_failure replaced-invocation + +setup_fixture health +jq '.total_api_errors=1' "$spool/health.json" >"$spool/health.tmp" \ + && mv "$spool/health.tmp" "$spool/health.json" +expect_failure historical-api-error + +setup_fixture discontinuity +second_manifest=$(find "$spool" -name reference.ndjson.manifest.json -type f | sort | sed -n '2p') +jq ".observed_at_ns=$((start_ns + 91000000000))" "$second_manifest" \ + >"$second_manifest.tmp" && mv "$second_manifest.tmp" "$second_manifest" +third_manifest=$(find "$spool" -name reference.ndjson.manifest.json -type f | sort | tail -n 1) +jq ".observed_at_ns=$((start_ns + 92000000000))" "$third_manifest" \ + >"$third_manifest.tmp" && mv "$third_manifest.tmp" "$third_manifest" +test_grace_seconds=100 +expect_failure artifact-discontinuity +unset test_grace_seconds + +setup_fixture verifier +printf 'drift\n' >>"$release/binance-usdm-reference-artifact-verifier" +expect_failure verifier-sha-drift + +printf '%s\n' 'Binance USD-M reference host gate tests passed' diff --git a/deployment/aliyun/test-binance-usdm-reference-release-contract.sh b/deployment/aliyun/test-binance-usdm-reference-release-contract.sh index 0c94720e5..2ad58cb4c 100755 --- a/deployment/aliyun/test-binance-usdm-reference-release-contract.sh +++ b/deployment/aliyun/test-binance-usdm-reference-release-contract.sh @@ -18,5 +18,6 @@ grep -F -- '/usr/local/bin/binance-usdm-reference-artifact-verifier' "$dockerfil grep -F -- 'artifact/binance-usdm-reference-artifact-verifier' "$workflow" >/dev/null grep -F -- 'binance-usdm-reference-artifact-verifier.sha256' "$workflow" >/dev/null grep -F -- 'verifier:{file:"binance-usdm-reference-artifact-verifier"' "$workflow" >/dev/null +grep -F -- 'binance-usdm-reference-shadow-gate.sh' "$workflow" >/dev/null printf '%s\n' 'Binance USD-M reference release contract tests passed' diff --git a/deployment/aliyun/test-binance-usdm-reference-shadow-gate.sh b/deployment/aliyun/test-binance-usdm-reference-shadow-gate.sh index 50618068e..aec875d60 100755 --- a/deployment/aliyun/test-binance-usdm-reference-shadow-gate.sh +++ b/deployment/aliyun/test-binance-usdm-reference-shadow-gate.sh @@ -102,6 +102,7 @@ grep -Fq 'ExecStart=/opt/monday/releases/binance-usdm-reference-collector/%i/bin grep -Fq -- '--output-root /data/monday/spool/binance-usdm-reference-shadow/%i' "$service" grep -Fq 'ReadWritePaths=/data/monday/spool/binance-usdm-reference-shadow/%i' "$service" grep -Fq 'binance-usdm-reference-collector-shadow@.service' "$workflow" +grep -Fq 'binance-usdm-reference-shadow-gate.sh' "$workflow" grep -Fq 'binance-usdm-reference-shadow-gate-policy.jq' "$workflow" grep -Fq 'binance-usdm-reference-control-assets.sha256' "$workflow" grep -Fq 'binance-usdm-reference-control.tar.gz' "$workflow" @@ -115,6 +116,7 @@ if grep -Fq 'binance-usdm-reference-' <<<"$shared_assets"; then exit 1 fi grep -Fq 'binance-usdm-reference-collector-shadow@.service' <<<"$reference_assets" +grep -Fq 'binance-usdm-reference-shadow-gate.sh' <<<"$reference_assets" grep -Fq 'binance-usdm-reference-shadow-gate-policy.jq' <<<"$reference_assets" grep -Fq 'control_manifest:{file:"binance-usdm-reference-control-assets.sha256"' \ <<<"$reference_release"