diff --git a/.github/workflows/acr-publish.yml b/.github/workflows/acr-publish.yml index 11b65e143..a30e844bf 100644 --- a/.github/workflows/acr-publish.yml +++ b/.github/workflows/acr-publish.yml @@ -26,13 +26,13 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 - name: Log in to ACR - uses: docker/login-action@v3 + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 with: registry: ${{ vars.ACR_REGISTRY }} username: ${{ vars.ACR_USERNAME }} @@ -40,13 +40,15 @@ jobs: - name: Build and push id: build - uses: docker/build-push-action@v6 + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 with: context: rust_hft file: ${{ matrix.file }} platforms: linux/amd64 push: true - tags: ${{ vars.ACR_REGISTRY }}/wildcard0923/${{ matrix.repository }}:${{ github.sha }} + tags: | + ${{ vars.ACR_REGISTRY }}/wildcard0923/${{ matrix.repository }}:${{ github.sha }} + ${{ vars.ACR_REGISTRY }}/wildcard0923/${{ matrix.repository }}:run-${{ github.run_id }}-${{ github.run_attempt }} cache-from: type=gha,scope=acr-${{ matrix.repository }} cache-to: type=gha,mode=max,scope=acr-${{ matrix.repository }} @@ -54,6 +56,86 @@ jobs: run: | echo '${{ matrix.repository }}=${{ vars.ACR_REGISTRY }}/wildcard0923/${{ matrix.repository }}@${{ steps.build.outputs.digest }}' >> "$GITHUB_STEP_SUMMARY" + - name: Package bare-metal trading host contract + if: matrix.repository == 'hft-trading' + env: + IMAGE_REPOSITORY: ${{ vars.ACR_REGISTRY }}/wildcard0923/hft-trading + IMAGE_DIGEST: ${{ steps.build.outputs.digest }} + run: | + set -euo pipefail + mkdir -p trading-host-artifact + trading_control_assets=( + hft-trading-ecs.service + trading-ecs-hostctl.sh + trading-ecs-paper-shadow-policy.jq + trading-ecs-runtime.sh + ) + ( + cd deployment/aliyun + sha256sum "${trading_control_assets[@]}" \ + > ../../trading-host-artifact/trading-ecs-control-assets.sha256 + tar --sort=name --mtime='UTC 1970-01-01' \ + --owner=0 --group=0 --numeric-owner \ + -czf ../../trading-host-artifact/trading-ecs-control.tar.gz \ + "${trading_control_assets[@]}" + ) + ( + cd trading-host-artifact + published_registry=${IMAGE_REPOSITORY%%/*} + repository_path=${IMAGE_REPOSITORY#*/} + case "$published_registry" in + *-vpc.ap-northeast-1.personal.cr.aliyuncs.com) + vpc_registry=$published_registry + published_registry=${published_registry/-vpc.ap-northeast-1.personal.cr.aliyuncs.com/.ap-northeast-1.personal.cr.aliyuncs.com} + ;; + *.ap-northeast-1.personal.cr.aliyuncs.com) + vpc_registry=${published_registry/.ap-northeast-1.personal.cr.aliyuncs.com/-vpc.ap-northeast-1.personal.cr.aliyuncs.com} + ;; + *) + echo "ACR_REGISTRY must be a Tokyo Personal Edition endpoint" >&2 + exit 1 + ;; + esac + published_repository="$published_registry/$repository_path" + vpc_repository="$vpc_registry/$repository_path" + control_manifest_sha=$(sha256sum trading-ecs-control-assets.sha256 \ + | awk '{print $1}') + control_archive_sha=$(sha256sum trading-ecs-control.tar.gz \ + | awk '{print $1}') + jq -S -n \ + --arg source_revision '${{ github.sha }}' \ + --arg published_repository "$published_repository" \ + --arg image_repository "$vpc_repository" \ + --arg image_digest "$IMAGE_DIGEST" \ + --arg image_reference "$vpc_repository@$IMAGE_DIGEST" \ + --arg control_manifest_sha256 "$control_manifest_sha" \ + --arg control_archive_sha256 "$control_archive_sha" \ + '{schema:"monday.hft_trading_ecs_release.v1", + source_revision:$source_revision, + image:{published_repository:$published_repository, + repository:$image_repository,digest:$image_digest, + reference:$image_reference}, + control_manifest:{file:"trading-ecs-control-assets.sha256", + sha256:$control_manifest_sha256}, + control_archive:{file:"trading-ecs-control.tar.gz", + sha256:$control_archive_sha256}, + platform:{region:"ap-northeast-1",host_os:"ubuntu", + host_version:"26.04",architecture:"amd64", + orchestrator:"none"}}' \ + > hft-trading-ecs-release.json + sha256sum hft-trading-ecs-release.json \ + > hft-trading-ecs-release.json.sha256 + ) + + - name: Upload bare-metal trading host contract + if: matrix.repository == 'hft-trading' + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: hft-trading-ecs-linux-amd64-${{ github.sha }} + path: trading-host-artifact/ + if-no-files-found: error + retention-days: 90 + - name: Extract collector binaries for bare-metal ECS if: matrix.repository == 'binance-lob-archiver' env: @@ -125,9 +207,18 @@ jobs: - name: Upload collector binary artifact bundle if: matrix.repository == 'binance-lob-archiver' - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: binance-lob-archiver-linux-amd64-${{ github.sha }} path: artifact/ if-no-files-found: error retention-days: 7 + + - name: Remove ACR credentials + if: always() + env: + ACR_REGISTRY: ${{ vars.ACR_REGISTRY }} + run: | + docker logout "$ACR_REGISTRY" >/dev/null 2>&1 || true + docker_config_root=${DOCKER_CONFIG:-$HOME/.docker} + rm -f -- "$docker_config_root/config.json" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d52285618..1711b3cfc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -261,6 +261,7 @@ jobs: ../deployment/aliyun/test-rust-lob-control-plane.sh ../deployment/aliyun/test-rust-lob-release-adoption.sh ../deployment/aliyun/test-polymarket-raw-ops-control-plane.sh + ../deployment/aliyun/test-trading-ecs-host-contract.sh cargo test -p hft-collector systemd_memory_envelope_covers_measured_polymarket_cold_start --locked - name: Build focused packages diff --git a/deployment/aliyun/README.md b/deployment/aliyun/README.md index 1bd9becad..1b5758d2d 100644 --- a/deployment/aliyun/README.md +++ b/deployment/aliyun/README.md @@ -1,5 +1,10 @@ # Aliyun Binance data host +The public-data collector host described below is not the trading host. The +future Tokyo bare-ECS trading contract is documented separately in +[`TRADING_ECS_HOST.md`](TRADING_ECS_HOST.md). Trading does not run on ACK, and +staging its ACR image does not enable or start a runtime. + The Tokyo ECS runs the Rust Binance LOB archiver as two public-market-data services. Neither service has trading credentials or submits orders. The legacy Python collector, its systemd template, and its deployment tests have diff --git a/deployment/aliyun/TRADING_ECS_HOST.md b/deployment/aliyun/TRADING_ECS_HOST.md new file mode 100644 index 000000000..0b16aeadc --- /dev/null +++ b/deployment/aliyun/TRADING_ECS_HOST.md @@ -0,0 +1,220 @@ +# Tokyo bare-ECS trading host contract + +This contract prepares the existing `monday/hft-trading` ACR image for a future +dedicated Tokyo ECS. It does not create an ECS, start a stopped ECS, add a public +IP, start ClickHouse, use ACK, enable `LiveSmall`, inject a real credential, or +send an order. + +## Supported boundary + +- Host: dedicated Alibaba Cloud ECS in `ap-northeast-1`, Ubuntu 26.04, amd64. +- Orchestrator: none. The trading runtime uses Docker and a static systemd unit + directly on the host; it is not an ACK workload. +- Host authority: the instance must expose the exact RAM role + `MondayTradingEcsRole` through ECS metadata v2. Attach only the permissions + needed to retrieve the reviewed secret objects and operational evidence. +- Image: Tokyo Personal Edition ACR repository `wildcard0923/hft-trading`, pulled + through its VPC endpoint and pinned as `repository@sha256:<64-hex>`. +- Activation: signed, content-addressed Paper or Shadow deployment artifacts. + Both execute through the runtime's simulated Paper path. `StartLiveSmall` is + rejected by both this host policy and `hft-live`. +- Boot state: stopped. `hft-trading-ecs.service` deliberately has no `[Install]` + section and `Restart=no`; staging cannot start it and an ECS reboot cannot + auto-start it. + +Personal ACR is sufficient for the current three-image scale. The host contract +does not trust a tag at runtime: every publication also creates a unique +`run--` retention tag so a later rebuild of the same +source SHA cannot make the reviewed manifest unreferenced. The workflow records +the pushed OCI digest and packages a deterministic control archive into +`hft-trading-ecs-linux-amd64-`. The release manifest binds the exact +source revision, public publish repository, VPC pull repository, image digest, +control manifest, control archive, architecture, region, and host version. +Checkout, Buildx setup, ACR login, image build/push, and artifact upload all use +reviewed full commit SHAs instead of mutable action tags. The publish job logs +out of ACR and removes its Docker credential file on every completion path. + +## Credentials and RAM role + +Repository files never contain ACR or venue credentials. An operator-controlled +Cloud Assistant command may use the instance RAM role to retrieve reviewed +secret material, but it must materialize only these ephemeral files: + +```text +/run/monday/trading-secrets/runtime.env +/run/monday/trading-secrets/feedback-signing-key.hex +``` + +`/run` must resolve to `tmpfs`. Staging creates the dedicated non-login system +account and group `mondayhft`; it rejects UID or GID `1000`, so the ordinary +Ubuntu login account can never inherit trading-runtime access. It also rejects +supplementary group members and any other passwd entry reusing the runtime UID +or primary GID. The secret root +is root:`mondayhft` mode `0750`, and both `runtime.env` and the feedback key are +root:`mondayhft` mode `0440`. The feedback key contains one 32-byte Ed25519 key +encoded as 64 lowercase hex characters. The runtime environment must contain a +gRPC token of at least 32 characters with no edge whitespace plus a venue +API-key/secret pair, matching the Rust gRPC server's own validation. The wrapper rejects +missing, empty, persistent, path-traversing, overly broad, or malformed secret +inputs before Docker starts. The filesystem is checked separately for the root +and each file, so a disk-backed file bind-mounted below a tmpfs directory is not +accepted. It bind-mounts `runtime.env` read-only and imports +each validated assignment inside the unprivileged container without evaluation; +it never uses Docker `--env-file`, so secret values are not copied into Docker +inspect or daemon metadata. The deployment-envelope signing key is never a host +input. Per-activation state is owned by `mondayhft` mode `0700`. Container and +systemd core dumps are disabled so process-environment credentials cannot be +persisted through a crash dump. + +The ACR login password is a separate, root-owned mode-`0400` one-line direct +child of the canonical mode-`0700` tmpfs directory `/run/monday/acr-auth/`. +Symlinks, `..`, alternate spellings, nested paths, and disk-backed files are +rejected. It is used only as +`docker login --password-stdin` with a one-run `DOCKER_CONFIG` on that same +tmpfs. The host then logs out and deletes the temporary Docker configuration; an +exit trap retries both actions on every failure path. Rotate the currently +exposed registry password before this path is used on a real host and update the +GitHub `ACR_PASSWORD` secret; never pass it in a command argument. + +## Activation bundle + +The operator supplies an absolute, canonical, root-owned activation directory +below the canonical activation root. Runtime validation walks every component +from `/` through the activation root to the final directory; every component +must be a real root-owned directory, never a symlink, with no group/world write +bit. `activation.sha256` must list every other regular file exactly once in +sorted order and at least contain: + +```text +config/system.yaml +deployment/bundle.json +deployment/envelope.json +deployment/policy.json +deployment/trusted-keys.json +``` + +Formula assets referenced by `bundle.json` live under the same directory +and must also be in `activation.sha256`. Only the root manifest excludes itself; +a nested file also named `activation.sha256` is ordinary bundle content and must +be listed and hashed. The signed envelope must contain exactly one JSON value, +exactly one `LoadFactor`, and exactly one of `StartPaper` or +`StartShadow`. The runtime policy must authorize that same artifact and the same +start intent, carry the matching approval class, and be non-paused. +`LoadAllocatorPolicy`, unknown intents, live-small intent, and live-small +approval all fail closed. `hft-live` still performs the authoritative signature, +policy, hash, scope, nonce, risk, and bundle checks. + +## Staging and cutover + +Download the release artifact for the reviewed source SHA, verify its GitHub +artifact provenance, then copy it into a root-owned, non-group/world-writable +child directory under `/opt/monday/incoming/hft-trading/`. The GitHub artifact is +retained for 90 days; the ACR digest itself remains the durable runtime +publication. On the stopped future ECS: + +```bash +sudo deployment/aliyun/trading-ecs-hostctl.sh stage \ + /opt/monday/incoming/hft-trading/ \ + '' \ + /run/monday/acr-auth/password +sudo rm -f /run/monday/acr-auth/password +``` + +Before it pulls an image, creates an account, or installs a file, `stage` rejects +any existing active runtime or unit state other than explicit `static`, +`disabled`, or a proven absent unit; `enabled`, `linked`, `alias`, `indirect`, +`generated`, and ambiguous states fail closed. It then verifies ECS metadata v2, +Tokyo, Ubuntu 26.04, amd64, absence of kubelet, +RAM role, release schema/checksums, the deterministic control archive, and the +pulled `RepoDigests`. It creates or validates the dedicated `mondayhft` account, +installs the unit, and requires `systemctl is-enabled` to report the exact +`static` state (systemd returns success for this state). It records `stage.json` +plus an adjacent `STAGED.sha256`; it never writes the current activation pointer +and never starts or enables the service. The installed runtime, hostctl, policy, +and systemd unit must each hash to the selected release's four-entry control +manifest. `ExecStartPre` repeats that binding and revalidates Ubuntu 26.04, +amd64, Tokyo metadata-v2 identity, the exact RAM role, and the absence of kubelet +on every manual start; a host or control-plane drift therefore cannot start the +container. Staging and `ExecStartPre` also reject every +`monday-hft-trading.service.d` directory and require systemd's effective +`FragmentPath`, empty `DropInPaths`, static state, `Restart=no`, empty +`ExecStartPost`, and exact preflight/run/stop command vectors to match the +selected unit. A stale or newly injected drop-in therefore cannot override the +hashed unit silently. + +After independent review of the signed activation and ephemeral secret injection: + +```bash +sudo /usr/local/sbin/monday-hft-trading-hostctl cutover \ + 'crpi-INSTANCE-vpc.ap-northeast-1.personal.cr.aliyuncs.com/wildcard0923/hft-trading@sha256:' \ + '' \ + /opt/monday/activations/ +``` + +Stage, cutover, and rollback share a non-blocking host lock, and every evidence +run uses an exclusively created directory. Cutover is accepted only while the +unit and exact-name Docker container are absent. It reruns the complete +preflight, starts the exact locally staged digest with `--pull never`, freezes +the systemd `InvocationID`, main PID, restart count, Docker container ID, running +state, image digest, and candidate-owned loopback ports. Docker publishes the +candidate's metrics port `9090` and authenticated gRPC port `9092` onto separate +random `127.0.0.1` host ports; neither service is publicly exposed, and health +never trusts an unrelated process already listening on a conventional host +port. Container identity acquisition is bounded to tolerate normal Docker port +publication latency. The gate rechecks that complete identity around two clean +`/health` and `/readiness` samples. Docker stop uses `SIGINT`, matching +`hft-live`'s graceful shutdown path so cancellation and reconciliation run +before the container is removed. It +rechecks once more immediately before atomically +committing the single-file `PASSED.sha256` marker. No marker means no successful +cutover; an uncommitted JSON file is explicitly marked `cutover.unconfirmed.json`. + +On any failure or shell `EXIT`, `TERM`, or `INT` after cutover becomes armed, the +candidate is stopped, the prior non-secret pointer is restored, and an atomic +`FAILED.sha256` evidence marker is written. Cleanup ignores a second HUP, INT, +or TERM after it begins. `FAILED.sha256` is emitted only after both stop and +pointer restoration are proven; otherwise the non-canonical +`EMERGENCY_FAILED_OPEN.sha256` marker blocks trading and demands immediate manual +recovery. (`SIGKILL` and host power loss cannot run userspace cleanup; absence of +`PASSED.sha256` still fails closed.) The prior +runtime is intentionally not auto-restarted: its nonce is consumed, so a +rollback restart requires a newly signed envelope and nonce. Explicit rollback +has the same stopped-state rule: + +```bash +sudo /usr/local/sbin/monday-hft-trading-hostctl rollback \ + /var/lib/monday/evidence/hft-trading/cutover/ +``` + +Successful cutover evidence includes root-only mode-`0400`, content-addressed +candidate and previous pointer snapshots and their hashes. Rollback first proves +that `PASSED.sha256` commits +that evidence, the current pointer still equals the candidate snapshot, and the +active systemd invocation, PID, restart count, container ID, digest, and bound +ports still equal the cutover identity. It repeats pointer and runtime identity +checks immediately before stop. Only then does it atomically append a +`PASSED.rollback-pending.sha256` revocation marker, stop the runtime, restore the +prior pointer if one existed, and append `PASSED.rolled-back.sha256` plus +content-addressed rollback evidence. The original `PASSED.sha256` remains +immutable historical evidence; either rollback marker revokes it as current +authorization. Legacy, tampered, or stale evidence cannot stop a newer runtime. +An interruption leaves the rollback-pending revocation visible. Rollback cannot +enable the service or resume trading. + +## Offline verification + +```bash +deployment/aliyun/test-trading-ecs-host-contract.sh +cargo test --manifest-path rust_hft/Cargo.toml \ + -p hft-live --no-default-features --test deployment_envelope --locked +cargo test --manifest-path rust_hft/Cargo.toml \ + -p hft-live --no-default-features --test deployment_artifacts --locked +``` + +The shell contract tests prove runtime rejection of tags, +public/non-Tokyo/wrong-repo references, multiple envelope JSON values, nested +unhashed files, mismatched Paper/Shadow intents, allocator/live-small authority, +drifted control assets, stale rollback lineage, and absent credentials. They +also pin the static unit, no-restart behavior, immutable ACR workflow output, +and rollback-marker semantics. These are offline proofs only; they do not +provision a host or prove venue acceptance. diff --git a/deployment/aliyun/hft-trading-ecs.service b/deployment/aliyun/hft-trading-ecs.service new file mode 100644 index 000000000..70b3c3885 --- /dev/null +++ b/deployment/aliyun/hft-trading-ecs.service @@ -0,0 +1,32 @@ +[Unit] +Description=Monday digest-pinned HFT Paper/Shadow runtime (bare ECS) +Documentation=https://github.com/proerror77/monday/tree/main/deployment/aliyun +Requires=docker.service +After=docker.service network-online.target +Wants=network-online.target +ConditionPathExists=/etc/monday/hft-trading-current.env +StartLimitIntervalSec=0 + +[Service] +Type=simple +User=root +Group=root +Environment=HFT_CURRENT_FILE=/etc/monday/hft-trading-current.env +ExecStartPre=/usr/local/libexec/monday-hft-trading-runtime preflight +ExecStart=/usr/local/libexec/monday-hft-trading-runtime run +ExecStop=/usr/local/libexec/monday-hft-trading-runtime stop +ExecStopPost=/usr/local/libexec/monday-hft-trading-runtime ensure-stopped +Restart=no +TimeoutStartSec=180 +TimeoutStopSec=75 +KillMode=process +UMask=0077 +LimitCORE=0 +NoNewPrivileges=yes +PrivateTmp=yes +ProtectHome=yes +ProtectSystem=strict +ReadWritePaths=/run/monday /var/lib/monday /var/run/docker.sock + +# Deliberately no [Install] section. The service is static and cannot start at +# boot. A reviewed hostctl cutover is the only supported start path. diff --git a/deployment/aliyun/test-trading-ecs-host-contract.sh b/deployment/aliyun/test-trading-ecs-host-contract.sh new file mode 100755 index 000000000..52529230c --- /dev/null +++ b/deployment/aliyun/test-trading-ecs-host-contract.sh @@ -0,0 +1,1297 @@ +#!/usr/bin/env bash +# shellcheck disable=SC1090,SC2016,SC2034,SC2094,SC2251,SC2317,SC2329 +set -Eeuo pipefail + +export LC_ALL=C + +SCRIPT_DIR=$(cd -- "$(dirname -- "$0")" && pwd) +readonly SCRIPT_DIR +readonly HOSTCTL=$SCRIPT_DIR/trading-ecs-hostctl.sh +readonly RUNTIME=$SCRIPT_DIR/trading-ecs-runtime.sh +readonly POLICY=$SCRIPT_DIR/trading-ecs-paper-shadow-policy.jq +readonly UNIT=$SCRIPT_DIR/hft-trading-ecs.service +readonly WORKFLOW=$SCRIPT_DIR/../../.github/workflows/acr-publish.yml + +for command in awk chmod cp find grep jq ln mkfifo mktemp rm sed sha256sum shellcheck sort; do + command -v "$command" >/dev/null 2>&1 \ + || { printf 'missing trading host contract test dependency: %s\n' "$command" >&2; exit 2; } +done + +shellcheck "$HOSTCTL" "$RUNTIME" "$0" +bash -n "$HOSTCTL" "$RUNTIME" "$0" +if printf '{}\n' | jq -e -s -f "$POLICY" --argjson policy '[]' \ + >/dev/null 2>&1; then + printf 'empty policy unexpectedly passed\n' >&2 + exit 1 +fi + +tmp_dir=$(mktemp -d "$SCRIPT_DIR/.host-contract-test.XXXXXX") +tmp_dir=$(cd -- "$tmp_dir" && pwd -P) +trap 'chmod -R u+w "$tmp_dir" 2>/dev/null || true; rm -rf "$tmp_dir"' EXIT + +valid_image="crpi-ygobwehhof7qs9m3-vpc.ap-northeast-1.personal.cr.aliyuncs.com/wildcard0923/hft-trading@sha256:$(printf 'a%.0s' {1..64})" + +( + source "$HOSTCTL" + valid_image_reference "$valid_image" + ! valid_image_reference "${valid_image%@*}:latest" + ! valid_image_reference "${valid_image%@*}:2b82d590" + ! valid_image_reference "${valid_image/-vpc/}" + ! valid_image_reference "${valid_image/ap-northeast-1/ap-southeast-1}" + ! valid_image_reference "${valid_image/hft-trading/research-runner}" + ! valid_image_reference "${valid_image%@*}:release@${valid_image##*@}" +) + +# A systemd static unit reports success from is-enabled. Accept exactly the +# textual static state, while rejecting enabled, linked, generated, or missing. +( + source "$HOSTCTL" + systemctl_state=static + systemctl() { + [[ $1 == is-enabled ]] || return 2 + [[ $systemctl_state != missing ]] || return 1 + printf '%s\n' "$systemctl_state" + } + assert_service_static + systemctl_state=enabled + ! assert_service_static + systemctl_state=linked + ! assert_service_static + systemctl_state=missing + ! assert_service_static +) + +# The selected unit file is insufficient by itself: systemd's loaded fragment, +# drop-ins, and effective command vectors must remain exactly fail-closed. +( + source "$HOSTCTL" + unit_path=$tmp_dir/effective.service + runtime_program=/usr/local/libexec/monday-hft-trading-runtime + mock_fragment=$unit_path + mock_dropins= + mock_restart=no + mock_start_post= + mock_start_pre="{ path=$runtime_program ; argv[]=$runtime_program preflight ; ignore_errors=no ; }" + mock_start="{ path=$runtime_program ; argv[]=$runtime_program run ; ignore_errors=no ; }" + mock_stop="{ path=$runtime_program ; argv[]=$runtime_program stop ; ignore_errors=no ; }" + mock_stop_post="{ path=$runtime_program ; argv[]=$runtime_program ensure-stopped ; ignore_errors=no ; }" + systemctl() { + case $1 in + is-enabled) printf 'static\n' ;; + show) + case ${3#--property=} in + FragmentPath) printf '%s\n' "$mock_fragment" ;; + DropInPaths) printf '%s\n' "$mock_dropins" ;; + Restart) printf '%s\n' "$mock_restart" ;; + ExecStartPost) printf '%s\n' "$mock_start_post" ;; + ExecStartPre) printf '%s\n' "$mock_start_pre" ;; + ExecStart) printf '%s\n' "$mock_start" ;; + ExecStop) printf '%s\n' "$mock_stop" ;; + ExecStopPost) printf '%s\n' "$mock_stop_post" ;; + *) return 2 ;; + esac + ;; + *) return 2 ;; + esac + } + assert_effective_service_contract "$unit_path" "$runtime_program" + mock_start_pre="{ path=$runtime_program ; argv[]=$runtime_program preflight ; ignore_errors=yes ; }" + ! assert_effective_service_contract "$unit_path" "$runtime_program" + mock_start_pre="{ path=$runtime_program ; argv[]=$runtime_program preflight ; ignore_errors=no ; }" + mock_dropins=/etc/systemd/system/monday-hft-trading.service.d/stale.conf + mock_start="{ path=/bin/false ; argv[]=/bin/false ; ignore_errors=no ; }" + ! assert_effective_service_contract "$unit_path" "$runtime_program" + mock_dropins= + ! assert_effective_service_contract "$unit_path" "$runtime_program" + mock_start="{ path=$runtime_program ; argv[]=$runtime_program run ; ignore_errors=no ; }" + mock_start_post="{ path=/bin/false ; argv[]=/bin/false ; ignore_errors=no ; }" + ! assert_effective_service_contract "$unit_path" "$runtime_program" + mock_start_post= + mock_restart=always + ! assert_effective_service_contract "$unit_path" "$runtime_program" + mock_restart=no + mock_fragment=/run/systemd/system/monday-hft-trading.service + ! assert_effective_service_contract "$unit_path" "$runtime_program" + dropin_root=$tmp_dir/dropin-root + mkdir -p "$dropin_root/$SERVICE.d" + ! assert_no_service_dropin_directories "$dropin_root" +) + +# Runtime carries the same effective-unit check because it is the actual +# ExecStartPre boundary, not merely a staging-time assertion. +( + source "$RUNTIME" + unit_path=/etc/systemd/system/monday-hft-trading.service + runtime_program=/usr/local/libexec/monday-hft-trading-runtime + mock_dropins= + systemctl() { + case $1 in + is-enabled) printf 'static\n' ;; + show) + case ${3#--property=} in + FragmentPath) printf '%s\n' "$unit_path" ;; + DropInPaths) printf '%s\n' "$mock_dropins" ;; + Restart) printf 'no\n' ;; + ExecStartPost) printf '\n' ;; + ExecStartPre|ExecStart|ExecStop|ExecStopPost) + case ${3#--property=} in + ExecStartPre) argument=preflight ;; + ExecStart) argument=run ;; + ExecStop) argument=stop ;; + ExecStopPost) argument=ensure-stopped ;; + esac + printf '{ path=%s ; argv[]=%s %s ; ignore_errors=no ; }\n' \ + "$runtime_program" "$runtime_program" "$argument" + ;; + *) return 2 ;; + esac + ;; + *) return 2 ;; + esac + } + assert_effective_service_contract "$unit_path" "$runtime_program" + mock_dropins=/run/systemd/system/monday-hft-trading.service.d/stale.conf + ! assert_effective_service_contract "$unit_path" "$runtime_program" +) + +( + source "$HOSTCTL" + unit_path=$tmp_dir/pre-stage.service + service_state=static + service_status=0 + systemctl() { + [[ $1 == is-enabled ]] || return 2 + [[ -z $service_state ]] || printf '%s\n' "$service_state" + return "$service_status" + } + assert_service_boot_disabled_before_stage "$unit_path" + service_state=disabled + service_status=1 + assert_service_boot_disabled_before_stage "$unit_path" + service_state=not-found + service_status=1 + assert_service_boot_disabled_before_stage "$unit_path" + service_state=enabled + service_status=0 + ! assert_service_boot_disabled_before_stage "$unit_path" + service_state=linked + ! assert_service_boot_disabled_before_stage "$unit_path" + service_state= + service_status=1 + : >"$unit_path" + ! assert_service_boot_disabled_before_stage "$unit_path" + rm "$unit_path" + ! assert_service_boot_disabled_before_stage "$unit_path" + service_state=static + ! assert_service_boot_disabled_before_stage "$unit_path" + service_state=disabled + service_status=0 + ! assert_service_boot_disabled_before_stage "$unit_path" +) + +( + source "$HOSTCTL" + docker_state=healthy + docker_names= + docker() { + case "${1:-} ${2:-}" in + 'info ') [[ $docker_state == healthy ]] ;; + 'container ls') + [[ $docker_state != list-error ]] || return 1 + [[ $3 == --all && $4 == --filter \ + && $5 == 'name=^/monday-hft-trading$' && $6 == --format \ + && $7 == '{{.Names}}' ]] || return 2 + printf '%s\n' "$docker_names" + ;; + *) return 2 ;; + esac + } + assert_no_orphan_container + docker_names=monday-hft-trading + ! assert_no_orphan_container + docker_names= + docker_state=list-error + ! assert_no_orphan_container + docker_state=down + ! assert_no_orphan_container +) + +( + source "$RUNTIME" + valid_image_reference "$valid_image" + [[ $(image_digest_hex "$valid_image") == "$(printf 'a%.0s' {1..64})" ]] + ! valid_image_reference "${valid_image%@*}:latest" + validate_bare_host_identity_values ubuntu 26.04 amd64 ap-northeast-1 \ + MondayTradingEcsRole i-example123 + ! validate_bare_host_identity_values ubuntu 24.04 amd64 ap-northeast-1 \ + MondayTradingEcsRole i-example123 + ! validate_bare_host_identity_values ubuntu 26.04 amd64 ap-northeast-1 \ + WrongRole i-example123 +) + +# Exercise the complete start-time host check with metadata-v2 and systemd +# behavior mocked at their command boundaries. +( + source "$RUNTIME" + mock_os=ubuntu + mock_version=26.04 + mock_arch=amd64 + mock_region=ap-northeast-1 + mock_role=MondayTradingEcsRole + mock_instance=i-example123 + kubelet_active=false + source() { + [[ $1 == /etc/os-release ]] || return 2 + ID=$mock_os + VERSION_ID=$mock_version + } + systemctl() { + [[ $1 == is-active && $2 == --quiet && $3 == kubelet ]] || return 2 + [[ $kubelet_active == true ]] + } + dpkg() { + [[ $1 == --print-architecture ]] || return 2 + printf '%s\n' "$mock_arch" + } + metadata_token() { printf 'metadata-v2-token\n'; } + metadata_get() { + case $2 in + region-id) printf '%s\n' "$mock_region" ;; + ram/security-credentials/) printf '%s\n' "$mock_role" ;; + instance-id) printf '%s\n' "$mock_instance" ;; + *) return 2 ;; + esac + } + verify_bare_tokyo_host + mock_region=ap-southeast-1 + ! verify_bare_tokyo_host + mock_region=ap-northeast-1 + mock_role=WrongRole + ! verify_bare_tokyo_host + mock_role=MondayTradingEcsRole + mock_version=24.04 + ! verify_bare_tokyo_host + mock_version=26.04 + mock_arch=arm64 + ! verify_bare_tokyo_host + mock_arch=amd64 + kubelet_active=true + ! verify_bare_tokyo_host +) + +# Absence is accepted only with an independently healthy Docker daemon. A +# daemon error, a stopped orphan, or an exact-name listing residue all fail. +( + source "$RUNTIME" + docker_scenario=daemon_down + docker() { + if [[ $1 == container && $2 == inspect ]]; then + if [[ $docker_scenario == stopped ]]; then + printf 'false\n' + return 0 + fi + return 1 + fi + if [[ $1 == info ]]; then + [[ $docker_scenario != daemon_down ]] + return + fi + if [[ $1 == container && $2 == ls ]]; then + [[ $docker_scenario == stale_listing ]] && printf '%s\n' "$CONTAINER_NAME" + return 0 + fi + return 2 + } + ! assert_stopped + docker_scenario=stopped + ! assert_stopped + docker_scenario=stale_listing + ! assert_stopped + docker_scenario=absent + assert_stopped +) + +acr_auth_root=$tmp_dir/acr-auth +mkdir "$acr_auth_root" +printf '%s\n' example_registry_password >"$acr_auth_root/password" +chmod 0700 "$acr_auth_root" +chmod 0400 "$acr_auth_root/password" +( + source "$HOSTCTL" + EXPECTED_ROOT_UID=$(id -u) + ACR_AUTH_ROOT=$acr_auth_root + findmnt() { printf '%s\n' tmpfs; } + verify_acr_password_file "$acr_auth_root/password" + chmod 0600 "$acr_auth_root/password" + printf '\n' >"$acr_auth_root/password" + chmod 0400 "$acr_auth_root/password" + ! (verify_acr_password_file "$acr_auth_root/password" 2>/dev/null) + chmod 0600 "$acr_auth_root/password" + printf '%s\n' example_registry_password >"$acr_auth_root/password" + chmod 0400 "$acr_auth_root/password" + ! (verify_acr_password_file \ + "$acr_auth_root/../${acr_auth_root##*/}/password" 2>/dev/null) + findmnt() { printf '%s\n' ext4; } + ! (verify_acr_password_file "$acr_auth_root/password" 2>/dev/null) + findmnt() { + local argument target= + for argument in "$@"; do target=$argument; done + if [[ $target == "$acr_auth_root/password" ]]; then + printf '%s\n' ext4 + else + printf '%s\n' tmpfs + fi + } + ! (verify_acr_password_file "$acr_auth_root/password" 2>/dev/null) +) + +activation_dir=$tmp_dir/activation +mkdir -p "$activation_dir/config" "$activation_dir/deployment" +cat >"$activation_dir/config/system.yaml" <<'YAML' +version: "2.0" +venues: [] +YAML +printf '%s\n' '{"bundle_id":"example"}' >"$activation_dir/deployment/bundle.json" +cat >"$activation_dir/deployment/envelope.json" <<'JSON' +{"envelope":{"allowed_intent_types":["LoadFactor","StartShadow"],"approval_class":"Shadow"},"key_id":"example-key","signature":"example-signature"} +JSON +cat >"$activation_dir/deployment/policy.json" <<'JSON' +{"account_id":"example","venue":"binance","allowed_instruments":["BTCUSDT"],"allowed_intent_types":["LoadFactor","StartShadow"],"runtime_paused":false,"approvals":[{"approval_class":"Shadow"}]} +JSON +printf '%s\n' '{"example-key":"example-public-key"}' \ + >"$activation_dir/deployment/trusted-keys.json" +( + cd "$activation_dir" + find . -type f ! -path ./activation.sha256 -print \ + | sed 's#^./##' | LC_ALL=C sort \ + | while IFS= read -r path; do sha256sum "$path"; done \ + >activation.sha256 +) +chmod 0444 "$activation_dir/activation.sha256" + +# The cutover-owned candidate snapshot must remain root-readable by runtime +# preflight while exposing no group/world bits. This crosses the hostctl writer +# and runtime reader rather than testing either mode rule in isolation. +candidate_mode_file=$tmp_dir/candidate-mode.env +( + source "$HOSTCTL" + write_current_file "$candidate_mode_file" "$valid_image" \ + "$(printf 'b%.0s' {1..64})" "$activation_dir" \ + "$(printf 'c%.0s' {1..40})" + chmod 0400 "$candidate_mode_file" +) +( + source "$RUNTIME" + EXPECTED_ROOT_UID=$(id -u) + HFT_CURRENT_FILE=$candidate_mode_file + load_current + [[ $HFT_TRADING_IMAGE == "$valid_image" ]] + [[ $HFT_ACTIVATION_DIR == "$activation_dir" ]] +) +chmod 0444 "$candidate_mode_file" +if ( + source "$RUNTIME" + EXPECTED_ROOT_UID=$(id -u) + HFT_CURRENT_FILE=$candidate_mode_file + load_current +) >/dev/null 2>&1; then + printf 'group/world-readable candidate pointer unexpectedly passed\n' >&2 + exit 1 +fi + +control_root=$tmp_dir/control +mkdir "$control_root" +cp "$POLICY" "$control_root/" + +( + source "$RUNTIME" + EXPECTED_ROOT_UID=0 + stat_uid() { printf '0\n'; } + ACTIVATION_ROOT=$tmp_dir + CONTROL_ROOT=$control_root + validate_activation_manifest "$activation_dir" + validate_paper_shadow_authority "$activation_dir" + path_root=$tmp_dir/activation-path-components + unsafe_parent=$path_root/world-writable + mkdir -p "$unsafe_parent" + cp -R "$activation_dir" "$unsafe_parent/candidate" + chmod 0777 "$unsafe_parent" + ACTIVATION_ROOT=$path_root + ! validate_activation_manifest "$unsafe_parent/candidate" + real_parent=$path_root/real-parent + mkdir -p "$real_parent" + cp -R "$activation_dir" "$real_parent/candidate" + ln -s "$real_parent" "$path_root/symlink-parent" + ! validate_activation_manifest "$path_root/symlink-parent/candidate" + ACTIVATION_ROOT=$tmp_dir + mkdir "$activation_dir/nested" + printf 'nested checksum input\n' >"$activation_dir/nested/activation.sha256" + ! validate_activation_manifest "$activation_dir" + chmod 0644 "$activation_dir/activation.sha256" + ( + cd "$activation_dir" + find . -type f ! -path ./activation.sha256 -print \ + | sed 's#^./##' | LC_ALL=C sort \ + | while IFS= read -r path; do sha256sum "$path"; done \ + >activation.sha256 + ) + chmod 0444 "$activation_dir/activation.sha256" + validate_activation_manifest "$activation_dir" + cp "$activation_dir/deployment/envelope.json" "$tmp_dir/valid-envelope.json" + { + cat "$tmp_dir/valid-envelope.json" + cat "$tmp_dir/valid-envelope.json" + } >"$activation_dir/deployment/envelope.json" + ! validate_paper_shadow_authority "$activation_dir" + cp "$tmp_dir/valid-envelope.json" "$activation_dir/deployment/envelope.json" + jq '.envelope.allowed_intent_types = ["LoadFactor","StartLiveSmall"] | + .envelope.approval_class = "HumanApprovedLiveSmall"' \ + "$activation_dir/deployment/envelope.json" >"$tmp_dir/live-envelope.json" + cp "$tmp_dir/live-envelope.json" "$activation_dir/deployment/envelope.json" + ! validate_paper_shadow_authority "$activation_dir" +) + +# Restore Shadow authority, then prove policy-side live permission also fails. +cat >"$activation_dir/deployment/envelope.json" <<'JSON' +{"envelope":{"allowed_intent_types":["LoadFactor","StartShadow"],"approval_class":"Shadow"},"key_id":"example-key","signature":"example-signature"} +JSON +( + source "$RUNTIME" + CONTROL_ROOT=$control_root + jq '.envelope.allowed_intent_types += ["StartWithdraw"]' \ + "$activation_dir/deployment/envelope.json" >"$tmp_dir/unknown-start-envelope.json" + cp "$tmp_dir/unknown-start-envelope.json" "$activation_dir/deployment/envelope.json" + ! validate_paper_shadow_authority "$activation_dir" +) +cat >"$activation_dir/deployment/envelope.json" <<'JSON' +{"envelope":{"allowed_intent_types":["LoadFactor","StartShadow"],"approval_class":"Shadow"},"key_id":"example-key","signature":"example-signature"} +JSON +jq '.allowed_intent_types = ["LoadFactor","StartPaper"]' \ + "$activation_dir/deployment/policy.json" >"$tmp_dir/wrong-start-policy.json" +cp "$tmp_dir/wrong-start-policy.json" "$activation_dir/deployment/policy.json" +( + source "$RUNTIME" + CONTROL_ROOT=$control_root + ! validate_paper_shadow_authority "$activation_dir" +) +cat >"$activation_dir/deployment/policy.json" <<'JSON' +{"account_id":"example","venue":"binance","allowed_instruments":["BTCUSDT"],"allowed_intent_types":["LoadFactor","StartShadow"],"runtime_paused":false,"approvals":[{"approval_class":"Shadow"}]} +JSON +jq '.envelope.allowed_intent_types += ["LoadAllocatorPolicy"]' \ + "$activation_dir/deployment/envelope.json" >"$tmp_dir/allocator-envelope.json" +cp "$tmp_dir/allocator-envelope.json" "$activation_dir/deployment/envelope.json" +( + source "$RUNTIME" + CONTROL_ROOT=$control_root + ! validate_paper_shadow_authority "$activation_dir" +) +cat >"$activation_dir/deployment/envelope.json" <<'JSON' +{"envelope":{"allowed_intent_types":["LoadFactor","StartShadow"],"approval_class":"Shadow"},"key_id":"example-key","signature":"example-signature"} +JSON +jq '.allowed_intent_types += ["LoadAllocatorPolicy"]' \ + "$activation_dir/deployment/policy.json" >"$tmp_dir/allocator-policy.json" +cp "$tmp_dir/allocator-policy.json" "$activation_dir/deployment/policy.json" +( + source "$RUNTIME" + CONTROL_ROOT=$control_root + ! validate_paper_shadow_authority "$activation_dir" +) +cat >"$activation_dir/deployment/policy.json" <<'JSON' +{"account_id":"example","venue":"binance","allowed_instruments":["BTCUSDT"],"allowed_intent_types":["LoadFactor","StartShadow"],"runtime_paused":false,"approvals":[{"approval_class":"Shadow"}]} +JSON +jq '.envelope.allowed_intent_types = ["LoadModel","StartShadow"]' \ + "$activation_dir/deployment/envelope.json" >"$tmp_dir/model-envelope.json" +cp "$tmp_dir/model-envelope.json" "$activation_dir/deployment/envelope.json" +jq '.allowed_intent_types = ["LoadModel","StartShadow"]' \ + "$activation_dir/deployment/policy.json" >"$tmp_dir/model-policy.json" +cp "$tmp_dir/model-policy.json" "$activation_dir/deployment/policy.json" +( + source "$RUNTIME" + CONTROL_ROOT=$control_root + ! validate_paper_shadow_authority "$activation_dir" +) +cat >"$activation_dir/deployment/envelope.json" <<'JSON' +{"envelope":{"allowed_intent_types":["LoadFactor","StartShadow"],"approval_class":"Shadow"},"key_id":"example-key","signature":"example-signature"} +JSON +cat >"$activation_dir/deployment/policy.json" <<'JSON' +{"account_id":"example","venue":"binance","allowed_instruments":["BTCUSDT"],"allowed_intent_types":["LoadFactor","StartShadow"],"runtime_paused":false,"approvals":[{"approval_class":"Shadow"}]} +JSON +jq '.allowed_intent_types += ["StartLiveSmall"]' \ + "$activation_dir/deployment/policy.json" >"$tmp_dir/live-policy.json" +cp "$tmp_dir/live-policy.json" "$activation_dir/deployment/policy.json" +( + source "$RUNTIME" + CONTROL_ROOT=$control_root + ! validate_paper_shadow_authority "$activation_dir" +) + +secret_root=$tmp_dir/secrets +mkdir "$secret_root" +cat >"$secret_root/runtime.env" <<'ENV' +HFT_GRPC_AUTH_TOKEN=example_token_with_at_least_32_chars +HFT_SECRET_BINANCE_API_KEY=example_key +HFT_SECRET_BINANCE_SECRET=example_secret +ENV +printf '%s\n' "$(printf 'b%.0s' {1..64})" >"$secret_root/feedback-signing-key.hex" +chmod 0750 "$secret_root" +chmod 0440 "$secret_root/runtime.env" +chmod 0440 "$secret_root/feedback-signing-key.hex" +( + source "$RUNTIME" + EXPECTED_ROOT_UID=$(id -u) + getent() { + case $1 in + passwd) printf 'mondayhft:x:991:991::/nonexistent:/usr/sbin/nologin\n' ;; + group) printf 'mondayhft:x:991:\n' ;; + *) return 2 ;; + esac + } + id() { + [[ $1 == -g && $2 == mondayhft ]] || return 2 + printf '991\n' + } + stat_gid() { printf '991\n'; } + SECRET_ROOT=$secret_root + findmnt() { printf '%s\n' tmpfs; } + validate_runtime_secrets + chmod 0640 "$secret_root/runtime.env" + sed -i.bak \ + 's/example_token_with_at_least_32_chars/short_token/' \ + "$secret_root/runtime.env" + rm "$secret_root/runtime.env.bak" + chmod 0440 "$secret_root/runtime.env" + ! validate_runtime_secrets + chmod 0640 "$secret_root/runtime.env" + sed -i.bak \ + 's/short_token/example_token_with_at_least_32_chars/' \ + "$secret_root/runtime.env" + rm "$secret_root/runtime.env.bak" + chmod 0440 "$secret_root/runtime.env" + validate_runtime_secrets + chmod 0600 "$secret_root/runtime.env" + printf '%s\n' 'HFT_EXECUTION_MODE=live' >>"$secret_root/runtime.env" + chmod 0440 "$secret_root/runtime.env" + ! validate_runtime_secrets + chmod 0600 "$secret_root/runtime.env" + sed -i.bak '/^HFT_EXECUTION_MODE=/d' "$secret_root/runtime.env" + rm "$secret_root/runtime.env.bak" + chmod 0440 "$secret_root/runtime.env" + SECRET_ROOT="$tmp_dir/../${tmp_dir##*/}/secrets" + ! validate_runtime_secrets + SECRET_ROOT=$secret_root + findmnt() { + local argument target= + for argument in "$@"; do target=$argument; done + if [[ $target == "$secret_root/runtime.env" ]]; then + printf '%s\n' ext4 + else + printf '%s\n' tmpfs + fi + } + ! validate_runtime_secrets + findmnt() { printf '%s\n' tmpfs; } + rm "$secret_root/runtime.env" + ! validate_runtime_secrets +) + +# The container identity must be a dedicated, non-login system account. UID or +# GID 1000 (the ordinary Ubuntu login account) is never accepted. +( + source "$RUNTIME" + account_uid=991 + account_gid=991 + account_members= + duplicate_passwd= + duplicate_group= + getent() { + case $1 in + passwd) + printf 'mondayhft:x:%s:%s::/nonexistent:/usr/sbin/nologin\n' \ + "$account_uid" "$account_gid" + if [[ -z ${2:-} && -n $duplicate_passwd ]]; then + printf '%s\n' "$duplicate_passwd" + fi + ;; + group) + printf 'mondayhft:x:%s:%s\n' "$account_gid" "$account_members" + if [[ -z ${2:-} && -n $duplicate_group ]]; then + printf '%s\n' "$duplicate_group" + fi + ;; + *) return 2 ;; + esac + } + id() { + [[ $1 == -g && $2 == mondayhft ]] || return 2 + printf '%s\n' "$account_gid" + } + account_ids=$(runtime_account_ids) + [[ $account_ids == $'991\n991' ]] + account_uid=1000 + ! runtime_account_ids >/dev/null + account_uid=991 + account_gid=1000 + ! runtime_account_ids >/dev/null + account_gid=991 + account_members=ubuntu + ! runtime_account_ids >/dev/null + account_members= + duplicate_passwd='ubuntu:x:991:1000::/home/ubuntu:/bin/bash' + ! runtime_account_ids >/dev/null + duplicate_passwd='other:x:992:991::/nonexistent:/usr/sbin/nologin' + ! runtime_account_ids >/dev/null + duplicate_passwd= + duplicate_group='ubuntu-runtime:x:991:ubuntu' + ! runtime_account_ids >/dev/null +) + +( + source "$HOSTCTL" + account_uid=991 + account_gid=991 + account_members= + duplicate_passwd= + duplicate_group= + getent() { + case $1 in + passwd) + printf 'mondayhft:x:%s:%s::/nonexistent:/usr/sbin/nologin\n' \ + "$account_uid" "$account_gid" + if [[ -z ${2:-} && -n $duplicate_passwd ]]; then + printf '%s\n' "$duplicate_passwd" + fi + ;; + group) + printf 'mondayhft:x:%s:%s\n' "$account_gid" "$account_members" + if [[ -z ${2:-} && -n $duplicate_group ]]; then + printf '%s\n' "$duplicate_group" + fi + ;; + *) return 2 ;; + esac + } + groupadd() { return 2; } + useradd() { return 2; } + id() { + [[ $1 == -g && $2 == mondayhft ]] || return 2 + printf '%s\n' "$account_gid" + } + ensure_runtime_account + account_members=ubuntu + ! ensure_runtime_account + account_members= + account_uid=1000 + ! ensure_runtime_account + account_uid=991 + duplicate_passwd='ubuntu:x:991:1000::/home/ubuntu:/bin/bash' + ! ensure_runtime_account + duplicate_passwd= + duplicate_group='ubuntu-runtime:x:991:ubuntu' + ! ensure_runtime_account +) + +artifact_dir=$tmp_dir/artifact +mkdir "$artifact_dir" +control_assets=( + hft-trading-ecs.service + trading-ecs-hostctl.sh + trading-ecs-paper-shadow-policy.jq + trading-ecs-runtime.sh +) +( + cd "$SCRIPT_DIR" + sha256sum "${control_assets[@]}" \ + >"$artifact_dir/trading-ecs-control-assets.sha256" + tar -czf "$artifact_dir/trading-ecs-control.tar.gz" "${control_assets[@]}" +) +control_manifest_sha=$(sha256sum "$artifact_dir/trading-ecs-control-assets.sha256" \ + | awk '{print $1}') +control_archive_sha=$(sha256sum "$artifact_dir/trading-ecs-control.tar.gz" \ + | awk '{print $1}') + +selected_control_dir=$tmp_dir/selected-control +mkdir "$selected_control_dir" +cp "$UNIT" "$selected_control_dir/hft-trading-ecs.service" +cp "$HOSTCTL" "$selected_control_dir/trading-ecs-hostctl.sh" +cp "$POLICY" "$selected_control_dir/trading-ecs-paper-shadow-policy.jq" +cp "$RUNTIME" "$selected_control_dir/trading-ecs-runtime.sh" +chmod 0444 "$selected_control_dir/hft-trading-ecs.service" \ + "$selected_control_dir/trading-ecs-paper-shadow-policy.jq" +chmod 0555 "$selected_control_dir/trading-ecs-hostctl.sh" \ + "$selected_control_dir/trading-ecs-runtime.sh" +( + cd "$selected_control_dir" + sha256sum "${control_assets[@]}" >trading-ecs-control-assets.sha256 +) +chmod 0444 "$selected_control_dir/trading-ecs-control-assets.sha256" +( + source "$RUNTIME" + EXPECTED_ROOT_UID=$(id -u) + validate_selected_control_assets \ + "$selected_control_dir/trading-ecs-control-assets.sha256" \ + "$selected_control_dir/trading-ecs-runtime.sh" \ + "$selected_control_dir/trading-ecs-hostctl.sh" \ + "$selected_control_dir/trading-ecs-paper-shadow-policy.jq" \ + "$selected_control_dir/hft-trading-ecs.service" + chmod 0755 "$selected_control_dir/trading-ecs-hostctl.sh" + printf '\n' >>"$selected_control_dir/trading-ecs-hostctl.sh" + chmod 0555 "$selected_control_dir/trading-ecs-hostctl.sh" + ! validate_selected_control_assets \ + "$selected_control_dir/trading-ecs-control-assets.sha256" \ + "$selected_control_dir/trading-ecs-runtime.sh" \ + "$selected_control_dir/trading-ecs-hostctl.sh" \ + "$selected_control_dir/trading-ecs-paper-shadow-policy.jq" \ + "$selected_control_dir/hft-trading-ecs.service" + chmod 0755 "$selected_control_dir/trading-ecs-hostctl.sh" + cp "$HOSTCTL" "$selected_control_dir/trading-ecs-hostctl.sh" + chmod 0555 "$selected_control_dir/trading-ecs-hostctl.sh" + chmod 0644 "$selected_control_dir/trading-ecs-control-assets.sha256" + printf '%s trading-ecs-runtime.sh\n' "$(printf 'f%.0s' {1..64})" \ + >>"$selected_control_dir/trading-ecs-control-assets.sha256" + chmod 0444 "$selected_control_dir/trading-ecs-control-assets.sha256" + ! validate_selected_control_assets \ + "$selected_control_dir/trading-ecs-control-assets.sha256" \ + "$selected_control_dir/trading-ecs-runtime.sh" \ + "$selected_control_dir/trading-ecs-hostctl.sh" \ + "$selected_control_dir/trading-ecs-paper-shadow-policy.jq" \ + "$selected_control_dir/hft-trading-ecs.service" +) +cat >"$artifact_dir/hft-trading-ecs-release.json" <hft-trading-ecs-release.json.sha256 +) +( + source "$HOSTCTL" + EXPECTED_ROOT_UID=$(id -u) + ARTIFACT_ROOT=$tmp_dir + verify_release_manifest "$artifact_dir" + mkdir "$tmp_dir/traversal" + ! verify_release_manifest "$tmp_dir/traversal/../artifact" + extract_dir=$tmp_dir/extracted-control + mkdir "$extract_dir" + verify_control_bundle "$artifact_dir" "$extract_dir" + malicious_artifact=$tmp_dir/nonregular-control-artifact + malicious_source=$tmp_dir/nonregular-control-source + malicious_extract=$tmp_dir/nonregular-control-extract + cp -R "$artifact_dir" "$malicious_artifact" + mkdir "$malicious_source" "$malicious_extract" + cp "$UNIT" "$malicious_source/hft-trading-ecs.service" + cp "$HOSTCTL" "$malicious_source/trading-ecs-hostctl.sh" + cp "$POLICY" "$malicious_source/trading-ecs-paper-shadow-policy.jq" + mkfifo "$malicious_source/trading-ecs-runtime.sh" + ( + cd "$malicious_source" + COPYFILE_DISABLE=1 tar -czf \ + "$malicious_artifact/trading-ecs-control.tar.gz" "${control_assets[@]}" + ) + malicious_archive_sha=$(sha256sum \ + "$malicious_artifact/trading-ecs-control.tar.gz" | awk '{print $1}') + jq --arg sha "$malicious_archive_sha" '.control_archive.sha256 = $sha' \ + "$malicious_artifact/hft-trading-ecs-release.json" \ + >"$tmp_dir/nonregular-release.json" + cp "$tmp_dir/nonregular-release.json" \ + "$malicious_artifact/hft-trading-ecs-release.json" + ! verify_control_bundle "$malicious_artifact" "$malicious_extract" + cp "$artifact_dir/hft-trading-ecs-release.json" "$tmp_dir/valid-release.json" + jq '.image.published_repository = "crpi-different.ap-northeast-1.personal.cr.aliyuncs.com/wildcard0923/hft-trading"' \ + "$tmp_dir/valid-release.json" >"$artifact_dir/hft-trading-ecs-release.json" + ( + cd "$artifact_dir" + sha256sum hft-trading-ecs-release.json >hft-trading-ecs-release.json.sha256 + ) + ! verify_release_manifest "$artifact_dir" + cp "$tmp_dir/valid-release.json" "$artifact_dir/hft-trading-ecs-release.json" + ( + cd "$artifact_dir" + sha256sum hft-trading-ecs-release.json >hft-trading-ecs-release.json.sha256 + ) + jq '.image.reference = .image.published_repository + "@" + .image.digest' \ + "$artifact_dir/hft-trading-ecs-release.json" >"$tmp_dir/public-release.json" + cp "$tmp_dir/public-release.json" "$artifact_dir/hft-trading-ecs-release.json" + ( + cd "$artifact_dir" + sha256sum hft-trading-ecs-release.json >hft-trading-ecs-release.json.sha256 + ) + ! verify_release_manifest "$artifact_dir" +) + +# A process/container replacement after a clean health response must invalidate +# the frozen systemd/container identity instead of producing success evidence. +( + source "$HOSTCTL" + identity_state=$tmp_dir/identity-state + curl_count=$tmp_dir/identity-curl-count + printf '%s\n' original >"$identity_state" + printf '0\n' >"$curl_count" + systemctl() { + case $1 in + is-active) return 0 ;; + is-enabled) printf 'static\n'; return 0 ;; + show) + case $3 in + --property=InvocationID) printf '%s\n' "$(printf '1%.0s' {1..32})" ;; + --property=MainPID) printf '4242\n' ;; + --property=NRestarts) printf '0\n' ;; + *) return 2 ;; + esac + ;; + *) return 2 ;; + esac + } + docker() { + [[ $1 == container && $2 == inspect && $3 == --format ]] || return 2 + case $4 in + '{{.Id}}') + if [[ $(<"$identity_state") == original ]]; then + printf '%s\n' "$(printf '2%.0s' {1..64})" + else + printf '%s\n' "$(printf '3%.0s' {1..64})" + fi + ;; + '{{.State.Running}}') printf 'true\n' ;; + '{{.Config.Image}}') printf '%s\n' "$valid_image" ;; + '{{with (index (index .NetworkSettings.Ports "9090/tcp") 0)}}{{.HostIp}}:{{.HostPort}}{{end}}') + printf '127.0.0.1:49152\n' + ;; + '{{with (index (index .NetworkSettings.Ports "9092/tcp") 0)}}{{.HostIp}}:{{.HostPort}}{{end}}') + printf '127.0.0.1:49153\n' + ;; + *) return 2 ;; + esac + } + curl() { + local count + count=$(<"$curl_count") + count=$((count + 1)) + printf '%s\n' "$count" >"$curl_count" + if (( count == 2 )); then + printf '%s\n' replaced >"$identity_state" + fi + return 0 + } + grpc_endpoint_ready() { return 0; } + sleep() { return 0; } + frozen_identity=$(capture_runtime_identity "$valid_image") + [[ $frozen_identity == *'|127.0.0.1:49152|127.0.0.1:49153' ]] + ! wait_for_health "$valid_image" "$frozen_identity" +) + +# Container creation and random port publication are asynchronous after the +# systemd start succeeds, so identity acquisition is bounded rather than single-shot. +( + source "$HOSTCTL" + identity_attempts=$tmp_dir/identity-attempts + printf '0\n' >"$identity_attempts" + capture_runtime_identity() { + local attempts + attempts=$(<"$identity_attempts") + attempts=$((attempts + 1)) + printf '%s\n' "$attempts" >"$identity_attempts" + (( attempts >= 3 )) || return 1 + printf 'ready-identity\n' + } + sleep() { return 0; } + [[ $(wait_for_runtime_identity "$valid_image") == ready-identity ]] + [[ $(<"$identity_attempts") == 3 ]] +) + +# Pointer installation must not let a failed durable write be masked by later +# successful commands when the helper itself is called from an `if !` context. +( + source "$HOSTCTL" + install() { return 1; } + sync() { return 0; } + mv() { return 0; } + ! install_current_pointer "$tmp_dir/not-used" "$tmp_dir/current.env" +) + +# Failure finalization is the common explicit, EXIT, TERM, and INT cleanup: +# stop the candidate, restore the previous pointer, and commit FAILED.sha256. +( + source "$HOSTCTL" + failure_dir=$tmp_dir/failure-evidence + failure_current=$tmp_dir/failure-current.env + failure_previous=$failure_dir/previous-current.env + failure_runtime=$tmp_dir/failure-runtime + mkdir -m 0700 "$failure_dir" + printf 'candidate\n' >"$failure_current" + printf 'previous\n' >"$failure_previous" + printf 'partial\n' >"$failure_dir/cutover.json.tmp" + cat >"$failure_runtime" <<'SH' +#!/usr/bin/env bash +[[ ${1:-} == ensure-stopped ]] +SH + chmod 0755 "$failure_runtime" + systemctl() { [[ $1 == stop ]]; } + restore_previous_pointer() { cp "$1" "$2"; } + mv() { + case ${1:-} in -T|-Tf|-fT) shift ;; esac + command mv "$@" + } + finalize_cutover_failure "$failure_dir" "$failure_previous" \ + "$failure_current" "$failure_runtime" "$valid_image" \ + "$(printf 'c%.0s' {1..40})" "$(printf 'd%.0s' {1..64})" \ + "$(printf 'e%.0s' {1..64})" interrupted_SIGTERM \ + "$failure_dir/cutover.json.tmp" "$failure_dir/PASSED.sha256.tmp" + [[ $(<"$failure_current") == previous ]] + verify_single_checksum_marker "$failure_dir" FAILED.sha256 cutover.failed.json + jq -e ' + .result == "failed_closed" and + .failure_reason == "interrupted_SIGTERM" and + .runtime_stopped == true and + .previous_pointer_restored == true and + .previous_runtime_restarted == false + ' "$failure_dir/cutover.failed.json" >/dev/null + [[ ! -e $failure_dir/cutover.json.tmp ]] +) + +# A stop failure must never mint the canonical fail-closed marker. +( + source "$HOSTCTL" + emergency_dir=$tmp_dir/emergency-evidence + emergency_current=$tmp_dir/emergency-current.env + emergency_previous=$emergency_dir/previous-current.env + emergency_runtime=$tmp_dir/emergency-runtime + mkdir -m 0700 "$emergency_dir" + printf 'candidate\n' >"$emergency_current" + printf 'previous\n' >"$emergency_previous" + cat >"$emergency_runtime" <<'SH' +#!/usr/bin/env bash +exit 1 +SH + chmod 0755 "$emergency_runtime" + systemctl() { [[ $1 == stop ]]; } + restore_previous_pointer() { cp "$1" "$2"; } + mv() { + [[ ${1:-} != -T ]] || shift + command mv "$@" + } + ! finalize_cutover_failure "$emergency_dir" "$emergency_previous" \ + "$emergency_current" "$emergency_runtime" "$valid_image" \ + "$(printf 'c%.0s' {1..40})" "$(printf 'd%.0s' {1..64})" \ + "$(printf 'e%.0s' {1..64})" health_gate_failed '' '' + [[ ! -e $emergency_dir/FAILED.sha256 ]] + verify_single_checksum_marker "$emergency_dir" \ + EMERGENCY_FAILED_OPEN.sha256 cutover.emergency.json + jq -e ' + .result == "emergency_failed_open" and + .runtime_stopped == false and + .trading_authority_blocked == true + ' "$emergency_dir/cutover.emergency.json" >/dev/null +) + +# Rollback authority is bound to immutable pointer snapshots and the exact +# active systemd/container identity. Legacy or stale evidence cannot stop a +# newer runtime. +rollback_evidence=$tmp_dir/rollback-lineage +rollback_current=$tmp_dir/rollback-current.env +mkdir -m 0700 "$rollback_evidence" +printf 'candidate pointer\n' >"$rollback_evidence/candidate-current.env" +printf 'previous pointer\n' >"$rollback_evidence/previous-current.env" +cp "$rollback_evidence/candidate-current.env" "$rollback_current" +chmod 0400 "$rollback_evidence/candidate-current.env" \ + "$rollback_evidence/previous-current.env" +chmod 0600 "$rollback_current" +rollback_candidate_sha=$(sha256sum "$rollback_evidence/candidate-current.env" \ + | awk '{print $1}') +rollback_previous_sha=$(sha256sum "$rollback_evidence/previous-current.env" \ + | awk '{print $1}') +jq -S -n --arg image "$valid_image" \ + --arg source "$(printf 'c%.0s' {1..40})" \ + --arg release "$(printf 'd%.0s' {1..64})" \ + --arg activation "$(printf 'e%.0s' {1..64})" \ + --arg invocation "$(printf '1%.0s' {1..32})" \ + --arg container "$(printf '2%.0s' {1..64})" \ + --arg candidate "$rollback_candidate_sha" \ + --arg previous "$rollback_previous_sha" ' + {schema:"monday.hft_trading_ecs_cutover.v1",result:"passed", + mode_boundary:"paper_or_shadow_only",image_reference:$image, + source_revision:$source,release_manifest_sha256:$release, + activation_manifest_sha256:$activation, + systemd_invocation_id:$invocation,main_pid:4242, + container_id:$container,health_endpoint:"127.0.0.1:49152", + grpc_endpoint:"127.0.0.1:49153",nrestarts:0, + health_samples:2,grpc_connect_samples:2, + candidate_pointer_file:"candidate-current.env", + candidate_pointer_sha256:$candidate,current_pointer_sha256:$candidate, + previous_pointer_present:true, + previous_pointer_file:"previous-current.env", + previous_pointer_sha256:$previous, + service_enabled:false,live_small_enabled:false} +' >"$rollback_evidence/cutover.json" +cp "$rollback_evidence/cutover.json" "$tmp_dir/valid-lineage-cutover.json" +( + cd "$rollback_evidence" + sha256sum cutover.json >PASSED.sha256 +) +chmod 0444 "$rollback_evidence/cutover.json" "$rollback_evidence/PASSED.sha256" +( + ROLLBACK_IMAGE= + ROLLBACK_EXPECTED_IDENTITY= + ROLLBACK_CANDIDATE_SHA= + ROLLBACK_PREVIOUS_SHA= + source "$HOSTCTL" + EXPECTED_ROOT_UID=$(id -u) + verify_single_checksum_marker "$rollback_evidence" PASSED.sha256 cutover.json + verify_rollback_lineage "$rollback_evidence" "$rollback_current" + [[ $ROLLBACK_IMAGE == "$valid_image" ]] + [[ $ROLLBACK_CANDIDATE_SHA == "$rollback_candidate_sha" ]] + [[ $ROLLBACK_PREVIOUS_SHA == "$rollback_previous_sha" ]] + [[ $ROLLBACK_EXPECTED_IDENTITY == \ + "$(printf '1%.0s' {1..32})|4242|0|$(printf '2%.0s' {1..64})|127.0.0.1:49152|127.0.0.1:49153" ]] + chmod 0644 "$rollback_evidence/candidate-current.env" + printf 'tampered candidate\n' >"$rollback_evidence/candidate-current.env" + chmod 0400 "$rollback_evidence/candidate-current.env" + ! verify_rollback_lineage "$rollback_evidence" "$rollback_current" + chmod 0644 "$rollback_evidence/candidate-current.env" + cp "$rollback_current" "$rollback_evidence/candidate-current.env" + chmod 0400 "$rollback_evidence/candidate-current.env" + chmod 0644 "$rollback_evidence/previous-current.env" + printf 'tampered previous\n' >"$rollback_evidence/previous-current.env" + chmod 0400 "$rollback_evidence/previous-current.env" + ! verify_rollback_lineage "$rollback_evidence" "$rollback_current" + chmod 0644 "$rollback_evidence/previous-current.env" + printf 'previous pointer\n' >"$rollback_evidence/previous-current.env" + chmod 0400 "$rollback_evidence/previous-current.env" + chmod 0644 "$rollback_current" + printf 'newer pointer\n' >"$rollback_current" + chmod 0600 "$rollback_current" + ! verify_rollback_lineage "$rollback_evidence" "$rollback_current" + cp "$rollback_evidence/candidate-current.env" "$rollback_current" + chmod 0600 "$rollback_current" + chmod 0644 "$rollback_evidence/cutover.json" "$rollback_evidence/PASSED.sha256" + jq 'del(.candidate_pointer_file,.candidate_pointer_sha256, + .current_pointer_sha256,.previous_pointer_present, + .previous_pointer_file,.previous_pointer_sha256)' \ + "$tmp_dir/valid-lineage-cutover.json" >"$rollback_evidence/cutover.json" + ( + cd "$rollback_evidence" + sha256sum cutover.json >PASSED.sha256 + ) + chmod 0444 "$rollback_evidence/cutover.json" "$rollback_evidence/PASSED.sha256" + ! verify_rollback_lineage "$rollback_evidence" "$rollback_current" +) +chmod 0644 "$rollback_evidence/cutover.json" "$rollback_evidence/PASSED.sha256" +cp "$tmp_dir/valid-lineage-cutover.json" "$rollback_evidence/cutover.json" +( + cd "$rollback_evidence" + sha256sum cutover.json >PASSED.sha256 +) +chmod 0444 "$rollback_evidence/cutover.json" "$rollback_evidence/PASSED.sha256" +stale_stop_log=$tmp_dir/stale-rollback-stop.log +rollback_test_uid=$(id -u) +if ( + source "$HOSTCTL" + EXPECTED_ROOT_UID=$rollback_test_uid + CURRENT_FILE=$rollback_current + id() { + [[ $1 == -u ]] || return 2 + printf '0\n' + } + runtime_identity_matches() { return 1; } + systemctl() { + if [[ $1 == stop ]]; then + printf 'unsafe stop\n' >"$stale_stop_log" + fi + return 0 + } + rollback_cutover "$rollback_evidence" +) >/dev/null 2>&1; then + printf 'stale rollback evidence unexpectedly succeeded\n' >&2 + exit 1 +fi +[[ ! -e $stale_stop_log ]] + +rollback_second_evidence=$tmp_dir/rollback-second-recheck +rollback_second_current=$tmp_dir/rollback-second-current.env +cp -R "$rollback_evidence" "$rollback_second_evidence" +cp "$rollback_current" "$rollback_second_current" +chmod 0700 "$rollback_second_evidence" +chmod 0600 "$rollback_second_current" +second_stop_log=$tmp_dir/second-rollback-stop.log +if ( + source "$HOSTCTL" + EXPECTED_ROOT_UID=$rollback_test_uid + CURRENT_FILE=$rollback_second_current + identity_checks=0 + id() { + [[ $1 == -u ]] || return 2 + printf '0\n' + } + runtime_identity_matches() { + identity_checks=$((identity_checks + 1)) + [[ $identity_checks -eq 1 ]] + } + systemctl() { + if [[ $1 == stop ]]; then + printf 'unsafe stop\n' >"$second_stop_log" + fi + return 0 + } + mv() { + [[ ${1:-} != -T ]] || shift + command mv "$@" + } + rollback_cutover "$rollback_second_evidence" +) >/dev/null 2>&1; then + printf 'changed identity after rollback intent unexpectedly passed\n' >&2 + exit 1 +fi +[[ ! -e $second_stop_log ]] +[[ -f $rollback_second_evidence/PASSED.sha256 ]] +[[ -f $rollback_second_evidence/PASSED.rollback-pending.sha256 ]] +( + cd "$rollback_second_evidence" + sha256sum --check --strict PASSED.sha256 >/dev/null + sha256sum --check --strict PASSED.rollback-pending.sha256 >/dev/null +) + +rollback_success_evidence=$tmp_dir/rollback-success +rollback_success_current=$tmp_dir/rollback-success-current.env +rollback_runtime=$tmp_dir/rollback-runtime +cp -R "$rollback_evidence" "$rollback_success_evidence" +cp "$rollback_current" "$rollback_success_current" +chmod 0700 "$rollback_success_evidence" +chmod 0600 "$rollback_success_current" +printf '#!/usr/bin/env bash\nexit 0\n' >"$rollback_runtime" +chmod 0700 "$rollback_runtime" +( + source "$HOSTCTL" + EXPECTED_ROOT_UID=$rollback_test_uid + CURRENT_FILE=$rollback_success_current + RUNTIME_PROGRAM=$rollback_runtime + id() { printf '0\n'; } + runtime_identity_matches() { return 0; } + systemctl() { return 0; } + install() { + local -a arguments=() + while (( $# > 0 )); do + case $1 in + -o|-g) shift 2 ;; + *) arguments+=("$1"); shift ;; + esac + done + command install "${arguments[@]}" + } + mv() { + case ${1:-} in -T|-Tf|-fT) shift ;; esac + command mv "$@" + } + rollback_cutover "$rollback_success_evidence" >/dev/null +) +cmp "$rollback_evidence/PASSED.sha256" "$rollback_success_evidence/PASSED.sha256" +( + cd "$rollback_success_evidence" + sha256sum --check --strict PASSED.sha256 >/dev/null + sha256sum --check --strict PASSED.rollback-pending.sha256 >/dev/null + sha256sum --check --strict PASSED.rolled-back.sha256 >/dev/null + sha256sum --check --strict ROLLED_BACK.sha256 >/dev/null +) + +grep -Fq \ + 'uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5' \ + "$WORKFLOW" +grep -Fq \ + 'uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f' \ + "$WORKFLOW" +grep -Fq \ + 'uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9' \ + "$WORKFLOW" +grep -Fq \ + 'uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8' \ + "$WORKFLOW" +[[ $(grep -Fc \ + 'uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02' \ + "$WORKFLOW") -eq 2 ]] +if grep -Eq \ + 'uses: (actions/(checkout|upload-artifact)|docker/(setup-buildx-action|login-action|build-push-action))@v[0-9]' \ + "$WORKFLOW"; then + printf 'workflow actions must be pinned to full commit SHAs\n' >&2 + exit 1 +fi +grep -Fq 'docker logout "$ACR_REGISTRY"' "$WORKFLOW" +grep -Fq 'rm -f -- "$docker_config_root/config.json"' "$WORKFLOW" +awk ' + /- name: Remove ACR credentials/ { + if (getline <= 0 || $0 !~ /^[[:space:]]+if: always\(\)$/) exit 1 + found = 1 + } + END { if (!found) exit 1 } +' "$WORKFLOW" +grep -Fq 'hft-trading-ecs-linux-amd64-${{ github.sha }}' "$WORKFLOW" +grep -Fq 'IMAGE_DIGEST: ${{ steps.build.outputs.digest }}' "$WORKFLOW" +grep -Fq ':run-${{ github.run_id }}-${{ github.run_attempt }}' "$WORKFLOW" +grep -Fq 'reference:$image_reference' "$WORKFLOW" +grep -Fq 'orchestrator:"none"' "$WORKFLOW" +grep -Fq -- '--pull never' "$RUNTIME" +grep -Fq -- '--publish 127.0.0.1::9090/tcp' "$RUNTIME" +grep -Fq -- '--publish 127.0.0.1::9092/tcp' "$RUNTIME" +grep -Fq -- '--stop-signal SIGINT' "$RUNTIME" +grep -Fq -- '--stop-timeout 60' "$RUNTIME" +grep -Fq -- '--ulimit core=0:0' "$RUNTIME" +grep -Fq -- '--entrypoint /bin/sh' "$RUNTIME" +grep -Fq '/run/secrets/hft/runtime.env' "$RUNTIME" +if grep -Fq -- '--env-file' "$RUNTIME"; then + printf 'Docker daemon metadata must not contain runtime secret values\n' >&2 + exit 1 +fi +if grep -Fq -- '--user 1000:1000' "$RUNTIME"; then + printf 'ordinary Ubuntu UID/GID 1000 must not run the trading container\n' >&2 + exit 1 +fi +grep -Fq 'StartLiveSmall' "$POLICY" +grep -Fq 'live_small_enabled:false' "$HOSTCTL" +grep -Fq 'service_enabled:false' "$HOSTCTL" +grep -Fq 'previous_runtime_restarted:false' "$HOSTCTL" +grep -Fq 'new signed envelope and nonce before restart' "$HOSTCTL" +grep -Fq 'PASSED.rolled-back.sha256' "$HOSTCTL" +grep -Fq 'PASSED.rollback-pending.sha256' "$HOSTCTL" +grep -Fq 'PASSED.sha256 is the commit point' "$HOSTCTL" +grep -Fq 'cutover.unconfirmed.json' "$HOSTCTL" +grep -Fq 'trap - EXIT; trap "" HUP INT TERM' "$HOSTCTL" +grep -Fq 'failure_reason=interrupted_SIGHUP' "$HOSTCTL" +grep -Fq 'failure_reason=interrupted_SIGINT' "$HOSTCTL" +grep -Fq 'failure_reason=interrupted_SIGTERM' "$HOSTCTL" +grep -Fq 'finalize_cutover_failure' "$HOSTCTL" +grep -Fq 'result=emergency_failed_open' "$HOSTCTL" +grep -Fq 'wait_for_runtime_identity' "$HOSTCTL" +grep -Fq 'grpc_endpoint_ready' "$HOSTCTL" +grep -Fq 'verify_rollback_lineage' "$HOSTCTL" +grep -Fq 'candidate_pointer_sha256' "$HOSTCTL" +grep -Fq 'flock -n 9' "$HOSTCTL" +grep -Fq "mkdir -m 0700 \"\$evidence_dir\"" "$HOSTCTL" +grep -Fq 'docker info >/dev/null 2>&1' "$HOSTCTL" +grep -Fq 'docker info >/dev/null 2>&1' "$RUNTIME" +grep -Fq 'DOCKER_CONFIG=$docker_config docker login' "$HOSTCTL" +grep -Fq 'assert_service_static' "$HOSTCTL" +grep -Fq 'assert_service_boot_disabled_before_stage' "$HOSTCTL" +grep -Fq 'useradd --system' "$HOSTCTL" +grep -Fq '[[ $state == absent ]]' "$RUNTIME" +grep -Fq 'verify_bare_tokyo_host || die' "$RUNTIME" +grep -Fq 'validate_selected_control_assets' "$RUNTIME" +grep -Fq 'assert_effective_service_contract "$UNIT_PATH" "${BASH_SOURCE[0]}"' "$RUNTIME" +grep -Fq 'assert_effective_service_contract "$unit_path" "$runtime_program"' "$HOSTCTL" +grep -Fq 'secure_directory_chain "$activation_dir"' "$RUNTIME" +grep -Fq '! -path ./activation.sha256' "$RUNTIME" +grep -Fq 'ExecStartPre=/usr/local/libexec/monday-hft-trading-runtime preflight' "$UNIT" +grep -Fq 'ExecStopPost=/usr/local/libexec/monday-hft-trading-runtime ensure-stopped' "$UNIT" +grep -Fq 'assert-stopped' "$HOSTCTL" +grep -Fq 'Restart=no' "$UNIT" +grep -Fq 'LimitCORE=0' "$UNIT" +if grep -Eq '^\[Install\]$' "$UNIT"; then + printf 'trading service must remain static and boot-disabled\n' >&2 + exit 1 +fi +if grep -Eq 'systemctl[[:space:]]+enable' "$HOSTCTL"; then + printf 'host contract must never enable the trading service\n' >&2 + exit 1 +fi +if grep -Eq 'hft-trading:(latest|[$][{]?[A-Za-z_])' "$HOSTCTL" "$RUNTIME" "$UNIT"; then + printf 'mutable trading image reference detected\n' >&2 + exit 1 +fi + +printf 'trading ECS host contract tests passed\n' diff --git a/deployment/aliyun/trading-ecs-hostctl.sh b/deployment/aliyun/trading-ecs-hostctl.sh new file mode 100755 index 000000000..3b0e41bcb --- /dev/null +++ b/deployment/aliyun/trading-ecs-hostctl.sh @@ -0,0 +1,1002 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +export LC_ALL=C +umask 0077 + +readonly EXPECTED_REGION=ap-northeast-1 +readonly EXPECTED_HOST_OS=ubuntu +readonly EXPECTED_HOST_VERSION=26.04 +readonly EXPECTED_ARCH=amd64 +readonly EXPECTED_RAM_ROLE_DEFAULT=MondayTradingEcsRole +readonly ARTIFACT_ROOT_DEFAULT=/opt/monday/incoming/hft-trading +readonly ACR_AUTH_ROOT_DEFAULT=/run/monday/acr-auth +readonly RELEASE_ROOT_DEFAULT=/opt/monday/releases/hft-trading +readonly CONTROL_ROOT_DEFAULT=/opt/monday/control/hft-trading +readonly EVIDENCE_ROOT_DEFAULT=/var/lib/monday/evidence/hft-trading +readonly CURRENT_FILE_DEFAULT=/etc/monday/hft-trading-current.env +readonly RUNTIME_PROGRAM_DEFAULT=/usr/local/libexec/monday-hft-trading-runtime +readonly HOSTCTL_PROGRAM_DEFAULT=/usr/local/sbin/monday-hft-trading-hostctl +readonly UNIT_PATH_DEFAULT=/etc/systemd/system/monday-hft-trading.service +readonly SERVICE=monday-hft-trading.service +readonly HOST_LOCK_DEFAULT=/run/monday/hft-trading-host.lock +readonly HFT_RUNTIME_USER_DEFAULT=mondayhft +readonly HFT_RUNTIME_GROUP_DEFAULT=mondayhft + +die() { + printf 'hft-trading host contract failed: %s\n' "$*" >&2 + exit 1 +} + +require_root() { + [[ $(id -u) -eq 0 ]] || die 'must run as root on the bare ECS host' +} + +stat_uid() { + if [[ $(uname -s) == Darwin ]]; then stat -f %u -- "$1"; else stat -c %u -- "$1"; fi +} + +stat_mode() { + if [[ $(uname -s) == Darwin ]]; then stat -f %Lp -- "$1"; else stat -c %a -- "$1"; fi +} + +secure_regular_file() { + local path=$1 max_mode=$2 expected_uid=${EXPECTED_ROOT_UID:-0} mode + [[ -f $path && ! -L $path ]] || return 1 + [[ $(stat_uid "$path") == "$expected_uid" ]] || return 1 + mode=$(stat_mode "$path") || return 1 + (( (8#$mode & ~8#$max_mode) == 0 )) +} + +secure_directory() { + local path=$1 expected_uid=${EXPECTED_ROOT_UID:-0} mode + [[ -d $path && ! -L $path ]] || return 1 + [[ $(stat_uid "$path") == "$expected_uid" ]] || return 1 + mode=$(stat_mode "$path") || return 1 + (( (8#$mode & 8#022) == 0 )) +} + +canonical_directory() { + local path=$1 canonical + [[ $path == /* ]] || return 1 + canonical=$(cd -- "$path" 2>/dev/null && pwd -P) || return 1 + [[ $path == "$canonical" ]] || return 1 + printf '%s\n' "$canonical" +} + +assert_service_static() { + local state + if ! state=$(systemctl is-enabled "$SERVICE" 2>/dev/null); then + return 1 + fi + [[ $state == static ]] +} + +assert_no_service_dropin_directories() { + local root + for root in "$@"; do + [[ ! -e $root/$SERVICE.d ]] || return 1 + done +} + +effective_exec_matches() { + local property=$1 program=$2 argument=$3 value after_open before_close + value=$(systemctl show "$SERVICE" --property="$property" --value) || return 1 + [[ $value == "{ path=$program ; argv[]=$program $argument ;"* \ + && $value == *'ignore_errors=no'* && $value == *' }' ]] || return 1 + after_open=${value#*\{} + before_close=${value%\}} + [[ $after_open != "$value" && $after_open != *'{'* \ + && $before_close != "$value" && $before_close != *'}'* ]] +} + +assert_effective_service_contract() { + local unit_path=$1 runtime_program=$2 fragment dropins restart start_post + assert_no_service_dropin_directories \ + /etc/systemd/system /run/systemd/system /usr/local/lib/systemd/system \ + /usr/lib/systemd/system /lib/systemd/system || return 1 + fragment=$(systemctl show "$SERVICE" --property=FragmentPath --value) || return 1 + dropins=$(systemctl show "$SERVICE" --property=DropInPaths --value) || return 1 + restart=$(systemctl show "$SERVICE" --property=Restart --value) || return 1 + start_post=$(systemctl show "$SERVICE" --property=ExecStartPost --value) || return 1 + [[ $fragment == "$unit_path" && -z $dropins && $restart == no \ + && -z $start_post ]] || return 1 + assert_service_static || return 1 + effective_exec_matches ExecStartPre "$runtime_program" preflight || return 1 + effective_exec_matches ExecStart "$runtime_program" run || return 1 + effective_exec_matches ExecStop "$runtime_program" stop || return 1 + effective_exec_matches ExecStopPost "$runtime_program" ensure-stopped +} + +assert_service_boot_disabled_before_stage() { + local unit_path=$1 state='' status=0 + state=$(systemctl is-enabled "$SERVICE" 2>/dev/null) || status=$? + case $state in + static) (( status == 0 )) ;; + disabled) (( status == 1 )) ;; + not-found) (( status == 1 )) && [[ ! -e $unit_path ]] ;; + *) return 1 ;; + esac +} + +assert_no_orphan_container() { + local names + docker info >/dev/null 2>&1 || return 1 + names=$(docker container ls --all --filter 'name=^/monday-hft-trading$' \ + --format '{{.Names}}') || return 1 + [[ -z $names ]] +} + +ensure_runtime_account() { + local runtime_user=$HFT_RUNTIME_USER_DEFAULT + local runtime_group=$HFT_RUNTIME_GROUP_DEFAULT + local passwd_record group_record all_passwd all_groups uid gid home shell + local group_gid group_members other_name other_uid other_gid + if ! getent group "$runtime_group" >/dev/null; then + groupadd --system "$runtime_group" || return 1 + fi + if ! getent passwd "$runtime_user" >/dev/null; then + useradd --system --gid "$runtime_group" --home-dir /nonexistent \ + --no-create-home --shell /usr/sbin/nologin "$runtime_user" || return 1 + fi + passwd_record=$(getent passwd "$runtime_user") || return 1 + group_record=$(getent group "$runtime_group") || return 1 + IFS=: read -r _ _ uid gid _ home shell <<<"$passwd_record" + IFS=: read -r _ _ group_gid group_members <<<"$group_record" + [[ $uid =~ ^[0-9]+$ && $gid =~ ^[0-9]+$ ]] || return 1 + [[ $group_gid =~ ^[0-9]+$ && $group_gid == "$gid" ]] || return 1 + [[ $uid != 0 && $uid != 1000 && $gid != 0 && $gid != 1000 ]] || return 1 + [[ $home == /nonexistent ]] || return 1 + [[ $shell == /usr/sbin/nologin || $shell == /bin/false ]] || return 1 + [[ ${group_record%%:*} == "$runtime_group" && -z $group_members ]] || return 1 + [[ $(id -g "$runtime_user") == "$gid" ]] || return 1 + all_passwd=$(getent passwd) || return 1 + while IFS=: read -r other_name _ other_uid other_gid _; do + [[ $other_name == "$runtime_user" ]] && continue + [[ $other_uid != "$uid" && $other_gid != "$gid" ]] || return 1 + done <<<"$all_passwd" + all_groups=$(getent group) || return 1 + while IFS=: read -r other_name _ other_gid _; do + [[ $other_name == "$runtime_group" ]] && continue + [[ $other_gid != "$gid" ]] || return 1 + done <<<"$all_groups" +} + +valid_image_reference() { + [[ $1 =~ ^crpi-[a-z0-9]+-vpc[.]ap-northeast-1[.]personal[.]cr[.]aliyuncs[.]com/wildcard0923/hft-trading@sha256:[0-9a-f]{64}$ ]] +} + +release_digest_hex() { + valid_image_reference "$1" || return 1 + printf '%s\n' "${1##*@sha256:}" +} + +verify_single_checksum_marker() { + local directory=$1 marker=$2 expected_file=$3 + local line digest + [[ -f $directory/$marker && ! -L $directory/$marker ]] || return 1 + [[ $(wc -l <"$directory/$marker") -eq 1 ]] || return 1 + line=$(<"$directory/$marker") || return 1 + digest=${line%% *} + [[ $digest =~ ^[0-9a-f]{64}$ ]] || return 1 + [[ $line == "$digest $expected_file" ]] || return 1 + (cd "$directory" && sha256sum --check --strict "$marker" >/dev/null) || return 1 +} + +verify_release_manifest() { + local artifact_dir=$1 + local manifest=$artifact_dir/hft-trading-ecs-release.json + local artifact_root=${ARTIFACT_ROOT:-$ARTIFACT_ROOT_DEFAULT} + local canonical_root canonical_artifact + canonical_root=$(canonical_directory "$artifact_root") || return 1 + canonical_artifact=$(canonical_directory "$artifact_dir") || return 1 + [[ $artifact_root == "$canonical_root" && $artifact_dir == "$canonical_artifact" \ + && $canonical_artifact == "$canonical_root"/* \ + && ${canonical_artifact#"$canonical_root/"} != */* ]] || return 1 + secure_directory "$canonical_root" || return 1 + secure_directory "$artifact_dir" || return 1 + secure_regular_file "$manifest" 0644 || return 1 + secure_regular_file "$artifact_dir/hft-trading-ecs-release.json.sha256" 0644 \ + || return 1 + verify_single_checksum_marker "$artifact_dir" \ + hft-trading-ecs-release.json.sha256 hft-trading-ecs-release.json || return 1 + jq -e -s ' + length == 1 and (.[0] | + .schema == "monday.hft_trading_ecs_release.v1" and + (keys | sort) == (["control_archive","control_manifest","image", + "platform","schema","source_revision"] | sort) and + (.source_revision | test("^[0-9a-f]{40}$")) and + (.image | keys | sort) == (["digest","published_repository", + "reference","repository"] | sort) and + (.image.digest | test("^sha256:[0-9a-f]{64}$")) and + (.image.repository | test("^crpi-[a-z0-9]+-vpc[.]ap-northeast-1[.]personal[.]cr[.]aliyuncs[.]com/wildcard0923/hft-trading$")) and + .image.reference == (.image.repository + "@" + .image.digest) and + (.image.published_repository | test("^crpi-[a-z0-9]+[.]ap-northeast-1[.]personal[.]cr[.]aliyuncs[.]com/wildcard0923/hft-trading$")) and + ((.image.published_repository | + capture("^crpi-(?[a-z0-9]+)[.]ap-northeast-1").id) == + (.image.repository | + capture("^crpi-(?[a-z0-9]+)-vpc[.]ap-northeast-1").id)) and + (.control_manifest | keys | sort) == ["file","sha256"] and + .control_manifest.file == "trading-ecs-control-assets.sha256" and + (.control_manifest.sha256 | test("^[0-9a-f]{64}$")) and + (.control_archive | keys | sort) == ["file","sha256"] and + .control_archive.file == "trading-ecs-control.tar.gz" and + (.control_archive.sha256 | test("^[0-9a-f]{64}$")) and + .platform == {region:"ap-northeast-1",host_os:"ubuntu", + host_version:"26.04",architecture:"amd64",orchestrator:"none"} + ) + ' "$manifest" >/dev/null || return 1 +} + +verify_control_bundle() { + local artifact_dir=$1 extract_dir=$2 + local manifest_sha archive_sha control_names + secure_regular_file "$artifact_dir/trading-ecs-control-assets.sha256" 0644 || return 1 + secure_regular_file "$artifact_dir/trading-ecs-control.tar.gz" 0644 || return 1 + manifest_sha=$(jq -er '.control_manifest.sha256' \ + "$artifact_dir/hft-trading-ecs-release.json") || return 1 + archive_sha=$(jq -er '.control_archive.sha256' \ + "$artifact_dir/hft-trading-ecs-release.json") || return 1 + [[ $(sha256sum "$artifact_dir/trading-ecs-control-assets.sha256" | awk '{print $1}') == "$manifest_sha" ]] \ + || return 1 + [[ $(sha256sum "$artifact_dir/trading-ecs-control.tar.gz" | awk '{print $1}') == "$archive_sha" ]] \ + || return 1 + control_names=$(awk ' + NF != 2 || $1 !~ /^[0-9a-f]{64}$/ || $2 !~ /^[A-Za-z0-9._-]+$/ { exit 2 } + { print $2 } + ' "$artifact_dir/trading-ecs-control-assets.sha256") || return 1 + [[ $control_names == $'hft-trading-ecs.service\ntrading-ecs-hostctl.sh\ntrading-ecs-paper-shadow-policy.jq\ntrading-ecs-runtime.sh' ]] \ + || return 1 + diff -u \ + <(printf '%s\n' hft-trading-ecs.service trading-ecs-hostctl.sh \ + trading-ecs-paper-shadow-policy.jq trading-ecs-runtime.sh) \ + <(tar -tzf "$artifact_dir/trading-ecs-control.tar.gz" | LC_ALL=C sort) || return 1 + tar -tvzf "$artifact_dir/trading-ecs-control.tar.gz" \ + | awk 'substr($1, 1, 1) != "-" { exit 1 } END { if (NR != 4) exit 1 }' \ + || return 1 + tar --no-same-owner --no-same-permissions \ + -xzf "$artifact_dir/trading-ecs-control.tar.gz" -C "$extract_dir" || return 1 + [[ -z $(find "$extract_dir" -type l -print -quit) ]] || return 1 + (cd "$extract_dir" && sha256sum --check --strict \ + "$artifact_dir/trading-ecs-control-assets.sha256" >/dev/null) || return 1 +} + +metadata_token() { + curl -fsS --connect-timeout 2 --max-time 5 -X PUT \ + -H 'X-aliyun-ecs-metadata-token-ttl-seconds: 300' \ + http://100.100.100.200/latest/api/token +} + +metadata_get() { + local token=$1 path=$2 + curl -fsS --connect-timeout 2 --max-time 5 \ + -H "X-aliyun-ecs-metadata-token: $token" \ + "http://100.100.100.200/latest/meta-data/$path" +} + +verify_bare_tokyo_host() { + local token role region instance_id + # shellcheck disable=SC1091 + source /etc/os-release + [[ ${ID:-} == "$EXPECTED_HOST_OS" && ${VERSION_ID:-} == "$EXPECTED_HOST_VERSION" ]] \ + || die 'host must be Ubuntu 26.04' + [[ $(dpkg --print-architecture) == "$EXPECTED_ARCH" ]] || die 'host must be amd64' + systemctl is-active --quiet kubelet && die 'trading host must not be an ACK node' + [[ ! -e /var/lib/kubelet/kubeconfig ]] || die 'kubelet state is forbidden on the trading host' + token=$(metadata_token) || die 'Alibaba ECS metadata v2 is required' + region=$(metadata_get "$token" region-id) || die 'cannot read ECS region' + [[ $region == "$EXPECTED_REGION" ]] || die 'trading host must be in Tokyo' + role=$(metadata_get "$token" ram/security-credentials/) || die 'cannot read ECS RAM role' + role=${role%%$'\n'*} + [[ $role == "$EXPECTED_RAM_ROLE_DEFAULT" ]] \ + || die 'unexpected or missing ECS RAM role' + instance_id=$(metadata_get "$token" instance-id) || die 'cannot read ECS instance id' + [[ $instance_id =~ ^i-[a-z0-9]+$ ]] || die 'invalid ECS instance identity' + printf '%s\n%s\n%s\n' "$region" "$role" "$instance_id" +} + +verify_acr_password_file() { + local password_file=$1 + local auth_root=${ACR_AUTH_ROOT:-$ACR_AUTH_ROOT_DEFAULT} fs_type password_fs canonical_root + local canonical_parent + canonical_root=$(canonical_directory "$auth_root") \ + || die 'ACR auth root must be an absolute canonical directory' + secure_directory "$auth_root" || die 'ACR auth root is unsafe' + [[ $(stat_mode "$auth_root") == 700 ]] || die 'ACR auth root must be mode 0700' + [[ $password_file == "$canonical_root"/* \ + && ${password_file#"$canonical_root/"} != */* ]] \ + || die 'ACR password file must be a direct child of the ephemeral auth root' + canonical_parent=$(cd -- "${password_file%/*}" 2>/dev/null && pwd -P) \ + || die 'cannot resolve the ACR password parent directory' + [[ $canonical_parent == "$canonical_root" \ + && $password_file == "$canonical_parent/${password_file##*/}" ]] \ + || die 'ACR password path traversal is forbidden' + fs_type=$(findmnt -n -o FSTYPE --target "$auth_root") \ + || die 'cannot verify the ACR auth filesystem' + [[ $fs_type == tmpfs ]] || die 'ACR auth root must be on tmpfs' + password_fs=$(findmnt -n -o FSTYPE --target "$password_file") \ + || die 'cannot verify the ACR password filesystem' + [[ $password_fs == tmpfs ]] || die 'ACR password file must be on tmpfs' + secure_regular_file "$password_file" 0400 || die 'ACR password file must be root-owned mode 0400' + [[ $(stat_mode "$password_file") == 400 ]] || die 'ACR password file must be mode 0400' + [[ -s $password_file && $(wc -l <"$password_file") -eq 1 ]] || die 'ACR password file must contain one non-empty line' + grep -q '[^[:space:]]' "$password_file" || die 'ACR password file must contain one non-empty line' +} + +verify_local_image() { + local image=$1 + valid_image_reference "$image" || return 1 + docker image inspect "$image" --format '{{json .RepoDigests}}' \ + | jq -e --arg image "$image" 'index($image) != null' >/dev/null +} + +stage_release() { + local artifact_dir=$1 acr_username=$2 password_file=$3 + local release_root=${RELEASE_ROOT:-$RELEASE_ROOT_DEFAULT} + local control_root=${CONTROL_ROOT:-$CONTROL_ROOT_DEFAULT} + local runtime_program=${RUNTIME_PROGRAM:-$RUNTIME_PROGRAM_DEFAULT} + local hostctl_program=${HOSTCTL_PROGRAM:-$HOSTCTL_PROGRAM_DEFAULT} + local unit_path=${UNIT_PATH:-$UNIT_PATH_DEFAULT} + local auth_root=${ACR_AUTH_ROOT:-$ACR_AUTH_ROOT_DEFAULT} + local extract_dir docker_config='' image registry digest_hex release_dir source_revision + local release_manifest_sha + local host_values region role instance_id evidence_parent evidence_dir staged_at stage_run_id + local registry_logged_in=false + require_root + for command in curl diff docker dpkg find findmnt getent groupadd id install jq \ + sha256sum systemctl tar timeout useradd; do + command -v "$command" >/dev/null 2>&1 || die "missing host dependency: $command" + done + [[ $artifact_dir == /* ]] || die 'artifact directory must be absolute' + [[ $acr_username =~ ^[^[:space:]]+$ ]] || die 'ACR username is required' + systemctl is-active --quiet "$SERVICE" \ + && die 'staging cannot mutate an active trading runtime' + assert_service_boot_disabled_before_stage "$unit_path" \ + || die 'staging refuses a service that is not explicitly boot-disabled' + assert_no_service_dropin_directories \ + /etc/systemd/system /run/systemd/system /usr/local/lib/systemd/system \ + /usr/lib/systemd/system /lib/systemd/system \ + || die 'staging refuses trading-service drop-in directories' + assert_no_orphan_container \ + || die 'Docker daemon is unavailable or an orphan trading container exists' + verify_release_manifest "$artifact_dir" || die 'release manifest is invalid' + verify_acr_password_file "$password_file" + mapfile -t host_values < <(verify_bare_tokyo_host) + [[ ${#host_values[@]} -eq 3 ]] || die 'bare-host verification did not complete' + region=${host_values[0]} + role=${host_values[1]} + instance_id=${host_values[2]} + ensure_runtime_account || die 'dedicated non-login runtime account is unsafe' + image=$(jq -er '.image.reference' "$artifact_dir/hft-trading-ecs-release.json") + valid_image_reference "$image" || die 'release image is not an immutable Tokyo VPC reference' + registry=${image%%/*} + digest_hex=$(release_digest_hex "$image") + source_revision=$(jq -er '.source_revision' "$artifact_dir/hft-trading-ecs-release.json") + release_manifest_sha=$(sha256sum "$artifact_dir/hft-trading-ecs-release.json" | awk '{print $1}') + release_dir=$release_root/$digest_hex/$release_manifest_sha + extract_dir=$(mktemp -d) + stage_cleanup() { + rm -rf -- "$extract_dir" 2>/dev/null || true + if [[ $registry_logged_in == true && -n $docker_config ]]; then + DOCKER_CONFIG=$docker_config docker logout "$registry" >/dev/null 2>&1 || true + fi + [[ -z $docker_config ]] || rm -rf -- "$docker_config" 2>/dev/null || true + } + trap stage_cleanup EXIT + docker_config=$(mktemp -d "$auth_root/docker-config.XXXXXX") + chmod 0700 "$docker_config" + verify_control_bundle "$artifact_dir" "$extract_dir" || die 'control bundle is invalid' + + DOCKER_CONFIG=$docker_config docker login --username "$acr_username" \ + --password-stdin "$registry" <"$password_file" >/dev/null + registry_logged_in=true + if ! DOCKER_CONFIG=$docker_config docker pull "$image" >/dev/null; then + die 'digest-pinned ACR pull failed' + fi + DOCKER_CONFIG=$docker_config docker logout "$registry" >/dev/null \ + || die 'ACR logout failed' + registry_logged_in=false + verify_local_image "$image" || die 'pulled image RepoDigests do not contain the requested digest' + + install -d -o root -g root -m 0755 "$release_dir" "$control_root" \ + "${runtime_program%/*}" "${hostctl_program%/*}" "${unit_path%/*}" + install -o root -g root -m 0444 \ + "$artifact_dir/hft-trading-ecs-release.json" \ + "$artifact_dir/hft-trading-ecs-release.json.sha256" \ + "$artifact_dir/trading-ecs-control-assets.sha256" \ + "$artifact_dir/trading-ecs-control.tar.gz" "$release_dir/" + install -o root -g root -m 0555 "$extract_dir/trading-ecs-runtime.sh" "$runtime_program" + install -o root -g root -m 0555 "$extract_dir/trading-ecs-hostctl.sh" "$hostctl_program" + install -o root -g root -m 0444 \ + "$extract_dir/trading-ecs-paper-shadow-policy.jq" "$control_root/" + install -o root -g root -m 0444 "$extract_dir/hft-trading-ecs.service" "$unit_path" + systemctl daemon-reload + systemctl is-active --quiet "$SERVICE" && die 'staged service unexpectedly became active' + assert_effective_service_contract "$unit_path" "$runtime_program" \ + || die 'effective systemd service does not match the selected static unit' + + evidence_parent=${EVIDENCE_ROOT:-$EVIDENCE_ROOT_DEFAULT}/stage/$digest_hex/$release_manifest_sha/runs + install -d -o root -g root -m 0700 "$evidence_parent" + stage_run_id=$(date -u +%Y%m%dT%H%M%S)-$$ + evidence_dir=$evidence_parent/$stage_run_id + mkdir -m 0700 "$evidence_dir" || die 'stage evidence run already exists' + staged_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) + jq -S -n --arg image "$image" --arg source "$source_revision" \ + --arg release_manifest "$release_manifest_sha" \ + --arg region "$region" --arg role "$role" --arg instance "$instance_id" \ + --arg staged_at "$staged_at" ' + {schema:"monday.hft_trading_ecs_stage.v1",result:"staged", + image_reference:$image,source_revision:$source, + release_manifest_sha256:$release_manifest,region:$region, + ram_role:$role,instance_id:$instance,host_os:"ubuntu", + host_version:"26.04",architecture:"amd64",orchestrator:"none", + service_started:false,service_enabled:false,staged_at:$staged_at} + ' >"$evidence_dir/stage.json" + (cd "$evidence_dir" && sha256sum stage.json >STAGED.sha256) + chmod 0444 "$evidence_dir/stage.json" "$evidence_dir/STAGED.sha256" + sync -f "$evidence_dir" + printf '%s\n' "$evidence_dir/stage.json" + stage_cleanup + trap - EXIT +} + +write_current_file() { + local destination=$1 image=$2 release_manifest_sha=$3 activation_dir=$4 source_revision=$5 + local activation_sha + activation_sha=$(sha256sum "$activation_dir/activation.sha256" | awk '{print $1}') + umask 077 + { + printf 'HFT_TRADING_IMAGE=%q\n' "$image" + printf 'HFT_RELEASE_MANIFEST_SHA256=%q\n' "$release_manifest_sha" + printf 'HFT_ACTIVATION_DIR=%q\n' "$activation_dir" + printf 'HFT_ACTIVATION_SHA256=%q\n' "$activation_sha" + printf 'HFT_SOURCE_REVISION=%q\n' "$source_revision" + } >"$destination" + chmod 0600 "$destination" +} + +install_current_pointer() { + local source=$1 destination=$2 + local temporary=$destination.tmp.$$ + if ! install -o root -g root -m 0600 "$source" "$temporary" \ + || ! sync -f "$temporary" \ + || ! mv -Tf "$temporary" "$destination"; then + rm -f -- "$temporary" 2>/dev/null || true + return 1 + fi + sync -f "${destination%/*}" || return 1 +} + +capture_runtime_identity() { + local image=$1 invocation_id main_pid nrestarts container_id + local container_running container_image health_endpoint grpc_endpoint + systemctl is-active --quiet "$SERVICE" || return 1 + assert_service_static || return 1 + invocation_id=$(systemctl show "$SERVICE" --property=InvocationID --value) || return 1 + main_pid=$(systemctl show "$SERVICE" --property=MainPID --value) || return 1 + nrestarts=$(systemctl show "$SERVICE" --property=NRestarts --value) || return 1 + container_id=$(docker container inspect --format '{{.Id}}' monday-hft-trading) || return 1 + container_running=$(docker container inspect --format '{{.State.Running}}' \ + monday-hft-trading) || return 1 + container_image=$(docker container inspect --format '{{.Config.Image}}' \ + monday-hft-trading) || return 1 + health_endpoint=$(docker container inspect --format \ + '{{with (index (index .NetworkSettings.Ports "9090/tcp") 0)}}{{.HostIp}}:{{.HostPort}}{{end}}' \ + monday-hft-trading) || return 1 + grpc_endpoint=$(docker container inspect --format \ + '{{with (index (index .NetworkSettings.Ports "9092/tcp") 0)}}{{.HostIp}}:{{.HostPort}}{{end}}' \ + monday-hft-trading) || return 1 + [[ $invocation_id =~ ^[0-9a-f]{32}$ ]] || return 1 + [[ $main_pid =~ ^[1-9][0-9]*$ ]] || return 1 + [[ $nrestarts == 0 ]] || return 1 + [[ $container_id =~ ^[0-9a-f]{64}$ ]] || return 1 + [[ $container_running == true && $container_image == "$image" ]] || return 1 + [[ $health_endpoint =~ ^127[.]0[.]0[.]1:[1-9][0-9]{0,4}$ ]] || return 1 + (( ${health_endpoint##*:} <= 65535 )) || return 1 + [[ $grpc_endpoint =~ ^127[.]0[.]0[.]1:[1-9][0-9]{0,4}$ ]] || return 1 + (( ${grpc_endpoint##*:} <= 65535 )) || return 1 + [[ $grpc_endpoint != "$health_endpoint" ]] || return 1 + printf '%s|%s|%s|%s|%s|%s\n' "$invocation_id" "$main_pid" "$nrestarts" \ + "$container_id" "$health_endpoint" "$grpc_endpoint" +} + +wait_for_runtime_identity() { + local image=$1 identity _ + for _ in {1..30}; do + if identity=$(capture_runtime_identity "$image"); then + printf '%s\n' "$identity" + return 0 + fi + sleep 1 + done + return 1 +} + +runtime_identity_matches() { + local image=$1 expected=$2 actual + actual=$(capture_runtime_identity "$image") || return 1 + [[ $actual == "$expected" ]] +} + +grpc_endpoint_ready() { + local endpoint=$1 host=${1%:*} port=${1##*:} + [[ $endpoint == "$host:$port" && $host == 127.0.0.1 \ + && $port =~ ^[1-9][0-9]{0,4}$ ]] || return 1 + # The positional parameters expand in the bounded child Bash, not here. + # shellcheck disable=SC2016 + timeout 3 bash -c 'exec 3<>"/dev/tcp/$1/$2"' bash "$host" "$port" \ + >/dev/null 2>&1 +} + +wait_for_health() { + local image=$1 expected_identity=$2 _ health_endpoint grpc_endpoint + local identity_without_grpc + grpc_endpoint=${expected_identity##*|} + identity_without_grpc=${expected_identity%|*} + health_endpoint=${identity_without_grpc##*|} + [[ $health_endpoint =~ ^127[.]0[.]0[.]1:[1-9][0-9]{0,4}$ ]] || return 1 + [[ $grpc_endpoint =~ ^127[.]0[.]0[.]1:[1-9][0-9]{0,4}$ ]] || return 1 + for _ in {1..24}; do + runtime_identity_matches "$image" "$expected_identity" || return 1 + if curl -fsS --max-time 3 "http://$health_endpoint/health" >/dev/null \ + && curl -fsS --max-time 3 "http://$health_endpoint/readiness" >/dev/null \ + && grpc_endpoint_ready "$grpc_endpoint"; then + runtime_identity_matches "$image" "$expected_identity" || return 1 + sleep 10 + runtime_identity_matches "$image" "$expected_identity" || return 1 + if curl -fsS --max-time 3 "http://$health_endpoint/health" >/dev/null \ + && curl -fsS --max-time 3 "http://$health_endpoint/readiness" >/dev/null \ + && grpc_endpoint_ready "$grpc_endpoint" \ + && runtime_identity_matches "$image" "$expected_identity"; then + return 0 + fi + fi + sleep 5 + done + return 1 +} + +restore_previous_pointer() { + local previous_file=$1 current_file=$2 + if [[ -f $previous_file ]]; then + install_current_pointer "$previous_file" "$current_file" || return 1 + else + rm -f -- "$current_file" || return 1 + sync -f "${current_file%/*}" || return 1 + fi +} + +finalize_cutover_failure() { + local evidence_dir=$1 previous_file=$2 current_file=$3 runtime_program=$4 + local image=$5 source_revision=$6 release_manifest_sha=$7 activation_sha=$8 + local failure_reason=$9 cutover_tmp=${10:-} marker_tmp=${11:-} + local runtime_stopped=false previous_pointer_restored=false + local result evidence_name marker_name operator_action + local evidence_tmp marker_tmp_failed evidence_sha + [[ -d $evidence_dir && ! -L $evidence_dir ]] || return 1 + [[ ! -e $evidence_dir/PASSED.sha256 ]] || return 1 + if [[ -e $evidence_dir/FAILED.sha256 ]]; then + verify_single_checksum_marker "$evidence_dir" FAILED.sha256 cutover.failed.json + return + fi + rm -f -- "$evidence_dir/EMERGENCY_FAILED_OPEN.sha256" 2>/dev/null || true + if [[ -f $evidence_dir/cutover.emergency.json ]]; then + mv -T "$evidence_dir/cutover.emergency.json" \ + "$evidence_dir/cutover.emergency.previous.$$.json" 2>/dev/null || true + fi + [[ -z $cutover_tmp ]] || rm -f -- "$cutover_tmp" 2>/dev/null || true + [[ -z $marker_tmp ]] || rm -f -- "$marker_tmp" 2>/dev/null || true + if [[ -f $evidence_dir/cutover.json ]]; then + mv -T "$evidence_dir/cutover.json" \ + "$evidence_dir/cutover.unconfirmed.json" 2>/dev/null || true + fi + if [[ -f $evidence_dir/cutover.failed.json ]]; then + mv -T "$evidence_dir/cutover.failed.json" \ + "$evidence_dir/cutover.failed.unconfirmed.$$.json" 2>/dev/null || true + fi + systemctl stop "$SERVICE" >/dev/null 2>&1 || true + if "$runtime_program" ensure-stopped; then + runtime_stopped=true + else + failure_reason=${failure_reason}_and_stop_failed + fi + if restore_previous_pointer "$previous_file" "$current_file"; then + previous_pointer_restored=true + else + failure_reason=${failure_reason}_and_pointer_restore_failed + fi + if [[ $runtime_stopped == true && $previous_pointer_restored == true ]]; then + result=failed_closed + evidence_name=cutover.failed.json + marker_name=FAILED.sha256 + operator_action='new signed envelope and nonce before restart' + else + result=emergency_failed_open + evidence_name=cutover.emergency.json + marker_name=EMERGENCY_FAILED_OPEN.sha256 + operator_action='IMMEDIATE MANUAL STOP AND POINTER RECOVERY REQUIRED; trading remains blocked' + fi + evidence_tmp=$evidence_dir/$evidence_name.tmp.$$ + marker_tmp_failed=$evidence_dir/$marker_name.tmp.$$ + if ! jq -S -n --arg image "$image" --arg source "$source_revision" \ + --arg release_manifest "$release_manifest_sha" \ + --arg activation "$activation_sha" --arg reason "$failure_reason" \ + --arg result "$result" --arg operator_action "$operator_action" \ + --argjson runtime_stopped "$runtime_stopped" \ + --argjson pointer_restored "$previous_pointer_restored" \ + --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" ' + {schema:"monday.hft_trading_ecs_cutover.v1",result:$result, + image_reference:$image,source_revision:$source, + release_manifest_sha256:$release_manifest, + activation_manifest_sha256:$activation,failure_reason:$reason, + runtime_stopped:$runtime_stopped, + previous_pointer_restored:$pointer_restored, + previous_runtime_restarted:false, + trading_authority_blocked:true, + operator_action_required:$operator_action, + failed_at:$at} + ' >"$evidence_tmp"; then + return 1 + fi + evidence_sha=$(sha256sum "$evidence_tmp" | awk '{print $1}') || return 1 + [[ $evidence_sha =~ ^[0-9a-f]{64}$ ]] || return 1 + printf '%s %s\n' "$evidence_sha" "$evidence_name" >"$marker_tmp_failed" \ + || return 1 + chmod 0444 "$evidence_tmp" "$marker_tmp_failed" || return 1 + sync -f "$evidence_tmp" || return 1 + sync -f "$marker_tmp_failed" || return 1 + mv -T "$evidence_tmp" "$evidence_dir/$evidence_name" || return 1 + sync -f "$evidence_dir" || return 1 + mv -T "$marker_tmp_failed" "$evidence_dir/$marker_name" || return 1 + sync -f "$evidence_dir" >/dev/null 2>&1 || true + [[ $result == failed_closed ]] +} + +cutover_release() { + local image=$1 release_manifest_sha=$2 activation_dir=$3 + local release_root=${RELEASE_ROOT:-$RELEASE_ROOT_DEFAULT} + local evidence_root=${EVIDENCE_ROOT:-$EVIDENCE_ROOT_DEFAULT} + local current_file=${CURRENT_FILE:-$CURRENT_FILE_DEFAULT} + local runtime_program=${RUNTIME_PROGRAM:-$RUNTIME_PROGRAM_DEFAULT} + local digest_hex release_file source_revision activation_sha run_id evidence_parent evidence_dir + local candidate_file previous_file invocation_id main_pid nrestarts container_id + local health_endpoint grpc_endpoint expected_identity cutover_tmp marker_tmp cutover_sha + local candidate_pointer_sha current_pointer_sha previous_pointer_sha='' + local previous_pointer_present=false + local failure_reason=unknown success_ready=false + local cleanup_armed=false success_committed=false cutover_status + require_root + valid_image_reference "$image" || die 'cutover requires an immutable Tokyo VPC ACR digest reference' + [[ $release_manifest_sha =~ ^[0-9a-f]{64}$ ]] || die 'cutover requires an immutable release manifest digest' + [[ $activation_dir == /* ]] || die 'activation directory must be absolute' + verify_bare_tokyo_host >/dev/null + systemctl is-active --quiet "$SERVICE" && die 'cutover requires the runtime to be stopped' + assert_service_static || die 'cutover requires a static, boot-disabled trading service' + digest_hex=$(release_digest_hex "$image") + release_file=$release_root/$digest_hex/$release_manifest_sha/hft-trading-ecs-release.json + secure_regular_file "$release_file" 0444 || die 'image was not staged by this contract' + [[ $(sha256sum "$release_file" | awk '{print $1}') == "$release_manifest_sha" ]] \ + || die 'staged release manifest digest mismatch' + source_revision=$(jq -er --arg image "$image" \ + 'select(.image.reference == $image) | .source_revision' "$release_file") \ + || die 'staged release identity mismatch' + activation_sha=$(sha256sum "$activation_dir/activation.sha256" | awk '{print $1}') + run_id=$(date -u +%Y%m%dT%H%M%SZ)-$$-$digest_hex-${release_manifest_sha:0:16}-$activation_sha + evidence_parent=$evidence_root/cutover + install -d -o root -g root -m 0700 "$evidence_parent" "${current_file%/*}" + evidence_dir=$evidence_parent/$run_id + mkdir -m 0700 "$evidence_dir" || die 'cutover evidence run already exists' + candidate_file=$evidence_dir/candidate-current.env + previous_file=$evidence_dir/previous-current.env + if [[ -e $current_file ]]; then + secure_regular_file "$current_file" 0600 \ + || die 'existing current pointer is unsafe' + install -o root -g root -m 0400 "$current_file" "$previous_file" + sync -f "$previous_file" + previous_pointer_sha=$(sha256sum "$previous_file" | awk '{print $1}') + [[ $previous_pointer_sha =~ ^[0-9a-f]{64}$ ]] \ + || die 'cannot bind the previous pointer snapshot' + previous_pointer_present=true + fi + write_current_file "$candidate_file" "$image" "$release_manifest_sha" \ + "$activation_dir" "$source_revision" + chmod 0400 "$candidate_file" + sync -f "$candidate_file" + candidate_pointer_sha=$(sha256sum "$candidate_file" | awk '{print $1}') + [[ $candidate_pointer_sha =~ ^[0-9a-f]{64}$ ]] \ + || die 'cannot bind the candidate pointer snapshot' + cleanup_armed=true + trap 'cutover_status=$?; trap - EXIT; trap "" HUP INT TERM; if [[ $cleanup_armed == true && $success_committed != true ]]; then finalize_cutover_failure "$evidence_dir" "$previous_file" "$current_file" "$runtime_program" "$image" "$source_revision" "$release_manifest_sha" "$activation_sha" "$failure_reason" "${cutover_tmp:-}" "${marker_tmp:-}" >/dev/null 2>&1 || true; fi; exit "$cutover_status"' EXIT + trap 'failure_reason=interrupted_SIGHUP; exit 129' HUP + trap 'failure_reason=interrupted_SIGINT; exit 130' INT + trap 'failure_reason=interrupted_SIGTERM; exit 143' TERM + if ! "$runtime_program" assert-stopped; then + failure_reason=orphan_container_detected + elif ! HFT_CURRENT_FILE="$candidate_file" "$runtime_program" preflight; then + failure_reason=preflight_failed + elif ! install_current_pointer "$candidate_file" "$current_file"; then + failure_reason=current_pointer_install_failed + elif ! systemctl start "$SERVICE"; then + failure_reason=systemd_start_failed + elif ! expected_identity=$(wait_for_runtime_identity "$image"); then + failure_reason=initial_runtime_identity_failed + elif ! wait_for_health "$image" "$expected_identity"; then + failure_reason=health_gate_failed + elif ! runtime_identity_matches "$image" "$expected_identity"; then + failure_reason=post_health_runtime_identity_failed + else + success_ready=true + fi + + if [[ $success_ready == true ]]; then + IFS='|' read -r invocation_id main_pid nrestarts container_id health_endpoint \ + grpc_endpoint <<<"$expected_identity" + cutover_tmp=$evidence_dir/cutover.json.tmp + marker_tmp=$evidence_dir/PASSED.sha256.tmp + if ! current_pointer_sha=$(sha256sum "$current_file" | awk '{print $1}'); then + failure_reason=current_pointer_hash_failed + elif [[ $current_pointer_sha != "$candidate_pointer_sha" ]]; then + failure_reason=current_pointer_lineage_mismatch + elif ! jq -S -n --arg image "$image" --arg source "$source_revision" \ + --arg release_manifest "$release_manifest_sha" \ + --arg activation "$activation_sha" --arg invocation "$invocation_id" \ + --arg container_id "$container_id" \ + --arg health_endpoint "$health_endpoint" \ + --arg grpc_endpoint "$grpc_endpoint" \ + --arg candidate_pointer_sha "$candidate_pointer_sha" \ + --arg previous_pointer_sha "$previous_pointer_sha" \ + --argjson previous_pointer_present "$previous_pointer_present" \ + --argjson main_pid "$main_pid" --argjson nrestarts "$nrestarts" \ + --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" ' + {schema:"monday.hft_trading_ecs_cutover.v1",result:"passed", + mode_boundary:"paper_or_shadow_only",image_reference:$image, + source_revision:$source,release_manifest_sha256:$release_manifest, + activation_manifest_sha256:$activation, + systemd_invocation_id:$invocation,main_pid:$main_pid, + container_id:$container_id,health_endpoint:$health_endpoint, + grpc_endpoint:$grpc_endpoint, + nrestarts:$nrestarts,health_samples:2, + grpc_connect_samples:2, + candidate_pointer_file:"candidate-current.env", + candidate_pointer_sha256:$candidate_pointer_sha, + current_pointer_sha256:$candidate_pointer_sha, + previous_pointer_present:$previous_pointer_present, + previous_pointer_file:(if $previous_pointer_present + then "previous-current.env" else null end), + previous_pointer_sha256:(if $previous_pointer_present + then $previous_pointer_sha else null end), + service_enabled:false,live_small_enabled:false,passed_at:$at} + ' >"$cutover_tmp"; then + failure_reason=cutover_evidence_write_failed + elif ! cutover_sha=$(sha256sum "$cutover_tmp" | awk '{print $1}'); then + failure_reason=cutover_evidence_hash_failed + elif [[ ! $cutover_sha =~ ^[0-9a-f]{64}$ ]]; then + failure_reason=cutover_evidence_hash_invalid + elif ! printf '%s cutover.json\n' "$cutover_sha" >"$marker_tmp"; then + failure_reason=cutover_marker_write_failed + elif ! chmod 0444 "$cutover_tmp" "$marker_tmp"; then + failure_reason=cutover_evidence_chmod_failed + elif ! sync -f "$cutover_tmp" || ! sync -f "$marker_tmp"; then + failure_reason=cutover_evidence_sync_failed + elif ! runtime_identity_matches "$image" "$expected_identity"; then + failure_reason=pre_evidence_runtime_identity_failed + elif ! mv -T "$cutover_tmp" "$evidence_dir/cutover.json"; then + failure_reason=cutover_evidence_commit_failed + elif ! sync -f "$evidence_dir"; then + failure_reason=cutover_evidence_directory_sync_failed + elif ! runtime_identity_matches "$image" "$expected_identity"; then + failure_reason=pre_marker_runtime_identity_failed + elif ! trap '' HUP INT TERM; then + failure_reason=cutover_signal_mask_failed + elif ! mv -T "$marker_tmp" "$evidence_dir/PASSED.sha256"; then + failure_reason=cutover_marker_commit_failed + else + # PASSED.sha256 is the commit point. Losing the best-effort directory sync + # can remove success after a crash, but can never manufacture success. + success_committed=true + cleanup_armed=false + trap - EXIT HUP INT TERM + sync -f "$evidence_dir" >/dev/null 2>&1 || true + printf '%s\n' "$evidence_dir/cutover.json" || true + return 0 + fi + fi + + trap '' HUP INT TERM + if ! finalize_cutover_failure "$evidence_dir" "$previous_file" "$current_file" \ + "$runtime_program" "$image" "$source_revision" "$release_manifest_sha" \ + "$activation_sha" "$failure_reason" "${cutover_tmp:-}" "${marker_tmp:-}"; then + failure_reason=${failure_reason}_and_failure_evidence_commit_failed + die "cutover emergency cleanup failed: $failure_reason" + fi + cleanup_armed=false + trap - EXIT HUP INT TERM + die "cutover failed closed: $failure_reason; evidence=$evidence_dir/cutover.failed.json" +} + +current_pointer_matches_sha() { + local current_file=$1 expected_sha=$2 + secure_regular_file "$current_file" 0600 || return 1 + [[ $expected_sha =~ ^[0-9a-f]{64}$ ]] || return 1 + [[ $(sha256sum "$current_file" | awk '{print $1}') == "$expected_sha" ]] +} + +verify_rollback_lineage() { + local evidence_dir=$1 current_file=$2 + local cutover_file=$evidence_dir/cutover.json + local candidate_file=$evidence_dir/candidate-current.env + local previous_file=$evidence_dir/previous-current.env + local invocation main_pid nrestarts container_id health_endpoint grpc_endpoint + ROLLBACK_IMAGE= + ROLLBACK_EXPECTED_IDENTITY= + ROLLBACK_CANDIDATE_SHA= + ROLLBACK_PREVIOUS_PRESENT= + ROLLBACK_PREVIOUS_SHA= + secure_regular_file "$cutover_file" 0444 || return 1 + [[ $(stat_mode "$cutover_file") == 444 ]] || return 1 + jq -e -s ' + length == 1 and (.[0] | + .schema == "monday.hft_trading_ecs_cutover.v1" and + .result == "passed" and + (.image_reference | type == "string") and + (.source_revision | test("^[0-9a-f]{40}$")) and + (.release_manifest_sha256 | test("^[0-9a-f]{64}$")) and + (.activation_manifest_sha256 | test("^[0-9a-f]{64}$")) and + (.systemd_invocation_id | test("^[0-9a-f]{32}$")) and + (.main_pid | type == "number" and . > 0 and floor == .) and + .nrestarts == 0 and + (.container_id | test("^[0-9a-f]{64}$")) and + (.health_endpoint | test("^127[.]0[.]0[.]1:[1-9][0-9]{0,4}$")) and + (.grpc_endpoint | test("^127[.]0[.]0[.]1:[1-9][0-9]{0,4}$")) and + .health_endpoint != .grpc_endpoint and + .health_samples >= 2 and .grpc_connect_samples >= 2 and + .candidate_pointer_file == "candidate-current.env" and + (.candidate_pointer_sha256 | test("^[0-9a-f]{64}$")) and + .current_pointer_sha256 == .candidate_pointer_sha256 and + (.previous_pointer_present | type == "boolean") and + (if .previous_pointer_present then + .previous_pointer_file == "previous-current.env" and + (.previous_pointer_sha256 | test("^[0-9a-f]{64}$")) + else + .previous_pointer_file == null and .previous_pointer_sha256 == null + end) and + .service_enabled == false and .live_small_enabled == false + ) + ' "$cutover_file" >/dev/null || return 1 + ROLLBACK_IMAGE=$(jq -er '.image_reference' "$cutover_file") || return 1 + valid_image_reference "$ROLLBACK_IMAGE" || return 1 + ROLLBACK_CANDIDATE_SHA=$(jq -er '.candidate_pointer_sha256' "$cutover_file") \ + || return 1 + ROLLBACK_PREVIOUS_PRESENT=$(jq -r '.previous_pointer_present' "$cutover_file") \ + || return 1 + secure_regular_file "$candidate_file" 0400 || return 1 + [[ $(stat_mode "$candidate_file") == 400 ]] || return 1 + [[ $(sha256sum "$candidate_file" | awk '{print $1}') == "$ROLLBACK_CANDIDATE_SHA" ]] \ + || return 1 + current_pointer_matches_sha "$current_file" "$ROLLBACK_CANDIDATE_SHA" || return 1 + if [[ $ROLLBACK_PREVIOUS_PRESENT == true ]]; then + ROLLBACK_PREVIOUS_SHA=$(jq -er '.previous_pointer_sha256' "$cutover_file") \ + || return 1 + secure_regular_file "$previous_file" 0400 || return 1 + [[ $(stat_mode "$previous_file") == 400 ]] || return 1 + [[ $(sha256sum "$previous_file" | awk '{print $1}') == "$ROLLBACK_PREVIOUS_SHA" ]] \ + || return 1 + else + [[ $ROLLBACK_PREVIOUS_PRESENT == false && ! -e $previous_file ]] || return 1 + fi + invocation=$(jq -er '.systemd_invocation_id' "$cutover_file") || return 1 + main_pid=$(jq -er '.main_pid | tostring' "$cutover_file") || return 1 + nrestarts=$(jq -er '.nrestarts | tostring' "$cutover_file") || return 1 + container_id=$(jq -er '.container_id' "$cutover_file") || return 1 + health_endpoint=$(jq -er '.health_endpoint' "$cutover_file") || return 1 + grpc_endpoint=$(jq -er '.grpc_endpoint' "$cutover_file") || return 1 + ROLLBACK_EXPECTED_IDENTITY=$invocation'|'$main_pid'|'$nrestarts'|'$container_id'|'$health_endpoint'|'$grpc_endpoint +} + +rollback_cutover() { + local evidence_dir=$1 current_file=${CURRENT_FILE:-$CURRENT_FILE_DEFAULT} + local runtime_program=${RUNTIME_PROGRAM:-$RUNTIME_PROGRAM_DEFAULT} + local previous_file=$evidence_dir/previous-current.env + local marker=$evidence_dir/PASSED.sha256 + local pending_marker=$evidence_dir/PASSED.rollback-pending.sha256 + local rolled_back_marker=$evidence_dir/PASSED.rolled-back.sha256 + local pending_tmp=$evidence_dir/.PASSED.rollback-pending.sha256.tmp.$$ + local rolled_back_tmp=$evidence_dir/.PASSED.rolled-back.sha256.tmp.$$ + local rollback_at cutover_sha + require_root + [[ $evidence_dir == /* && -d $evidence_dir && ! -L $evidence_dir ]] \ + || die 'cutover evidence directory must be absolute' + secure_directory "$evidence_dir" || die 'cutover evidence directory is unsafe' + verify_single_checksum_marker "$evidence_dir" PASSED.sha256 cutover.json \ + || die 'canonical cutover success marker is invalid' + [[ ! -e $pending_marker && ! -e $rolled_back_marker ]] \ + || die 'rollback evidence already exists' + verify_rollback_lineage "$evidence_dir" "$current_file" \ + || die 'cutover evidence is stale, legacy, or has invalid pointer lineage' + runtime_identity_matches "$ROLLBACK_IMAGE" "$ROLLBACK_EXPECTED_IDENTITY" \ + || die 'active runtime does not match this cutover evidence' + install -m 0444 "$marker" "$pending_tmp" + mv -T "$pending_tmp" "$pending_marker" + sync -f "$evidence_dir" + if ! current_pointer_matches_sha "$current_file" "$ROLLBACK_CANDIDATE_SHA" \ + || ! runtime_identity_matches "$ROLLBACK_IMAGE" "$ROLLBACK_EXPECTED_IDENTITY"; then + die "rollback lineage changed; candidate was not stopped; marker held at $pending_marker" + fi + systemctl stop "$SERVICE" >/dev/null 2>&1 || true + "$runtime_program" ensure-stopped \ + || die "rollback stop failed; canonical marker invalidated at $pending_marker" + if [[ $ROLLBACK_PREVIOUS_PRESENT == true ]]; then + install_current_pointer "$previous_file" "$current_file" + current_pointer_matches_sha "$current_file" "$ROLLBACK_PREVIOUS_SHA" \ + || die "rollback pointer verification failed; marker held at $pending_marker" + else + rm -f -- "$current_file" + sync -f "${current_file%/*}" + [[ ! -e $current_file ]] \ + || die "rollback pointer removal failed; marker held at $pending_marker" + fi + install -m 0444 "$pending_marker" "$rolled_back_tmp" + mv -T "$rolled_back_tmp" "$rolled_back_marker" + rollback_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) + cutover_sha=$(sha256sum "$evidence_dir/cutover.json" | awk '{print $1}') + jq -S -n --arg cutover "$cutover_sha" \ + --arg candidate "$ROLLBACK_CANDIDATE_SHA" \ + --arg previous "$ROLLBACK_PREVIOUS_SHA" \ + --argjson previous_present "$ROLLBACK_PREVIOUS_PRESENT" \ + --arg at "$rollback_at" ' + {schema:"monday.hft_trading_ecs_rollback.v1",result:"rolled_back_stopped", + cutover_sha256:$cutover,runtime_stopped:true, + candidate_pointer_sha256:$candidate, + previous_pointer_present:$previous_present, + previous_pointer_sha256:(if $previous_present then $previous else null end), + previous_pointer_restored:true,previous_runtime_restarted:false, + operator_action_required:"new signed envelope and nonce before restart", + rolled_back_at:$at} + ' >"$evidence_dir/rollback.json" + (cd "$evidence_dir" && sha256sum rollback.json >ROLLED_BACK.sha256) + chmod 0444 "$rolled_back_marker" \ + "$evidence_dir/rollback.json" "$evidence_dir/ROLLED_BACK.sha256" + sync -f "$evidence_dir" + printf '%s\n' "$evidence_dir/rollback.json" +} + +usage() { + cat <<'USAGE' +Usage: + trading-ecs-hostctl.sh stage ARTIFACT_DIR ACR_USERNAME ACR_PASSWORD_FILE + trading-ecs-hostctl.sh cutover IMAGE_REFERENCE RELEASE_MANIFEST_SHA256 ACTIVATION_DIR + trading-ecs-hostctl.sh rollback CUTOVER_EVIDENCE_DIR + +The host must be a Tokyo Ubuntu 26.04 amd64 ECS with MondayTradingEcsRole. +stage never starts or enables the service. cutover accepts Paper/Shadow only. +USAGE +} + +with_host_lock() { + local lock_file=${HOST_LOCK:-$HOST_LOCK_DEFAULT} + require_root + command -v flock >/dev/null 2>&1 || die 'flock is required for host operations' + install -d -o root -g root -m 0755 "${lock_file%/*}" + exec 9>"$lock_file" + chmod 0600 "$lock_file" + flock -n 9 || die 'another trading host operation holds the lock' + "$@" +} + +if [[ ${BASH_SOURCE[0]} == "$0" ]]; then + case ${1:-} in + stage) + [[ $# -eq 4 ]] || { usage; exit 2; } + with_host_lock stage_release "$2" "$3" "$4" + ;; + cutover) + [[ $# -eq 4 ]] || { usage; exit 2; } + with_host_lock cutover_release "$2" "$3" "$4" + ;; + rollback) + [[ $# -eq 2 ]] || { usage; exit 2; } + with_host_lock rollback_cutover "$2" + ;; + *) usage; exit 2 ;; + esac +fi diff --git a/deployment/aliyun/trading-ecs-paper-shadow-policy.jq b/deployment/aliyun/trading-ecs-paper-shadow-policy.jq new file mode 100644 index 000000000..b2b568b00 --- /dev/null +++ b/deployment/aliyun/trading-ecs-paper-shadow-policy.jq @@ -0,0 +1,45 @@ +def one_json_value: + select(type == "array" and length == 1) | .[0]; + +def activation_intents($intents): + [$intents[] | select( + . == "StartPaper" or . == "StartShadow" or . == "StartLiveSmall" + )]; + +one_json_value as $signed +| activation_intents($signed.envelope.allowed_intent_types) as $signed_starts +| activation_intents($policy[0].allowed_intent_types) as $policy_starts +| ([$signed.envelope.allowed_intent_types[] + | select(. == "LoadFactor")]) as $signed_artifacts +| ($policy | length == 1) and + ($signed | type == "object") and + ($signed.envelope | type == "object") and + ($signed.envelope.allowed_intent_types | type == "array") and + ($policy[0] | type == "object") and + ($policy[0].allowed_intent_types | type == "array") and + ($policy[0].approvals | type == "array") and + ($signed.envelope.allowed_intent_types | all( + . == "LoadFactor" or + . == "StartPaper" or . == "StartShadow" + )) and + ($policy[0].allowed_intent_types | all( + . == "LoadFactor" or + . == "StartPaper" or . == "StartShadow" + )) and + ($signed_starts | length == 1) and + ($signed_artifacts | length == 1) and + ($policy_starts == [$signed_starts[0]]) and + (($policy[0].allowed_intent_types | index($signed_artifacts[0])) != null) and + ($signed.envelope.allowed_intent_types | index("StartLiveSmall") | not) and + ($policy[0].allowed_intent_types | index("StartLiveSmall") | not) and + ($policy[0].runtime_paused == false) and + ( + (($signed.envelope.allowed_intent_types | index("StartPaper")) != null and + $signed.envelope.approval_class == "Paper") + or + (($signed.envelope.allowed_intent_types | index("StartShadow")) != null and + $signed.envelope.approval_class == "Shadow") + ) and + ([$policy[0].approvals[]?.approval_class + | select(. == "HumanApprovedLiveSmall" or . == "SameClassAutoLiveSmall")] + | length == 0) diff --git a/deployment/aliyun/trading-ecs-runtime.sh b/deployment/aliyun/trading-ecs-runtime.sh new file mode 100755 index 000000000..8058fff37 --- /dev/null +++ b/deployment/aliyun/trading-ecs-runtime.sh @@ -0,0 +1,530 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +export LC_ALL=C + +readonly CURRENT_FILE_DEFAULT=/etc/monday/hft-trading-current.env +readonly ACTIVATION_ROOT_DEFAULT=/opt/monday/activations +readonly RELEASE_ROOT_DEFAULT=/opt/monday/releases/hft-trading +readonly CONTROL_ROOT_DEFAULT=/opt/monday/control/hft-trading +readonly STATE_ROOT_DEFAULT=/var/lib/monday/hft-trading +readonly SECRET_ROOT_DEFAULT=/run/monday/trading-secrets +readonly CONTAINER_NAME=monday-hft-trading +readonly EXPECTED_ROOT_UID_DEFAULT=0 +readonly HFT_RUNTIME_USER_DEFAULT=mondayhft +readonly HFT_RUNTIME_GROUP_DEFAULT=mondayhft +readonly EXPECTED_REGION=ap-northeast-1 +readonly EXPECTED_HOST_OS=ubuntu +readonly EXPECTED_HOST_VERSION=26.04 +readonly EXPECTED_ARCH=amd64 +readonly EXPECTED_RAM_ROLE=MondayTradingEcsRole +readonly HOSTCTL_PROGRAM=/usr/local/sbin/monday-hft-trading-hostctl +readonly UNIT_PATH=/etc/systemd/system/monday-hft-trading.service +readonly SERVICE=monday-hft-trading.service + +die() { + printf 'hft-trading runtime preflight failed: %s\n' "$*" >&2 + exit 1 +} + +stat_uid() { + if [[ $(uname -s) == Darwin ]]; then + stat -f %u -- "$1" + else + stat -c %u -- "$1" + fi +} + +stat_gid() { + if [[ $(uname -s) == Darwin ]]; then + stat -f %g -- "$1" + else + stat -c %g -- "$1" + fi +} + +stat_mode() { + if [[ $(uname -s) == Darwin ]]; then + stat -f %Lp -- "$1" + else + stat -c %a -- "$1" + fi +} + +secure_regular_file() { + local path=$1 max_mode=$2 expected_uid=${EXPECTED_ROOT_UID:-$EXPECTED_ROOT_UID_DEFAULT} + local mode + [[ -f $path && ! -L $path ]] || return 1 + [[ $(stat_uid "$path") == "$expected_uid" ]] || return 1 + mode=$(stat_mode "$path") || return 1 + (( (8#$mode & ~8#$max_mode) == 0 )) +} + +secure_directory() { + local path=$1 expected_uid=${EXPECTED_ROOT_UID:-$EXPECTED_ROOT_UID_DEFAULT} + local mode + [[ -d $path && ! -L $path ]] || return 1 + [[ $(stat_uid "$path") == "$expected_uid" ]] || return 1 + mode=$(stat_mode "$path") || return 1 + (( (8#$mode & 8#022) == 0 )) +} + +canonical_directory() { + local path=$1 canonical + [[ $path == /* ]] || return 1 + canonical=$(cd -- "$path" 2>/dev/null && pwd -P) || return 1 + [[ $path == "$canonical" ]] || return 1 + printf '%s\n' "$canonical" +} + +secure_directory_chain() { + local path=$1 current=/ component + local -a components + [[ $path == /* ]] || return 1 + secure_directory / || return 1 + [[ $path == / ]] && return 0 + IFS=/ read -r -a components <<<"${path#/}" + for component in "${components[@]}"; do + [[ -n $component && $component != . && $component != .. ]] || return 1 + current=${current%/}/$component + secure_directory "$current" || return 1 + done +} + +assert_no_service_dropin_directories() { + local root + for root in "$@"; do + [[ ! -e $root/$SERVICE.d ]] || return 1 + done +} + +effective_exec_matches() { + local property=$1 program=$2 argument=$3 value after_open before_close + value=$(systemctl show "$SERVICE" --property="$property" --value) || return 1 + [[ $value == "{ path=$program ; argv[]=$program $argument ;"* \ + && $value == *'ignore_errors=no'* && $value == *' }' ]] || return 1 + after_open=${value#*\{} + before_close=${value%\}} + [[ $after_open != "$value" && $after_open != *'{'* \ + && $before_close != "$value" && $before_close != *'}'* ]] +} + +assert_service_static() { + local state + state=$(systemctl is-enabled "$SERVICE" 2>/dev/null) || return 1 + [[ $state == static ]] +} + +assert_effective_service_contract() { + local unit_path=$1 runtime_program=$2 fragment dropins restart start_post + assert_no_service_dropin_directories \ + /etc/systemd/system /run/systemd/system /usr/local/lib/systemd/system \ + /usr/lib/systemd/system /lib/systemd/system || return 1 + fragment=$(systemctl show "$SERVICE" --property=FragmentPath --value) || return 1 + dropins=$(systemctl show "$SERVICE" --property=DropInPaths --value) || return 1 + restart=$(systemctl show "$SERVICE" --property=Restart --value) || return 1 + start_post=$(systemctl show "$SERVICE" --property=ExecStartPost --value) || return 1 + [[ $fragment == "$unit_path" && -z $dropins && $restart == no \ + && -z $start_post ]] || return 1 + assert_service_static || return 1 + effective_exec_matches ExecStartPre "$runtime_program" preflight || return 1 + effective_exec_matches ExecStart "$runtime_program" run || return 1 + effective_exec_matches ExecStop "$runtime_program" stop || return 1 + effective_exec_matches ExecStopPost "$runtime_program" ensure-stopped +} + +validate_bare_host_identity_values() { + local host_os=$1 host_version=$2 architecture=$3 region=$4 role=$5 instance_id=$6 + [[ $host_os == "$EXPECTED_HOST_OS" \ + && $host_version == "$EXPECTED_HOST_VERSION" \ + && $architecture == "$EXPECTED_ARCH" \ + && $region == "$EXPECTED_REGION" \ + && $role == "$EXPECTED_RAM_ROLE" \ + && $instance_id =~ ^i-[a-z0-9]+$ ]] +} + +metadata_token() { + curl -fsS --connect-timeout 2 --max-time 5 -X PUT \ + -H 'X-aliyun-ecs-metadata-token-ttl-seconds: 300' \ + http://100.100.100.200/latest/api/token +} + +metadata_get() { + local token=$1 path=$2 + curl -fsS --connect-timeout 2 --max-time 5 \ + -H "X-aliyun-ecs-metadata-token: $token" \ + "http://100.100.100.200/latest/meta-data/$path" +} + +verify_bare_tokyo_host() { + local token role region instance_id architecture + # shellcheck disable=SC1091 + source /etc/os-release + systemctl is-active --quiet kubelet && return 1 + [[ ! -e /var/lib/kubelet/kubeconfig ]] || return 1 + architecture=$(dpkg --print-architecture) || return 1 + token=$(metadata_token) || return 1 + region=$(metadata_get "$token" region-id) || return 1 + role=$(metadata_get "$token" ram/security-credentials/) || return 1 + role=${role%%$'\n'*} + instance_id=$(metadata_get "$token" instance-id) || return 1 + validate_bare_host_identity_values "${ID:-}" "${VERSION_ID:-}" \ + "$architecture" "$region" "$role" "$instance_id" +} + +validate_selected_control_assets() { + local manifest=$1 runtime_path=$2 hostctl_path=$3 policy_path=$4 unit_path=$5 + local control_names runtime_sha hostctl_sha policy_sha unit_sha + secure_regular_file "$manifest" 0444 || return 1 + secure_regular_file "$runtime_path" 0555 || return 1 + secure_regular_file "$hostctl_path" 0555 || return 1 + secure_regular_file "$policy_path" 0444 || return 1 + secure_regular_file "$unit_path" 0444 || return 1 + [[ $(stat_mode "$runtime_path") == 555 \ + && $(stat_mode "$hostctl_path") == 555 \ + && $(stat_mode "$policy_path") == 444 \ + && $(stat_mode "$unit_path") == 444 ]] || return 1 + control_names=$(awk ' + NF != 2 || $1 !~ /^[0-9a-f]{64}$/ || $2 !~ /^[A-Za-z0-9._-]+$/ { exit 2 } + { print $2 } + ' "$manifest") || return 1 + [[ $control_names == $'hft-trading-ecs.service\ntrading-ecs-hostctl.sh\ntrading-ecs-paper-shadow-policy.jq\ntrading-ecs-runtime.sh' ]] \ + || return 1 + runtime_sha=$(awk '$2 == "trading-ecs-runtime.sh" { print $1 }' "$manifest") + hostctl_sha=$(awk '$2 == "trading-ecs-hostctl.sh" { print $1 }' "$manifest") + policy_sha=$(awk '$2 == "trading-ecs-paper-shadow-policy.jq" { print $1 }' "$manifest") + unit_sha=$(awk '$2 == "hft-trading-ecs.service" { print $1 }' "$manifest") + [[ $runtime_sha =~ ^[0-9a-f]{64}$ \ + && $hostctl_sha =~ ^[0-9a-f]{64}$ \ + && $policy_sha =~ ^[0-9a-f]{64}$ \ + && $unit_sha =~ ^[0-9a-f]{64}$ ]] || return 1 + [[ $(sha256sum "$runtime_path" | awk '{print $1}') == "$runtime_sha" \ + && $(sha256sum "$hostctl_path" | awk '{print $1}') == "$hostctl_sha" \ + && $(sha256sum "$policy_path" | awk '{print $1}') == "$policy_sha" \ + && $(sha256sum "$unit_path" | awk '{print $1}') == "$unit_sha" ]] +} + +runtime_account_ids() { + local runtime_user=$HFT_RUNTIME_USER_DEFAULT + local runtime_group=$HFT_RUNTIME_GROUP_DEFAULT + local passwd_record group_record all_passwd all_groups uid gid home shell + local group_gid group_members other_name other_uid other_gid + passwd_record=$(getent passwd "$runtime_user") || return 1 + group_record=$(getent group "$runtime_group") || return 1 + IFS=: read -r _ _ uid gid _ home shell <<<"$passwd_record" + IFS=: read -r _ _ group_gid group_members <<<"$group_record" + [[ $uid =~ ^[0-9]+$ && $gid =~ ^[0-9]+$ ]] || return 1 + [[ $group_gid =~ ^[0-9]+$ && $group_gid == "$gid" ]] || return 1 + [[ $uid != 0 && $uid != 1000 && $gid != 0 && $gid != 1000 ]] || return 1 + [[ $home == /nonexistent ]] || return 1 + [[ $shell == /usr/sbin/nologin || $shell == /bin/false ]] || return 1 + [[ ${group_record%%:*} == "$runtime_group" && -z $group_members ]] || return 1 + [[ $(id -g "$runtime_user") == "$gid" ]] || return 1 + all_passwd=$(getent passwd) || return 1 + while IFS=: read -r other_name _ other_uid other_gid _; do + [[ $other_name == "$runtime_user" ]] && continue + [[ $other_uid != "$uid" && $other_gid != "$gid" ]] || return 1 + done <<<"$all_passwd" + all_groups=$(getent group) || return 1 + while IFS=: read -r other_name _ other_gid _; do + [[ $other_name == "$runtime_group" ]] && continue + [[ $other_gid != "$gid" ]] || return 1 + done <<<"$all_groups" + printf '%s\n%s\n' "$uid" "$gid" +} + +valid_image_reference() { + local image=$1 + [[ $image =~ ^crpi-[a-z0-9]+-vpc[.]ap-northeast-1[.]personal[.]cr[.]aliyuncs[.]com/wildcard0923/hft-trading@sha256:[0-9a-f]{64}$ ]] +} + +image_digest_hex() { + valid_image_reference "$1" || return 1 + printf '%s\n' "${1##*@sha256:}" +} + +validate_activation_manifest() { + local activation_dir=$1 + local manifest=$activation_dir/activation.sha256 + local activation_root=${ACTIVATION_ROOT:-$ACTIVATION_ROOT_DEFAULT} + local actual_names manifest_names required path expected_uid mode + local canonical_root canonical_activation + expected_uid=${EXPECTED_ROOT_UID:-$EXPECTED_ROOT_UID_DEFAULT} + canonical_root=$(canonical_directory "$activation_root") || return 1 + canonical_activation=$(canonical_directory "$activation_dir") || return 1 + [[ $canonical_root == "$activation_root" \ + && $canonical_activation == "$activation_dir" \ + && $activation_dir == "$activation_root"/* ]] || return 1 + secure_directory_chain "$activation_dir" || return 1 + secure_regular_file "$manifest" 0444 || return 1 + [[ -z $(find "$activation_dir" -type l -print -quit) ]] || return 1 + while IFS= read -r -d '' path; do + [[ $(stat_uid "$path") == "$expected_uid" ]] || return 1 + mode=$(stat_mode "$path") || return 1 + (( (8#$mode & 8#022) == 0 )) || return 1 + done < <(find "$activation_dir" \( -type f -o -type d \) -print0) + + manifest_names=$(awk ' + NF != 2 || $1 !~ /^[0-9a-f]{64}$/ || $2 !~ /^[A-Za-z0-9._\/-]+$/ || + $2 ~ /^\// || $2 ~ /(^|\/)\.\.?(\/|$)/ { exit 2 } + { print $2 } + ' "$manifest") || return 1 + [[ -n $manifest_names ]] || return 1 + [[ $(printf '%s\n' "$manifest_names" | LC_ALL=C sort -u) == "$manifest_names" ]] \ + || return 1 + actual_names=$(cd "$activation_dir" && find . -type f \ + ! -path ./activation.sha256 -print | sed 's#^./##' | LC_ALL=C sort) || return 1 + [[ $actual_names == "$manifest_names" ]] || return 1 + for required in \ + config/system.yaml \ + deployment/bundle.json \ + deployment/envelope.json \ + deployment/policy.json \ + deployment/trusted-keys.json; do + printf '%s\n' "$manifest_names" | grep -Fxq "$required" || return 1 + done + while IFS= read -r path; do + [[ -f $activation_dir/$path && ! -L $activation_dir/$path ]] || return 1 + done <<<"$manifest_names" + (cd "$activation_dir" && sha256sum --check --strict activation.sha256 >/dev/null) \ + || return 1 +} + +validate_paper_shadow_authority() { + local activation_dir=$1 control_root=${CONTROL_ROOT:-$CONTROL_ROOT_DEFAULT} + jq -e -s --slurpfile policy "$activation_dir/deployment/policy.json" \ + -f "$control_root/trading-ecs-paper-shadow-policy.jq" \ + "$activation_dir/deployment/envelope.json" >/dev/null +} + +validate_runtime_secrets() { + local secret_root=${SECRET_ROOT:-$SECRET_ROOT_DEFAULT} + local runtime_env=$secret_root/runtime.env + local feedback_key=$secret_root/feedback-signing-key.hex + local fs_type runtime_env_fs feedback_key_fs api_prefixes secret_prefixes grpc_token + local expected_uid runtime_uid runtime_gid + local runtime_ids + expected_uid=${EXPECTED_ROOT_UID:-$EXPECTED_ROOT_UID_DEFAULT} + canonical_directory "$secret_root" >/dev/null || return 1 + secure_directory "$secret_root" || return 1 + [[ $(stat_mode "$secret_root") == 750 ]] || return 1 + runtime_ids=$(runtime_account_ids) || return 1 + [[ $runtime_ids == *$'\n'* && ${runtime_ids#*$'\n'} != *$'\n'* ]] || return 1 + runtime_uid=${runtime_ids%%$'\n'*} + runtime_gid=${runtime_ids#*$'\n'} + [[ $runtime_uid =~ ^[0-9]+$ && $runtime_gid =~ ^[0-9]+$ ]] || return 1 + [[ $runtime_uid != 0 && $runtime_uid != 1000 \ + && $runtime_gid != 0 && $runtime_gid != 1000 ]] || return 1 + [[ $(stat_uid "$secret_root") == "$expected_uid" ]] || return 1 + [[ $(stat_gid "$secret_root") == "$runtime_gid" ]] || return 1 + fs_type=$(findmnt -n -o FSTYPE --target "$secret_root") || return 1 + [[ $fs_type == tmpfs ]] || return 1 + runtime_env_fs=$(findmnt -n -o FSTYPE --target "$runtime_env") || return 1 + feedback_key_fs=$(findmnt -n -o FSTYPE --target "$feedback_key") || return 1 + [[ $runtime_env_fs == tmpfs && $feedback_key_fs == tmpfs ]] || return 1 + secure_regular_file "$runtime_env" 0440 || return 1 + secure_regular_file "$feedback_key" 0440 || return 1 + [[ ${runtime_env%/*} == "$secret_root" && ${feedback_key%/*} == "$secret_root" ]] \ + || return 1 + [[ $(stat_mode "$runtime_env") == 440 ]] || return 1 + [[ $(stat_mode "$feedback_key") == 440 ]] || return 1 + [[ $(stat_gid "$runtime_env") == "$runtime_gid" ]] || return 1 + [[ $(stat_gid "$feedback_key") == "$runtime_gid" ]] || return 1 + [[ $(wc -l <"$feedback_key") -eq 1 ]] || return 1 + grep -Eq '^[0-9a-f]{64}$' "$feedback_key" || return 1 + grep -Eq '^HFT_GRPC_AUTH_TOKEN=.+$' "$runtime_env" || return 1 + grep -Eq '^HFT_SECRET_[A-Z0-9_]+_API_KEY=.+$' "$runtime_env" || return 1 + grep -Eq '^HFT_SECRET_[A-Z0-9_]+_SECRET=.+$' "$runtime_env" || return 1 + if ! awk -F= ' + !/^[A-Z_][A-Z0-9_]*=.+$/ { exit 1 } + seen[$1]++ { exit 1 } + $1 == "HFT_GRPC_AUTH_TOKEN" { grpc++; next } + $1 ~ /^HFT_SECRET_[A-Z0-9][A-Z0-9_]*_(API_KEY|SECRET)$/ { next } + { exit 1 } + END { if (grpc != 1) exit 1 } + ' "$runtime_env"; then + return 1 + fi + api_prefixes=$(sed -n 's/^HFT_SECRET_\([A-Z0-9_]*\)_API_KEY=.*/\1/p' \ + "$runtime_env" | LC_ALL=C sort -u) || return 1 + secret_prefixes=$(sed -n 's/^HFT_SECRET_\([A-Z0-9_]*\)_SECRET=.*/\1/p' \ + "$runtime_env" | LC_ALL=C sort -u) || return 1 + [[ -n $api_prefixes && $api_prefixes == "$secret_prefixes" ]] || return 1 + if grep -Eq '^[A-Za-z_][A-Za-z0-9_]*=$|^[[:space:]]|[[:space:]]$' "$runtime_env"; then + return 1 + fi + grpc_token=$(sed -n 's/^HFT_GRPC_AUTH_TOKEN=//p' "$runtime_env") || return 1 + [[ ${#grpc_token} -ge 32 \ + && ! $grpc_token =~ ^[[:space:]] \ + && ! $grpc_token =~ [[:space:]]$ ]] || return 1 +} + +load_current() { + local current_file=${HFT_CURRENT_FILE:-$CURRENT_FILE_DEFAULT} + secure_regular_file "$current_file" 0600 || die "invalid current release file" + # This file is root-owned, contains identifiers only, and is generated by hostctl. + # shellcheck disable=SC1090 + source "$current_file" + : "${HFT_TRADING_IMAGE:?missing HFT_TRADING_IMAGE}" + : "${HFT_RELEASE_MANIFEST_SHA256:?missing HFT_RELEASE_MANIFEST_SHA256}" + : "${HFT_ACTIVATION_DIR:?missing HFT_ACTIVATION_DIR}" + : "${HFT_ACTIVATION_SHA256:?missing HFT_ACTIVATION_SHA256}" + : "${HFT_SOURCE_REVISION:?missing HFT_SOURCE_REVISION}" +} + +preflight() { + local release_root=${RELEASE_ROOT:-$RELEASE_ROOT_DEFAULT} + local control_root=${CONTROL_ROOT:-$CONTROL_ROOT_DEFAULT} + local digest_hex release_file release_dir activation_sha + local control_manifest expected_control_manifest + assert_effective_service_contract "$UNIT_PATH" "${BASH_SOURCE[0]}" \ + || die "effective systemd service no longer matches the reviewed static unit" + verify_bare_tokyo_host || die "host identity no longer matches the reviewed Tokyo ECS contract" + load_current + valid_image_reference "$HFT_TRADING_IMAGE" || die "image is not the Tokyo VPC ACR hft-trading digest reference" + [[ $HFT_SOURCE_REVISION =~ ^[0-9a-f]{40}$ ]] || die "invalid source revision" + [[ $HFT_RELEASE_MANIFEST_SHA256 =~ ^[0-9a-f]{64}$ ]] || die "invalid release manifest identity" + digest_hex=$(image_digest_hex "$HFT_TRADING_IMAGE") + release_file=$release_root/$digest_hex/$HFT_RELEASE_MANIFEST_SHA256/hft-trading-ecs-release.json + release_dir=${release_file%/*} + secure_regular_file "$release_file" 0444 || die "missing immutable staged release" + secure_regular_file "$release_file.sha256" 0444 || die "missing staged release checksum" + [[ $(wc -l <"$release_file.sha256") -eq 1 ]] || die "invalid staged release checksum" + [[ $(sha256sum "$release_file" | awk '{print $1}') == "$HFT_RELEASE_MANIFEST_SHA256" ]] \ + || die "staged release identity mismatch" + [[ $(<"$release_file.sha256") == "$HFT_RELEASE_MANIFEST_SHA256 hft-trading-ecs-release.json" ]] \ + || die "staged release checksum mismatch" + jq -e --arg image "$HFT_TRADING_IMAGE" --arg source "$HFT_SOURCE_REVISION" ' + .schema == "monday.hft_trading_ecs_release.v1" and + .image.reference == $image and .source_revision == $source + ' "$release_file" >/dev/null || die "staged release identity mismatch" + control_manifest=$release_dir/trading-ecs-control-assets.sha256 + secure_regular_file "$control_manifest" 0444 || die "missing staged control manifest" + expected_control_manifest=$(jq -er '.control_manifest.sha256' "$release_file") \ + || die "release has no control manifest identity" + [[ $(sha256sum "$control_manifest" | awk '{print $1}') == "$expected_control_manifest" ]] \ + || die "staged control manifest mismatch" + validate_selected_control_assets "$control_manifest" "${BASH_SOURCE[0]}" \ + "$HOSTCTL_PROGRAM" "$control_root/trading-ecs-paper-shadow-policy.jq" \ + "$UNIT_PATH" || die "installed control assets do not match the selected release" + validate_activation_manifest "$HFT_ACTIVATION_DIR" || die "activation manifest is invalid" + activation_sha=$(sha256sum "$HFT_ACTIVATION_DIR/activation.sha256" | awk '{print $1}') + [[ $activation_sha == "$HFT_ACTIVATION_SHA256" ]] || die "activation identity mismatch" + validate_paper_shadow_authority "$HFT_ACTIVATION_DIR" || die "activation is not fail-closed Paper/Shadow" + validate_runtime_secrets || die "RAM-role injected tmpfs secrets are absent or unsafe" + docker image inspect "$HFT_TRADING_IMAGE" --format '{{json .RepoDigests}}' \ + | jq -e --arg image "$HFT_TRADING_IMAGE" 'index($image) != null' >/dev/null \ + || die "digest-pinned image is not staged locally" +} + +run_container() { + local state_root=${STATE_ROOT:-$STATE_ROOT_DEFAULT} + local secret_root=${SECRET_ROOT:-$SECRET_ROOT_DEFAULT} + local activation_id state_dir runtime_uid runtime_gid runtime_ids + preflight + runtime_ids=$(runtime_account_ids) \ + || die "dedicated runtime account is absent or unsafe" + [[ $runtime_ids == *$'\n'* && ${runtime_ids#*$'\n'} != *$'\n'* ]] \ + || die "dedicated runtime account is invalid" + runtime_uid=${runtime_ids%%$'\n'*} + runtime_gid=${runtime_ids#*$'\n'} + activation_id=$HFT_ACTIVATION_SHA256 + state_dir=$state_root/$activation_id + install -d -o root -g root -m 0750 "$state_root" + install -d -o "$runtime_uid" -g "$runtime_gid" -m 0700 "$state_dir" + exec docker run --rm --pull never \ + --name "$CONTAINER_NAME" \ + --publish 127.0.0.1::9090/tcp \ + --publish 127.0.0.1::9092/tcp \ + --read-only \ + --user "$runtime_uid:$runtime_gid" \ + --cap-drop ALL \ + --security-opt no-new-privileges:true \ + --pids-limit 512 \ + --ulimit nofile=65536:65536 \ + --ulimit core=0:0 \ + --stop-signal SIGINT \ + --stop-timeout 60 \ + --tmpfs "/tmp:rw,noexec,nosuid,nodev,size=64m,mode=0700,uid=$runtime_uid,gid=$runtime_gid" \ + --log-driver journald \ + --env HFT_ENV=production \ + --mount "type=bind,src=$HFT_ACTIVATION_DIR,dst=/activation,readonly" \ + --mount "type=bind,src=$state_dir,dst=/app/state" \ + --mount "type=bind,src=$secret_root/runtime.env,dst=/run/secrets/hft/runtime.env,readonly" \ + --mount "type=bind,src=$secret_root/feedback-signing-key.hex,dst=/run/secrets/hft/feedback-signing-key.hex,readonly" \ + --entrypoint /bin/sh \ + "$HFT_TRADING_IMAGE" \ + -euc 'while IFS= read -r secret; do export "$secret"; done < /run/secrets/hft/runtime.env; unset secret; exec /usr/local/bin/hft-live "$@"' \ + hft-live \ + --config /activation/config/system.yaml \ + --deployment-envelope /activation/deployment/envelope.json \ + --strategy-bundle /activation/deployment/bundle.json \ + --deployment-policy /activation/deployment/policy.json \ + --deployment-trusted-keys /activation/deployment/trusted-keys.json \ + --deployment-nonce-ledger /app/state/nonces.jsonl \ + --deployment-audit-log /app/state/audit.jsonl \ + --deployment-feedback-log /app/state/feedback.jsonl \ + --deployment-feedback-signing-key /run/secrets/hft/feedback-signing-key.hex \ + --deployment-feedback-key-id runtime-feedback-1 \ + --metrics-port 9090 +} + +container_state() { + local running names + if running=$(docker container inspect --format '{{.State.Running}}' \ + "$CONTAINER_NAME" 2>/dev/null); then + [[ $running == true || $running == false ]] || return 1 + printf '%s\n' "$running" + return 0 + fi + + # An inspect failure is safe only when the daemon is healthy and an exact-name + # listing independently proves that no stale container exists. + docker info >/dev/null 2>&1 || return 1 + names=$(docker container ls --all --filter "name=^/${CONTAINER_NAME}$" \ + --format '{{.Names}}') || return 1 + [[ -z $names ]] || return 1 + printf 'absent\n' +} + +assert_stopped() { + local state + state=$(container_state) || return 1 + # Cutover preflight requires absence, not merely a stopped orphan. + [[ $state == absent ]] +} + +stop_container() { + local state + state=$(container_state) || return 1 + case $state in + absent) return 0 ;; + true) docker stop --time 60 "$CONTAINER_NAME" >/dev/null || return 1 ;; + false) ;; + *) return 1 ;; + esac + # docker run --rm normally removes the container after stop. ExecStopPost is + # still the fail-closed cleanup backstop if removal has not completed yet. + state=$(container_state) || return 1 + [[ $state == absent || $state == false ]] +} + +ensure_stopped() { + local state + state=$(container_state) || return 1 + if [[ $state != absent ]]; then + docker rm --force "$CONTAINER_NAME" >/dev/null || return 1 + fi + assert_stopped +} + +if [[ ${BASH_SOURCE[0]} == "$0" ]]; then + case ${1:-} in + preflight) preflight ;; + run) run_container ;; + stop) stop_container ;; + ensure-stopped) ensure_stopped ;; + assert-stopped) assert_stopped ;; + *) die "usage: $0 preflight|run|stop|ensure-stopped|assert-stopped" ;; + esac +fi diff --git a/rust_hft/deployment/PRODUCTION_DEPLOYMENT.md b/rust_hft/deployment/PRODUCTION_DEPLOYMENT.md index 756631af6..22b5d14ef 100644 --- a/rust_hft/deployment/PRODUCTION_DEPLOYMENT.md +++ b/rust_hft/deployment/PRODUCTION_DEPLOYMENT.md @@ -12,6 +12,12 @@ This directory deploys the deterministic Rust runtime. It does not deploy an LLM Market-data and sentinel manifests remain separate runtime services. The research LoopRun is invoked separately and cannot mutate these manifests or runtime secrets. +The Kubernetes manifests are not the Monday live-host target. The future Tokyo +bare-ECS image path is defined by +[`../../deployment/aliyun/TRADING_ECS_HOST.md`](../../deployment/aliyun/TRADING_ECS_HOST.md). +That path is digest-only, static/boot-disabled, and currently permits signed +Paper or Shadow activation only. + ## Build From `rust_hft/`: