diff --git a/.gitignore b/.gitignore index 8010fdd0c..bcf7e9d7f 100644 --- a/.gitignore +++ b/.gitignore @@ -56,6 +56,7 @@ models/ *.jsonl *.json *.csv +!rust_hft/data/backtest/*.manifest.json !rust_hft/prediction-markets/**/*.json !rust_hft/prediction-markets/**/*.jsonl !rust_hft/prediction-markets/**/*.csv diff --git a/deploy/Dockerfile.hft b/deploy/Dockerfile.hft index de8f01ea9..cf207fc48 100644 --- a/deploy/Dockerfile.hft +++ b/deploy/Dockerfile.hft @@ -60,7 +60,7 @@ ENV HFT_TARGET=${TARGET} # 健康檢查 HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \ - CMD if [ "$HFT_TARGET" = "live" ]; then curl --fail --silent http://localhost:9090/readiness; else exit 0; fi + CMD if [ "$HFT_TARGET" = "live" ]; then curl --fail --silent http://localhost:9090/health; else exit 0; fi # 默認暴露端口 EXPOSE 9090 9092 diff --git a/rust_hft/Cargo.lock b/rust_hft/Cargo.lock index 85c4b2daf..9570f2555 100644 --- a/rust_hft/Cargo.lock +++ b/rust_hft/Cargo.lock @@ -4914,11 +4914,14 @@ dependencies = [ "chrono", "clap", "csv", + "hex", + "hft-collector", "itertools 0.12.1", "ordered-float 3.9.2", "serde", "serde_json", "serde_yaml", + "sha2", "tracing", "tracing-subscriber", ] @@ -5065,6 +5068,20 @@ dependencies = [ "url", ] +[[package]] +name = "hft-data-adapter-binance-prediction" +version = "0.1.0" +dependencies = [ + "async-trait", + "binance-sdk", + "futures", + "hft-core", + "hft-ports", + "rust_decimal", + "serde", + "tokio", +] + [[package]] name = "hft-data-adapter-bitget" version = "0.1.0" @@ -5935,6 +5952,7 @@ dependencies = [ "hft-data-adapter-asterdex", "hft-data-adapter-backpack", "hft-data-adapter-binance", + "hft-data-adapter-binance-prediction", "hft-data-adapter-bitget", "hft-data-adapter-bybit", "hft-data-adapter-grvt", diff --git a/rust_hft/Cargo.toml b/rust_hft/Cargo.toml index 79486e155..e422b7ba8 100644 --- a/rust_hft/Cargo.toml +++ b/rust_hft/Cargo.toml @@ -27,6 +27,7 @@ members = [ "data-pipelines/adapters-common", "data-pipelines/adapters/adapter-bitget", "data-pipelines/adapters/adapter-binance", + "data-pipelines/adapters/adapter-binance-prediction", "data-pipelines/adapters/adapter-backpack", "data-pipelines/adapters/adapter-mock", "data-pipelines/adapters/adapter-replay", diff --git a/rust_hft/alpha-harness/app/src/loop_control.rs b/rust_hft/alpha-harness/app/src/loop_control.rs index 5f1813083..759359a83 100644 --- a/rust_hft/alpha-harness/app/src/loop_control.rs +++ b/rust_hft/alpha-harness/app/src/loop_control.rs @@ -787,6 +787,10 @@ mod tests { metrics: BTreeMap::from([ ("gross_pnl_coverage_complete".to_string(), 1.0), ("mark_coverage_complete".to_string(), 1.0), + ("authoritative_account_snapshot_coverage".to_string(), 1.0), + ("venue_reconciliation_complete".to_string(), 1.0), + ("venue_reconciliation_healthy".to_string(), 1.0), + ("venue_reconciliation_age_us".to_string(), 1_000.0), ]), reason: None, observed_at: now + Duration::seconds(1), @@ -1130,7 +1134,12 @@ mod tests { account_id: Some(account_id.clone()), venue: Some(venue.clone()), symbol: None, - metrics: BTreeMap::new(), + metrics: BTreeMap::from([ + ("authoritative_account_snapshot_coverage".to_string(), 1.0), + ("venue_reconciliation_complete".to_string(), 1.0), + ("venue_reconciliation_healthy".to_string(), 1.0), + ("venue_reconciliation_age_us".to_string(), 1_000.0), + ]), reason: None, observed_at: now, }, @@ -1862,7 +1871,12 @@ mod tests { account_id: Some("account-1".to_string()), venue: Some("binance".to_string()), symbol: None, - metrics: BTreeMap::new(), + metrics: BTreeMap::from([ + ("authoritative_account_snapshot_coverage".to_string(), 1.0), + ("venue_reconciliation_complete".to_string(), 1.0), + ("venue_reconciliation_healthy".to_string(), 1.0), + ("venue_reconciliation_age_us".to_string(), 1_000.0), + ]), reason: None, observed_at: now, }, diff --git a/rust_hft/alpha-harness/domain/src/lib.rs b/rust_hft/alpha-harness/domain/src/lib.rs index f00808a63..0d6c6d533 100644 --- a/rust_hft/alpha-harness/domain/src/lib.rs +++ b/rust_hft/alpha-harness/domain/src/lib.rs @@ -1357,6 +1357,7 @@ pub fn runtime_stage_is_healthy( } if event.outcome == AttributionOutcome::Healthy && event.kind == AttributionKind::PortfolioSnapshot + && portfolio_snapshot_has_authoritative_truth(event) { if let Some(strategy_id) = event.strategy_id.as_ref() { health @@ -1387,6 +1388,27 @@ pub fn runtime_stage_is_healthy( }) } +const MAX_RUNTIME_RECONCILIATION_AGE_US: f64 = 30_000_000.0; + +fn portfolio_snapshot_has_authoritative_truth(event: &RuntimeAttributionEvent) -> bool { + let metric_is_one = |name: &str| { + event + .metrics + .get(name) + .is_some_and(|value| value.is_finite() && *value >= 1.0) + }; + let age_is_fresh = event + .metrics + .get("venue_reconciliation_age_us") + .is_some_and(|value| { + value.is_finite() && *value >= 0.0 && *value <= MAX_RUNTIME_RECONCILIATION_AGE_US + }); + metric_is_one("authoritative_account_snapshot_coverage") + && metric_is_one("venue_reconciliation_complete") + && metric_is_one("venue_reconciliation_healthy") + && age_is_fresh +} + #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct LearningDirective { pub directive_id: String, @@ -2614,6 +2636,24 @@ mod tests { AttributionOutcome::Healthy, Some("strategy-1"), )); + assert!(!runtime_stage_is_healthy( + &events, + "candidate-1", + AttributionMode::Shadow + )); + let snapshot = events.last_mut().unwrap(); + snapshot + .metrics + .insert("authoritative_account_snapshot_coverage".to_string(), 1.0); + snapshot + .metrics + .insert("venue_reconciliation_complete".to_string(), 1.0); + snapshot + .metrics + .insert("venue_reconciliation_healthy".to_string(), 1.0); + snapshot + .metrics + .insert("venue_reconciliation_age_us".to_string(), 1_000.0); assert!(runtime_stage_is_healthy( &events, "candidate-1", diff --git a/rust_hft/apps/backtest/Cargo.toml b/rust_hft/apps/backtest/Cargo.toml index c4f8bc898..f6ea7e42a 100644 --- a/rust_hft/apps/backtest/Cargo.toml +++ b/rust_hft/apps/backtest/Cargo.toml @@ -15,6 +15,8 @@ ordered-float = "3.9" serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } serde_yaml = "0.9" +sha2 = { workspace = true } +hex = { workspace = true } +hft-collector = { path = "../../tools/collector", default-features = false } tracing = { workspace = true } tracing-subscriber = { workspace = true } - diff --git a/rust_hft/apps/backtest/src/config.rs b/rust_hft/apps/backtest/src/config.rs index de1452d17..7f4b40f24 100644 --- a/rust_hft/apps/backtest/src/config.rs +++ b/rust_hft/apps/backtest/src/config.rs @@ -1,8 +1,15 @@ -use std::fs; +use std::fs::{self, File}; +use std::io::{BufRead, BufReader, Read, Seek, SeekFrom}; use std::path::Path; +use std::process::{Command, Stdio}; -use anyhow::Context; +use anyhow::{bail, Context}; +use hft_collector::lob_archiver::{ + source_revision, Market, ReplaySequenceEvent, ReplaySequenceValidator, +}; use serde::Deserialize; +use sha2::{Digest, Sha256}; +use std::collections::HashSet; #[derive(Debug, Clone, Deserialize)] pub struct BacktestConfig { @@ -66,6 +73,67 @@ impl BacktestConfig { self.strategy.resistance_count = 3; } } + + pub fn validate_data_artifact(&self) -> anyhow::Result<()> { + if !self.data.format.eq_ignore_ascii_case("ndjson") { + bail!("unsupported backtest data format: {}", self.data.format); + } + if !self.data.require_sequence { + bail!("backtests require data.require_sequence=true"); + } + let manifest_path = self + .data + .manifest_path + .as_deref() + .context("backtests require data.manifest_path")?; + let expected_manifest_sha256 = valid_sha256( + self.data + .manifest_sha256 + .as_deref() + .context("backtests require data.manifest_sha256")?, + "data.manifest_sha256", + )?; + let manifest_bytes = fs::read(resolve_path(manifest_path)) + .with_context(|| format!("无法读取回测数据 manifest: {manifest_path}"))?; + let actual_manifest_sha256 = hex::encode(Sha256::digest(&manifest_bytes)); + if actual_manifest_sha256 != expected_manifest_sha256 { + bail!( + "backtest manifest SHA-256 mismatch: expected {expected_manifest_sha256}, actual {actual_manifest_sha256}" + ); + } + let manifest: BacktestDataManifest = + serde_json::from_slice(&manifest_bytes).context("无法解析回测数据 manifest")?; + manifest.validate()?; + + let bytes = fs::read(resolve_path(&self.data.path)) + .with_context(|| format!("无法读取回测数据: {}", self.data.path))?; + let actual = hex::encode(Sha256::digest(&bytes)); + if actual != manifest.artifact_sha256 { + bail!( + "backtest data SHA-256 mismatch: expected {}, actual {actual}", + manifest.artifact_sha256 + ); + } + if canonical_path(&self.data.path)? != canonical_path(&manifest.artifact_path)? { + bail!("data.path does not match manifest artifact_path"); + } + validate_source_segments(&manifest, &bytes)?; + validate_event_tape(&self.data.path, &manifest)?; + self.validate_execution_model() + } + + fn validate_execution_model(&self) -> anyhow::Result<()> { + if !self.execution.fee_bps.is_finite() || self.execution.fee_bps < 0.0 { + bail!("execution.fee_bps must be finite and non-negative"); + } + if !self.execution.max_fill_ratio.is_finite() + || !(0.0..=1.0).contains(&self.execution.max_fill_ratio) + || self.execution.max_fill_ratio == 0.0 + { + bail!("execution.max_fill_ratio must be in (0, 1]"); + } + Ok(()) + } } #[derive(Debug, Clone, Deserialize)] @@ -80,11 +148,765 @@ pub struct DataConfig { #[serde(default = "default_depth_levels")] pub max_depth_levels: usize, #[serde(default)] + pub manifest_path: Option, + #[serde(default)] + pub manifest_sha256: Option, + #[serde(default)] + pub require_sequence: bool, + #[serde(default)] pub start_ts: Option, #[serde(default)] pub end_ts: Option, } +#[derive(Debug, Deserialize)] +struct BacktestDataManifest { + dataset_kind: String, + schema_version: String, + mission_id: String, + market: String, + symbol: String, + dataset: String, + source_revision: String, + source_segments: Vec, + rows: usize, + first_event_time_us: i64, + last_event_time_us: i64, + sequence_start: u64, + sequence_end: u64, + artifact_path: String, + artifact_sha256: String, + point_in_time: bool, +} + +#[derive(Debug, Deserialize)] +struct SourceSegmentEvidence { + path: String, + sha256: String, + collector_manifest_path: String, + collector_manifest_sha256: String, + success_marker_path: String, + start_received_at_ns: u64, + end_received_at_ns: u64, + events: u64, +} + +#[derive(Debug, Deserialize)] +struct CollectorRawManifest { + schema: String, + venue: String, + market: String, + dataset: String, + symbols: Vec, + mode: String, + replay_scope: String, + events: u64, + bytes: u64, + event_types: std::collections::HashMap, + has_replay_safe_checkpoint: bool, + all_symbols_bridged: bool, + start_received_at_ns: u64, + end_received_at_ns: u64, + file: String, + sha256: String, +} + +impl BacktestDataManifest { + fn validate(&self) -> anyhow::Result<()> { + if self.dataset_kind != "backtest_point_in_time_event_tape" + || self.schema_version != "backtest-pit-v1" + || self.mission_id.trim().is_empty() + || self.market.trim().is_empty() + || self.symbol.trim().is_empty() + || self.dataset.trim().is_empty() + || !self.point_in_time + || self.rows == 0 + || self.source_segments.is_empty() + { + bail!("backtest manifest is not a complete point-in-time event tape"); + } + valid_sha256(&self.artifact_sha256, "manifest.artifact_sha256")?; + valid_sha256(&self.source_revision, "manifest.source_revision")?; + if self.first_event_time_us > self.last_event_time_us + || self.sequence_start != 1 + || self.sequence_end < self.sequence_start + || self.sequence_end - self.sequence_start + 1 != self.rows as u64 + { + bail!("backtest manifest time/sequence coverage is inconsistent"); + } + Ok(()) + } +} + +fn validate_source_segments( + manifest: &BacktestDataManifest, + artifact_bytes: &[u8], +) -> anyhow::Result<()> { + let mut hashes = Vec::with_capacity(manifest.source_segments.len()); + let mut unique = HashSet::new(); + let mut previous_segment_end = None; + let market = manifest + .market + .parse::() + .map_err(anyhow::Error::msg)?; + let mut replay = ReplaySequenceValidator::new(market, &manifest.symbol)?; + let artifact_rows = parse_ndjson_values(artifact_bytes, "backtest event tape")?; + let mut materialized_rows = Vec::with_capacity(artifact_rows.len()); + for segment in &manifest.source_segments { + let expected = valid_sha256(&segment.sha256, "source segment sha256")?; + if !unique.insert(expected.to_string()) + || segment.events == 0 + || segment.start_received_at_ns > segment.end_received_at_ns + || previous_segment_end.is_some_and(|previous| segment.start_received_at_ns < previous) + { + bail!("backtest source segment evidence is incomplete or duplicated"); + } + previous_segment_end = Some(segment.end_received_at_ns); + let source_path = resolve_path(&segment.path); + let (actual, source, source_bytes) = open_hashed_source(&source_path) + .with_context(|| format!("无法读取源数据 segment: {}", segment.path))?; + if actual != expected { + bail!( + "source segment SHA-256 mismatch for {}: expected {expected}, actual {actual}", + segment.path + ); + } + if canonical_path(&segment.path)? == canonical_path(&manifest.artifact_path)? { + bail!("collector raw segment cannot also be the backtest artifact"); + } + + let expected_collector_manifest_sha = valid_sha256( + &segment.collector_manifest_sha256, + "source collector manifest sha256", + )?; + let collector_manifest_bytes = fs::read(resolve_path(&segment.collector_manifest_path)) + .with_context(|| { + format!( + "无法读取 collector manifest: {}", + segment.collector_manifest_path + ) + })?; + let actual_collector_manifest_sha = hex::encode(Sha256::digest(&collector_manifest_bytes)); + if actual_collector_manifest_sha != expected_collector_manifest_sha { + bail!( + "collector manifest SHA-256 mismatch: expected {expected_collector_manifest_sha}, actual {actual_collector_manifest_sha}" + ); + } + let collector: CollectorRawManifest = serde_json::from_slice(&collector_manifest_bytes) + .context("无法解析 collector raw manifest")?; + validate_collector_manifest(manifest, segment, &collector, expected, source_bytes)?; + + let success = + fs::read_to_string(resolve_path(&segment.success_marker_path)).with_context(|| { + format!( + "无法读取 collector success marker: {}", + segment.success_marker_path + ) + })?; + if success.trim() != expected { + bail!("collector success marker is not bound to the raw segment digest"); + } + + let mut raw_count = 0_u64; + let mut previous_received_at = None; + let mut observed_types = std::collections::HashMap::::new(); + visit_collector_rows(&segment.path, source, |raw| { + raw_count = raw_count.checked_add(1).context("collector row overflow")?; + let mut raw = raw + .as_object() + .cloned() + .context("collector raw row must be a JSON object")?; + let received_at_ns = raw + .remove("received_at_ns") + .and_then(|value| value.as_u64()) + .context("collector raw row is missing received_at_ns")?; + if received_at_ns < segment.start_received_at_ns + || received_at_ns > segment.end_received_at_ns + || previous_received_at.is_some_and(|previous| received_at_ns < previous) + { + bail!("collector raw rows are not point-in-time ordered"); + } + previous_received_at = Some(received_at_ns); + let event_type = raw + .remove("type") + .and_then(|value| value.as_str().map(str::to_owned)) + .context("collector raw row is missing type")?; + *observed_types.entry(event_type.clone()).or_default() += 1; + validate_collector_event_type(&event_type)?; + let replay_events = replay.observe(&event_type, &raw, received_at_ns)?; + if event_type == "checkpoint" + && (raw.get("replay_safe").and_then(serde_json::Value::as_bool) != Some(true) + || raw.get("synced").and_then(serde_json::Value::as_bool) != Some(true) + || raw.get("bridged").and_then(serde_json::Value::as_bool) != Some(true) + || raw + .get("symbol") + .and_then(serde_json::Value::as_str) + .map(str::is_empty) + .unwrap_or(true)) + { + bail!("collector replay checkpoint is incomplete"); + } + for event in replay_events { + materialized_rows.push(materialize_replay_event( + event, + materialized_rows.len() as u64 + 1, + )?); + } + Ok(()) + })?; + if raw_count != segment.events { + bail!("collector raw row count does not match source evidence"); + } + if observed_types != collector.event_types { + bail!("collector raw event types do not match its manifest"); + } + hashes.push(expected.to_string()); + } + replay.finish()?; + let actual_revision = source_revision(hashes.iter().map(String::as_str)); + if actual_revision != manifest.source_revision { + bail!( + "source revision mismatch: expected {}, actual {actual_revision}", + manifest.source_revision + ); + } + if materialized_rows != artifact_rows { + bail!("backtest event tape is not the deterministic point-in-time materialization of its collector sources"); + } + Ok(()) +} + +fn validate_collector_manifest( + manifest: &BacktestDataManifest, + segment: &SourceSegmentEvidence, + collector: &CollectorRawManifest, + expected_source_sha: &str, + source_bytes: u64, +) -> anyhow::Result<()> { + let source_file = Path::new(&segment.path) + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or_default(); + if collector.schema != "binance.lob_tape.v2" + || collector.venue != "binance" + || collector.market != manifest.market + || collector.dataset != manifest.dataset + || !collector + .symbols + .iter() + .any(|symbol| symbol == &manifest.symbol) + || collector.mode != "diff" + || collector.replay_scope != "captured_snapshot_seed_plus_sequence_checked_diffs" + || !collector.has_replay_safe_checkpoint + || !collector.all_symbols_bridged + || collector.events != segment.events + || collector.bytes != source_bytes + || collector.start_received_at_ns != segment.start_received_at_ns + || collector.end_received_at_ns != segment.end_received_at_ns + || collector.file != source_file + || collector.sha256 != expected_source_sha + || collector.event_types.values().sum::() != collector.events + || collector + .event_types + .get("checkpoint") + .copied() + .unwrap_or(0) + == 0 + || collector + .event_types + .get("sequence_gap") + .copied() + .unwrap_or(0) + != 0 + { + bail!("collector raw manifest is incomplete or does not match source evidence"); + } + Ok(()) +} + +fn materialize_replay_event( + replay_event: ReplaySequenceEvent, + sequence: u64, +) -> anyhow::Result { + let (event, received_at_ns, bids, asks) = match replay_event { + ReplaySequenceEvent::Snapshot { + received_at_ns, + bids, + asks, + } => ("snapshot", received_at_ns, bids, asks), + ReplaySequenceEvent::Diff { + received_at_ns, + bids, + asks, + } => ("l2_update", received_at_ns, bids, asks), + }; + let received_at_us = received_at_ns / 1_000 + u64::from(!received_at_ns.is_multiple_of(1_000)); + let timestamp = i64::try_from(received_at_us).context("receive time exceeds i64")?; + Ok(serde_json::json!({ + "timestamp": timestamp, + "sequence": sequence, + "event": event, + "bids": normalize_replay_levels(bids, "bids")?, + "asks": normalize_replay_levels(asks, "asks")?, + })) +} + +fn validate_collector_event_type(event_type: &str) -> anyhow::Result<()> { + match event_type { + "snapshot" | "diff" | "checkpoint" => Ok(()), + "trade" => bail!("binance.lob_tape.v2 does not contain trade events"), + unsupported => bail!("unsupported collector event type: {unsupported}"), + } +} + +fn normalize_replay_levels(levels: Vec<[String; 2]>, field: &str) -> anyhow::Result> { + levels + .into_iter() + .map(|[price, quantity]| { + let price = price + .parse::() + .with_context(|| format!("{field} contains a non-numeric price"))?; + let quantity = quantity + .parse::() + .with_context(|| format!("{field} contains a non-numeric quantity"))?; + if !price.is_finite() || !quantity.is_finite() || price <= 0.0 || quantity < 0.0 { + bail!("{field} contains an invalid price or quantity"); + } + Ok([price, quantity]) + }) + .collect() +} + +fn parse_ndjson_values(bytes: &[u8], label: &str) -> anyhow::Result> { + std::str::from_utf8(bytes) + .with_context(|| format!("{label} is not UTF-8"))? + .lines() + .filter(|line| !line.trim().is_empty()) + .map(|line| serde_json::from_str(line).with_context(|| format!("invalid {label} row"))) + .collect() +} + +fn open_hashed_source(path: &Path) -> anyhow::Result<(String, File, u64)> { + let mut source = File::open(path)?; + let bytes = source.metadata()?.len(); + let mut digest = Sha256::new(); + let mut buffer = [0_u8; 1024 * 1024]; + loop { + let read = source.read(&mut buffer)?; + if read == 0 { + break; + } + digest.update(&buffer[..read]); + } + source.seek(SeekFrom::Start(0))?; + Ok((hex::encode(digest.finalize()), source, bytes)) +} + +fn visit_collector_rows( + path: &str, + source: File, + mut visitor: impl FnMut(serde_json::Value) -> anyhow::Result<()>, +) -> anyhow::Result<()> { + if path.ends_with(".zst") { + let mut child = Command::new("zstd") + .args(["-q", "-dc"]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn() + .with_context(|| format!("failed to start zstd for collector segment: {path}"))?; + let mut stdin = child.stdin.take().context("zstd stdin unavailable")?; + let stdout = child.stdout.take().context("zstd stdout unavailable")?; + std::thread::scope(|scope| -> anyhow::Result<()> { + let writer = scope.spawn(move || std::io::copy(&mut &source, &mut stdin)); + let visit_result = visit_ndjson(BufReader::new(stdout), &mut visitor); + if visit_result.is_err() { + let _ = child.kill(); + } + let status = child.wait()?; + writer + .join() + .map_err(|_| anyhow::anyhow!("zstd input writer panicked"))??; + visit_result?; + if !status.success() { + bail!("zstd failed for collector segment {path}: {status}"); + } + Ok(()) + })?; + } else { + visit_ndjson(BufReader::new(source), &mut visitor)?; + } + Ok(()) +} + +fn visit_ndjson( + reader: impl BufRead, + visitor: &mut impl FnMut(serde_json::Value) -> anyhow::Result<()>, +) -> anyhow::Result<()> { + for line in reader.lines() { + let line = line.context("failed to read collector raw segment")?; + if line.trim().is_empty() { + continue; + } + visitor(serde_json::from_str(&line).context("invalid collector raw segment row")?)?; + } + Ok(()) +} + +fn validate_event_tape(path: &str, manifest: &BacktestDataManifest) -> anyhow::Result<()> { + let contents = fs::read_to_string(resolve_path(path))?; + let mut rows = 0_usize; + let mut first_time = None; + let mut last_time = None; + let mut first_sequence = None; + let mut last_sequence = None; + for line in contents.lines().filter(|line| !line.trim().is_empty()) { + let value: serde_json::Value = serde_json::from_str(line)?; + let timestamp = value + .get("timestamp") + .and_then(serde_json::Value::as_i64) + .context("event tape row is missing integer timestamp")?; + let sequence = value + .get("sequence") + .and_then(serde_json::Value::as_u64) + .context("event tape row is missing integer sequence")?; + first_time.get_or_insert(timestamp); + first_sequence.get_or_insert(sequence); + last_time = Some(timestamp); + last_sequence = Some(sequence); + rows += 1; + } + if rows != manifest.rows + || first_time != Some(manifest.first_event_time_us) + || last_time != Some(manifest.last_event_time_us) + || first_sequence != Some(manifest.sequence_start) + || last_sequence != Some(manifest.sequence_end) + { + bail!("event tape coverage does not match its manifest"); + } + Ok(()) +} + +fn valid_sha256<'a>(value: &'a str, field: &str) -> anyhow::Result<&'a str> { + if value.len() != 64 || !value.bytes().all(|byte| byte.is_ascii_hexdigit()) { + bail!("{field} must be a 64-character hex SHA-256"); + } + Ok(value) +} + +fn canonical_path(path: &str) -> anyhow::Result { + fs::canonicalize(resolve_path(path)).with_context(|| format!("无法解析路径: {path}")) +} + +fn resolve_path(path: &str) -> std::path::PathBuf { + let path = std::path::PathBuf::from(path); + if path.is_absolute() || path.exists() { + path + } else { + std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../..") + .join(path) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn fixture_manifest() -> BacktestDataManifest { + serde_json::from_value(fixture_manifest_value()).unwrap() + } + + fn fixture_manifest_value() -> serde_json::Value { + serde_json::from_slice( + &fs::read(resolve_path("data/backtest/sample.manifest.json")).unwrap(), + ) + .unwrap() + } + + fn fixture_collector_manifest_value() -> serde_json::Value { + serde_json::from_slice( + &fs::read(resolve_path("data/backtest/sample.raw.manifest.json")).unwrap(), + ) + .unwrap() + } + + #[test] + fn default_backtest_fixture_has_verified_pit_provenance() { + let config = + BacktestConfig::from_file(resolve_path("config/backtest/default.yaml")).unwrap(); + config.validate_data_artifact().unwrap(); + + let manifest = fixture_manifest(); + assert_ne!( + canonical_path(&manifest.source_segments[0].path).unwrap(), + canonical_path(&manifest.artifact_path).unwrap() + ); + assert!(!manifest.source_segments[0] + .collector_manifest_path + .is_empty()); + let raw = fs::read_to_string(resolve_path(&manifest.source_segments[0].path)).unwrap(); + assert!(raw.contains("\"type\":\"diff\"")); + assert!(raw.contains("\"frame\":{\"stream\"")); + assert!(raw.contains("\"snapshot\":{\"lastUpdateId\"")); + assert!(!raw.contains("\"type\":\"trade\"")); + + let tape = fs::read_to_string(resolve_path(&manifest.artifact_path)).unwrap(); + assert!(tape.contains("\"event\":\"snapshot\"")); + assert!(tape.contains("\"event\":\"l2_update\"")); + assert!(!tape.contains("\"event\":\"trade\"")); + } + + #[test] + fn rejects_unknown_backtest_schema() { + let mut manifest = fixture_manifest_value(); + manifest["schema_version"] = "backtest-pit-v2".into(); + let manifest: BacktestDataManifest = serde_json::from_value(manifest).unwrap(); + + assert!(manifest.validate().is_err()); + } + + #[test] + fn collector_identity_is_bound_to_backtest_manifest() { + let manifest = fixture_manifest(); + let segment = &manifest.source_segments[0]; + for (field, replacement) in [ + ("market", serde_json::json!("usdm")), + ("dataset", serde_json::json!("other-dataset")), + ("symbols", serde_json::json!(["ETHUSDT"])), + ("bytes", serde_json::json!(999)), + ] { + let mut collector = fixture_collector_manifest_value(); + collector[field] = replacement; + let collector: CollectorRawManifest = serde_json::from_value(collector).unwrap(); + assert!(validate_collector_manifest( + &manifest, + segment, + &collector, + &segment.sha256, + fs::metadata(resolve_path(&segment.path)).unwrap().len(), + ) + .is_err()); + } + } + + #[test] + fn collector_trade_events_are_not_fabricated_into_lob_tape() { + let error = validate_collector_event_type("trade").unwrap_err(); + assert!(error.to_string().contains("does not contain trade events")); + } + + #[test] + fn rejects_collector_sequence_gap() { + let rows = parse_ndjson_values( + &fs::read(resolve_path("data/backtest/sample.raw.ndjson")).unwrap(), + "fixture", + ) + .unwrap(); + let mut replay = ReplaySequenceValidator::new(Market::Spot, "BTCUSDT").unwrap(); + let snapshot = rows[0].as_object().unwrap(); + replay.observe("snapshot", snapshot, 100).unwrap(); + let mut gap = rows[1].clone(); + gap["frame"]["data"]["U"] = 105.into(); + gap["frame"]["data"]["u"] = 105.into(); + + assert!(replay + .observe("diff", gap.as_object().unwrap(), 200) + .is_err()); + } + + #[test] + fn governed_replay_buffers_pre_snapshot_diff_and_skips_stale_diff() { + let rows = parse_ndjson_values( + &fs::read(resolve_path("data/backtest/sample.raw.ndjson")).unwrap(), + "fixture", + ) + .unwrap(); + let mut replay = ReplaySequenceValidator::new(Market::Spot, "BTCUSDT").unwrap(); + let diff = rows[1].as_object().unwrap(); + assert!(replay.observe("diff", diff, 50).unwrap().is_empty()); + + let emitted = replay + .observe("snapshot", rows[0].as_object().unwrap(), 100) + .unwrap(); + assert!(matches!( + emitted.as_slice(), + [ + ReplaySequenceEvent::Snapshot { + received_at_ns: 100, + .. + }, + ReplaySequenceEvent::Diff { + received_at_ns: 100, + .. + } + ] + )); + + let mut stale = rows[1].clone(); + stale["frame"]["data"]["U"] = 90.into(); + stale["frame"]["data"]["u"] = 100.into(); + assert!(replay + .observe("diff", stale.as_object().unwrap(), 200) + .unwrap() + .is_empty()); + } + + #[test] + fn governed_replay_rejects_checkpoint_book_mismatch() { + let rows = parse_ndjson_values( + &fs::read(resolve_path("data/backtest/sample.raw.ndjson")).unwrap(), + "fixture", + ) + .unwrap(); + let mut replay = ReplaySequenceValidator::new(Market::Spot, "BTCUSDT").unwrap(); + replay + .observe("snapshot", rows[0].as_object().unwrap(), 100) + .unwrap(); + replay + .observe("diff", rows[1].as_object().unwrap(), 200) + .unwrap(); + let mut checkpoint = rows[2].clone(); + checkpoint["bids"][0][1] = "999".into(); + + assert!(replay + .observe("checkpoint", checkpoint.as_object().unwrap(), 300) + .is_err()); + } + + #[test] + fn receive_time_is_never_materialized_early() { + let received_at_ns = 1_700_000_000_100_000_001; + let materialized = materialize_replay_event( + ReplaySequenceEvent::Snapshot { + received_at_ns, + bids: vec![["100".to_string(), "1".to_string()]], + asks: vec![["101".to_string(), "1".to_string()]], + }, + 1, + ) + .unwrap(); + assert_eq!(materialized["timestamp"], 1_700_000_000_100_001_i64); + } + + #[test] + fn reads_real_zstd_collector_segment() { + let id = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + let directory = std::env::temp_dir().join(format!( + "monday-backtest-collector-{}-{id}", + std::process::id() + )); + fs::create_dir_all(&directory).unwrap(); + let raw_path = resolve_path("data/backtest/sample.raw.ndjson"); + let compressed_path = directory.join("part-1.jsonl.zst"); + assert!(Command::new("zstd") + .args(["-q", "-f"]) + .arg(&raw_path) + .arg("-o") + .arg(&compressed_path) + .status() + .unwrap() + .success()); + let (_, source, _) = open_hashed_source(&compressed_path).unwrap(); + fs::rename(&compressed_path, directory.join("verified.zst")).unwrap(); + fs::write(&compressed_path, b"replaced after verified read").unwrap(); + let mut rows = Vec::new(); + visit_collector_rows(compressed_path.to_str().unwrap(), source, |row| { + rows.push(row); + Ok(()) + }) + .unwrap(); + assert_eq!(rows.len(), 5); + assert_eq!(rows[0]["snapshot"]["lastUpdateId"], 100); + assert_eq!(rows[1]["frame"]["data"]["s"], "BTCUSDT"); + fs::remove_dir_all(directory).unwrap(); + } + + #[test] + fn compressed_collector_segment_passes_full_governed_chain() { + let id = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + let directory = + std::env::temp_dir().join(format!("monday-backtest-chain-{}-{id}", std::process::id())); + fs::create_dir_all(&directory).unwrap(); + let segment = directory.join("part-1.jsonl.zst"); + assert!(Command::new("zstd") + .args(["-q", "-f"]) + .arg(resolve_path("data/backtest/sample.raw.ndjson")) + .arg("-o") + .arg(&segment) + .status() + .unwrap() + .success()); + let segment_bytes = fs::read(&segment).unwrap(); + let segment_sha = hex::encode(Sha256::digest(&segment_bytes)); + + let collector_manifest_path = directory.join("part-1.jsonl.zst.manifest.json"); + let mut collector = fixture_collector_manifest_value(); + collector["file"] = "part-1.jsonl.zst".into(); + collector["bytes"] = segment_bytes.len().into(); + collector["sha256"] = segment_sha.clone().into(); + let collector_bytes = serde_json::to_vec(&collector).unwrap(); + fs::write(&collector_manifest_path, &collector_bytes).unwrap(); + let collector_sha = hex::encode(Sha256::digest(&collector_bytes)); + let success = directory.join("part-1.jsonl.zst._SUCCESS"); + fs::write(&success, format!("{segment_sha}\n")).unwrap(); + + let artifact = directory.join("backtest.ndjson"); + let artifact_bytes = fs::read(resolve_path("data/backtest/sample.ndjson")).unwrap(); + fs::write(&artifact, &artifact_bytes).unwrap(); + let artifact_sha = hex::encode(Sha256::digest(&artifact_bytes)); + let manifest = directory.join("backtest.manifest.json"); + let manifest_bytes = serde_json::to_vec(&serde_json::json!({ + "dataset_kind": "backtest_point_in_time_event_tape", + "schema_version": "backtest-pit-v1", + "mission_id": "compressed-fixture", + "market": "spot", + "symbol": "BTCUSDT", + "dataset": "binance_spot_lob", + "source_revision": source_revision([segment_sha.as_str()]), + "source_segments": [{ + "path": segment, + "sha256": segment_sha, + "collector_manifest_path": collector_manifest_path, + "collector_manifest_sha256": collector_sha, + "success_marker_path": success, + "start_received_at_ns": 1_700_000_000_100_000_000_u64, + "end_received_at_ns": 1_700_000_000_500_000_000_u64, + "events": 5 + }], + "rows": 4, + "first_event_time_us": 1_700_000_000_100_000_i64, + "last_event_time_us": 1_700_000_000_500_000_i64, + "sequence_start": 1, + "sequence_end": 4, + "artifact_path": artifact, + "artifact_sha256": artifact_sha, + "point_in_time": true + })) + .unwrap(); + fs::write(&manifest, &manifest_bytes).unwrap(); + let manifest_sha = hex::encode(Sha256::digest(&manifest_bytes)); + let yaml = format!( + "data:\n path: {}\n format: ndjson\n manifest_path: {}\n manifest_sha256: {}\n require_sequence: true\nstrategy: {{}}\nexecution: {{}}\nrisk: {{}}\noutput: {{}}\n", + artifact.display(), + manifest.display(), + manifest_sha, + ); + let config = BacktestConfig::from_yaml_str(&yaml, "compressed fixture").unwrap(); + + config.validate_data_artifact().unwrap(); + fs::write(&segment, b"corrupt").unwrap(); + assert!(config.validate_data_artifact().is_err()); + fs::remove_dir_all(directory).unwrap(); + } +} + fn default_format() -> String { "ndjson".to_string() } @@ -173,6 +995,10 @@ pub struct ExecutionConfig { pub take_profit_ticks: f64, #[serde(default)] pub hold_secs: Option, + #[serde(default)] + pub fee_bps: f64, + #[serde(default = "default_max_fill_ratio")] + pub max_fill_ratio: f64, } impl Default for ExecutionConfig { @@ -184,6 +1010,8 @@ impl Default for ExecutionConfig { stop_loss_ticks: default_slippage_ticks(), take_profit_ticks: default_slippage_ticks(), hold_secs: Some(900.0), + fee_bps: 0.0, + max_fill_ratio: default_max_fill_ratio(), } } } @@ -200,6 +1028,10 @@ fn default_slippage_ticks() -> f64 { 2.0 } +fn default_max_fill_ratio() -> f64 { + 0.1 +} + #[derive(Debug, Clone, Deserialize)] pub struct RiskConfig { #[serde(default = "default_inventory_limit")] diff --git a/rust_hft/apps/backtest/src/engine.rs b/rust_hft/apps/backtest/src/engine.rs index 69e40fb85..48b6df6be 100644 --- a/rust_hft/apps/backtest/src/engine.rs +++ b/rust_hft/apps/backtest/src/engine.rs @@ -41,16 +41,12 @@ impl BacktestEngine { } pub fn run(&mut self) -> Result { - if self.cfg.data.format.to_lowercase() != "ndjson" { - warn!( - "資料格式 {} 尚未實作專用解析器，將以 ndjson 模式處理", - self.cfg.data.format - ); - } + self.cfg.validate_data_artifact()?; let stream = open_event_stream( &self.cfg.data.path, self.cfg.data.start_ts, self.cfg.data.end_ts, + self.cfg.data.require_sequence, )?; self.run_with_stream(stream) } @@ -66,13 +62,22 @@ impl BacktestEngine { // 平倉殘餘持倉 if self.execution.has_position() { - if let Some(mid) = self.order_book.mid_price() { + if let Some((fill_qty, fill_price)) = self.order_book.executable_exit( + self.execution.position.side.unwrap(), + self.execution.position.qty, + self.cfg.execution.max_fill_ratio, + ) { let ts = self .last_ts .map(|t| t as f64 / MICROS_IN_SECOND) .unwrap_or(0.0); - self.execution - .exit_position(ts, mid, ExitReason::SessionEnd, &mut self.stats); + self.execution.exit_position( + ts, + fill_price, + fill_qty, + ExitReason::SessionEnd, + &mut self.stats, + ); } } @@ -150,15 +155,19 @@ impl BacktestEngine { let ofi_condition = ofi <= -self.cfg.strategy.ofi_threshold.abs().max(1e-9); if price_condition && depth_condition && cvd_condition && ofi_condition { - if let Some((bid_price, _)) = self.order_book.best_bid() { - let entry_price = - bid_price - self.cfg.execution.max_slippage_ticks * self.cfg.data.tick_size; - let qty = self.execution.calc_short_qty( - level.depth, - tt_vol_down, - self.cfg.execution.base_qty, - ); - if qty > 0.0 && self.execution.can_enter(qty) { + let requested_qty = self.execution.calc_short_qty( + level.depth, + tt_vol_down, + self.cfg.execution.base_qty, + ); + if let Some((qty, entry_price)) = self.order_book.executable_entry( + PositionSide::Short, + requested_qty, + self.cfg.execution.max_fill_ratio, + self.cfg.execution.max_slippage_ticks, + self.cfg.data.tick_size, + ) { + if self.execution.can_enter(qty) { self.execution.enter_short( ts_sec, entry_price, @@ -182,15 +191,19 @@ impl BacktestEngine { let ofi_condition = ofi >= self.cfg.strategy.ofi_threshold.abs().max(1e-9); if price_condition && depth_condition && cvd_condition && ofi_condition { - if let Some((ask_price, _)) = self.order_book.best_ask() { - let entry_price = - ask_price + self.cfg.execution.max_slippage_ticks * self.cfg.data.tick_size; - let qty = self.execution.calc_long_qty( - level.depth, - tt_vol_up, - self.cfg.execution.base_qty, - ); - if qty > 0.0 && self.execution.can_enter(qty) { + let requested_qty = self.execution.calc_long_qty( + level.depth, + tt_vol_up, + self.cfg.execution.base_qty, + ); + if let Some((qty, entry_price)) = self.order_book.executable_entry( + PositionSide::Long, + requested_qty, + self.cfg.execution.max_fill_ratio, + self.cfg.execution.max_slippage_ticks, + self.cfg.data.tick_size, + ) { + if self.execution.can_enter(qty) { self.execution.enter_long( ts_sec, entry_price, @@ -204,8 +217,14 @@ impl BacktestEngine { } } - self.execution - .evaluate_exit(ts_sec, mid, ofi, cvd_delta, &mut self.stats); + self.execution.evaluate_exit( + ts_sec, + mid, + ofi, + cvd_delta, + &self.order_book, + &mut self.stats, + ); Ok(()) } @@ -213,7 +232,9 @@ impl BacktestEngine { fn finish(&mut self) -> BacktestResult { BacktestResult { trades: mem::take(&mut self.execution.trades), - summary: self.stats.clone_into_summary(), + summary: self + .stats + .clone_into_summary(self.execution.position.qty.abs()), } } } @@ -306,6 +327,114 @@ impl OrderBook { } } + fn executable_exit( + &self, + position_side: PositionSide, + requested_qty: f64, + max_fill_ratio: f64, + ) -> Option<(f64, f64)> { + let participation = max_fill_ratio.clamp(0.0, 1.0); + if requested_qty <= 0.0 || participation <= 0.0 { + return None; + } + let levels: Box + '_> = match position_side { + PositionSide::Long => Box::new( + self.bids + .iter() + .rev() + .map(|(price, qty)| (price.into_inner(), *qty)), + ), + PositionSide::Short => Box::new( + self.asks + .iter() + .map(|(price, qty)| (price.into_inner(), *qty)), + ), + }; + let mut remaining = requested_qty; + let mut filled = 0.0; + let mut notional = 0.0; + for (price, displayed_qty) in levels { + let level_fill = remaining.min(displayed_qty.max(0.0) * participation); + if level_fill <= 0.0 { + continue; + } + filled += level_fill; + notional += level_fill * price; + remaining -= level_fill; + if remaining <= 1e-12 { + break; + } + } + (filled > 0.0).then_some((filled, notional / filled)) + } + + fn executable_entry( + &self, + position_side: PositionSide, + requested_qty: f64, + max_fill_ratio: f64, + max_slippage_ticks: f64, + tick_size: f64, + ) -> Option<(f64, f64)> { + let participation = max_fill_ratio.clamp(0.0, 1.0); + if requested_qty <= 0.0 + || participation <= 0.0 + || max_slippage_ticks < 0.0 + || tick_size <= 0.0 + { + return None; + } + let (levels, worst_price): (Box + '_>, f64) = + match position_side { + PositionSide::Long => { + let best = self.best_ask()?.0; + ( + Box::new( + self.asks + .iter() + .map(|(price, qty)| (price.into_inner(), *qty)), + ), + best + max_slippage_ticks * tick_size, + ) + } + PositionSide::Short => { + let best = self.best_bid()?.0; + ( + Box::new( + self.bids + .iter() + .rev() + .map(|(price, qty)| (price.into_inner(), *qty)), + ), + best - max_slippage_ticks * tick_size, + ) + } + }; + let mut remaining = requested_qty; + let mut filled = 0.0; + let mut notional = 0.0; + for (price, displayed_qty) in levels { + let outside_slippage = match position_side { + PositionSide::Long => price > worst_price + 1e-12, + PositionSide::Short => price < worst_price - 1e-12, + }; + if outside_slippage { + break; + } + let level_fill = remaining.min(displayed_qty.max(0.0) * participation); + if level_fill <= 0.0 { + continue; + } + filled += level_fill; + notional += level_fill * price; + remaining -= level_fill; + if remaining <= 1e-12 { + break; + } + } + (filled > 0.0).then_some((filled, notional / filled)) + } + fn snapshot(&self, max_levels: usize) -> DepthSnapshot { let bids = self .bids @@ -734,6 +863,8 @@ pub struct TradeRecord { pub entry_price: f64, pub exit_price: f64, pub pnl: f64, + pub gross_pnl: f64, + pub fees: f64, pub reason: ExitReason, pub reference_level: f64, pub reference_depth: f64, @@ -787,7 +918,10 @@ impl ExecutionManager { return 0.0; } let ratio = (vol / depth).min(self.cfg.max_position / base_qty); - (base_qty * ratio).min(self.cfg.max_position).max(0.0) + (base_qty * ratio) + .min(self.cfg.max_position) + .min(depth * self.cfg.max_fill_ratio.clamp(0.0, 1.0)) + .max(0.0) } fn calc_long_qty(&self, depth: f64, vol: f64, base_qty: f64) -> f64 { @@ -795,7 +929,10 @@ impl ExecutionManager { return 0.0; } let ratio = (vol / depth).min(self.cfg.max_position / base_qty); - (base_qty * ratio).min(self.cfg.max_position).max(0.0) + (base_qty * ratio) + .min(self.cfg.max_position) + .min(depth * self.cfg.max_fill_ratio.clamp(0.0, 1.0)) + .max(0.0) } fn enter_short( @@ -877,6 +1014,7 @@ impl ExecutionManager { mid: f64, ofi: f64, cvd_delta: f64, + order_book: &OrderBook, stats: &mut BacktestStats, ) { if self.position.side.is_none() { @@ -935,7 +1073,13 @@ impl ExecutionManager { }; if let Some(reason) = reason { - self.exit_position(ts, mid, reason, stats); + if let Some((fill_qty, fill_price)) = order_book.executable_exit( + self.position.side.unwrap(), + self.position.qty, + self.cfg.max_fill_ratio, + ) { + self.exit_position(ts, fill_price, fill_qty, reason, stats); + } } } @@ -943,22 +1087,30 @@ impl ExecutionManager { &mut self, ts: f64, price: f64, + fill_qty: f64, reason: ExitReason, stats: &mut BacktestStats, ) { if self.position.side.is_none() || self.position.qty == 0.0 { return; } - let qty = self.position.qty; + let qty = fill_qty.min(self.position.qty).max(0.0); + if qty <= 0.0 { + return; + } let entry_price = self.position.entry_price; let side = self.position.side.unwrap(); - let pnl = match side { + let gross_pnl = match side { PositionSide::Short => (entry_price - price) * qty, PositionSide::Long => (price - entry_price) * qty, }; + let fees = + (entry_price.abs() + price.abs()) * qty.abs() * self.cfg.fee_bps.max(0.0) / 10_000.0; + let pnl = gross_pnl - fees; + let turnover = (entry_price.abs() + price.abs()) * qty.abs(); self.pnl += pnl; self.equity_curve.push((ts, self.pnl)); - stats.update(pnl, self.pnl); + stats.update(pnl, gross_pnl, fees, turnover, self.pnl); if pnl < 0.0 { self.consecutive_losses += 1; @@ -990,12 +1142,17 @@ impl ExecutionManager { entry_price, exit_price: price, pnl, + gross_pnl, + fees, reason: exit_reason, reference_level: self.position.reference_level, reference_depth: self.position.reference_depth, }); - self.position.reset(); + self.position.qty -= qty; + if self.position.qty <= 1e-12 { + self.position.reset(); + } } } @@ -1003,6 +1160,9 @@ impl ExecutionManager { #[derive(Default)] struct BacktestStats { pub total_pnl: f64, + pub gross_pnl: f64, + pub total_fees: f64, + pub turnover: f64, pub wins: usize, pub losses: usize, pub max_drawdown: f64, @@ -1012,8 +1172,11 @@ struct BacktestStats { } impl BacktestStats { - fn update(&mut self, pnl: f64, equity: f64) { + fn update(&mut self, pnl: f64, gross_pnl: f64, fees: f64, turnover: f64, equity: f64) { self.total_pnl += pnl; + self.gross_pnl += gross_pnl; + self.total_fees += fees; + self.turnover += turnover; if pnl >= 0.0 { self.wins += 1; } else { @@ -1024,7 +1187,7 @@ impl BacktestStats { self.max_drawdown = self.max_drawdown.max(drawdown); } - fn clone_into_summary(&self) -> SummaryMetrics { + fn clone_into_summary(&self, open_position_qty: f64) -> SummaryMetrics { let total_trades = self.wins + self.losses; let win_rate = if total_trades > 0 { self.wins as f64 / total_trades as f64 @@ -1033,10 +1196,14 @@ impl BacktestStats { }; SummaryMetrics { total_pnl: self.total_pnl, + gross_pnl: self.gross_pnl, + total_fees: self.total_fees, + turnover: self.turnover, trades: total_trades, win_rate, max_drawdown: self.max_drawdown, max_position: self.max_position, + open_position_qty, } } } @@ -1044,10 +1211,14 @@ impl BacktestStats { #[derive(Debug, Clone, Serialize)] pub struct SummaryMetrics { pub total_pnl: f64, + pub gross_pnl: f64, + pub total_fees: f64, + pub turnover: f64, pub trades: usize, pub win_rate: f64, pub max_drawdown: f64, pub max_position: f64, + pub open_position_qty: f64, } #[cfg(test)] @@ -1065,6 +1236,9 @@ mod tests { tick_size: 0.01, lot_size: 0.01, max_depth_levels: 5, + manifest_path: None, + manifest_sha256: None, + require_sequence: false, start_ts: None, end_ts: None, }, @@ -1091,6 +1265,7 @@ mod tests { let stream = vec![ Ok(EventEnvelope { ts: 1_000_000, + sequence: None, payload: EventPayload::Snapshot { bids: vec![Level { price: 100.0, @@ -1104,6 +1279,7 @@ mod tests { }), Ok(EventEnvelope { ts: 1_100_000, + sequence: None, payload: EventPayload::L2Update { bids: vec![Level { price: 100.1, @@ -1117,6 +1293,7 @@ mod tests { }), Ok(EventEnvelope { ts: 1_200_000, + sequence: None, payload: EventPayload::Trade { side: TradeSide::Buy, price: 100.1, @@ -1132,4 +1309,102 @@ mod tests { assert_eq!(result.summary.trades, result.trades.len()); assert_eq!(engine.stats.last_trade_price, Some(100.1)); } + + #[test] + fn execution_fees_are_deducted_from_backtest_pnl() { + let mut execution = ExecutionManager::new( + ExecutionConfig { + fee_bps: 10.0, + ..ExecutionConfig::default() + }, + RiskConfig::default(), + 0.01, + ); + let mut stats = BacktestStats::default(); + execution.enter_long(1.0, 100.0, 1.0, 100.0, 10.0, &mut stats); + execution.exit_position(2.0, 110.0, 1.0, ExitReason::SessionEnd, &mut stats); + + assert_eq!(execution.trades.len(), 1); + assert_eq!(execution.trades[0].gross_pnl, 10.0); + assert!((execution.trades[0].fees - 0.21).abs() < 1e-9); + assert!((execution.trades[0].pnl - 9.79).abs() < 1e-9); + } + + #[test] + fn exit_respects_displayed_depth_and_leaves_residual_position() { + let mut execution = ExecutionManager::new( + ExecutionConfig { + max_fill_ratio: 0.5, + ..ExecutionConfig::default() + }, + RiskConfig::default(), + 0.01, + ); + let mut stats = BacktestStats::default(); + execution.enter_long(1.0, 100.0, 2.0, 100.0, 10.0, &mut stats); + let mut book = OrderBook::new(5); + book.apply_snapshot( + 2, + &[Level { + price: 101.0, + quantity: 1.0, + }], + &[Level { + price: 102.0, + quantity: 1.0, + }], + ); + let (fill_qty, fill_price) = book + .executable_exit(PositionSide::Long, execution.position.qty, 0.5) + .unwrap(); + execution.exit_position( + 2.0, + fill_price, + fill_qty, + ExitReason::SessionEnd, + &mut stats, + ); + + assert_eq!(execution.trades[0].qty, 0.5); + assert_eq!(execution.position.qty, 1.5); + } + + #[test] + fn entry_walks_current_l2_and_respects_slippage_band() { + let mut book = OrderBook::new(5); + book.apply_snapshot( + 1, + &[ + Level { + price: 99.9, + quantity: 1.0, + }, + Level { + price: 99.8, + quantity: 4.0, + }, + ], + &[ + Level { + price: 100.0, + quantity: 1.0, + }, + Level { + price: 100.1, + quantity: 4.0, + }, + Level { + price: 100.2, + quantity: 10.0, + }, + ], + ); + + let (fill_qty, fill_price) = book + .executable_entry(PositionSide::Long, 10.0, 0.5, 1.0, 0.1) + .unwrap(); + + assert_eq!(fill_qty, 2.5); + assert!((fill_price - 100.08).abs() < 1e-9); + } } diff --git a/rust_hft/apps/backtest/src/event.rs b/rust_hft/apps/backtest/src/event.rs index ff52e14a3..e54b2817b 100644 --- a/rust_hft/apps/backtest/src/event.rs +++ b/rust_hft/apps/backtest/src/event.rs @@ -9,6 +9,8 @@ use serde::Deserialize; pub struct EventEnvelope { #[serde(alias = "timestamp")] pub ts: i64, + #[serde(default)] + pub sequence: Option, #[serde(flatten)] pub payload: EventPayload, } @@ -77,15 +79,26 @@ pub struct EventStream { line_no: usize, start_ts: Option, end_ts: Option, + require_sequence: bool, + next_sequence: Option, + last_ts: Option, } impl EventStream { - pub fn new(reader: R, start_ts: Option, end_ts: Option) -> Self { + pub fn new( + reader: R, + start_ts: Option, + end_ts: Option, + require_sequence: bool, + ) -> Self { Self { reader: reader.lines(), line_no: 0, start_ts, end_ts, + require_sequence, + next_sequence: None, + last_ts: None, } } @@ -93,10 +106,16 @@ impl EventStream { path: P, start_ts: Option, end_ts: Option, + require_sequence: bool, ) -> anyhow::Result>> { let file = File::open(&path) .with_context(|| format!("無法開啟事件檔案: {}", path.as_ref().display()))?; - Ok(EventStream::new(BufReader::new(file), start_ts, end_ts)) + Ok(EventStream::new( + BufReader::new(file), + start_ts, + end_ts, + require_sequence, + )) } } @@ -118,6 +137,46 @@ impl Iterator for EventStream { } }; + if self.require_sequence { + let sequence = match event.sequence { + Some(sequence) => sequence, + None => { + return Some(Err(anyhow::anyhow!( + "事件缺少 sequence (line {})", + self.line_no + ))) + } + }; + if let Some(expected) = self.next_sequence { + if sequence != expected { + return Some(Err(anyhow::anyhow!( + "事件 sequence gap (line {}): expected {}, actual {}", + self.line_no, + expected, + sequence + ))); + } + } + self.next_sequence = match sequence.checked_add(1) { + Some(next) => Some(next), + None => { + return Some(Err(anyhow::anyhow!( + "事件 sequence overflow (line {})", + self.line_no + ))) + } + }; + } + if self.last_ts.is_some_and(|last| event.ts < last) { + return Some(Err(anyhow::anyhow!( + "事件时间倒退 (line {}): previous {}, actual {}", + self.line_no, + self.last_ts.unwrap_or_default(), + event.ts + ))); + } + self.last_ts = Some(event.ts); + if let Some(start) = self.start_ts { if event.ts < start { continue; @@ -144,6 +203,29 @@ pub fn open_event_stream>( path: P, start_ts: Option, end_ts: Option, + require_sequence: bool, ) -> anyhow::Result>> { - EventStream::>::from_path(path, start_ts, end_ts) + EventStream::>::from_path(path, start_ts, end_ts, require_sequence) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Cursor; + + #[test] + fn required_sequence_rejects_gaps() { + let rows = concat!( + "{\"ts\":1,\"sequence\":4,\"event\":\"snapshot\",\"bids\":[[1,1]],\"asks\":[[2,1]]}\n", + "{\"ts\":2,\"sequence\":6,\"event\":\"trade\",\"side\":\"buy\",\"price\":1,\"quantity\":1}\n" + ); + let mut stream = EventStream::new(Cursor::new(rows), None, None, true); + assert!(stream.next().expect("first row").is_ok()); + assert!(stream + .next() + .expect("gap row") + .expect_err("gap must fail") + .to_string() + .contains("sequence gap")); + } } diff --git a/rust_hft/apps/backtest/src/main.rs b/rust_hft/apps/backtest/src/main.rs index 11feaed4d..f41c7002d 100644 --- a/rust_hft/apps/backtest/src/main.rs +++ b/rust_hft/apps/backtest/src/main.rs @@ -151,6 +151,8 @@ fn write_trades_csv(path: &Path, trades: &[TradeRecord]) -> anyhow::Result<()> { "qty", "entry_price", "exit_price", + "gross_pnl", + "fees", "pnl", "reason", "reference_level", @@ -164,6 +166,8 @@ fn write_trades_csv(path: &Path, trades: &[TradeRecord]) -> anyhow::Result<()> { format!("{:.6}", trade.qty), format!("{:.6}", trade.entry_price), format!("{:.6}", trade.exit_price), + format!("{:.6}", trade.gross_pnl), + format!("{:.6}", trade.fees), format!("{:.6}", trade.pnl), format!("{:?}", trade.reason), format!("{:.6}", trade.reference_level), @@ -179,17 +183,25 @@ fn write_summary_csv(path: &Path, summary: &SummaryMetrics) -> anyhow::Result<() .with_context(|| format!("無法寫入摘要檔案: {}", path.display()))?; writer.write_record([ "total_pnl", + "gross_pnl", + "total_fees", + "turnover", "trades", "win_rate", "max_drawdown", "max_position", + "open_position_qty", ])?; writer.write_record([ format!("{:.6}", summary.total_pnl), + format!("{:.6}", summary.gross_pnl), + format!("{:.6}", summary.total_fees), + format!("{:.6}", summary.turnover), summary.trades.to_string(), format!("{:.4}", summary.win_rate), format!("{:.6}", summary.max_drawdown), format!("{:.6}", summary.max_position), + format!("{:.6}", summary.open_position_qty), ])?; writer.flush()?; Ok(()) @@ -205,8 +217,12 @@ fn write_metrics_json(path: &Path, summary: &SummaryMetrics) -> anyhow::Result<( fn print_summary(summary: &SummaryMetrics) { info!("===== 回測摘要 ====="); info!("總損益 (PnL): {:.6}", summary.total_pnl); + info!("總毛利: {:.6}", summary.gross_pnl); + info!("總費用: {:.6}", summary.total_fees); + info!("成交額: {:.6}", summary.turnover); info!("交易筆數: {}", summary.trades); info!("勝率: {:.2}%", summary.win_rate * 100.0); info!("最大回撤: {:.6}", summary.max_drawdown); info!("最高持倉: {:.6}", summary.max_position); + info!("未平倉殘量: {:.6}", summary.open_position_qty); } diff --git a/rust_hft/apps/live/src/helpers/metrics.rs b/rust_hft/apps/live/src/helpers/metrics.rs index 77b63bdfb..0c7ac27eb 100644 --- a/rust_hft/apps/live/src/helpers/metrics.rs +++ b/rust_hft/apps/live/src/helpers/metrics.rs @@ -41,10 +41,8 @@ async fn run_axum_metrics_server( interval.tick().await; if let Ok(engine) = sync_engine_arc.try_lock() { - let latency_stats = engine.get_latency_stats(); - if !latency_stats.is_empty() { - infra_metrics::MetricsRegistry::global() - .update_from_latency_monitor(&latency_stats); + if engine.get_statistics().is_running { + engine.sync_latency_metrics_to_prometheus(); } } } diff --git a/rust_hft/apps/live/src/helpers/sentinel.rs b/rust_hft/apps/live/src/helpers/sentinel.rs index 924e34918..2f8ec2701 100644 --- a/rust_hft/apps/live/src/helpers/sentinel.rs +++ b/rust_hft/apps/live/src/helpers/sentinel.rs @@ -76,6 +76,7 @@ async fn run_sentinel_loop( let mut interval = tokio::time::interval(Duration::from_millis(check_interval_ms)); let mut last_state = SentinelState::Normal; + let mut last_orders_submitted = 0_u64; loop { interval.tick().await; @@ -88,25 +89,24 @@ async fn run_sentinel_loop( // 從引擎獲取真實 PnL、延遲和回撤統計 (drawdown 現在由 Portfolio 計算) let sentinel_stats = engine.get_sentinel_stats(); - // 估算活躍訂單數：提交 - 完成 - 取消 - 拒絕 - let active_orders = engine_stats - .orders_submitted - .saturating_sub(engine_stats.orders_filled) - .saturating_sub(engine_stats.orders_canceled) - .saturating_sub(engine_stats.orders_rejected); - let stats = SystemStats { latency_p99_us: sentinel_stats.latency_p99_us, latency_p50_us: sentinel_stats.latency_p50_us, drawdown_pct: sentinel_stats.drawdown_pct, pnl: sentinel_stats.pnl, high_water_mark: sentinel_stats.high_water_mark, - position_count: active_orders as i64, - notional_value: 0.0, - order_rate: 0.0, - ws_reconnect_count: 0, - data_gap_count: 0, + position_count: sentinel_stats.position_count, + notional_value: sentinel_stats.notional_value, + order_rate: engine_stats + .orders_submitted + .saturating_sub(last_orders_submitted) as f64 + / (check_interval_ms.max(1) as f64 / 1_000.0), + data_gap_count: engine_stats + .market_events_dropped + .saturating_add(engine_stats.snapshot_publish_failed) + .saturating_add(engine_stats.data_integrity_gaps), }; + last_orders_submitted = engine_stats.orders_submitted; (stats, engine_stats.is_running) }; diff --git a/rust_hft/apps/live/src/main.rs b/rust_hft/apps/live/src/main.rs index 6e32db514..9ce96cf1d 100644 --- a/rust_hft/apps/live/src/main.rs +++ b/rust_hft/apps/live/src/main.rs @@ -195,13 +195,20 @@ async fn main() -> Result<(), Box> { let mut system = builder.build(); let attribution_observer = if let Some(deployment) = activation.as_ref() { let feedback_log = open_feedback_log(&args)?; - let (receiver, market_reader) = { + let (receiver, market_reader, account_reader, runtime_truth_reader) = { let engine = system.engine.lock().await; - (engine.subscribe_execution_events(), engine.market_reader()) + ( + engine.subscribe_execution_events(), + engine.market_reader(), + engine.account_reader(), + engine.runtime_truth_reader(), + ) }; Some(RuntimeAttributionObserver::new( receiver, market_reader, + account_reader, + runtime_truth_reader, deployment.request.clone(), feedback_log, market_stale_us, diff --git a/rust_hft/apps/live/src/runtime_attribution.rs b/rust_hft/apps/live/src/runtime_attribution.rs index 9cbd584d2..21f755fd6 100644 --- a/rust_hft/apps/live/src/runtime_attribution.rs +++ b/rust_hft/apps/live/src/runtime_attribution.rs @@ -7,9 +7,9 @@ use crate::deployment_envelope::{ }; use alpha_domain::{AttributionKind, AttributionMode, AttributionOutcome, RuntimeAttributionEvent}; use chrono::{DateTime, Utc}; -use engine::aggregation::MarketView; +use engine::{aggregation::MarketView, RuntimeTruthStatus}; use hft_core::{Side, Symbol, VenueId, VenueSymbol}; -use ports::ExecutionEvent; +use ports::{AccountView, ExecutionEvent}; use rust_decimal::prelude::ToPrimitive; use rust_decimal::Decimal; use snapshot::SnapshotReader; @@ -22,6 +22,8 @@ const COVERAGE_MISSING: f64 = 0.0; pub struct RuntimeAttributionObserver { receiver: broadcast::Receiver, market_reader: Arc>, + account_reader: Arc>, + runtime_truth_reader: Arc>, activation: ActivationRequest, feedback_log: RuntimeFeedbackLog, stale_us: u64, @@ -81,6 +83,8 @@ impl RuntimeAttributionObserver { pub fn new( receiver: broadcast::Receiver, market_reader: Arc>, + account_reader: Arc>, + runtime_truth_reader: Arc>, activation: ActivationRequest, feedback_log: RuntimeFeedbackLog, stale_us: u64, @@ -88,6 +92,8 @@ impl RuntimeAttributionObserver { Self { receiver, market_reader, + account_reader, + runtime_truth_reader, activation, feedback_log, stale_us, @@ -120,10 +126,14 @@ impl RuntimeAttributionObserver { }, _ = snapshots.tick() => { let market = self.market_reader.load(); + let account = self.account_reader.load(); + let runtime_truth = self.runtime_truth_reader.load(); for event in portfolio_attribution( &self.activation, &mut state, market.as_ref(), + account.as_ref(), + runtime_truth.as_ref(), Utc::now(), self.stale_us, )? { @@ -133,10 +143,14 @@ impl RuntimeAttributionObserver { _ = &mut shutdown => { self.drain_pending(&mut state)?; let market = self.market_reader.load(); + let account = self.account_reader.load(); + let runtime_truth = self.runtime_truth_reader.load(); for event in portfolio_attribution( &self.activation, &mut state, market.as_ref(), + account.as_ref(), + runtime_truth.as_ref(), Utc::now(), self.stale_us, )? { @@ -457,6 +471,8 @@ fn portfolio_attribution( activation: &ActivationRequest, state: &mut AttributionState, market: &MarketView, + account: &AccountView, + runtime_truth: &RuntimeTruthStatus, observed_at: DateTime, stale_us: u64, ) -> anyhow::Result> { @@ -545,12 +561,85 @@ fn portfolio_attribution( )?, } }; + let mut event = event; + attach_authoritative_account_metrics(&mut event, account, runtime_truth, observed_at)?; events.push(event); } Ok(events) } +fn attach_authoritative_account_metrics( + event: &mut RuntimeAttributionEvent, + account: &AccountView, + runtime_truth: &RuntimeTruthStatus, + observed_at: DateTime, +) -> anyhow::Result<()> { + for (name, value) in [ + ("authoritative_account_cash_balance", account.cash_balance), + ("authoritative_account_realized_pnl", account.realized_pnl), + ( + "authoritative_account_unrealized_pnl", + account.unrealized_pnl, + ), + ("authoritative_account_total_pnl", account.total_pnl()), + ("authoritative_account_equity", account.equity()), + ] { + event + .metrics + .insert(name.to_string(), decimal_metric(name, value)?); + } + event.metrics.insert( + "authoritative_account_open_positions".to_string(), + account.positions.len() as f64, + ); + event.metrics.insert( + "authoritative_account_snapshot_coverage".to_string(), + if runtime_truth.reconciliation_complete && runtime_truth.reconciliation_healthy { + COVERAGE_COMPLETE + } else { + COVERAGE_MISSING + }, + ); + event.metrics.insert( + "venue_reconciliation_complete".to_string(), + if runtime_truth.reconciliation_complete { + COVERAGE_COMPLETE + } else { + COVERAGE_MISSING + }, + ); + event.metrics.insert( + "venue_reconciliation_healthy".to_string(), + if runtime_truth.reconciliation_healthy { + COVERAGE_COMPLETE + } else { + COVERAGE_MISSING + }, + ); + let observed_at_us = u64::try_from(observed_at.timestamp_micros()).unwrap_or_default(); + let reconciliation_age_us = observed_at_us.saturating_sub(runtime_truth.observed_at_us); + event.metrics.insert( + "venue_reconciliation_age_us".to_string(), + reconciliation_age_us as f64, + ); + const MAX_RECONCILIATION_AGE_US: u64 = 30_000_000; + if event.kind == AttributionKind::PortfolioSnapshot + && event.outcome == AttributionOutcome::Healthy + && (!runtime_truth.reconciliation_complete + || !runtime_truth.reconciliation_healthy + || runtime_truth.observed_at_us == 0 + || reconciliation_age_us > MAX_RECONCILIATION_AGE_US) + { + event.outcome = AttributionOutcome::Decayed; + event.reason = Some( + "authoritative venue reconciliation is missing, unhealthy, or stale; portfolio evidence withheld from promotion" + .to_string(), + ); + } + Ok(()) +} + fn stream_gap_attribution( activation: &ActivationRequest, state: &mut AttributionState, @@ -1029,7 +1118,20 @@ mod tests { state: &mut AttributionState, market: &MarketView, ) -> Vec { - portfolio_attribution(activation, state, market, execution_time(NOW_US), u64::MAX).unwrap() + portfolio_attribution( + activation, + state, + market, + &AccountView::default(), + &RuntimeTruthStatus { + reconciliation_complete: true, + reconciliation_healthy: true, + observed_at_us: NOW_US, + }, + execution_time(NOW_US), + u64::MAX, + ) + .unwrap() } fn event_for<'a>( @@ -1419,6 +1521,11 @@ mod tests { assert_eq!(event.symbol, None); assert_eq!(event.metrics["gross_total_pnl"], 15.0); assert_eq!(event.metrics["session_equity"], 1015.0); + assert_eq!( + event.metrics["authoritative_account_snapshot_coverage"], + 1.0 + ); + assert_eq!(event.metrics["authoritative_account_equity"], 0.0); } #[test] @@ -1534,6 +1641,28 @@ mod tests { assert_eq!(events[0].outcome, AttributionOutcome::Healthy); } + #[test] + fn portfolio_snapshot_decays_without_authoritative_reconciliation() { + let activation = activation(); + let mut state = AttributionState::new(&activation).unwrap(); + let events = portfolio_attribution( + &activation, + &mut state, + &market_with_mid(&[("BTCUSDT", 100.0)]), + &AccountView::default(), + &RuntimeTruthStatus::default(), + execution_time(NOW_US), + u64::MAX, + ) + .unwrap(); + + assert_eq!(events[0].outcome, AttributionOutcome::Decayed); + assert_eq!( + events[0].metrics["authoritative_account_snapshot_coverage"], + 0.0 + ); + } + #[test] fn multi_instrument_portfolio_requires_each_mark_to_be_fresh() { let activation = formula_activation(&["BTCUSDT", "ETHUSDT"]); @@ -1541,29 +1670,55 @@ mod tests { let observed_at = Utc::now(); let now_us = observed_at.timestamp_micros().max(0) as u64; let stale_us = 1_000; + let runtime_truth = RuntimeTruthStatus { + reconciliation_complete: true, + reconciliation_healthy: true, + observed_at_us: now_us, + }; let one_stale = market_with_timed_mids(&[ ("BTCUSDT", 100.0, now_us), ("ETHUSDT", 50.0, now_us - stale_us - 1), ]); - let events = - portfolio_attribution(&activation, &mut state, &one_stale, observed_at, stale_us) - .unwrap(); + let events = portfolio_attribution( + &activation, + &mut state, + &one_stale, + &AccountView::default(), + &runtime_truth, + observed_at, + stale_us, + ) + .unwrap(); assert!(events.is_empty()); let complete = market_with_timed_mids(&[("BTCUSDT", 100.0, now_us), ("ETHUSDT", 50.0, now_us)]); - let events = - portfolio_attribution(&activation, &mut state, &complete, observed_at, stale_us) - .unwrap(); + let events = portfolio_attribution( + &activation, + &mut state, + &complete, + &AccountView::default(), + &runtime_truth, + observed_at, + stale_us, + ) + .unwrap(); assert_eq!(events.len(), 2); assert!(events .iter() .all(|event| event.outcome == AttributionOutcome::Healthy)); - let events = - portfolio_attribution(&activation, &mut state, &one_stale, observed_at, stale_us) - .unwrap(); + let events = portfolio_attribution( + &activation, + &mut state, + &one_stale, + &AccountView::default(), + &runtime_truth, + observed_at, + stale_us, + ) + .unwrap(); assert_eq!(events.len(), 2); assert!(events .iter() @@ -1684,6 +1839,13 @@ mod tests { let observer = RuntimeAttributionObserver::new( execution_tx.subscribe(), market.reader(), + snapshot::SnapshotContainer::new(AccountView::default()).reader(), + snapshot::SnapshotContainer::new(RuntimeTruthStatus { + reconciliation_complete: true, + reconciliation_healthy: true, + observed_at_us: hft_core::now_micros(), + }) + .reader(), activation, feedback_log, u64::MAX, diff --git a/rust_hft/apps/live/tests/deployment_artifacts.rs b/rust_hft/apps/live/tests/deployment_artifacts.rs index e4f76fe8f..94c820406 100644 --- a/rust_hft/apps/live/tests/deployment_artifacts.rs +++ b/rust_hft/apps/live/tests/deployment_artifacts.rs @@ -206,13 +206,18 @@ fn docker_artifacts_enforce_the_live_runtime_contract() { "curl", "protobuf-compiler", "--mount=type=cache,target=/usr/local/cargo/registry", - "/readiness", + "/health", "ENTRYPOINT [\"/usr/local/bin/hft-live\"]", "EXPOSE 9090 9092", "USER hft", "clickhouse,redis,grpc", ]; - let runtime_image_forbidden = ["hft-collector", "EXPOSE 9090 9091 9092", "|| true"]; + let runtime_image_forbidden = [ + "hft-collector", + "EXPOSE 9090 9091 9092", + "|| true", + "/readiness", + ]; for (label, content) in [ ("rust_hft/docker/Dockerfile", RUST_HFT_DOCKERFILE), @@ -234,11 +239,11 @@ fn docker_artifacts_enforce_the_live_runtime_contract() { ROOT_DOCKERFILE, &[ "curl", - "http://localhost:9090/readiness", + "http://localhost:9090/health", "EXPOSE 9090 9092", "USER hft", ], - &["EXPOSE 9090 9091 9092"], + &["EXPOSE 9090 9091 9092", "/readiness"], ); assert_text_contract( "deploy/docker-compose.yml", @@ -276,6 +281,7 @@ fn kubernetes_artifacts_enforce_the_live_runtime_contract() { "rust_hft/deployment/k8s/trading-engine.yaml", K8S_TRADING_ENGINE, &[ + "path: /health", "path: /readiness", "--deployment-envelope", "--strategy-bundle", @@ -302,6 +308,8 @@ fn kubernetes_artifacts_enforce_the_live_runtime_contract() { ], &["path: /ready", "containerPort: 9091", "BITGET_API_SECRET"], ); + assert_eq!(K8S_TRADING_ENGINE.matches("path: /health").count(), 2); + assert_eq!(K8S_TRADING_ENGINE.matches("path: /readiness").count(), 1); assert_text_contract( "rust_hft/deployment/k8s/configmaps.yaml", K8S_CONFIG_MAPS, diff --git a/rust_hft/apps/live/tests/deployment_envelope.rs b/rust_hft/apps/live/tests/deployment_envelope.rs index dfd095be6..7a4948ad5 100644 --- a/rust_hft/apps/live/tests/deployment_envelope.rs +++ b/rust_hft/apps/live/tests/deployment_envelope.rs @@ -387,12 +387,13 @@ async fn shadow_activation_waits_for_market_then_produces_loop_consumable_eviden .register_strategies_from_config_strict() .unwrap() .build(); - let (execution_receiver, mut diagnostic_receiver, market_reader, notify) = { + let (execution_receiver, mut diagnostic_receiver, market_reader, account_reader, notify) = { let engine = system.engine.lock().await; ( engine.subscribe_execution_events(), engine.subscribe_execution_events(), engine.market_reader(), + engine.account_reader(), engine.get_wakeup_notify(), ) }; @@ -422,9 +423,17 @@ async fn shadow_activation_waits_for_market_then_produces_loop_consumable_eviden observed_at: now, }) .unwrap(); + let runtime_truth_reader = snapshot::SnapshotContainer::new(engine::RuntimeTruthStatus { + reconciliation_complete: true, + reconciliation_healthy: true, + observed_at_us: hft_core::now_micros(), + }) + .reader(); let observer = RuntimeAttributionObserver::new( execution_receiver, market_reader, + account_reader, + runtime_truth_reader, request.clone(), feedback_log, u64::MAX, diff --git a/rust_hft/config/backtest/default.yaml b/rust_hft/config/backtest/default.yaml index 953706586..c97fb420c 100644 --- a/rust_hft/config/backtest/default.yaml +++ b/rust_hft/config/backtest/default.yaml @@ -4,6 +4,9 @@ data: tick_size: 0.5 lot_size: 0.01 max_depth_levels: 20 + manifest_path: data/backtest/sample.manifest.json + manifest_sha256: 0892719a8b5bbc460fac93e307e56b62234bbc8c6f323ad8c08257908b60b4c7 + require_sequence: true strategy: liquidity_window_secs: 900 # ΔT = 15 分鐘 @@ -22,6 +25,8 @@ execution: stop_loss_ticks: 8.0 take_profit_ticks: 12.0 hold_secs: 600.0 + fee_bps: 10.0 + max_fill_ratio: 0.1 risk: inventory_limit: 0.1 @@ -32,4 +37,3 @@ output: trades_csv: backtest_trades.csv summary_csv: backtest_summary.csv metrics_json: backtest_metrics.json - diff --git a/rust_hft/config/dev/binance_prediction_live.yaml.example b/rust_hft/config/dev/binance_prediction_live.yaml.example index 889a3cebf..f90efbed0 100644 --- a/rust_hft/config/dev/binance_prediction_live.yaml.example +++ b/rust_hft/config/dev/binance_prediction_live.yaml.example @@ -30,8 +30,9 @@ venues: funding_source: CEX timeout_ms: 3000 -# Execution-only venue: a strategy must target BINANCE_PREDICTION explicitly and use the -# outcome token ID as OrderIntent.symbol. Live orders also require envelope max_slippage_bps. +# A strategy must target BINANCE_PREDICTION explicitly and use the outcome token ID as +# OrderIntent.symbol. For REST order-book collection, add data_config.outcomes with the token, +# market ID, and vendor. Live orders also require envelope max_slippage_bps. strategies: [] # Zero limits keep this example fail-closed until an approved deployment supplies real limits. diff --git a/rust_hft/config/dev/binance_prediction_quotes_only.yaml.example b/rust_hft/config/dev/binance_prediction_quotes_only.yaml.example new file mode 100644 index 000000000..526573046 --- /dev/null +++ b/rust_hft/config/dev/binance_prediction_quotes_only.yaml.example @@ -0,0 +1,44 @@ +schema_version: v2 +quotes_only: true + +engine: + queue_capacity: 1024 + stale_us: 1000000 + top_n: 20 + ack_timeout_ms: 3000 + reconcile_interval_ms: 5000 + balance_reconcile_tolerance_usd: 1 + auto_cancel_exchange_only: false + +venues: + - name: binance-prediction-quotes + venue_type: BinancePrediction + rest: "https://api.binance.com" + api_key: "${BINANCE_PREDICTION_API_KEY}" + secret: "${BINANCE_PREDICTION_API_SECRET}" + execution_mode: Paper + capabilities: + ws_order_placement: false + snapshot_crc: false + all_in_one_topics: false + private_ws_heartbeat: false + use_incremental_books: false + simulate_execution: false + symbol_catalog: + - "REPLACE_WITH_OUTCOME_TOKEN_ID@BINANCE_PREDICTION" + data_config: + poll_interval_ms: 1000 + outcomes: + - token_id: "REPLACE_WITH_OUTCOME_TOKEN_ID" + market_id: 1 + vendor: predict_fun + +strategies: [] + +risk: + risk_type: Default + global_position_limit: 0 + global_notional_limit: 0 + max_daily_trades: 0 + max_orders_per_second: 0 + staleness_threshold_us: 1000000 diff --git a/rust_hft/data-pipelines/adapters/adapter-binance-prediction/Cargo.toml b/rust_hft/data-pipelines/adapters/adapter-binance-prediction/Cargo.toml new file mode 100644 index 000000000..a938eb593 --- /dev/null +++ b/rust_hft/data-pipelines/adapters/adapter-binance-prediction/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "hft-data-adapter-binance-prediction" +version = "0.1.0" +edition = "2021" +description = "Binance Prediction public order-book market data adapter" + +[dependencies] +hft-core = { package = "hft-core", path = "../../../market-core/core" } +ports = { package = "hft-ports", path = "../../../market-core/ports" } +async-trait = { workspace = true } +binance-sdk = { version = "61.0.0", default-features = false, features = ["rustls-tls", "w3w_prediction"] } +futures = { workspace = true } +rust_decimal = { workspace = true } +serde = { workspace = true } +tokio = { workspace = true, features = ["rt", "sync", "time"] } + +[lib] +name = "adapter_binance_prediction_data" diff --git a/rust_hft/data-pipelines/adapters/adapter-binance-prediction/src/lib.rs b/rust_hft/data-pipelines/adapters/adapter-binance-prediction/src/lib.rs new file mode 100644 index 000000000..3e83a6cca --- /dev/null +++ b/rust_hft/data-pipelines/adapters/adapter-binance-prediction/src/lib.rs @@ -0,0 +1,413 @@ +//! Binance Prediction REST order-book market data adapter. +//! +//! Binance's Prediction Trading API exposes REST order books rather than a public websocket. +//! This adapter polls only explicitly configured outcome tokens and never fabricates quotes. + +use std::collections::{HashMap, HashSet}; +use std::sync::{ + atomic::{AtomicBool, AtomicU64, Ordering}, + Arc, +}; +use std::time::Duration; + +use async_trait::async_trait; +use binance_sdk::{ + config::ConfigurationRestApi, + w3w_prediction::{ + rest_api::{QueryOrderBookParams, RestApi}, + W3WPredictionRestApi, + }, +}; +use hft_core::{HftError, HftResult, Price, Quantity, Symbol, VenueId}; +use ports::{BookLevel, BoxStream, ConnectionHealth, MarketEvent, MarketSnapshot, MarketStream}; +use rust_decimal::Decimal; +use serde::Deserialize; +use tokio::sync::mpsc; +use tokio::time::{interval, MissedTickBehavior}; + +const EVENT_QUEUE_CAPACITY: usize = 1_024; +const DEFAULT_POLL_INTERVAL_MS: u64 = 1_000; + +#[derive(Debug, Clone, Deserialize)] +pub struct PredictionOutcomeConfig { + pub token_id: String, + pub market_id: i64, + #[serde(default = "default_vendor")] + pub vendor: String, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct BinancePredictionMarketDataConfig { + #[serde(default)] + pub api_key: String, + #[serde(default)] + pub api_secret: String, + #[serde(default = "default_rest_base_url")] + pub rest_base_url: String, + #[serde(default = "default_poll_interval_ms")] + pub poll_interval_ms: u64, + pub outcomes: Vec, +} + +#[derive(Default)] +struct ConnectionState { + connected: AtomicBool, + last_heartbeat: AtomicU64, +} + +pub struct BinancePredictionMarketStream { + config: BinancePredictionMarketDataConfig, + state: Arc, +} + +impl BinancePredictionMarketStream { + pub fn new(config: BinancePredictionMarketDataConfig) -> HftResult { + if config.api_key.trim().is_empty() || config.api_secret.trim().is_empty() { + return Err(HftError::Authentication( + "Binance Prediction market data requires API credentials".to_string(), + )); + } + if config.outcomes.is_empty() { + return Err(HftError::Config( + "Binance Prediction market data requires at least one configured outcome" + .to_string(), + )); + } + if config.poll_interval_ms == 0 { + return Err(HftError::Config( + "Binance Prediction poll_interval_ms must be positive".to_string(), + )); + } + if config.outcomes.iter().any(|outcome| { + outcome.token_id.trim().is_empty() + || outcome.market_id <= 0 + || outcome.vendor.trim().is_empty() + }) { + return Err(HftError::Config( + "Binance Prediction outcomes require token_id, positive market_id, and vendor" + .to_string(), + )); + } + let unique_token_ids: HashSet<_> = config + .outcomes + .iter() + .map(|outcome| outcome.token_id.as_str()) + .collect(); + if unique_token_ids.len() != config.outcomes.len() { + return Err(HftError::Config( + "Binance Prediction outcome token_id values must be unique".to_string(), + )); + } + Ok(Self { + config, + state: Arc::new(ConnectionState::default()), + }) + } + + fn api(&self) -> HftResult { + let config = ConfigurationRestApi::builder() + .api_key(self.config.api_key.clone()) + .api_secret(self.config.api_secret.clone()) + .base_path(self.config.rest_base_url.clone()) + .timeout(self.config.poll_interval_ms.max(1_000)) + .keep_alive(true) + .retries(1) + .build() + .map_err(|error| HftError::Config(error.to_string()))?; + Ok(W3WPredictionRestApi::from_config(config)) + } +} + +fn default_vendor() -> String { + "predict_fun".to_string() +} +fn default_rest_base_url() -> String { + "https://api.binance.com".to_string() +} +const fn default_poll_interval_ms() -> u64 { + DEFAULT_POLL_INTERVAL_MS +} + +fn timestamp_micros(timestamp_ms: Option) -> HftResult { + let timestamp = timestamp_ms.ok_or_else(|| { + HftError::Parse("Binance Prediction order-book timestamp is missing".to_string()) + })?; + u64::try_from(timestamp) + .ok() + .and_then(|value| value.checked_mul(1_000)) + .ok_or_else(|| HftError::Parse("Binance Prediction timestamp is invalid".to_string())) +} + +fn parse_levels( + rows: impl IntoIterator, Option)>, + side: &str, +) -> HftResult> { + rows.into_iter() + .map(|(price, size)| { + level( + price.as_deref().ok_or_else(|| { + HftError::Parse(format!("Binance Prediction {side} is missing price")) + })?, + size.as_deref().ok_or_else(|| { + HftError::Parse(format!("Binance Prediction {side} is missing size")) + })?, + ) + }) + .collect() +} + +fn level(price: &str, size: &str) -> HftResult { + let price = price.parse::().map_err(|error| { + HftError::Parse(format!( + "Binance Prediction order-book price is invalid: {error}" + )) + })?; + let size = size.parse::().map_err(|error| { + HftError::Parse(format!( + "Binance Prediction order-book size is invalid: {error}" + )) + })?; + if !(Decimal::ZERO..=Decimal::ONE).contains(&price) || size < Decimal::ZERO { + return Err(HftError::Parse( + "Binance Prediction order-book level is out of range".to_string(), + )); + } + Ok(BookLevel { + price: Price(price), + quantity: Quantity(size), + }) +} + +fn snapshot_from_levels( + symbol: Symbol, + sequence: u64, + timestamp_ms: Option, + mut bids: Vec, + mut asks: Vec, +) -> HftResult { + if bids.is_empty() || asks.is_empty() { + return Err(HftError::Parse( + "Binance Prediction order book is missing a bid or ask side".to_string(), + )); + } + bids.sort_by_key(|level| std::cmp::Reverse(level.price)); + asks.sort_by_key(|level| level.price); + Ok(MarketSnapshot { + symbol, + timestamp: timestamp_micros(timestamp_ms)?, + bids, + asks, + sequence, + source_venue: Some(VenueId::BINANCE_PREDICTION), + }) +} + +#[async_trait] +impl MarketStream for BinancePredictionMarketStream { + async fn subscribe(&self, symbols: Vec) -> HftResult> { + if symbols.is_empty() { + return Err(HftError::Config( + "Binance Prediction requires at least one configured outcome token".to_string(), + )); + } + let outcomes: HashMap<_, _> = self + .config + .outcomes + .iter() + .map(|outcome| (outcome.token_id.as_str(), outcome.clone())) + .collect(); + let selected: Vec<_> = symbols + .into_iter() + .map(|symbol| { + let token_id = symbol.as_str().to_string(); + let outcome = outcomes.get(token_id.as_str()).cloned().ok_or_else(|| { + HftError::Config(format!( + "Binance Prediction token {token_id} is missing from data_config.outcomes" + )) + })?; + Ok((symbol, outcome)) + }) + .collect::>()?; + let api = self.api()?; + let state = Arc::clone(&self.state); + let poll_interval = Duration::from_millis(self.config.poll_interval_ms); + let (tx, mut rx) = mpsc::channel(EVENT_QUEUE_CAPACITY); + tokio::spawn(async move { + let mut sequence = 0_u64; + let mut ticker = interval(poll_interval); + ticker.set_missed_tick_behavior(MissedTickBehavior::Skip); + loop { + ticker.tick().await; + for (symbol, outcome) in &selected { + let params = match QueryOrderBookParams::builder( + outcome.vendor.clone(), + outcome.market_id, + outcome.token_id.clone(), + ) + .build() + { + Ok(params) => params, + Err(error) => { + let _ = tx.send(Err(HftError::Config(error.to_string()))).await; + continue; + } + }; + let response = match api.query_order_book(params).await { + Ok(response) => response + .data() + .await + .map_err(|error| HftError::Exchange(error.to_string())), + Err(error) => Err(HftError::Exchange(error.to_string())), + }; + match response { + Ok(book) => { + let bids = parse_levels( + book.bids + .unwrap_or_default() + .into_iter() + .map(|row| (row.price, row.size)), + "bid", + ); + let asks = parse_levels( + book.asks + .unwrap_or_default() + .into_iter() + .map(|row| (row.price, row.size)), + "ask", + ); + match bids.and_then(|bids| { + asks.and_then(|asks| { + snapshot_from_levels( + symbol.clone(), + sequence.saturating_add(1), + book.timestamp, + bids, + asks, + ) + }) + }) { + Ok(snapshot) => { + sequence = sequence.saturating_add(1); + state.connected.store(true, Ordering::SeqCst); + state + .last_heartbeat + .store(hft_core::now_micros(), Ordering::SeqCst); + if tx.send(Ok(MarketEvent::Snapshot(snapshot))).await.is_err() { + return; + } + } + Err(error) => { + let _ = tx.send(Err(error)).await; + } + } + } + Err(error) => { + state.connected.store(false, Ordering::SeqCst); + if tx.send(Err(error)).await.is_err() { + return; + } + } + } + } + } + }); + Ok(Box::pin(futures::stream::poll_fn(move |cx| { + rx.poll_recv(cx) + }))) + } + + async fn health(&self) -> ConnectionHealth { + ConnectionHealth { + connected: self.state.connected.load(Ordering::SeqCst), + latency_ms: None, + last_heartbeat: self.state.last_heartbeat.load(Ordering::SeqCst), + } + } + async fn connect(&mut self) -> HftResult<()> { + Ok(()) + } + async fn disconnect(&mut self) -> HftResult<()> { + self.state.connected.store(false, Ordering::SeqCst); + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_a_prediction_order_book_into_a_snapshot() { + let snapshot = snapshot_from_levels( + Symbol::new("token-yes"), + 1, + Some(1_750_000_000_000), + vec![level("0.42", "10").unwrap()], + vec![level("0.55", "8").unwrap()], + ) + .unwrap(); + assert_eq!(snapshot.symbol.as_str(), "token-yes"); + assert_eq!(snapshot.source_venue, Some(VenueId::BINANCE_PREDICTION)); + assert_eq!(snapshot.bids[0].price.to_string(), "0.42"); + assert_eq!(snapshot.asks[0].price.to_string(), "0.55"); + } + + #[test] + fn rejects_one_sided_order_books() { + assert!(snapshot_from_levels( + Symbol::new("token-yes"), + 1, + Some(1_750_000_000_000), + vec![level("0.42", "10").unwrap()], + vec![], + ) + .is_err()); + } + + #[test] + fn rejects_empty_or_uncredentialed_config() { + let config = BinancePredictionMarketDataConfig { + api_key: String::new(), + api_secret: String::new(), + rest_base_url: default_rest_base_url(), + poll_interval_ms: 1_000, + outcomes: vec![], + }; + assert!(BinancePredictionMarketStream::new(config).is_err()); + } + + #[test] + fn rejects_duplicate_outcome_token_ids() { + let config = BinancePredictionMarketDataConfig { + api_key: "test-key".to_string(), + api_secret: "test-secret".to_string(), + rest_base_url: default_rest_base_url(), + poll_interval_ms: 1_000, + outcomes: vec![ + PredictionOutcomeConfig { + token_id: "token-yes".to_string(), + market_id: 1, + vendor: default_vendor(), + }, + PredictionOutcomeConfig { + token_id: "token-yes".to_string(), + market_id: 2, + vendor: default_vendor(), + }, + ], + }; + assert!(BinancePredictionMarketStream::new(config).is_err()); + } + + #[test] + fn rejects_missing_exchange_timestamp() { + assert!(snapshot_from_levels( + Symbol::new("token-yes"), + 1, + None, + vec![level("0.42", "10").unwrap()], + vec![level("0.55", "8").unwrap()], + ) + .is_err()); + } +} diff --git a/rust_hft/data/backtest/sample.manifest.json b/rust_hft/data/backtest/sample.manifest.json new file mode 100644 index 000000000..0088f282a --- /dev/null +++ b/rust_hft/data/backtest/sample.manifest.json @@ -0,0 +1,29 @@ +{ + "dataset_kind": "backtest_point_in_time_event_tape", + "schema_version": "backtest-pit-v1", + "mission_id": "sample-backtest-fixture", + "market": "spot", + "symbol": "BTCUSDT", + "dataset": "binance_spot_lob", + "source_revision": "1fd99199bb65251b6af5af2ac0726a464467e5c0e73450c57f5e0672deda922f", + "source_segments": [ + { + "path": "data/backtest/sample.raw.ndjson", + "sha256": "def1428c8e1f21914a0eb7fac6ea6b02309f5ebcd3212006363cff4186a6418c", + "collector_manifest_path": "data/backtest/sample.raw.manifest.json", + "collector_manifest_sha256": "e7c503b63c6cbfbdecf9df4217330e65190de676cba374720371cd9b678e52d8", + "success_marker_path": "data/backtest/sample.raw.ndjson._SUCCESS", + "start_received_at_ns": 1700000000100000000, + "end_received_at_ns": 1700000000500000000, + "events": 5 + } + ], + "rows": 4, + "first_event_time_us": 1700000000100000, + "last_event_time_us": 1700000000500000, + "sequence_start": 1, + "sequence_end": 4, + "artifact_path": "data/backtest/sample.ndjson", + "artifact_sha256": "fb23300c1a947a39aff4848104d44c014b12e599f461647afa206235dece4027", + "point_in_time": true +} diff --git a/rust_hft/data/backtest/sample.ndjson b/rust_hft/data/backtest/sample.ndjson index 10ccfc1d3..b4823ac8c 100644 --- a/rust_hft/data/backtest/sample.ndjson +++ b/rust_hft/data/backtest/sample.ndjson @@ -1,8 +1,4 @@ -{"timestamp":1700000000000000,"event":"snapshot","bids":[[29990.0,1.5],[29989.5,1.2],[29989.0,1.0]],"asks":[[29990.5,1.3],[29991.0,1.1],[29991.5,1.0]]} -{"timestamp":1700000000500000,"event":"trade","side":"sell","price":29989.5,"quantity":0.8} -{"timestamp":1700000000600000,"event":"l2_update","bids":[[29989.0,1.8],[29988.5,2.0],[29988.0,1.5]],"asks":[[29990.5,1.0],[29991.0,0.8],[29991.5,0.6]]} -{"timestamp":1700000000700000,"event":"trade","side":"sell","price":29988.5,"quantity":1.0} -{"timestamp":1700000000800000,"event":"trade","side":"buy","price":29990.0,"quantity":0.3} -{"timestamp":1700000000900000,"event":"l2_update","bids":[[29987.5,2.5],[29987.0,2.1],[29986.5,2.0]],"asks":[[29989.5,0.9],[29990.0,0.8],[29990.5,0.7]]} -{"timestamp":1700000001200000,"event":"trade","side":"sell","price":29987.0,"quantity":1.2} -{"timestamp":1700000001800000,"event":"trade","side":"buy","price":29989.0,"quantity":0.6} +{"timestamp":1700000000100000,"sequence":1,"event":"snapshot","bids":[[29990.0,1.5],[29989.5,1.2]],"asks":[[29990.5,1.3],[29991.0,1.1]]} +{"timestamp":1700000000200000,"sequence":2,"event":"l2_update","bids":[[29990.0,1.7]],"asks":[]} +{"timestamp":1700000000400000,"sequence":3,"event":"l2_update","bids":[],"asks":[[29990.5,0.9]]} +{"timestamp":1700000000500000,"sequence":4,"event":"l2_update","bids":[[29989.5,0.0]],"asks":[[29991.0,1.4]]} diff --git a/rust_hft/data/backtest/sample.raw.manifest.json b/rust_hft/data/backtest/sample.raw.manifest.json new file mode 100644 index 000000000..54dd19217 --- /dev/null +++ b/rust_hft/data/backtest/sample.raw.manifest.json @@ -0,0 +1,22 @@ +{ + "schema": "binance.lob_tape.v2", + "venue": "binance", + "market": "spot", + "dataset": "binance_spot_lob", + "symbols": ["BTCUSDT"], + "mode": "diff", + "replay_scope": "captured_snapshot_seed_plus_sequence_checked_diffs", + "events": 5, + "bytes": 1240, + "event_types": { + "snapshot": 1, + "checkpoint": 1, + "diff": 3 + }, + "has_replay_safe_checkpoint": true, + "all_symbols_bridged": true, + "start_received_at_ns": 1700000000100000000, + "end_received_at_ns": 1700000000500000000, + "file": "sample.raw.ndjson", + "sha256": "def1428c8e1f21914a0eb7fac6ea6b02309f5ebcd3212006363cff4186a6418c" +} diff --git a/rust_hft/data/backtest/sample.raw.ndjson b/rust_hft/data/backtest/sample.raw.ndjson new file mode 100644 index 000000000..3f27bdc73 --- /dev/null +++ b/rust_hft/data/backtest/sample.raw.ndjson @@ -0,0 +1,5 @@ +{"received_at_ns":1700000000100000000,"type":"snapshot","session_id":"session-1","symbol":"BTCUSDT","request_started_at_ns":1700000000000000000,"snapshot":{"lastUpdateId":100,"bids":[["29990.0","1.5"],["29989.5","1.2"]],"asks":[["29990.5","1.3"],["29991.0","1.1"]]}} +{"received_at_ns":1700000000200000000,"type":"diff","session_id":"session-1","frame":{"stream":"btcusdt@depth@100ms","data":{"e":"depthUpdate","E":1700000000200,"s":"BTCUSDT","U":101,"u":101,"b":[["29990.0","1.7"]],"a":[]}}} +{"received_at_ns":1700000000300000000,"type":"checkpoint","reason":"scheduled","replay_safe":true,"session_id":"session-1","symbol":"BTCUSDT","last_update_id":101,"synced":true,"bridged":true,"bids":[["29990.0","1.7"],["29989.5","1.2"]],"asks":[["29990.5","1.3"],["29991.0","1.1"]]} +{"received_at_ns":1700000000400000000,"type":"diff","session_id":"session-1","frame":{"stream":"btcusdt@depth@100ms","data":{"e":"depthUpdate","E":1700000000400,"s":"BTCUSDT","U":102,"u":102,"b":[],"a":[["29990.5","0.9"]]}}} +{"received_at_ns":1700000000500000000,"type":"diff","session_id":"session-1","frame":{"stream":"btcusdt@depth@100ms","data":{"e":"depthUpdate","E":1700000000500,"s":"BTCUSDT","U":103,"u":103,"b":[["29989.5","0"]],"a":[["29991.0","1.4"]]}}} diff --git a/rust_hft/data/backtest/sample.raw.ndjson._SUCCESS b/rust_hft/data/backtest/sample.raw.ndjson._SUCCESS new file mode 100644 index 000000000..a5475e9e2 --- /dev/null +++ b/rust_hft/data/backtest/sample.raw.ndjson._SUCCESS @@ -0,0 +1 @@ +def1428c8e1f21914a0eb7fac6ea6b02309f5ebcd3212006363cff4186a6418c diff --git a/rust_hft/deployment/docker/Dockerfile.trading b/rust_hft/deployment/docker/Dockerfile.trading index e1a3e096e..387a8cc2e 100644 --- a/rust_hft/deployment/docker/Dockerfile.trading +++ b/rust_hft/deployment/docker/Dockerfile.trading @@ -39,7 +39,7 @@ ENV RUST_LOG=info EXPOSE 9090 9092 HEALTHCHECK --interval=30s --timeout=10s --start-period=10s --retries=3 \ - CMD curl --fail --silent http://127.0.0.1:9090/readiness || exit 1 + CMD curl --fail --silent http://127.0.0.1:9090/health || exit 1 ENTRYPOINT ["/usr/local/bin/hft-live"] CMD ["--config", "/app/config/system.yaml"] diff --git a/rust_hft/docker/Dockerfile b/rust_hft/docker/Dockerfile index e1a3e096e..387a8cc2e 100644 --- a/rust_hft/docker/Dockerfile +++ b/rust_hft/docker/Dockerfile @@ -39,7 +39,7 @@ ENV RUST_LOG=info EXPOSE 9090 9092 HEALTHCHECK --interval=30s --timeout=10s --start-period=10s --retries=3 \ - CMD curl --fail --silent http://127.0.0.1:9090/readiness || exit 1 + CMD curl --fail --silent http://127.0.0.1:9090/health || exit 1 ENTRYPOINT ["/usr/local/bin/hft-live"] CMD ["--config", "/app/config/system.yaml"] diff --git a/rust_hft/execution-gateway/adapters/adapter-ondo-perps/src/lib.rs b/rust_hft/execution-gateway/adapters/adapter-ondo-perps/src/lib.rs index 962c41b24..f031b9162 100644 --- a/rust_hft/execution-gateway/adapters/adapter-ondo-perps/src/lib.rs +++ b/rust_hft/execution-gateway/adapters/adapter-ondo-perps/src/lib.rs @@ -558,6 +558,7 @@ mod tests { jurisdiction: Some("SG".into()), eligibility_confirmed: true, allow_tokenized_securities: true, + ..Default::default() }, side: Side::Buy, quantity: Quantity(Decimal::from(2)), diff --git a/rust_hft/execution-gateway/adapters/adapter-polymarket/src/lib.rs b/rust_hft/execution-gateway/adapters/adapter-polymarket/src/lib.rs index 7502c9685..c3f7c1c29 100644 --- a/rust_hft/execution-gateway/adapters/adapter-polymarket/src/lib.rs +++ b/rust_hft/execution-gateway/adapters/adapter-polymarket/src/lib.rs @@ -4107,6 +4107,7 @@ impl ExecutionClient for PolymarketExecutionClient { quantity: Quantity(position.size), avg_price: Price(position.avg_price), unrealized_pnl: position.cash_pnl, + realized_pnl: position.realized_pnl, }) .collect()) } diff --git a/rust_hft/infra-services/core/metrics/src/http_server.rs b/rust_hft/infra-services/core/metrics/src/http_server.rs index da98d0025..0311e8397 100644 --- a/rust_hft/infra-services/core/metrics/src/http_server.rs +++ b/rust_hft/infra-services/core/metrics/src/http_server.rs @@ -214,9 +214,37 @@ mod tests { } #[tokio::test] - async fn test_health_handler() { + async fn health_remains_live_when_runtime_is_fail_closed() { + MetricsRegistry::global().update_engine_statistics(&crate::EngineStatisticsExport { + cycle_count: 1, + execution_events_processed: 1, + orders_submitted: 0, + orders_ack: 0, + orders_filled: 0, + orders_rejected: 0, + orders_canceled: 0, + runtime_truth_observed_at_us: crate::now_micros(), + reconciliation_complete: false, + reconciliation_healthy: false, + risk_halted: true, + data_integrity_gaps: 1, + }); let response = health_handler().await.into_response(); assert_eq!(response.status(), StatusCode::OK); + let body = axum::body::to_bytes(response.into_body(), usize::MAX) + .await + .expect("health response body"); + let body: serde_json::Value = serde_json::from_slice(&body).expect("health response JSON"); + assert_eq!(body["status"], "healthy"); + + let readiness = readiness_handler(State(Arc::new(MetricsServerConfig { + readiness_max_idle_secs: u64::MAX, + readiness_max_utilization: 1.0, + ..Default::default() + }))) + .await + .into_response(); + assert_eq!(readiness.status(), StatusCode::SERVICE_UNAVAILABLE); } // 集成测试：启动服务器并测试端点 diff --git a/rust_hft/infra-services/core/metrics/src/lib.rs b/rust_hft/infra-services/core/metrics/src/lib.rs index 4593182f0..5a05f943c 100644 --- a/rust_hft/infra-services/core/metrics/src/lib.rs +++ b/rust_hft/infra-services/core/metrics/src/lib.rs @@ -2,7 +2,7 @@ //! //! 為 HFT 系統提供分段延遲監控、隊列利用率、事件計數等關鍵指標 -use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::OnceLock; use tracing::debug; @@ -70,10 +70,19 @@ pub struct MetricsRegistry { pub engine_orders_filled: Gauge, pub engine_orders_rejected: Gauge, pub engine_orders_canceled: Gauge, + pub runtime_reconciliation_complete: Gauge, + pub runtime_reconciliation_healthy: Gauge, + pub runtime_risk_halted: Gauge, + pub runtime_data_integrity_gaps: Gauge, // 本地就緒狀態跟蹤（不依賴 Prometheus 讀取，使 /readiness 更輕量） last_activity_micros: AtomicU64, last_queue_utilization_ppm: AtomicU64, // 以百萬分位儲存（ppm），避免 f64 原子 + runtime_truth_observed_at_us: AtomicU64, + reconciliation_complete: AtomicBool, + reconciliation_healthy: AtomicBool, + risk_halted: AtomicBool, + data_integrity_gaps: AtomicU64, } /// 引擎統計快照（由 engine 匯出，用於更新 gauges） @@ -86,6 +95,11 @@ pub struct EngineStatisticsExport { pub orders_filled: u64, pub orders_rejected: u64, pub orders_canceled: u64, + pub runtime_truth_observed_at_us: u64, + pub reconciliation_complete: bool, + pub reconciliation_healthy: bool, + pub risk_halted: bool, + pub data_integrity_gaps: u64, } impl MetricsRegistry { @@ -274,6 +288,24 @@ impl MetricsRegistry { let engine_orders_canceled = Gauge::new("hft_engine_orders_canceled", "已撤銷訂單數（當前快照）") .expect("創建 engine_orders_canceled 失敗"); + let runtime_reconciliation_complete = Gauge::new( + "hft_runtime_reconciliation_complete", + "交易所持倉與成交對帳是否完整", + ) + .expect("創建 runtime_reconciliation_complete 失敗"); + let runtime_reconciliation_healthy = Gauge::new( + "hft_runtime_reconciliation_healthy", + "交易所持倉與成交對帳是否健康", + ) + .expect("創建 runtime_reconciliation_healthy 失敗"); + let runtime_risk_halted = + Gauge::new("hft_runtime_risk_halted", "風控是否已暫停或緊急停止交易") + .expect("創建 runtime_risk_halted 失敗"); + let runtime_data_integrity_gaps = Gauge::new( + "hft_runtime_data_integrity_gaps", + "運行期偵測到的市場資料完整性斷層", + ) + .expect("創建 runtime_data_integrity_gaps 失敗"); // 對帳指標 let reconcile_runs = IntCounter::new("hft_reconcile_runs_total", "對帳執行次數") @@ -382,6 +414,18 @@ impl MetricsRegistry { registry .register(Box::new(engine_orders_canceled.clone())) .expect("註冊 engine_orders_canceled 失敗"); + registry + .register(Box::new(runtime_reconciliation_complete.clone())) + .expect("註冊 runtime_reconciliation_complete 失敗"); + registry + .register(Box::new(runtime_reconciliation_healthy.clone())) + .expect("註冊 runtime_reconciliation_healthy 失敗"); + registry + .register(Box::new(runtime_risk_halted.clone())) + .expect("註冊 runtime_risk_halted 失敗"); + registry + .register(Box::new(runtime_data_integrity_gaps.clone())) + .expect("註冊 runtime_data_integrity_gaps 失敗"); registry .register(Box::new(reconcile_runs.clone())) .expect("註冊對帳次數指標失敗"); @@ -454,8 +498,17 @@ impl MetricsRegistry { engine_orders_filled, engine_orders_rejected, engine_orders_canceled, + runtime_reconciliation_complete, + runtime_reconciliation_healthy, + runtime_risk_halted, + runtime_data_integrity_gaps, last_activity_micros: AtomicU64::new(now), last_queue_utilization_ppm: AtomicU64::new(0), + runtime_truth_observed_at_us: AtomicU64::new(0), + reconciliation_complete: AtomicBool::new(false), + reconciliation_healthy: AtomicBool::new(false), + risk_halted: AtomicBool::new(false), + data_integrity_gaps: AtomicU64::new(0), } } @@ -629,6 +682,23 @@ impl MetricsRegistry { self.engine_orders_filled.set(s.orders_filled as f64); self.engine_orders_rejected.set(s.orders_rejected as f64); self.engine_orders_canceled.set(s.orders_canceled as f64); + self.runtime_reconciliation_complete + .set(if s.reconciliation_complete { 1.0 } else { 0.0 }); + self.runtime_reconciliation_healthy + .set(if s.reconciliation_healthy { 1.0 } else { 0.0 }); + self.runtime_risk_halted + .set(if s.risk_halted { 1.0 } else { 0.0 }); + self.runtime_data_integrity_gaps + .set(s.data_integrity_gaps as f64); + self.runtime_truth_observed_at_us + .store(s.runtime_truth_observed_at_us, Ordering::Relaxed); + self.reconciliation_complete + .store(s.reconciliation_complete, Ordering::Relaxed); + self.reconciliation_healthy + .store(s.reconciliation_healthy, Ordering::Relaxed); + self.risk_halted.store(s.risk_halted, Ordering::Relaxed); + self.data_integrity_gaps + .store(s.data_integrity_gaps, Ordering::Relaxed); self.note_activity(); } @@ -720,7 +790,24 @@ impl MetricsRegistry { let last = self.last_activity_micros.load(Ordering::Relaxed); let idle_secs = (now.saturating_sub(last)) as f64 / 1_000_000.0; let util = self.queue_utilization_value(); - let ready = idle_secs <= max_idle_secs as f64 && util <= max_utilization; + let truth_observed_at_us = self.runtime_truth_observed_at_us.load(Ordering::Relaxed); + let truth_age_secs = (truth_observed_at_us > 0) + .then(|| now.saturating_sub(truth_observed_at_us) as f64 / 1_000_000.0); + let reconciliation_complete = self.reconciliation_complete.load(Ordering::Relaxed); + let reconciliation_healthy = self.reconciliation_healthy.load(Ordering::Relaxed); + let risk_halted = self.risk_halted.load(Ordering::Relaxed); + let data_integrity_gaps = self.data_integrity_gaps.load(Ordering::Relaxed); + let reconciliation_ready = match truth_age_secs { + None => false, + Some(age) => { + reconciliation_complete && reconciliation_healthy && age <= max_idle_secs as f64 + } + }; + let ready = idle_secs <= max_idle_secs as f64 + && util <= max_utilization + && reconciliation_ready + && !risk_halted + && data_integrity_gaps == 0; ( ready, serde_json::json!({ @@ -728,11 +815,47 @@ impl MetricsRegistry { "queue_utilization": util, "max_idle_secs": max_idle_secs, "max_utilization": max_utilization, + "reconciliation_complete": reconciliation_complete, + "reconciliation_healthy": reconciliation_healthy, + "reconciliation_age_secs": truth_age_secs, + "risk_halted": risk_halted, + "data_integrity_gaps": data_integrity_gaps, }), ) } } +#[cfg(test)] +mod runtime_readiness_tests { + use super::*; + + #[test] + fn readiness_fails_closed_on_unhealthy_runtime_truth() { + let registry = MetricsRegistry::create_with_prometheus(); + assert!(!registry.assess_readiness(1.0, 60).0); + registry.update_engine_statistics(&EngineStatisticsExport { + cycle_count: 1, + execution_events_processed: 1, + orders_submitted: 0, + orders_ack: 0, + orders_filled: 0, + orders_rejected: 0, + orders_canceled: 0, + runtime_truth_observed_at_us: now_micros(), + reconciliation_complete: false, + reconciliation_healthy: false, + risk_halted: true, + data_integrity_gaps: 1, + }); + + let (ready, detail) = registry.assess_readiness(1.0, 60); + + assert!(!ready); + assert_eq!(detail["risk_halted"], true); + assert_eq!(detail["data_integrity_gaps"], 1); + } +} + /// 便利宏：記錄分段延遲 #[inline] pub fn now_micros() -> u64 { diff --git a/rust_hft/market-core/core/src/types.rs b/rust_hft/market-core/core/src/types.rs index 3f4d1c123..4611419ac 100644 --- a/rust_hft/market-core/core/src/types.rs +++ b/rust_hft/market-core/core/src/types.rs @@ -421,6 +421,24 @@ pub struct ComplianceContext { pub eligibility_confirmed: bool, #[serde(default)] pub allow_tokenized_securities: bool, + /// Executable top-of-book depth captured with the order decision. + #[serde(default)] + pub top_depth_usd: Option, + /// Spread captured with the order decision, in basis points. + #[serde(default)] + pub spread_bps: Option, + /// Corporate-action state from an authoritative reference-data source. + #[serde(default)] + pub corporate_action_active: Option, + /// Identity of the authoritative compliance/market-quality evidence producer. + #[serde(default)] + pub evidence_source: Option, + /// Venue for which the evidence was produced. + #[serde(default)] + pub evidence_venue: Option, + /// Local observation timestamp of the evidence, in microseconds since epoch. + #[serde(default)] + pub evidence_observed_at: Option, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] diff --git a/rust_hft/market-core/engine/src/execution_control.rs b/rust_hft/market-core/engine/src/execution_control.rs index 64d2843f5..25eba5cef 100644 --- a/rust_hft/market-core/engine/src/execution_control.rs +++ b/rust_hft/market-core/engine/src/execution_control.rs @@ -16,6 +16,7 @@ pub struct RuntimeReconciliationReport { pub order_report: OrderReconciliationReport, pub balance_report: Option, pub position_report: Option, + pub fill_report: Option, pub complete: bool, pub healthy: bool, } @@ -49,6 +50,15 @@ pub struct PositionReconciliationReport { pub healthy: bool, } +#[derive(Debug, Clone)] +pub struct FillReconciliationReport { + /// Authoritative venue fills that are absent from the local accounting ledger. + pub exchange_only_fill_ids: Vec, + pub client_errors: Vec, + pub complete: bool, + pub healthy: bool, +} + /// Shared control plane for Sentinel, IPC, and gRPC. /// /// The engine lock is held only while changing mode or taking an OMS snapshot. @@ -362,13 +372,23 @@ impl ExecutionControlHandle { &self, include_balances: bool, ) -> HftResult { + if include_balances { + self.engine + .lock() + .await + .publish_runtime_truth_status(crate::RuntimeTruthStatus { + reconciliation_complete: false, + reconciliation_healthy: false, + observed_at_us: hft_core::now_micros(), + }); + } let worker_tx = self.worker_sender()?; let (reply_tx, reply_rx) = oneshot::channel(); worker_tx .send(ControlCommand::Reconcile { include_balances, include_positions: include_balances, - include_recent_fills: false, + include_recent_fills: include_balances, reply: reply_tx, }) .map_err(|_| HftError::Execution("execution worker control channel closed".into()))?; @@ -384,8 +404,10 @@ impl ExecutionControlHandle { let engine = self.engine.lock().await; engine.reconcile_open_orders(&exchange_orders) }; - let (balance_report, position_report) = if include_balances { - let account_view = self.engine.lock().await.get_account_view(); + let (balance_report, position_report, fill_report) = if include_balances { + let engine = self.engine.lock().await; + let account_view = engine.get_account_view(); + let portfolio_state = engine.export_portfolio_state(); ( Some(reconcile_balances( &worker_snapshot, @@ -393,17 +415,27 @@ impl ExecutionControlHandle { self.balance_tolerance_usd, )), reconcile_positions(&worker_snapshot, &account_view.positions), + Some(reconcile_recent_fills( + &worker_snapshot, + &portfolio_state.processed_fill_ids, + )), ) } else { - (None, None) + (None, None, None) }; let complete = worker_snapshot.is_complete() + && (!include_balances + || worker_snapshot.clients.iter().all(|client| { + client.positions.as_ref().is_some_and(Result::is_ok) + && client.recent_fills.as_ref().is_some_and(Result::is_ok) + })) && balance_report .as_ref() .is_none_or(|balances| balances.complete) && position_report .as_ref() - .is_none_or(|positions| positions.complete); + .is_none_or(|positions| positions.complete) + && fill_report.as_ref().is_none_or(|fills| fills.complete); let healthy = complete && !order_report.has_discrepancies() && balance_report @@ -411,16 +443,29 @@ impl ExecutionControlHandle { .is_none_or(|balances| balances.healthy) && position_report .as_ref() - .is_none_or(|positions| positions.healthy); + .is_none_or(|positions| positions.healthy) + && fill_report.as_ref().is_none_or(|fills| fills.healthy); - Ok(RuntimeReconciliationReport { + let report = RuntimeReconciliationReport { worker_snapshot, order_report, balance_report, position_report, + fill_report, complete, healthy, - }) + }; + if include_balances { + self.engine + .lock() + .await + .publish_runtime_truth_status(crate::RuntimeTruthStatus { + reconciliation_complete: report.complete, + reconciliation_healthy: report.healthy, + observed_at_us: hft_core::now_micros(), + }); + } + Ok(report) } /// Quiesce both strategy production and worker intake before taking an authoritative @@ -514,6 +559,9 @@ impl ExecutionControlHandle { let mut cash_balance = Decimal::ZERO; let mut positions = std::collections::HashMap::new(); + let mut baseline_processed_fill_ids = + std::collections::HashMap::>::new(); + let mut baseline_recent_accounting_event_ids = Vec::new(); let mut saw_balances = false; for client in &snapshot.clients { let balances = client.balances.as_ref().ok_or_else(|| { @@ -562,6 +610,38 @@ impl ExecutionControlHandle { ))); } } + + let recent_fills = client.recent_fills.as_ref().ok_or_else(|| { + HftError::Execution(format!( + "execution client {} did not provide recent fills for account bootstrap", + client.client_index + )) + })?; + for fill in recent_fills.as_ref().map_err(|error| { + HftError::Execution(format!( + "execution client {} recent-fill bootstrap failed: {error}", + client.client_index + )) + })? { + if fill.fill_id.is_empty() { + return Err(HftError::Execution(format!( + "execution client {} returned a recent fill without fill_id", + client.client_index + ))); + } + let inserted = baseline_processed_fill_ids + .entry(fill.order_id.clone()) + .or_default() + .insert(fill.fill_id.clone()); + if !inserted { + return Err(HftError::Execution(format!( + "account bootstrap found duplicate recent fill {}:{}", + fill.order_id.0, fill.fill_id + ))); + } + baseline_recent_accounting_event_ids + .push((fill.order_id.clone(), format!("fill:{}", fill.fill_id))); + } } if !saw_balances { return Err(HftError::Execution( @@ -599,8 +679,8 @@ impl ExecutionControlHandle { account_view, order_meta: current.order_meta, market_prices: current.market_prices, - processed_fill_ids: current.processed_fill_ids, - recent_accounting_event_ids: current.recent_accounting_event_ids, + processed_fill_ids: baseline_processed_fill_ids, + recent_accounting_event_ids: baseline_recent_accounting_event_ids, })?; Ok(true) } @@ -724,18 +804,70 @@ fn reconcile_balances( } } +fn reconcile_recent_fills( + snapshot: &WorkerReconcileSnapshot, + processed_fill_ids: &std::collections::HashMap>, +) -> FillReconciliationReport { + let mut exchange_only_fill_ids = Vec::new(); + let mut client_errors = Vec::new(); + let mut observed = std::collections::HashSet::new(); + + if snapshot.clients.is_empty() { + client_errors.push("no execution clients returned recent-fill snapshots".to_string()); + } + for client in &snapshot.clients { + match &client.recent_fills { + None => client_errors.push(format!( + "client={} does not support authoritative recent fills", + client.client_index + )), + Some(Err(error)) => client_errors.push(format!( + "client={} recent-fill snapshot failed: {}", + client.client_index, error + )), + Some(Ok(fills)) => { + for fill in fills { + let identity = (fill.order_id.clone(), fill.fill_id.clone()); + if fill.fill_id.is_empty() { + client_errors.push(format!( + "client={} returned a recent fill without fill_id", + client.client_index + )); + continue; + } + if !observed.insert(identity.clone()) { + client_errors.push(format!( + "duplicate authoritative fill identity order={} fill={}", + fill.order_id.0, fill.fill_id + )); + continue; + } + let locally_processed = processed_fill_ids + .get(&fill.order_id) + .is_some_and(|ids| ids.contains(&fill.fill_id)); + if !locally_processed { + exchange_only_fill_ids + .push(format!("{}:{}", fill.order_id.0, fill.fill_id)); + } + } + } + } + } + exchange_only_fill_ids.sort(); + let complete = client_errors.is_empty(); + let healthy = complete && exchange_only_fill_ids.is_empty(); + FillReconciliationReport { + exchange_only_fill_ids, + client_errors, + complete, + healthy, + } +} + fn reconcile_positions( snapshot: &WorkerReconcileSnapshot, local_positions: &std::collections::HashMap, ) -> Option { - if snapshot - .clients - .iter() - .all(|client| client.positions.is_none()) - { - return None; - } - #[derive(Default)] struct VenuePositions { quantities: BTreeMap, @@ -1132,6 +1264,52 @@ mod tests { worker.await.expect("worker task"); } + #[tokio::test] + async fn order_only_reconciliation_does_not_publish_authoritative_account_truth() { + let engine = Arc::new(Mutex::new(Engine::new(EngineConfig::default()))); + let runtime_truth = engine.lock().await.runtime_truth_reader(); + let (worker_tx, mut worker_rx) = mpsc::unbounded_channel(); + let control = ExecutionControlHandle::new(engine, Some(worker_tx), true); + let worker = tokio::spawn(async move { + match worker_rx.recv().await.expect("control command") { + ControlCommand::Reconcile { + include_balances, + include_positions, + include_recent_fills, + reply, + } => { + assert!(!include_balances); + assert!(!include_positions); + assert!(!include_recent_fills); + reply + .send(WorkerReconcileSnapshot { + clients: vec![crate::execution_worker::ClientReconcileSnapshot { + client_index: 0, + venue: Some(VenueId::POLYMARKET), + account_id: None, + open_orders: Ok(Vec::new()), + balances: None, + positions: None, + recent_fills: None, + }], + }) + .expect("send reconciliation"); + } + _ => panic!("unexpected control command"), + } + }); + + let report = control.reconcile(false).await.expect("reconcile"); + worker.await.expect("worker task"); + + assert!(report.complete); + assert!(report.healthy); + let truth = runtime_truth.load(); + assert!(!truth.reconciliation_complete); + assert!(!truth.reconciliation_healthy); + assert_eq!(truth.observed_at_us, 0); + } + #[tokio::test] async fn guarded_reconciliation_quiesces_before_snapshot_and_resumes_only_when_healthy() { let engine = Arc::new(Mutex::new(Engine::new(EngineConfig::default()))); @@ -1274,10 +1452,13 @@ mod tests { match worker_rx.recv().await.expect("control command") { ControlCommand::Reconcile { include_balances, + include_positions, + include_recent_fills, reply, - .. } => { assert!(include_balances); + assert!(include_positions); + assert!(include_recent_fills); reply .send(WorkerReconcileSnapshot { clients: vec![crate::execution_worker::ClientReconcileSnapshot { @@ -1286,8 +1467,8 @@ mod tests { account_id: None, open_orders: Ok(Vec::new()), balances: Some(balances), - positions: None, - recent_fills: None, + positions: Some(Ok(Vec::new())), + recent_fills: Some(Ok(Vec::new())), }], }) .expect("send reconciliation"); @@ -1358,6 +1539,80 @@ mod tests { assert_eq!(balances.missing_valuations, vec!["client=0 asset=BTC"]); } + #[test] + fn recent_fill_reconciliation_detects_unaccounted_exchange_fill() { + let order_id = OrderId("order-1".to_string()); + let snapshot = WorkerReconcileSnapshot { + clients: vec![crate::execution_worker::ClientReconcileSnapshot { + client_index: 0, + venue: Some(VenueId::POLYMARKET), + account_id: None, + open_orders: Ok(Vec::new()), + balances: None, + positions: None, + recent_fills: Some(Ok(vec![ports::AccountFill { + fill_id: "fill-1".to_string(), + order_id: order_id.clone(), + symbol: Symbol::new("123"), + side: Side::Buy, + price: Price(Decimal::new(5, 1)), + quantity: Quantity(Decimal::ONE), + fee: None, + timestamp: 1, + }])), + }], + }; + + let missing = reconcile_recent_fills(&snapshot, &HashMap::new()); + assert!(missing.complete); + assert!(!missing.healthy); + assert_eq!(missing.exchange_only_fill_ids, vec!["order-1:fill-1"]); + + let processed = HashMap::from([( + order_id, + std::collections::HashSet::from(["fill-1".to_string()]), + )]); + let matched = reconcile_recent_fills(&snapshot, &processed); + assert!(matched.healthy); + } + + #[tokio::test] + async fn live_reconciliation_is_incomplete_without_positions_and_recent_fills() { + let (worker_tx, mut worker_rx) = mpsc::unbounded_channel(); + let control = + ExecutionControlHandle::new(engine_with_equity(Decimal::ZERO), Some(worker_tx), true); + let worker = tokio::spawn(async move { + match worker_rx.recv().await.expect("control command") { + ControlCommand::Reconcile { reply, .. } => reply + .send(WorkerReconcileSnapshot { + clients: vec![crate::execution_worker::ClientReconcileSnapshot { + client_index: 0, + venue: Some(VenueId::BINANCE), + account_id: None, + open_orders: Ok(Vec::new()), + balances: Some(Ok(Vec::new())), + positions: None, + recent_fills: None, + }], + }) + .expect("send reconciliation"), + _ => panic!("unexpected control command"), + } + }); + + let report = control.reconcile(true).await.expect("reconcile"); + worker.await.expect("worker task"); + + assert!(!report.complete); + assert!(!report.healthy); + assert!(report + .position_report + .expect("position report") + .client_errors + .iter() + .any(|error| error.contains("does not support"))); + } + #[test] fn position_reconciliation_detects_exchange_only_and_quantity_mismatch() { let token_a = Symbol::new("111"); @@ -1369,6 +1624,7 @@ mod tests { quantity: Quantity(Decimal::from(2)), avg_price: Price(Decimal::new(45, 2)), unrealized_pnl: Decimal::ZERO, + realized_pnl: Decimal::ZERO, }, )]); let snapshot = WorkerReconcileSnapshot { @@ -1384,12 +1640,14 @@ mod tests { quantity: Quantity(Decimal::from(3)), avg_price: Price(Decimal::new(45, 2)), unrealized_pnl: Decimal::ZERO, + realized_pnl: Decimal::ZERO, }, ports::Position { symbol: token_b.clone(), quantity: Quantity(Decimal::ONE), avg_price: Price(Decimal::new(55, 2)), unrealized_pnl: Decimal::ZERO, + realized_pnl: Decimal::ZERO, }, ])), recent_fills: None, @@ -1415,6 +1673,7 @@ mod tests { quantity: Quantity(Decimal::ONE), avg_price: Price(Decimal::new(50, 2)), unrealized_pnl: Decimal::ZERO, + realized_pnl: Decimal::ZERO, }, )]); let snapshot = WorkerReconcileSnapshot { @@ -1462,6 +1721,7 @@ mod tests { quantity: Quantity(Decimal::ONE), avg_price: Price(Decimal::from(100)), unrealized_pnl: Decimal::ZERO, + realized_pnl: Decimal::ZERO, }, )]); let snapshot = WorkerReconcileSnapshot { @@ -1544,12 +1804,22 @@ mod tests { Some(Decimal::from(80)), )])), positions: Some(Ok(vec![ports::Position { - symbol: token, + symbol: token.clone(), quantity: Quantity(Decimal::from(2)), avg_price: Price(Decimal::new(45, 2)), unrealized_pnl: Decimal::new(10, 2), + realized_pnl: Decimal::ZERO, + }])), + recent_fills: Some(Ok(vec![ports::AccountFill { + fill_id: "historical-fill-1".to_string(), + order_id: OrderId("historical-order-1".to_string()), + symbol: token, + side: Side::Buy, + price: Price(Decimal::new(45, 2)), + quantity: Quantity(Decimal::from(2)), + fee: None, + timestamp: 1, }])), - recent_fills: None, }], } } @@ -1572,6 +1842,12 @@ mod tests { assert_eq!(account.unrealized_pnl, Decimal::new(10, 2)); assert_eq!(account.high_water_mark, account.equity()); assert!(account.session_start_us > 0); + drop(account); + let state = engine.lock().await.export_portfolio_state(); + assert!(state + .processed_fill_ids + .get(&OrderId("historical-order-1".to_string())) + .is_some_and(|ids| ids.contains("historical-fill-1"))); assert!(!control .bootstrap_pristine_account(&polymarket_account_snapshot(Vec::new())) diff --git a/rust_hft/market-core/engine/src/execution_queues.rs b/rust_hft/market-core/engine/src/execution_queues.rs index 8dee7e150..490688acf 100644 --- a/rust_hft/market-core/engine/src/execution_queues.rs +++ b/rust_hft/market-core/engine/src/execution_queues.rs @@ -129,6 +129,7 @@ pub fn create_execution_queues(config: ExecutionQueueConfig) -> (EngineQueues, W impl EngineQueues { /// 发送订单意图到执行 worker (非阻塞) + #[allow(clippy::result_large_err)] // Returning ownership avoids a heap allocation on the hot path. pub fn send_intent(&mut self, intent: OrderIntent) -> Result<(), OrderIntent> { let now = hft_core::now_micros(); let envelope = OrderIntentEnvelope::new( diff --git a/rust_hft/market-core/engine/src/lib.rs b/rust_hft/market-core/engine/src/lib.rs index 09a566121..a71c81ce2 100644 --- a/rust_hft/market-core/engine/src/lib.rs +++ b/rust_hft/market-core/engine/src/lib.rs @@ -57,6 +57,13 @@ pub enum TradingMode { Emergency, } +#[derive(Debug, Clone, Default)] +pub struct RuntimeTruthStatus { + pub reconciliation_complete: bool, + pub reconciliation_healthy: bool, + pub observed_at_us: u64, +} + /// 引擎運行統計（內部狀態） #[derive(Debug, Default)] pub struct EngineStats { @@ -72,6 +79,7 @@ pub struct EngineStats { pub market_events_dropped: u64, pub intents_dropped: u64, pub snapshot_publish_failed: u64, + pub data_integrity_gaps: u64, // Sentinel 控制 pub trading_mode: TradingMode, /// 引擎啟動時間 (微秒時間戳) @@ -231,6 +239,7 @@ pub struct Engine { market_snapshots: SnapshotContainer, /// 帳戶快照發佈容器 account_snapshots: SnapshotContainer, + runtime_truth_snapshots: SnapshotContainer, /// 註冊的策略 strategies: Vec>, /// 風控管理器（可選） @@ -321,6 +330,7 @@ impl Engine { aggregation_engine, market_snapshots: SnapshotContainer::new(initial_market_view), account_snapshots: SnapshotContainer::new(initial_account_view), + runtime_truth_snapshots: SnapshotContainer::new(RuntimeTruthStatus::default()), strategies: Vec::new(), risk_manager: None, venue_specs: VenueSpec::build_default_venue_specs(), @@ -341,6 +351,7 @@ impl Engine { market_events_dropped: 0, intents_dropped: 0, snapshot_publish_failed: 0, + data_integrity_gaps: 0, trading_mode: TradingMode::Normal, start_time_us: now_micros(), }, @@ -622,6 +633,7 @@ impl Engine { } // 同步引擎統計（以 Gauge） let s = self.get_statistics(); + let runtime_truth = self.runtime_truth_snapshots.load(); let export = infra_metrics::EngineStatisticsExport { cycle_count: s.cycle_count, execution_events_processed: s.execution_events_processed, @@ -630,6 +642,14 @@ impl Engine { orders_filled: s.orders_filled, orders_rejected: s.orders_rejected, orders_canceled: s.orders_canceled, + runtime_truth_observed_at_us: runtime_truth.observed_at_us, + reconciliation_complete: runtime_truth.reconciliation_complete, + reconciliation_healthy: runtime_truth.reconciliation_healthy, + risk_halted: matches!( + self.stats.trading_mode, + TradingMode::Paused | TradingMode::Emergency + ), + data_integrity_gaps: s.data_integrity_gaps, }; infra_metrics::MetricsRegistry::global().update_engine_statistics(&export); } @@ -815,6 +835,14 @@ impl Engine { self.account_snapshots.reader() } + pub fn runtime_truth_reader(&self) -> Arc> { + self.runtime_truth_snapshots.reader() + } + + pub(crate) fn publish_runtime_truth_status(&mut self, status: RuntimeTruthStatus) { + self.runtime_truth_snapshots.store(Arc::new(status)); + } + /// 提交訂單意圖到執行隊列（用於 dry-run 測試） pub fn submit_order_intent(&mut self, intent: ports::OrderIntent) -> Result<(), HftError> { self.ensure_accepting_new_intents()?; @@ -979,6 +1007,10 @@ impl Engine { } for event in aggregation_events.drain(..) { + if matches!(event, ports::MarketEvent::Disconnect { .. }) { + self.stats.data_integrity_gaps = + self.stats.data_integrity_gaps.saturating_add(1); + } let book = Self::event_venue_symbol(&event).and_then(|key| { self.aggregation_engine .orderbooks @@ -1772,20 +1804,36 @@ impl Engine { .unwrap_or((0, 0)); // 從 Portfolio 獲取 PnL 和回撤（如果有） - let (pnl, unrealized_pnl, drawdown_pct, max_drawdown_pct, high_water_mark) = - if let Some(pm) = &self.portfolio_manager { - let av = pm.reader().load(); - let total_pnl = av.total_pnl(); - ( - total_pnl.to_string().parse::().unwrap_or(0.0), - av.unrealized_pnl.to_string().parse::().unwrap_or(0.0), - av.drawdown_pct, - av.max_drawdown_pct, - av.high_water_mark.to_string().parse::().unwrap_or(0.0), - ) - } else { - (0.0, 0.0, 0.0, 0.0, 0.0) - }; + let ( + pnl, + unrealized_pnl, + drawdown_pct, + max_drawdown_pct, + high_water_mark, + position_count, + notional_value, + ) = if let Some(pm) = &self.portfolio_manager { + let av = pm.reader().load(); + let total_pnl = av.total_pnl(); + let notional = av + .positions + .values() + .map(|position| { + (position.avg_price.0 * position.quantity.0 + position.unrealized_pnl).abs() + }) + .sum::(); + ( + total_pnl.to_string().parse::().unwrap_or(0.0), + av.unrealized_pnl.to_string().parse::().unwrap_or(0.0), + av.drawdown_pct, + av.max_drawdown_pct, + av.high_water_mark.to_string().parse::().unwrap_or(0.0), + av.positions.len() as i64, + notional.to_string().parse::().unwrap_or(0.0), + ) + } else { + (0.0, 0.0, 0.0, 0.0, 0.0, 0, 0.0) + }; SentinelStats { latency_p99_us, @@ -1795,6 +1843,8 @@ impl Engine { drawdown_pct, max_drawdown_pct, high_water_mark, + position_count, + notional_value, } } @@ -1901,6 +1951,10 @@ impl Engine { orders_filled: self.stats.orders_filled, orders_rejected: self.stats.orders_rejected, orders_canceled: self.stats.orders_canceled, + market_events_dropped: self.stats.market_events_dropped, + intents_dropped: self.stats.intents_dropped, + snapshot_publish_failed: self.stats.snapshot_publish_failed, + data_integrity_gaps: self.stats.data_integrity_gaps, latency, uptime_seconds, } @@ -2058,6 +2112,10 @@ pub struct SentinelStats { pub max_drawdown_pct: f64, /// 高水位標記 pub high_water_mark: f64, + /// Number of authoritative open positions. + pub position_count: i64, + /// Gross marked notional across authoritative positions. + pub notional_value: f64, } /// 引擎延遲統計信息 @@ -2097,6 +2155,10 @@ pub struct EngineStatistics { pub orders_filled: u64, pub orders_rejected: u64, pub orders_canceled: u64, + pub market_events_dropped: u64, + pub intents_dropped: u64, + pub snapshot_publish_failed: u64, + pub data_integrity_gaps: u64, /// 延遲統計 pub latency: EngineLatencyStats, /// 引擎運行時間（秒） diff --git a/rust_hft/market-core/engine/tests/tokenized_security_execution_e2e.rs b/rust_hft/market-core/engine/tests/tokenized_security_execution_e2e.rs index 470e0eac9..0e11ffcc9 100644 --- a/rust_hft/market-core/engine/tests/tokenized_security_execution_e2e.rs +++ b/rust_hft/market-core/engine/tests/tokenized_security_execution_e2e.rs @@ -8,9 +8,10 @@ use engine::{ use futures::stream; use hft_core::{ AssetClass, ComplianceContext, HftResult, OrderId, OrderType, Price, ProductType, Quantity, - RegulatoryProfile, Side, Symbol, TimeInForce, + RegulatoryProfile, Side, Symbol, TimeInForce, VenueId, }; use ports::{BoxStream, ConnectionHealth, ExecutionClient, ExecutionEvent, OpenOrder, OrderIntent}; +use rust_decimal::Decimal; use tokio::sync::Mutex; #[derive(Default)] @@ -95,6 +96,12 @@ fn tokenized_security_intent() -> OrderIntent { jurisdiction: Some("AE".to_string()), eligibility_confirmed: true, allow_tokenized_securities: true, + top_depth_usd: Some(Decimal::from(100_000)), + spread_bps: Some(Decimal::ONE), + corporate_action_active: Some(false), + evidence_source: Some("paper-reference-feed".to_string()), + evidence_venue: Some(VenueId::BINANCE_TOKENIZED_SECURITIES), + evidence_observed_at: Some(hft_core::now_micros()), }) } diff --git a/rust_hft/market-core/ports/src/traits.rs b/rust_hft/market-core/ports/src/traits.rs index d070703d8..7be4c3c63 100644 --- a/rust_hft/market-core/ports/src/traits.rs +++ b/rust_hft/market-core/ports/src/traits.rs @@ -242,6 +242,9 @@ pub struct Position { pub quantity: Quantity, pub avg_price: Price, pub unrealized_pnl: rust_decimal::Decimal, + /// Realized PnL accumulated while the current position lifecycle is open. + #[serde(default)] + pub realized_pnl: rust_decimal::Decimal, } /// 策略接口 @@ -557,8 +560,11 @@ pub trait RiskManager: Send + Sync { account: &AccountView, venue_specs: &std::collections::HashMap, ) -> Vec { - // 默認實現：批量調用 review()，根據 target_venue 或 symbol 查找對應 VenueSpec + // Keep a projected account across the whole batch. Calling review() with the same + // account snapshot for every intent lets individually-valid orders exceed aggregate + // position/notional limits when they are emitted in one engine tick. let mut approved_intents = Vec::new(); + let mut projected_account = account.clone(); for intent in intents { if matches!(intent.asset_class, AssetClass::TokenizedSecurity) @@ -586,7 +592,27 @@ pub trait RiskManager: Send + Sync { }; if let Some(spec) = venue_spec { - let reviewed = self.review(vec![intent], account, spec); + let reviewed = self.review(vec![intent], &projected_account, spec); + for approved in &reviewed { + let signed_quantity = match approved.side { + Side::Buy => approved.quantity.0, + Side::Sell => -approved.quantity.0, + }; + let position = projected_account + .positions + .entry(approved.symbol.clone()) + .or_insert_with(|| Position { + symbol: approved.symbol.clone(), + quantity: Quantity::zero(), + avg_price: approved.price.unwrap_or_else(Price::zero), + unrealized_pnl: rust_decimal::Decimal::ZERO, + realized_pnl: rust_decimal::Decimal::ZERO, + }); + position.quantity.0 += signed_quantity; + if let Some(price) = approved.price { + position.avg_price = price; + } + } approved_intents.extend(reviewed); } else { // 沒有找到對應的 VenueSpec，拒絕此訂單 diff --git a/rust_hft/market-core/runtime/Cargo.toml b/rust_hft/market-core/runtime/Cargo.toml index 4606441b8..9c23cad78 100644 --- a/rust_hft/market-core/runtime/Cargo.toml +++ b/rust_hft/market-core/runtime/Cargo.toml @@ -24,6 +24,7 @@ portfolio_core = { package = "hft-portfolio-core", path = "../../risk-control/po # Data adapters (optional) adapter-bitget-data = { package = "hft-data-adapter-bitget", path = "../../data-pipelines/adapters/adapter-bitget", optional = true } adapter-binance-data = { package = "hft-data-adapter-binance", path = "../../data-pipelines/adapters/adapter-binance", optional = true } +adapter-binance-prediction-data = { package = "hft-data-adapter-binance-prediction", path = "../../data-pipelines/adapters/adapter-binance-prediction", optional = true } adapter-backpack-data = { package = "hft-data-adapter-backpack", path = "../../data-pipelines/adapters/adapter-backpack", optional = true } adapter-bybit-data = { package = "hft-data-adapter-bybit", path = "../../data-pipelines/adapters/adapter-bybit", optional = true } adapter-mock-data = { package = "hft-data-adapter-mock", path = "../../data-pipelines/adapters/adapter-mock", optional = true } @@ -108,6 +109,7 @@ json-simd = [ # Data adapters adapter-bitget-data = ["dep:adapter-bitget-data"] adapter-binance-data = ["dep:adapter-binance-data"] +adapter-binance-prediction-data = ["dep:adapter-binance-prediction-data"] adapter-backpack-data = ["dep:adapter-backpack-data"] adapter-bybit-data = ["dep:adapter-bybit-data"] adapter-mock-data = ["dep:adapter-mock-data"] @@ -150,7 +152,7 @@ infra-secrets = ["dep:infra-secrets"] # Convenience features bitget = ["adapter-bitget-data", "adapter-bitget-execution"] binance = ["adapter-binance-data", "adapter-binance-execution"] -binance-prediction = ["adapter-binance-prediction-execution"] +binance-prediction = ["adapter-binance-prediction-data", "adapter-binance-prediction-execution"] backpack = ["adapter-backpack-data", "adapter-backpack-execution"] bybit = ["adapter-bybit-data", "adapter-bybit-execution"] okx = ["adapter-okx-execution"] diff --git a/rust_hft/market-core/runtime/src/exposure_projection.rs b/rust_hft/market-core/runtime/src/exposure_projection.rs new file mode 100644 index 000000000..3f6226ce0 --- /dev/null +++ b/rust_hft/market-core/runtime/src/exposure_projection.rs @@ -0,0 +1,178 @@ +use std::collections::HashMap; + +use hft_core::{ProductType, Symbol, VenueId}; +use ports::{AccountView, OrderIntent}; +use rust_decimal::Decimal; + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +struct ExposureKey { + venue: Option, + product_type: ProductType, + symbol: Symbol, +} + +#[derive(Debug, Clone)] +pub(crate) struct ExposureProjection { + pub symbol_gross_quantity: Decimal, + pub symbol_gross_notional: Decimal, + pub gross_notional: Decimal, +} + +/// Conservative projector for a batch of intents. Existing positions do not carry venue/product +/// attribution, so they cannot be proven reducible by an intent for a particular venue. They are +/// therefore retained as gross exposure; new exposure is isolated by venue + product + symbol. +#[derive(Debug, Clone)] +pub(crate) struct ExposureProjector { + remaining_quantity: HashMap, + remaining_notional: HashMap, + keyed_quantity: HashMap, + keyed_notional: HashMap, + gross_notional: Decimal, +} + +impl ExposureProjector { + pub(crate) fn new(account: &AccountView) -> Self { + let remaining_quantity = account + .positions + .iter() + .map(|(symbol, position)| (symbol.clone(), position.quantity.0)) + .collect(); + let remaining_notional: HashMap<_, _> = account + .positions + .iter() + .map(|(symbol, position)| { + ( + symbol.clone(), + (position.avg_price.0 * position.quantity.0 + position.unrealized_pnl).abs(), + ) + }) + .collect(); + let gross_notional = remaining_notional.values().copied().sum(); + Self { + remaining_quantity, + remaining_notional, + keyed_quantity: HashMap::new(), + keyed_notional: HashMap::new(), + gross_notional, + } + } + + pub(crate) fn project( + &mut self, + intent: &OrderIntent, + ) -> Result { + let price = intent + .price + .map(|price| price.0) + .filter(|price| *price > Decimal::ZERO) + .ok_or("projected exposure requires a positive executable price")?; + let incoming = match intent.side { + hft_core::Side::Buy => intent.quantity.0, + hft_core::Side::Sell => -intent.quantity.0, + }; + if incoming.is_zero() { + return Err("projected exposure requires non-zero quantity"); + } + + if !incoming.is_zero() { + let key = ExposureKey { + venue: intent.target_venue, + product_type: intent.product_type, + symbol: intent.symbol.clone(), + }; + let old_quantity = self.keyed_quantity.get(&key).copied().unwrap_or_default(); + let next_quantity = old_quantity + incoming; + let old_notional = self.keyed_notional.get(&key).copied().unwrap_or_default(); + let next_notional = next_quantity.abs() * price; + self.gross_notional += next_notional - old_notional; + self.keyed_quantity.insert(key.clone(), next_quantity); + self.keyed_notional.insert(key, next_notional); + } + + let unattributed = self + .remaining_quantity + .get(&intent.symbol) + .copied() + .unwrap_or_default() + .abs(); + let keyed = self + .keyed_quantity + .iter() + .filter(|(key, _)| key.symbol == intent.symbol) + .map(|(_, quantity)| quantity.abs()) + .sum::(); + let symbol_gross_notional = self + .remaining_notional + .get(&intent.symbol) + .copied() + .unwrap_or_default() + + self + .keyed_notional + .iter() + .filter(|(key, _)| key.symbol == intent.symbol) + .map(|(_, notional)| *notional) + .sum::(); + Ok(ExposureProjection { + symbol_gross_quantity: unattributed + keyed, + symbol_gross_notional, + gross_notional: self.gross_notional, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use hft_core::{OrderType, Price, Quantity, Side, TimeInForce}; + + fn intent(venue: VenueId, side: Side, quantity: i64) -> OrderIntent { + OrderIntent::crypto_spot( + Symbol::new("BTCUSDT"), + side, + Quantity(Decimal::from(quantity)), + OrderType::Limit, + Some(Price(Decimal::ONE)), + TimeInForce::GTC, + "alpha".to_string(), + Some(venue), + ) + } + + #[test] + fn opposite_orders_on_different_venues_do_not_net() { + let mut projector = ExposureProjector::new(&AccountView::default()); + projector + .project(&intent(VenueId::BINANCE, Side::Buy, 60)) + .unwrap(); + let projected = projector + .project(&intent(VenueId::BITGET, Side::Sell, 60)) + .unwrap(); + + assert_eq!(projected.symbol_gross_quantity, Decimal::from(120)); + assert_eq!(projected.gross_notional, Decimal::from(120)); + } + + #[test] + fn unattributed_existing_position_cannot_be_net_reduced() { + let symbol = Symbol::new("BTCUSDT"); + let mut account = AccountView::default(); + account.positions.insert( + symbol.clone(), + ports::Position { + symbol, + quantity: Quantity(Decimal::from(60)), + avg_price: Price(Decimal::ONE), + unrealized_pnl: Decimal::ZERO, + realized_pnl: Decimal::ZERO, + }, + ); + let mut projector = ExposureProjector::new(&account); + + let projected = projector + .project(&intent(VenueId::BITGET, Side::Sell, 60)) + .unwrap(); + + assert_eq!(projected.symbol_gross_quantity, Decimal::from(120)); + assert_eq!(projected.gross_notional, Decimal::from(120)); + } +} diff --git a/rust_hft/market-core/runtime/src/ipc_handler.rs b/rust_hft/market-core/runtime/src/ipc_handler.rs index 4fa5e455e..14765b08d 100644 --- a/rust_hft/market-core/runtime/src/ipc_handler.rs +++ b/rust_hft/market-core/runtime/src/ipc_handler.rs @@ -235,9 +235,7 @@ impl CommandHandler for SystemCommandHandler { average_price: pos.avg_price.0, market_value: pos.avg_price.0 * pos.quantity.0 + pos.unrealized_pnl, unrealized_pnl: pos.unrealized_pnl, - realized_pnl: rust_decimal::Decimal::ZERO, // TODO: 逐部位已实现损益追踪需要扩展 ports::Position 结构 - // 当前系统仅在 AccountView 层面追踪总已实现损益 - // 未来改进：在 Position 中添加 realized_pnl 字段，并在 PortfolioCore 中追踪每个仓位的平仓损益 + realized_pnl: pos.realized_pnl, last_update: std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap_or_default() @@ -1072,6 +1070,7 @@ mod tests { quantity: Quantity(Decimal::from(2)), avg_price: Price(Decimal::new(51, 2)), unrealized_pnl: Decimal::new(5, 2), + realized_pnl: Decimal::ZERO, }])), recent_fills: Some(Ok(vec![ports::AccountFill { fill_id: "fill-1".to_string(), diff --git a/rust_hft/market-core/runtime/src/lib.rs b/rust_hft/market-core/runtime/src/lib.rs index 56a25370d..0f67d9f7b 100644 --- a/rust_hft/market-core/runtime/src/lib.rs +++ b/rust_hft/market-core/runtime/src/lib.rs @@ -8,6 +8,7 @@ //! - Risk managers //! - Event consumers +mod exposure_projection; pub mod ipc_handler; pub mod portfolio_manager; pub mod risk_manager_factory; diff --git a/rust_hft/market-core/runtime/src/portfolio_manager.rs b/rust_hft/market-core/runtime/src/portfolio_manager.rs index 8450d2e54..ebf3b1ffe 100644 --- a/rust_hft/market-core/runtime/src/portfolio_manager.rs +++ b/rust_hft/market-core/runtime/src/portfolio_manager.rs @@ -1,7 +1,11 @@ use std::collections::HashMap; +use hft_core::VenueId; +use ports::RiskManager; +use rust_decimal::Decimal; use tracing::warn; +use crate::exposure_projection::ExposureProjector; use crate::system_builder::{PortfolioSpec, StrategyConfig}; /// 聚合策略與組合設定的管理器 @@ -53,3 +57,206 @@ impl PortfolioManager { .unwrap_or_default() } } + +/// Applies configured cross-strategy portfolio budgets before the venue/global risk manager. +/// Until positions carry strategy attribution, existing account exposure is conservatively +/// charged to every portfolio instead of assuming that un-attributed exposure is harmless. +pub struct PortfolioBudgetRiskManager { + base_risk_manager: Box, + manager: PortfolioManager, +} + +impl PortfolioBudgetRiskManager { + pub fn new( + base_risk_manager: Box, + definitions: Vec, + strategies: &[StrategyConfig], + ) -> Self { + Self { + base_risk_manager, + manager: PortfolioManager::new(definitions, strategies), + } + } + + fn filter( + &self, + intents: Vec, + account: &ports::AccountView, + ) -> Vec { + let specs: HashMap<&str, &PortfolioSpec> = self + .manager + .portfolio_specs() + .iter() + .map(|spec| (spec.name.as_str(), spec)) + .collect(); + let mut projectors: HashMap = HashMap::new(); + let mut approved = Vec::with_capacity(intents.len()); + + for intent in intents { + let portfolio_names = self.manager.portfolios_for_strategy(&intent.strategy_id); + if portfolio_names.is_empty() { + approved.push(intent); + continue; + } + + let mut projections = Vec::with_capacity(portfolio_names.len()); + let mut reject_reason = None; + + for portfolio_name in &portfolio_names { + let Some(spec) = specs.get(portfolio_name.as_str()) else { + reject_reason = Some("missing portfolio definition"); + break; + }; + let mut projector = projectors + .get(portfolio_name) + .cloned() + .unwrap_or_else(|| ExposureProjector::new(account)); + let projected = match projector.project(&intent) { + Ok(projected) => projected, + Err(reason) => { + reject_reason = Some(reason); + break; + } + }; + if spec + .max_position + .is_some_and(|limit| projected.symbol_gross_quantity > limit) + { + reject_reason = Some("portfolio position budget exceeded"); + break; + } + if spec + .max_notional + .is_some_and(|limit| projected.gross_notional > limit) + { + reject_reason = Some("portfolio notional budget exceeded"); + break; + } + projections.push((portfolio_name.clone(), projector)); + } + + if let Some(reason) = reject_reason { + warn!(strategy = %intent.strategy_id, symbol = %intent.symbol, %reason, "投资组合预算拒绝订单意图"); + continue; + } + for (portfolio_name, projector) in projections { + projectors.insert(portfolio_name, projector); + } + approved.push(intent); + } + + approved + } +} + +#[cfg(test)] +mod tests { + use super::*; + use hft_core::{OrderType, Price, Quantity, Side, Symbol, TimeInForce}; + + #[test] + fn shared_portfolio_budget_accumulates_across_strategies() { + let base = crate::RiskManagerFactory::create_risk_manager(&crate::RiskConfig { + risk_type: "Default".to_string(), + global_position_limit: Decimal::from(1_000), + global_notional_limit: Decimal::from(100_000), + max_orders_per_second: 100, + staleness_threshold_us: u64::MAX, + max_daily_loss: Decimal::from(10_000), + max_drawdown_pct: 5.0, + ..Default::default() + }); + let manager = PortfolioBudgetRiskManager::new( + base, + vec![PortfolioSpec { + name: "shared".to_string(), + strategies: vec!["alpha-a".to_string(), "alpha-b".to_string()], + max_notional: Some(Decimal::from(100)), + max_position: None, + ..Default::default() + }], + &[], + ); + let intent = |strategy: &str| { + ports::OrderIntent::crypto_spot( + Symbol::new("BTCUSDT"), + Side::Buy, + Quantity(Decimal::from(60)), + OrderType::Limit, + Some(Price(Decimal::ONE)), + TimeInForce::GTC, + strategy.to_string(), + Some(VenueId::BINANCE), + ) + }; + + let approved = manager.filter( + vec![intent("alpha-a"), intent("alpha-b")], + &ports::AccountView::default(), + ); + + assert_eq!(approved.len(), 1); + } +} + +impl RiskManager for PortfolioBudgetRiskManager { + fn review_orders( + &mut self, + intents: Vec, + account: &ports::AccountView, + venue_specs: &HashMap, + ) -> Vec { + let filtered = self.filter(intents, account); + self.base_risk_manager + .review_orders(filtered, account, venue_specs) + } + + fn review( + &mut self, + intents: Vec, + account: &ports::AccountView, + venue: &ports::VenueSpec, + ) -> Vec { + let filtered = self.filter(intents, account); + self.base_risk_manager.review(filtered, account, venue) + } + + fn review_with_venue_specs( + &mut self, + intents: Vec, + account: &ports::AccountView, + venue_specs: &HashMap, + ) -> Vec { + let filtered = self.filter(intents, account); + self.base_risk_manager + .review_with_venue_specs(filtered, account, venue_specs) + } + + fn on_execution_event(&mut self, event: &ports::ExecutionEvent) { + self.base_risk_manager.on_execution_event(event) + } + + fn emergency_stop(&mut self) -> Result<(), hft_core::HftError> { + self.base_risk_manager.emergency_stop() + } + + fn get_risk_metrics(&self) -> HashMap { + self.base_risk_manager.get_risk_metrics() + } + + fn should_halt_trading(&self, account: &ports::AccountView) -> bool { + self.base_risk_manager.should_halt_trading(account) + } + + fn risk_metrics(&self) -> ports::RiskMetrics { + self.base_risk_manager.risk_metrics() + } + + fn update_config(&mut self, update: ports::RiskConfigUpdate) -> Result<(), hft_core::HftError> { + self.base_risk_manager.update_config(update) + } + + fn get_config_snapshot(&self) -> ports::RiskConfigSnapshot { + self.base_risk_manager.get_config_snapshot() + } +} diff --git a/rust_hft/market-core/runtime/src/risk_manager_factory.rs b/rust_hft/market-core/runtime/src/risk_manager_factory.rs index 45116165a..fdc474a89 100644 --- a/rust_hft/market-core/runtime/src/risk_manager_factory.rs +++ b/rust_hft/market-core/runtime/src/risk_manager_factory.rs @@ -4,7 +4,7 @@ //! including support for per-strategy risk overrides. use chrono::{DateTime, Utc, Weekday}; -use hft_core::Quantity; +use hft_core::{AssetClass, ProductType, Quantity, VenueId}; use ports::RiskManager; use risk::{ DefaultRiskManager, EnhancedRiskConfig, EnhancedRiskManager, RiskConfig, TradingWindow, @@ -12,7 +12,11 @@ use risk::{ use std::collections::HashMap; use tracing::{debug, info, warn}; -use crate::{RiskConfig as SystemRiskConfig, StrategyRiskOverride, TradingWindowConfig}; +use crate::exposure_projection::ExposureProjector; +use crate::{ + RiskConfig as SystemRiskConfig, StrategyRiskOverride, TokenizedSecuritiesRiskConfig, + TradingWindowConfig, +}; /// Risk Manager Factory that creates risk managers with per-strategy overrides pub struct RiskManagerFactory; @@ -80,8 +84,19 @@ impl RiskManagerFactory { system_risk_config: &SystemRiskConfig, ) -> Box { let base_risk_manager = Self::create_risk_manager(system_risk_config); + let max_position = system_risk_config + .enhanced + .as_ref() + .map(|config| config.max_position_per_symbol) + .filter(|limit| *limit > rust_decimal::Decimal::ZERO) + .unwrap_or(system_risk_config.global_position_limit); + let base_risk_manager: Box = Box::new(ProjectedExposureRiskManager::new( + base_risk_manager, + max_position, + system_risk_config.global_notional_limit, + )); - if system_risk_config.strategy_overrides.is_empty() { + let strategy_aware = if system_risk_config.strategy_overrides.is_empty() { // No overrides, return base manager base_risk_manager } else { @@ -90,7 +105,13 @@ impl RiskManagerFactory { base_risk_manager, system_risk_config.strategy_overrides.clone(), )) - } + }; + + Box::new(TokenizedSecuritiesRiskManager::new( + strategy_aware, + system_risk_config.tokenized_securities.clone(), + system_risk_config.staleness_threshold_us, + )) } /// Convert trading window configuration @@ -125,6 +146,274 @@ impl RiskManagerFactory { } } +/// Enforces batch exposure on venue + product + symbol identities before legacy account views can +/// net same-named instruments across venues. +pub struct ProjectedExposureRiskManager { + base_risk_manager: Box, + max_position_per_symbol: rust_decimal::Decimal, + max_global_notional: rust_decimal::Decimal, +} + +impl ProjectedExposureRiskManager { + fn new( + base_risk_manager: Box, + max_position_per_symbol: rust_decimal::Decimal, + max_global_notional: rust_decimal::Decimal, + ) -> Self { + Self { + base_risk_manager, + max_position_per_symbol, + max_global_notional, + } + } + + fn filter( + &self, + intents: Vec, + account: &ports::AccountView, + ) -> Vec { + let mut projector = ExposureProjector::new(account); + let mut approved = Vec::with_capacity(intents.len()); + for intent in intents { + let mut next_projector = projector.clone(); + let Ok(projected) = next_projector.project(&intent) else { + warn!(symbol = %intent.symbol, "全局敞口无法投影，拒绝订单意图"); + continue; + }; + if projected.symbol_gross_quantity > self.max_position_per_symbol + || projected.gross_notional > self.max_global_notional + { + warn!(symbol = %intent.symbol, "跨 venue projected exposure 超过全局限额"); + continue; + } + projector = next_projector; + approved.push(intent); + } + approved + } +} + +impl RiskManager for ProjectedExposureRiskManager { + fn review_orders( + &mut self, + intents: Vec, + account: &ports::AccountView, + venue_specs: &HashMap, + ) -> Vec { + let filtered = self.filter(intents, account); + self.base_risk_manager + .review_orders(filtered, account, venue_specs) + } + + fn review( + &mut self, + intents: Vec, + account: &ports::AccountView, + venue: &ports::VenueSpec, + ) -> Vec { + let filtered = self.filter(intents, account); + self.base_risk_manager.review(filtered, account, venue) + } + + fn review_with_venue_specs( + &mut self, + intents: Vec, + account: &ports::AccountView, + venue_specs: &HashMap, + ) -> Vec { + let filtered = self.filter(intents, account); + self.base_risk_manager + .review_with_venue_specs(filtered, account, venue_specs) + } + + fn on_execution_event(&mut self, event: &ports::ExecutionEvent) { + self.base_risk_manager.on_execution_event(event) + } + + fn emergency_stop(&mut self) -> Result<(), hft_core::HftError> { + self.base_risk_manager.emergency_stop() + } + + fn get_risk_metrics(&self) -> HashMap { + self.base_risk_manager.get_risk_metrics() + } + + fn should_halt_trading(&self, account: &ports::AccountView) -> bool { + self.base_risk_manager.should_halt_trading(account) + } + + fn risk_metrics(&self) -> ports::RiskMetrics { + self.base_risk_manager.risk_metrics() + } + + fn update_config(&mut self, update: ports::RiskConfigUpdate) -> Result<(), hft_core::HftError> { + self.base_risk_manager.update_config(update) + } + + fn get_config_snapshot(&self) -> ports::RiskConfigSnapshot { + self.base_risk_manager.get_config_snapshot() + } +} + +/// Fail-closed policy layer for securities-like tokens. Market-quality and corporate-action +/// evidence must travel with the intent so the execution path cannot silently use stale UI data. +pub struct TokenizedSecuritiesRiskManager { + base_risk_manager: Box, + config: TokenizedSecuritiesRiskConfig, + evidence_max_age_us: u64, +} + +impl TokenizedSecuritiesRiskManager { + pub fn new( + base_risk_manager: Box, + config: TokenizedSecuritiesRiskConfig, + evidence_max_age_us: u64, + ) -> Self { + Self { + base_risk_manager, + config, + evidence_max_age_us, + } + } + + fn is_tokenized(intent: &ports::OrderIntent) -> bool { + matches!(intent.asset_class, AssetClass::TokenizedSecurity) + || matches!(intent.product_type, ProductType::TokenizedSecuritySpot) + } + + fn filter( + &self, + intents: Vec, + account: &ports::AccountView, + ) -> Vec { + let mut approved = Vec::with_capacity(intents.len()); + // AccountView does not retain asset-class attribution. The shared projector therefore + // conservatively charges every existing position to this securities-token budget. + let mut projector = ExposureProjector::new(account); + + for intent in intents { + if !Self::is_tokenized(&intent) { + approved.push(intent); + continue; + } + + let context = &intent.compliance_context; + let jurisdiction_restricted = context.jurisdiction.as_ref().is_some_and(|value| { + self.config + .restricted_jurisdictions + .iter() + .any(|restricted| restricted.eq_ignore_ascii_case(value)) + }); + let market_quality_ok = context + .top_depth_usd + .is_some_and(|depth| depth >= self.config.min_top_depth_usd) + && context + .spread_bps + .is_some_and(|spread| spread <= self.config.max_spread_bps); + let corporate_action_ok = !self.config.freeze_on_corporate_action + || matches!(context.corporate_action_active, Some(false)); + let evidence_ok = context + .evidence_source + .as_deref() + .is_some_and(|source| !source.trim().is_empty()) + && context.evidence_venue == intent.target_venue + && context.evidence_observed_at.is_some_and(|observed_at| { + hft_core::now_micros().saturating_sub(observed_at) <= self.evidence_max_age_us + }); + + let mut next_projector = projector.clone(); + let projected = match next_projector.project(&intent) { + Ok(projected) => projected, + Err(reason) => { + warn!(symbol = %intent.symbol, %reason, "拒绝证券 token 意图：无法投影敞口"); + continue; + } + }; + + if !self.config.allow_trading + || !context.allow_tokenized_securities + || !context.eligibility_confirmed + || jurisdiction_restricted + || !market_quality_ok + || !corporate_action_ok + || !evidence_ok + || projected.symbol_gross_notional > self.config.max_notional_per_symbol + || projected.gross_notional > self.config.max_asset_class_notional + { + warn!(symbol = %intent.symbol, "证券 token 风控证据或预算不满足，拒绝意图"); + continue; + } + + projector = next_projector; + approved.push(intent); + } + + approved + } +} + +impl RiskManager for TokenizedSecuritiesRiskManager { + fn review_orders( + &mut self, + intents: Vec, + account: &ports::AccountView, + venue_specs: &HashMap, + ) -> Vec { + let filtered = self.filter(intents, account); + self.base_risk_manager + .review_orders(filtered, account, venue_specs) + } + + fn review( + &mut self, + intents: Vec, + account: &ports::AccountView, + venue: &ports::VenueSpec, + ) -> Vec { + let filtered = self.filter(intents, account); + self.base_risk_manager.review(filtered, account, venue) + } + + fn review_with_venue_specs( + &mut self, + intents: Vec, + account: &ports::AccountView, + venue_specs: &HashMap, + ) -> Vec { + let filtered = self.filter(intents, account); + self.base_risk_manager + .review_with_venue_specs(filtered, account, venue_specs) + } + + fn on_execution_event(&mut self, event: &ports::ExecutionEvent) { + self.base_risk_manager.on_execution_event(event) + } + + fn emergency_stop(&mut self) -> Result<(), hft_core::HftError> { + self.base_risk_manager.emergency_stop() + } + + fn get_risk_metrics(&self) -> HashMap { + self.base_risk_manager.get_risk_metrics() + } + + fn should_halt_trading(&self, account: &ports::AccountView) -> bool { + self.base_risk_manager.should_halt_trading(account) + } + + fn risk_metrics(&self) -> ports::RiskMetrics { + self.base_risk_manager.risk_metrics() + } + + fn update_config(&mut self, update: ports::RiskConfigUpdate) -> Result<(), hft_core::HftError> { + self.base_risk_manager.update_config(update) + } + + fn get_config_snapshot(&self) -> ports::RiskConfigSnapshot { + self.base_risk_manager.get_config_snapshot() + } +} + /// Strategy-aware risk manager wrapper that applies per-strategy overrides pub struct StrategyAwareRiskManager { base_risk_manager: Box, @@ -292,6 +581,9 @@ impl RiskManager for StrategyAwareRiskManager { mod tests { use super::*; use crate::EnhancedRiskSettings; + use hft_core::{ + ComplianceContext, OrderType, Price, RegulatoryProfile, Side, Symbol, TimeInForce, + }; use rust_decimal::Decimal; use std::collections::HashMap; @@ -388,4 +680,145 @@ mod tests { assert_eq!(config_with_overrides.strategy_overrides.len(), 1); let _ = strategy_aware_manager as Box; } + + #[test] + fn tokenized_security_batch_requires_evidence_and_projected_budget() { + let risk_config = SystemRiskConfig { + risk_type: "Default".to_string(), + global_position_limit: Decimal::from(1000), + global_notional_limit: Decimal::from(100_000), + max_daily_trades: 100, + max_orders_per_second: 10, + staleness_threshold_us: u64::MAX, + max_daily_loss: Decimal::from(10_000), + max_drawdown_pct: 5.0, + enhanced: None, + strategy_overrides: HashMap::new(), + tokenized_securities: TokenizedSecuritiesRiskConfig { + allow_trading: true, + max_notional_per_symbol: Decimal::from(1_000), + max_asset_class_notional: Decimal::from(2_000), + min_top_depth_usd: Decimal::from(10_000), + max_spread_bps: Decimal::from(10), + freeze_on_corporate_action: true, + restricted_jurisdictions: vec!["US".to_string()], + }, + }; + let context = ComplianceContext { + regulatory_profile: RegulatoryProfile::AdgmTokenizedSecurity, + jurisdiction: Some("AE".to_string()), + eligibility_confirmed: true, + allow_tokenized_securities: true, + top_depth_usd: Some(Decimal::from(20_000)), + spread_bps: Some(Decimal::from(5)), + corporate_action_active: Some(false), + evidence_source: Some("licensed-reference-feed".to_string()), + evidence_venue: Some(VenueId::BINANCE_TOKENIZED_SECURITIES), + evidence_observed_at: Some(hft_core::now_micros()), + }; + let intent = ports::OrderIntent::crypto_spot( + Symbol::new("TSLAUSDT"), + Side::Buy, + Quantity(Decimal::from(6)), + OrderType::Limit, + Some(Price(Decimal::from(100))), + TimeInForce::GTC, + "token-alpha".to_string(), + Some(VenueId::BINANCE_TOKENIZED_SECURITIES), + ) + .tokenized_security_spot(context); + let specs = HashMap::from([( + VenueId::BINANCE_TOKENIZED_SECURITIES, + ports::VenueSpec::binance_spot(), + )]); + let mut manager = RiskManagerFactory::create_strategy_aware_risk_manager(&risk_config); + + let approved = manager.review_with_venue_specs( + vec![intent.clone(), intent], + &ports::AccountView::default(), + &specs, + ); + + assert_eq!(approved.len(), 1); + + let mut short_account = ports::AccountView::default(); + short_account.positions.insert( + Symbol::new("TSLAUSDT"), + ports::Position { + symbol: Symbol::new("TSLAUSDT"), + quantity: Quantity(Decimal::from(-6)), + avg_price: Price(Decimal::from(100)), + unrealized_pnl: Decimal::ZERO, + realized_pnl: Decimal::ZERO, + }, + ); + let short_add = ports::OrderIntent::crypto_spot( + Symbol::new("TSLAUSDT"), + Side::Sell, + Quantity(Decimal::from(6)), + OrderType::Limit, + Some(Price(Decimal::from(100))), + TimeInForce::GTC, + "token-alpha".to_string(), + Some(VenueId::BINANCE_TOKENIZED_SECURITIES), + ) + .tokenized_security_spot(ComplianceContext { + regulatory_profile: RegulatoryProfile::AdgmTokenizedSecurity, + jurisdiction: Some("AE".to_string()), + eligibility_confirmed: true, + allow_tokenized_securities: true, + top_depth_usd: Some(Decimal::from(20_000)), + spread_bps: Some(Decimal::from(5)), + corporate_action_active: Some(false), + evidence_source: Some("licensed-reference-feed".to_string()), + evidence_venue: Some(VenueId::BINANCE_TOKENIZED_SECURITIES), + evidence_observed_at: Some(hft_core::now_micros()), + }); + let mut manager = RiskManagerFactory::create_strategy_aware_risk_manager(&risk_config); + assert!(manager + .review_with_venue_specs(vec![short_add], &short_account, &specs) + .is_empty()); + } + + #[test] + fn projected_exposure_does_not_net_opposite_orders_across_venues() { + let config = SystemRiskConfig { + risk_type: "Default".to_string(), + global_position_limit: Decimal::from(100), + global_notional_limit: Decimal::from(1_000), + max_orders_per_second: 100, + staleness_threshold_us: u64::MAX, + max_daily_loss: Decimal::from(10_000), + max_drawdown_pct: 5.0, + ..Default::default() + }; + let base = RiskManagerFactory::create_risk_manager(&config); + let manager = ProjectedExposureRiskManager::new( + base, + config.global_position_limit, + config.global_notional_limit, + ); + let intent = |venue, side| { + ports::OrderIntent::crypto_spot( + Symbol::new("BTCUSDT"), + side, + Quantity(Decimal::from(60)), + OrderType::Limit, + Some(Price(Decimal::ONE)), + TimeInForce::GTC, + "cross-venue".to_string(), + Some(venue), + ) + }; + + let approved = manager.filter( + vec![ + intent(VenueId::BINANCE, Side::Buy), + intent(VenueId::BITGET, Side::Sell), + ], + &ports::AccountView::default(), + ); + + assert_eq!(approved.len(), 1); + } } diff --git a/rust_hft/market-core/runtime/src/system_builder.rs b/rust_hft/market-core/runtime/src/system_builder.rs index 14d337e52..d2b49da66 100644 --- a/rust_hft/market-core/runtime/src/system_builder.rs +++ b/rust_hft/market-core/runtime/src/system_builder.rs @@ -663,6 +663,15 @@ impl SystemBuilder { // Create configurable risk manager using factory let risk_manager = crate::RiskManagerFactory::create_strategy_aware_risk_manager(&self.config.risk); + let risk_manager: Box = if self.config.portfolios.is_empty() { + risk_manager + } else { + Box::new(crate::PortfolioBudgetRiskManager::new( + risk_manager, + self.config.portfolios.clone(), + &self.config.strategies, + )) + }; engine.register_risk_manager_boxed(risk_manager); info!( "已注册风控管理器 (类型: {}, 策略覆盖数: {})", @@ -881,6 +890,22 @@ impl SystemRuntime { self.engine.lock().await.register_event_consumer(consumer); } } + VenueType::BinancePrediction => { + #[cfg(feature = "adapter-binance-prediction-data")] + { + let config = parse_binance_prediction_market_config( + &venue_type, + venue_cfg.as_ref(), + )?; + let stream = + adapter_binance_prediction_data::BinancePredictionMarketStream::new( + config, + )?; + let consumer = bridge.bridge_instrument_stream(stream, instruments).await?; + self.engine.lock().await.register_event_consumer(consumer); + info!("Binance Prediction REST order books bridged into the engine"); + } + } VenueType::Bybit => { #[cfg(feature = "adapter-bybit-data")] { @@ -1180,7 +1205,7 @@ impl SystemRuntime { return Ok::<(), HftError>(()); } return Err(HftError::Execution(format!( - "initial live reconciliation failed: complete={}, exchange_only={}, local_only={}, quantity_mismatch={}, balance_complete={}, balance_difference_usd={:?}, position_complete={}, position_exchange_only={}, position_local_only={}, position_quantity_mismatch={}", + "initial live reconciliation failed: complete={}, exchange_only={}, local_only={}, quantity_mismatch={}, balance_complete={}, balance_difference_usd={:?}, position_complete={}, position_exchange_only={}, position_local_only={}, position_quantity_mismatch={}, fill_complete={}, exchange_only_fills={}", report.complete, report.order_report.exchange_only.len(), report.order_report.local_only.len(), @@ -1209,6 +1234,14 @@ impl SystemRuntime { .position_report .as_ref() .map_or(0, |positions| positions.quantity_mismatch.len()), + report + .fill_report + .as_ref() + .is_none_or(|fills| fills.complete), + report + .fill_report + .as_ref() + .map_or(0, |fills| fills.exchange_only_fill_ids.len()), ))); } if operator_control_only { @@ -1300,6 +1333,14 @@ impl SystemRuntime { .position_report .as_ref() .map_or(0, |positions| positions.quantity_mismatch.len()), + fill_complete = report + .fill_report + .as_ref() + .is_none_or(|fills| fills.complete), + exchange_only_fills = report + .fill_report + .as_ref() + .map_or(0, |fills| fills.exchange_only_fill_ids.len()), "runtime order reconciliation unhealthy; pausing new intents" ); } @@ -1320,6 +1361,8 @@ impl SystemRuntime { interval.tick().await; let (cash, pos, unr, rlz, stats) = { let eng = engine_arc.lock().await; + #[cfg(feature = "metrics")] + eng.sync_latency_metrics_to_prometheus(); let av = eng.get_account_view(); let st = eng.get_statistics(); ( @@ -1330,21 +1373,6 @@ impl SystemRuntime { st, ) }; - // 導出引擎統計到 Prometheus（僅在 metrics feature 啟用時） - #[cfg(feature = "metrics")] - { - infra_metrics::MetricsRegistry::global().update_engine_statistics( - &infra_metrics::EngineStatisticsExport { - cycle_count: stats.cycle_count, - execution_events_processed: stats.execution_events_processed, - orders_submitted: stats.orders_submitted, - orders_ack: stats.orders_ack, - orders_filled: stats.orders_filled, - orders_rejected: stats.orders_rejected, - orders_canceled: stats.orders_canceled, - }, - ); - } // 當引擎停止時，狀態任務退出，避免 Ctrl-C 卡住 if !stats.is_running { break; @@ -1467,6 +1495,38 @@ impl SystemRuntime { // Legacy 方法已移至 runtime_management 模組 } +#[cfg(feature = "adapter-binance-prediction-data")] +fn parse_binance_prediction_market_config( + venue: &VenueType, + venue_cfg: Option<&VenueConfig>, +) -> HftResult { + if *venue != VenueType::BinancePrediction { + return Err(HftError::Config( + "Binance Prediction market config was requested for another venue".to_string(), + )); + } + let venue_cfg = venue_cfg.ok_or_else(|| { + HftError::Config("Binance Prediction market data requires venue config".to_string()) + })?; + let mut config: adapter_binance_prediction_data::BinancePredictionMarketDataConfig = venue_cfg + .data_config + .clone() + .ok_or_else(|| { + HftError::Config( + "Binance Prediction market data requires data_config.outcomes".to_string(), + ) + }) + .and_then(|value| { + serde_yaml::from_value(value).map_err(|error| HftError::Config(error.to_string())) + })?; + config.api_key = venue_cfg.api_key.clone().unwrap_or_default(); + config.api_secret = venue_cfg.secret.clone().unwrap_or_default(); + if let Some(rest) = &venue_cfg.rest { + config.rest_base_url = rest.clone(); + } + Ok(config) +} + #[cfg(feature = "adapter-binance-data")] fn parse_binance_capabilities( venue_cfg: &VenueConfig, diff --git a/rust_hft/market-core/runtime/src/system_builder/config_loader.rs b/rust_hft/market-core/runtime/src/system_builder/config_loader.rs index 32e22d161..07bff24f7 100644 --- a/rust_hft/market-core/runtime/src/system_builder/config_loader.rs +++ b/rust_hft/market-core/runtime/src/system_builder/config_loader.rs @@ -1314,7 +1314,7 @@ risk: } #[test] - fn loads_binance_prediction_as_an_execution_only_venue() { + fn loads_binance_prediction_with_an_explicit_execution_boundary() { let config = load_config_from_str( r#" engine: @@ -1361,6 +1361,26 @@ strategies: [] ); } + #[test] + fn binance_prediction_quotes_example_keeps_execution_disabled() { + let content = + include_str!("../../../../config/dev/binance_prediction_quotes_only.yaml.example") + .replace("${BINANCE_PREDICTION_API_KEY}", "test-key") + .replace("${BINANCE_PREDICTION_API_SECRET}", "test-secret") + .replace("REPLACE_WITH_OUTCOME_TOKEN_ID", "112233"); + let config = + load_config_from_str(&content).expect("load Binance Prediction quotes example"); + + assert!(config.quotes_only); + assert_eq!(config.venues[0].venue_type, VenueType::BinancePrediction); + assert_eq!( + config.venues[0].symbol_catalog[0].venue_id(), + Some(VenueId::BINANCE_PREDICTION) + ); + assert_eq!(config.venues[0].execution_mode.as_deref(), Some("Paper")); + assert!(config.strategies.is_empty()); + } + #[test] fn binance_prediction_live_example_stays_loadable() { let content = include_str!("../../../../config/dev/binance_prediction_live.yaml.example") diff --git a/rust_hft/market-core/runtime/src/system_builder/venue_registry.rs b/rust_hft/market-core/runtime/src/system_builder/venue_registry.rs index 44aa3d888..961cc232b 100644 --- a/rust_hft/market-core/runtime/src/system_builder/venue_registry.rs +++ b/rust_hft/market-core/runtime/src/system_builder/venue_registry.rs @@ -57,8 +57,12 @@ impl SystemBuilder { venue: &VenueConfig, instruments: &[InstrumentSpec], ) -> Self { - if venue.venue_type == VenueType::BinancePrediction { - info!("Binance Prediction is execution-only; skipping streaming market adapter"); + if venue.venue_type == VenueType::BinancePrediction + && (venue.symbol_catalog.is_empty() || venue.data_config.is_none()) + { + info!( + "Binance Prediction needs an explicit outcome catalog and data_config; keeping the venue execution-only" + ); return self; } let venue_id = to_venue_id(&venue.venue_type); @@ -128,6 +132,7 @@ fn instrument_for_venue(symbol: Symbol, venue: VenueId) -> InstrumentSpec { } VenueId::ONDO_PERPS => InstrumentSpec::ondo_perp(symbol), VenueId::POLYMARKET => InstrumentSpec::polymarket_outcome(symbol), + VenueId::BINANCE_PREDICTION => InstrumentSpec::prediction_market_outcome(symbol), _ => InstrumentSpec::crypto_spot(symbol, venue), } } @@ -283,6 +288,105 @@ mod tests { ); } + #[test] + fn binance_prediction_catalog_drives_prediction_market_plan() { + let mut config = SystemConfig::default(); + config.venues.push(VenueConfig { + name: "binance-prediction".into(), + account_id: None, + venue_type: VenueType::BinancePrediction, + ws_public: None, + ws_private: None, + rest: None, + api_key: None, + secret: None, + passphrase: None, + execution_mode: Some("Paper".into()), + capabilities: VenueCapabilities::default(), + inst_type: None, + simulate_execution: false, + symbol_catalog: vec![InstrumentId::new("112233@BINANCE_PREDICTION")], + data_config: Some( + serde_yaml::from_str( + "outcomes:\n - token_id: '112233'\n market_id: 1\n vendor: predict_fun\n", + ) + .expect("valid Binance Prediction outcome config"), + ), + execution_config: None, + secret_ref_api_key: None, + secret_ref_secret: None, + secret_ref_passphrase: None, + }); + + let builder = SystemBuilder::new(config).register_market_streams_from_config(); + let (venue, _, instruments) = &builder.market_stream_plans[0]; + assert_eq!(*venue, VenueType::BinancePrediction); + assert_eq!( + instruments, + &[InstrumentSpec::prediction_market_outcome(Symbol::new( + "112233" + ))] + ); + } + + #[test] + fn binance_prediction_without_an_outcome_catalog_stays_execution_only() { + let mut config = SystemConfig::default(); + config.venues.push(VenueConfig { + name: "binance-prediction".into(), + account_id: None, + venue_type: VenueType::BinancePrediction, + ws_public: None, + ws_private: None, + rest: None, + api_key: None, + secret: None, + passphrase: None, + execution_mode: Some("Live".into()), + capabilities: VenueCapabilities::default(), + inst_type: None, + simulate_execution: false, + symbol_catalog: Vec::new(), + data_config: None, + execution_config: None, + secret_ref_api_key: None, + secret_ref_secret: None, + secret_ref_passphrase: None, + }); + + let builder = SystemBuilder::new(config).register_market_streams_from_config(); + assert!(builder.market_stream_plans.is_empty()); + } + + #[test] + fn binance_prediction_without_quote_data_stays_execution_only() { + let mut config = SystemConfig::default(); + config.venues.push(VenueConfig { + name: "binance-prediction".into(), + account_id: None, + venue_type: VenueType::BinancePrediction, + ws_public: None, + ws_private: None, + rest: None, + api_key: None, + secret: None, + passphrase: None, + execution_mode: Some("Live".into()), + capabilities: VenueCapabilities::default(), + inst_type: None, + simulate_execution: false, + symbol_catalog: vec![InstrumentId::new("112233@BINANCE_PREDICTION")], + data_config: None, + execution_config: None, + secret_ref_api_key: None, + secret_ref_secret: None, + secret_ref_passphrase: None, + }); + + let builder = SystemBuilder::new(config).register_market_streams_from_config(); + assert!(builder.market_stream_plans.is_empty()); + } + #[test] fn polymarket_catalog_preserves_outcome_token_identity() { let mut config = SystemConfig::default(); diff --git a/rust_hft/risk-control/portfolio-core/src/lib.rs b/rust_hft/risk-control/portfolio-core/src/lib.rs index 8ef92ae8e..b545f6642 100644 --- a/rust_hft/risk-control/portfolio-core/src/lib.rs +++ b/rust_hft/risk-control/portfolio-core/src/lib.rs @@ -244,6 +244,7 @@ impl Portfolio { quantity: Quantity::zero(), avg_price: Price::zero(), unrealized_pnl: Decimal::ZERO, + realized_pnl: Decimal::ZERO, }); let old_qty = pos.quantity.0; @@ -267,7 +268,9 @@ impl Portfolio { } else { -Decimal::ONE }; - self.view.realized_pnl += (price.0 - pos.avg_price.0) * closed_quantity * direction; + let realized_delta = (price.0 - pos.avg_price.0) * closed_quantity * direction; + pos.realized_pnl += realized_delta; + self.view.realized_pnl += realized_delta; if new_qty == Decimal::ZERO { pos.avg_price = Price::zero(); @@ -454,6 +457,7 @@ mod tests { let position = view.positions.get(&symbol).unwrap(); assert_eq!(position.quantity.0, Decimal::from(2)); assert_eq!(position.avg_price.0, Decimal::from(110)); + assert_eq!(position.realized_pnl, Decimal::from(30)); assert_eq!(view.realized_pnl, Decimal::from(30)); fill(&mut portfolio, "S-2", "f4", &symbol, Side::Sell, 90, 2); @@ -485,6 +489,7 @@ mod tests { let position = view.positions.get(&symbol).unwrap(); assert_eq!(position.quantity.0, Decimal::from(-1)); assert_eq!(position.avg_price.0, Decimal::from(110)); + assert_eq!(position.realized_pnl, Decimal::from(20)); assert_eq!(view.realized_pnl, Decimal::from(20)); fill(&mut portfolio, "B-2", "f4", &symbol, Side::Buy, 100, 2); @@ -493,6 +498,7 @@ mod tests { let position = view.positions.get(&symbol).unwrap(); assert_eq!(position.quantity.0, Decimal::ONE); assert_eq!(position.avg_price.0, Decimal::from(100)); + assert_eq!(position.realized_pnl, Decimal::from(30)); assert_eq!(view.realized_pnl, Decimal::from(30)); assert_eq!(view.cash_balance, Decimal::from(930)); assert_eq!(view.equity(), Decimal::from(1030)); diff --git a/rust_hft/risk-control/risk/src/default_risk_manager.rs b/rust_hft/risk-control/risk/src/default_risk_manager.rs index 9f59c61d4..563ac2fc1 100644 --- a/rust_hft/risk-control/risk/src/default_risk_manager.rs +++ b/rust_hft/risk-control/risk/src/default_risk_manager.rs @@ -176,10 +176,7 @@ impl DefaultRiskManager { if position.quantity.0 == Decimal::ZERO { None } else { - Some( - position.avg_price.0 - + position.unrealized_pnl / position.quantity.0, - ) + Some(position.avg_price.0 + position.unrealized_pnl / position.quantity.0) } }) }) @@ -192,8 +189,7 @@ impl DefaultRiskManager { (position.avg_price.0 * position.quantity.0 + position.unrealized_pnl).abs() }) .sum::(); - let projected_notional = - other_notional + reference_price.abs() * new_position.0.abs(); + let projected_notional = other_notional + reference_price.abs() * new_position.0.abs(); if projected_notional > self.config.max_global_notional { return Err(format!( @@ -260,6 +256,7 @@ impl RiskManager for DefaultRiskManager { self.last_account = Some(account.clone()); self.last_account_update_us = Self::current_time_us(); let mut approved_intents = Vec::new(); + let mut projected_account = account.clone(); for mut intent in intents { // 第一步：精度标准化 @@ -285,7 +282,7 @@ impl RiskManager for DefaultRiskManager { let checks = vec![ venue_check, rate_check, - self.check_position_limits(&intent, account), + self.check_position_limits(&intent, &projected_account), self.check_daily_loss(account), self.check_drawdown(account), ]; @@ -307,6 +304,24 @@ impl RiskManager for DefaultRiskManager { ); self.update_state(&intent.symbol); + let signed_quantity = match intent.side { + hft_core::Side::Buy => intent.quantity.0, + hft_core::Side::Sell => -intent.quantity.0, + }; + let position = projected_account + .positions + .entry(intent.symbol.clone()) + .or_insert_with(|| ports::Position { + symbol: intent.symbol.clone(), + quantity: Quantity::zero(), + avg_price: intent.price.unwrap_or(Price(Decimal::ZERO)), + unrealized_pnl: Decimal::ZERO, + realized_pnl: Decimal::ZERO, + }); + position.quantity.0 += signed_quantity; + if let Some(price) = intent.price { + position.avg_price = price; + } approved_intents.push(intent); } else { // 风控拒绝 @@ -614,6 +629,7 @@ mod tests { quantity: Quantity::from_f64(0.5).unwrap(), avg_price: Price::from_f64(67000.0).unwrap(), unrealized_pnl: Decimal::from(100), + realized_pnl: Decimal::ZERO, }, ); @@ -739,6 +755,28 @@ mod tests { assert!(result.unwrap_err().contains("全局名义价值限额")); } + #[test] + fn batch_review_uses_projected_account_exposure() { + let config = RiskConfig { + max_global_notional: Decimal::from(100), + max_position_per_symbol: Quantity::from_f64(100.0).unwrap(), + aggressive_mode: true, + ..Default::default() + }; + let mut mgr = DefaultRiskManager::new(config); + let account = AccountView::default(); + let venue = VenueSpec { + min_notional: Decimal::ZERO, + ..VenueSpec::default() + }; + let first = create_test_intent("BTCUSDT", Side::Buy, 0.6, Some(100.0)); + let second = create_test_intent("BTCUSDT", Side::Buy, 0.6, Some(100.0)); + + let approved = mgr.review(vec![first, second], &account, &venue); + + assert_eq!(approved.len(), 1); + } + #[test] fn reduce_and_close_orders_bypass_an_already_breached_global_cap() { let config = RiskConfig { @@ -756,6 +794,7 @@ mod tests { quantity: Quantity::from_f64(11.0).unwrap(), avg_price: Price::from_f64(100.0).unwrap(), unrealized_pnl: Decimal::ZERO, + realized_pnl: Decimal::ZERO, }, ); @@ -766,8 +805,7 @@ mod tests { assert!(mgr.check_position_limits(&close, &account).is_ok()); assert!(mgr.check_position_limits(&increase, &account).is_err()); - account.positions.get_mut(&symbol).unwrap().quantity = - Quantity::from_f64(-11.0).unwrap(); + account.positions.get_mut(&symbol).unwrap().quantity = Quantity::from_f64(-11.0).unwrap(); let cover = create_test_intent("BTCUSDT", Side::Buy, 1.0, Some(100.0)); assert!(mgr.check_position_limits(&cover, &account).is_ok()); } @@ -860,7 +898,7 @@ mod tests { let result = PrecisionNormalizer::validate_order_minimums( &intent, Quantity::from_f64(0.0001).unwrap(), // qty check passes - Decimal::from(10), // notional check fails + Decimal::from(10), // notional check fails ); assert!(result.is_err()); assert!(result.unwrap_err().contains("名义价值低于最小值")); @@ -913,7 +951,10 @@ mod tests { } let metrics = mgr.get_risk_metrics(); - assert_eq!(*metrics.get("total_orders_today").unwrap(), Decimal::from(5)); + assert_eq!( + *metrics.get("total_orders_today").unwrap(), + Decimal::from(5) + ); } #[test] diff --git a/rust_hft/risk-control/risk/src/sentinel.rs b/rust_hft/risk-control/risk/src/sentinel.rs index 3dcbe2637..df12d95b8 100644 --- a/rust_hft/risk-control/risk/src/sentinel.rs +++ b/rust_hft/risk-control/risk/src/sentinel.rs @@ -56,15 +56,15 @@ impl Default for SentinelConfig { fn default() -> Self { Self { // 延遲閾值 - latency_warn_us: 15_000, // 15ms 警告 - latency_degrade_us: 25_000, // 25ms 降頻 - latency_stop_us: 50_000, // 50ms 停止 + latency_warn_us: 15_000, // 15ms 警告 + latency_degrade_us: 25_000, // 25ms 降頻 + latency_stop_us: 50_000, // 50ms 停止 // 回撤閾值 - drawdown_warn_pct: 2.0, // 2% 警告 - drawdown_degrade_pct: 3.0, // 3% 降頻 - drawdown_stop_pct: 5.0, // 5% 停止 - drawdown_emergency_pct: 7.0, // 7% 緊急平倉 + drawdown_warn_pct: 2.0, // 2% 警告 + drawdown_degrade_pct: 3.0, // 3% 降頻 + drawdown_stop_pct: 5.0, // 5% 停止 + drawdown_emergency_pct: 7.0, // 7% 緊急平倉 // 降頻策略 degrade_reduce_position_pct: 50.0, @@ -151,10 +151,8 @@ pub struct SystemStats { pub notional_value: f64, /// 訂單提交率 (orders/second) pub order_rate: f64, - /// WebSocket 重連次數 - pub ws_reconnect_count: u32, /// 數據間隙次數 - pub data_gap_count: u32, + pub data_gap_count: u64, } /// Sentinel 哨兵 - 自動化風控核心 @@ -167,6 +165,7 @@ pub struct Sentinel { // 回撤追蹤 consecutive_drawdown_violations: u32, + last_data_gap_count: u64, // 恢復追蹤 last_violation_time: Option, @@ -187,6 +186,7 @@ impl Sentinel { state: SentinelState::Normal, consecutive_latency_violations: 0, consecutive_drawdown_violations: 0, + last_data_gap_count: 0, last_violation_time: None, degraded_since: None, total_checks: 0, @@ -229,18 +229,32 @@ impl Sentinel { // 檢查回撤 let drawdown_action = self.check_drawdown(stats); + // A newly dropped market event or failed account snapshot invalidates the trading view. + // Stop immediately and require operator-controlled recovery. + let data_action = if stats.data_gap_count > self.last_data_gap_count { + error!( + "Market-data integrity gap detected: {} -> {}", + self.last_data_gap_count, stats.data_gap_count + ); + self.last_data_gap_count = stats.data_gap_count; + SentinelAction::Stop + } else { + SentinelAction::Continue + }; + // 合併動作（取更嚴重的） - let action = latency_action.merge(drawdown_action); + let action = latency_action.merge(drawdown_action).merge(data_action); // 更新狀態 self.update_state(action); // 檢查恢復條件 if (self.state == SentinelState::Degraded || self.state == SentinelState::Recovering) - && self.should_recover(stats) { - self.recover(); - return SentinelAction::Continue; - } + && self.should_recover(stats) + { + self.recover(); + return SentinelAction::Continue; + } action } @@ -274,7 +288,10 @@ impl Sentinel { return SentinelAction::Degrade; } } else if latency >= self.config.latency_warn_us { - warn!("Latency warning: {}us >= {}us", latency, self.config.latency_warn_us); + warn!( + "Latency warning: {}us >= {}us", + latency, self.config.latency_warn_us + ); self.total_warnings += 1; return SentinelAction::Warn; } else { @@ -494,6 +511,18 @@ mod tests { assert_eq!(sentinel.state(), SentinelState::Emergency); } + #[test] + fn test_sentinel_stops_on_new_data_gap() { + let mut sentinel = Sentinel::with_defaults(); + let action = sentinel.check(&SystemStats { + data_gap_count: 1, + ..Default::default() + }); + + assert_eq!(action, SentinelAction::Stop); + assert_eq!(sentinel.state(), SentinelState::Stopped); + } + #[test] fn test_action_merge() { assert_eq!( diff --git a/rust_hft/tools/collector/src/bin/lob-pit-materializer.rs b/rust_hft/tools/collector/src/bin/lob-pit-materializer.rs index 44c0e5c64..7a4f0e974 100644 --- a/rust_hft/tools/collector/src/bin/lob-pit-materializer.rs +++ b/rust_hft/tools/collector/src/bin/lob-pit-materializer.rs @@ -1,7 +1,12 @@ use anyhow::{anyhow, bail, Context, Result}; use chrono::{DateTime, Utc}; use clap::{Parser, ValueEnum}; -use hft_collector::{DataModality, PointInTimeFeatureRow}; +use hft_collector::{ + lob_archiver::{ + source_revision as governed_source_revision, Market as LobMarket, ReplaySequenceValidator, + }, + DataModality, PointInTimeFeatureRow, +}; use rust_decimal::{prelude::ToPrimitive, Decimal}; use serde::{Deserialize, Serialize}; use serde_json::Value; @@ -151,11 +156,19 @@ struct Replay { next_bucket_ns: Option, series_id: u64, saw_seed: bool, + sequence_validator: ReplaySequenceValidator, } impl Replay { - fn new(market: Market, symbol: String, bucket_ns: u64, depth: usize) -> Self { - Self { + fn new(market: Market, symbol: String, bucket_ns: u64, depth: usize) -> Result { + let sequence_validator = ReplaySequenceValidator::new( + match market { + Market::Spot => LobMarket::Spot, + Market::Usdm => LobMarket::Usdm, + }, + &symbol, + )?; + Ok(Self { market, symbol, bucket_ns, @@ -166,7 +179,8 @@ impl Replay { next_bucket_ns: None, series_id: 0, saw_seed: false, - } + sequence_validator, + }) } fn start_series(&mut self, state: BookState, received_at_ns: u64) -> Result<()> { @@ -226,11 +240,20 @@ impl Replay { fn process_event(&mut self, event: Value) -> Result<()> { let received_at_ns = json_u64(&event, "received_at_ns")?; - match event.get("type").and_then(Value::as_str) { - Some("sequence_gap") => bail!("LOB tape contains a sequence gap event"), - Some("snapshot") => self.process_snapshot(event, received_at_ns), - Some("checkpoint") => self.process_checkpoint(event, received_at_ns), - Some("diff") => self.process_diff(event, received_at_ns), + let event_type = event + .get("type") + .and_then(Value::as_str) + .context("event has no type")?; + self.sequence_validator.observe( + event_type, + event.as_object().context("event is not an object")?, + received_at_ns, + )?; + match event_type { + "sequence_gap" => bail!("LOB tape contains a sequence gap event"), + "snapshot" => self.process_snapshot(event, received_at_ns), + "checkpoint" => self.process_checkpoint(event, received_at_ns), + "diff" => self.process_diff(event, received_at_ns), _ => Ok(()), } } @@ -337,7 +360,7 @@ fn materialize(args: &Args) -> Result { .bucket_ms .checked_mul(1_000_000) .context("bucket size overflow")?; - let mut replay = Replay::new(args.market, symbol.clone(), bucket_ns, args.top_depth); + let mut replay = Replay::new(args.market, symbol.clone(), bucket_ns, args.top_depth)?; for segment in &segments { replay_segment(segment.path(), &symbol, &mut replay)?; } @@ -347,6 +370,7 @@ fn materialize(args: &Args) -> Result { if replay.state.as_ref().is_some_and(|state| !state.bridged) { bail!("snapshot-only replay series never received a valid first diff"); } + replay.sequence_validator.finish()?; let revision = source_revision(&segments); let created_at = Utc::now(); @@ -791,12 +815,11 @@ fn publish_immutable(path: &Path, bytes: &[u8]) -> Result<()> { } fn source_revision(segments: &[VerifiedSegment]) -> String { - let mut digest = Sha256::new(); - for segment in segments { - digest.update(segment.evidence.sha256.as_bytes()); - digest.update(b"\n"); - } - hex::encode(digest.finalize()) + governed_source_revision( + segments + .iter() + .map(|segment| segment.evidence.sha256.as_str()), + ) } fn sha256_file(path: &Path) -> Result { @@ -1011,7 +1034,9 @@ mod tests { .status() .unwrap() .success()); - std::fs::remove_file(raw).unwrap(); + if raw.exists() { + std::fs::remove_file(raw).unwrap(); + } let hash = sha256_file(&data).unwrap(); let event_types = events.iter().fold(BTreeMap::new(), |mut counts, event| { *counts diff --git a/rust_hft/tools/collector/src/lob_archiver.rs b/rust_hft/tools/collector/src/lob_archiver.rs index 545a753ff..2d09b24af 100644 --- a/rust_hft/tools/collector/src/lob_archiver.rs +++ b/rust_hft/tools/collector/src/lob_archiver.rs @@ -84,6 +84,265 @@ impl DepthDiff { } } +pub fn source_revision<'a>(segment_hashes: impl IntoIterator) -> String { + let mut digest = Sha256::new(); + for hash in segment_hashes { + digest.update(hash.as_bytes()); + digest.update(b"\n"); + } + hex::encode(digest.finalize()) +} + +#[derive(Debug)] +pub struct ReplaySequenceValidator { + market: Market, + symbol: String, + state: Option, + pending: Vec, +} + +#[derive(Debug)] +struct ReplaySequenceState { + session_id: String, + last_update_id: u64, + bridged: bool, + bids: HashMap, + asks: HashMap, +} + +#[derive(Debug)] +struct ReplaySequenceDiff { + session_id: String, + diff: DepthDiff, + received_at_ns: u64, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ReplaySequenceEvent { + Snapshot { + received_at_ns: u64, + bids: Vec<[String; 2]>, + asks: Vec<[String; 2]>, + }, + Diff { + received_at_ns: u64, + bids: Vec<[String; 2]>, + asks: Vec<[String; 2]>, + }, +} + +impl ReplaySequenceValidator { + pub fn new(market: Market, symbol: impl Into) -> anyhow::Result { + let symbol = symbol.into(); + if symbol.is_empty() { + anyhow::bail!("replay symbol is empty"); + } + Ok(Self { + market, + symbol, + state: None, + pending: Vec::new(), + }) + } + + pub fn observe( + &mut self, + event_type: &str, + raw: &serde_json::Map, + received_at_ns: u64, + ) -> anyhow::Result> { + let mut events = Vec::new(); + match event_type { + "snapshot" if required_string(raw, "symbol")? == self.symbol => { + if self.state.as_ref().is_some_and(|state| !state.bridged) { + anyhow::bail!("snapshot replaced an unbridged replay series"); + } + let snapshot = raw + .get("snapshot") + .and_then(Value::as_object) + .ok_or_else(|| anyhow::anyhow!("snapshot event has no nested payload"))?; + self.state = Some(ReplaySequenceState { + session_id: required_string(raw, "session_id")?.to_string(), + last_update_id: snapshot + .get("lastUpdateId") + .and_then(Value::as_u64) + .ok_or_else(|| anyhow::anyhow!("snapshot payload has no lastUpdateId"))?, + bridged: false, + bids: parse_snapshot_side(snapshot.get("bids"))?, + asks: parse_snapshot_side(snapshot.get("asks"))?, + }); + events.push(ReplaySequenceEvent::Snapshot { + received_at_ns, + bids: parse_replay_levels(snapshot.get("bids"), "snapshot bids")?, + asks: parse_replay_levels(snapshot.get("asks"), "snapshot asks")?, + }); + for pending in std::mem::take(&mut self.pending) { + let effective_time = received_at_ns.max(pending.received_at_ns); + if self.apply_diff(&pending)? { + events.push(ReplaySequenceEvent::Diff { + received_at_ns: effective_time, + bids: pending.diff.bids, + asks: pending.diff.asks, + }); + } + } + } + "diff" => { + let frame = raw + .get("frame") + .ok_or_else(|| anyhow::anyhow!("diff event has no nested frame"))?; + let diff = DepthDiff::from_frame(frame)?; + if diff.symbol != self.symbol { + return Ok(events); + } + let diff = ReplaySequenceDiff { + session_id: required_string(raw, "session_id")?.to_string(), + diff, + received_at_ns, + }; + if self.state.is_none() { + self.pending.push(diff); + } else if self.state.as_ref().expect("checked state").session_id != diff.session_id + { + if !self.state.as_ref().expect("checked state").bridged { + anyhow::bail!("diff replaced an unbridged replay series"); + } + self.state = None; + self.pending = vec![diff]; + } else { + if self.apply_diff(&diff)? { + events.push(ReplaySequenceEvent::Diff { + received_at_ns, + bids: diff.diff.bids, + asks: diff.diff.asks, + }); + } + } + } + "checkpoint" if required_string(raw, "symbol")? == self.symbol => { + let checkpoint = ReplaySequenceState { + session_id: required_string(raw, "session_id")?.to_string(), + last_update_id: raw + .get("last_update_id") + .and_then(Value::as_u64) + .ok_or_else(|| anyhow::anyhow!("checkpoint has no last update id"))?, + bridged: raw + .get("bridged") + .and_then(Value::as_bool) + .ok_or_else(|| anyhow::anyhow!("checkpoint has no bridged state"))?, + bids: parse_snapshot_side(raw.get("bids"))?, + asks: parse_snapshot_side(raw.get("asks"))?, + }; + match self.state.as_ref() { + None if !self.pending.is_empty() => { + anyhow::bail!("diffs arrived before replay seed") + } + None => { + self.state = Some(checkpoint); + events.push(replay_checkpoint_seed(raw, received_at_ns)?); + } + Some(state) if state.session_id != checkpoint.session_id => { + if !state.bridged { + anyhow::bail!("checkpoint replaced an unbridged replay series"); + } + self.state = Some(checkpoint); + events.push(replay_checkpoint_seed(raw, received_at_ns)?); + } + Some(state) + if state.last_update_id != checkpoint.last_update_id + || state.bridged != checkpoint.bridged + || state.bids != checkpoint.bids + || state.asks != checkpoint.asks => + { + anyhow::bail!("checkpoint does not match replayed update state") + } + Some(_) => {} + } + } + _ => {} + } + Ok(events) + } + + fn apply_diff(&mut self, update: &ReplaySequenceDiff) -> anyhow::Result { + let state = self + .state + .as_mut() + .ok_or_else(|| anyhow::anyhow!("diff has no replay seed"))?; + if update.session_id != state.session_id { + anyhow::bail!("diff session does not match replay state"); + } + if update.diff.final_update_id <= state.last_update_id { + return Ok(false); + } + let expected = state + .last_update_id + .checked_add(u64::from(self.market == Market::Spot)) + .ok_or_else(|| anyhow::anyhow!("update id overflow"))?; + let accepted = if self.market == Market::Usdm { + update.diff.previous_update_id == Some(state.last_update_id) + || !state.bridged + && update.diff.first_update_id <= expected + && expected <= update.diff.final_update_id + } else { + update.diff.first_update_id <= expected && expected <= update.diff.final_update_id + }; + if !accepted { + anyhow::bail!( + "Binance sequence gap: expected {expected}, received {}-{}", + update.diff.first_update_id, + update.diff.final_update_id + ); + } + state.last_update_id = update.diff.final_update_id; + state.bridged = true; + update_side(&mut state.bids, &update.diff.bids); + update_side(&mut state.asks, &update.diff.asks); + Ok(true) + } + + pub fn finish(&self) -> anyhow::Result<()> { + if !self.pending.is_empty() + || self.state.is_none() + || self.state.as_ref().is_some_and(|state| !state.bridged) + { + anyhow::bail!("collector replay did not finish in a bridged state"); + } + Ok(()) + } +} + +fn replay_checkpoint_seed( + raw: &serde_json::Map, + received_at_ns: u64, +) -> anyhow::Result { + Ok(ReplaySequenceEvent::Snapshot { + received_at_ns, + bids: parse_replay_levels(raw.get("bids"), "checkpoint bids")?, + asks: parse_replay_levels(raw.get("asks"), "checkpoint asks")?, + }) +} + +fn parse_replay_levels(value: Option<&Value>, field: &str) -> anyhow::Result> { + serde_json::from_value( + value + .cloned() + .ok_or_else(|| anyhow::anyhow!("{field} are missing"))?, + ) + .map_err(Into::into) +} + +fn required_string<'a>( + object: &'a serde_json::Map, + field: &str, +) -> anyhow::Result<&'a str> { + object + .get(field) + .and_then(Value::as_str) + .filter(|value| !value.is_empty()) + .ok_or_else(|| anyhow::anyhow!("collector payload is missing {field}")) +} + fn validate_levels(levels: &[[String; 2]]) -> anyhow::Result<()> { for [price, quantity] in levels { // Reuse market-core's non-floating parser on its supported fast-path.