From fb354504b1be0df601987a820a16cefc80ced978 Mon Sep 17 00:00:00 2001 From: Sonic Shih Date: Fri, 24 Jul 2026 00:31:27 +0800 Subject: [PATCH] fix(polymarket): admit event-local sparse references --- .../src/polymarket_research_import.rs | 39 ++++++++++++++++--- .../src/polymarket_research_normalize.rs | 5 ++- .../src/polymarket_research_select.rs | 39 ++++++++++++++----- 3 files changed, 67 insertions(+), 16 deletions(-) diff --git a/rust_hft/tools/collector/src/polymarket_research_import.rs b/rust_hft/tools/collector/src/polymarket_research_import.rs index 298348db6..1927844da 100644 --- a/rust_hft/tools/collector/src/polymarket_research_import.rs +++ b/rust_hft/tools/collector/src/polymarket_research_import.rs @@ -726,8 +726,15 @@ fn validate_reference_policy(references: &[Value]) -> Result<()> { } Ok(()) } -pub(crate) fn with_validated_research_segments( +#[derive(Clone, Copy, PartialEq, Eq)] +enum ReferenceHourPolicy { + Consecutive, + Nondecreasing, +} + +fn with_validated_research_segments_policy( config: &ResearchSegmentValidationConfig, + hour_policy: ReferenceHourPolicy, consume: impl FnOnce(&ResearchSegmentValidationReport, &Path, &Path) -> Result, ) -> Result { if config.references.is_empty() || config.references.len() > MAX_REFERENCE_SEGMENTS { @@ -757,7 +764,10 @@ pub(crate) fn with_validated_research_segments( if let Some(previous) = references.last() { let previous = segment_hour(&previous.identity)?; let current = segment_hour(&reference.identity)?; - if current < previous || current > previous + TimeDelta::hours(1) { + if current < previous + || (hour_policy == ReferenceHourPolicy::Consecutive + && current > previous + TimeDelta::hours(1)) + { bail!("reference segments must be same or consecutive UTC hours"); } } @@ -830,6 +840,20 @@ pub(crate) fn with_validated_research_segments( Ok(result) } +pub(crate) fn with_validated_research_segments( + config: &ResearchSegmentValidationConfig, + consume: impl FnOnce(&ResearchSegmentValidationReport, &Path, &Path) -> Result, +) -> Result { + with_validated_research_segments_policy(config, ReferenceHourPolicy::Consecutive, consume) +} + +pub(crate) fn with_event_local_validated_research_segments( + config: &ResearchSegmentValidationConfig, + consume: impl FnOnce(&ResearchSegmentValidationReport, &Path, &Path) -> Result, +) -> Result { + with_validated_research_segments_policy(config, ReferenceHourPolicy::Nondecreasing, consume) +} + pub fn validate_research_segments( config: &ResearchSegmentValidationConfig, ) -> Result { @@ -1299,7 +1323,7 @@ mod tests { } #[test] - fn merged_trade_union_matches_later_completion_proof() { + fn event_local_trade_union_allows_sparse_reference_hours() { let temp = tempfile::tempdir().unwrap(); let root = fs::canonicalize(temp.path()).unwrap(); let market = triplet(&root, "crypto_expiry", &market_rows()); @@ -1324,7 +1348,7 @@ mod tests { ), ]; for record in &mut second { - record["recorded_at"] = json!("2026-07-17T05:02:00Z"); + record["recorded_at"] = json!("2026-07-17T07:02:00Z"); } let second = triplet(&root, "crypto_expiry_reference", &second); let config = crate::polymarket_research_normalize::PolymarketEvidenceConfig { @@ -1337,6 +1361,11 @@ mod tests { market_ids: vec!["market-1".to_owned()], }; + let error = crate::polymarket_research_select::select_research_contracts(&config) + .unwrap_err() + .to_string(); + assert!(error.contains("same or consecutive UTC hours")); + let normalized = crate::polymarket_research_normalize::normalize_polymarket_evidence(&config).unwrap(); let trades = String::from_utf8(normalized.ndjson) @@ -1360,7 +1389,7 @@ mod tests { ); assert_eq!( trades[missing_trade["record_id"].as_str().unwrap()], - "2026-07-17T05:02:00Z" + "2026-07-17T07:02:00Z" ); } diff --git a/rust_hft/tools/collector/src/polymarket_research_normalize.rs b/rust_hft/tools/collector/src/polymarket_research_normalize.rs index ebf271006..1b27d6a71 100644 --- a/rust_hft/tools/collector/src/polymarket_research_normalize.rs +++ b/rust_hft/tools/collector/src/polymarket_research_normalize.rs @@ -1,7 +1,8 @@ use crate::polymarket_research_import::{ResearchSegmentValidationReport, TradeCompletionIdentity}; use crate::polymarket_research_select::{ decimal_text, json_strings, required, timestamp, utc_text, visit, - with_selected_research_contracts, ResearchSelectionConfig, SelectedContract, WINDOW_SECS, + with_event_local_selected_research_contracts, ResearchSelectionConfig, SelectedContract, + WINDOW_SECS, }; use crate::polymarket_upload::{ trade_record_ids_sha256, validate_canonical_trade, validate_market_settlement, @@ -716,7 +717,7 @@ fn normalize_raw( pub fn normalize_polymarket_evidence( config: &PolymarketEvidenceConfig, ) -> Result { - with_selected_research_contracts( + with_event_local_selected_research_contracts( config, |inputs, market_path, reference_path, contracts, start, end| { normalize_raw(inputs, market_path, reference_path, contracts, start, end) diff --git a/rust_hft/tools/collector/src/polymarket_research_select.rs b/rust_hft/tools/collector/src/polymarket_research_select.rs index 79fb6bb7b..30f74e7d0 100644 --- a/rust_hft/tools/collector/src/polymarket_research_select.rs +++ b/rust_hft/tools/collector/src/polymarket_research_select.rs @@ -1,6 +1,6 @@ use crate::polymarket_research_import::{ - with_validated_research_segments, ResearchSegmentValidationConfig, - ResearchSegmentValidationReport, + with_event_local_validated_research_segments, with_validated_research_segments, + ResearchSegmentValidationConfig, ResearchSegmentValidationReport, }; use crate::polymarket_upload::validate_market_metadata; use anyhow::{anyhow, bail, Context, Result}; @@ -396,8 +396,9 @@ fn enrich_metadata(path: &Path, contracts: &mut BTreeMap( +fn with_selected_research_contracts_policy( config: &ResearchSelectionConfig, + event_local: bool, consume: impl FnOnce( &ResearchSegmentValidationReport, &Path, @@ -408,17 +409,37 @@ pub(crate) fn with_selected_research_contracts( ) -> Result, ) -> Result { let (start, end, market_ids) = selection(config)?; - with_validated_research_segments(&config.segments, |inputs, market_path, reference_path| { - let mut contracts = discover(market_path, start, end, &market_ids)?; - enrich_metadata(reference_path, &mut contracts)?; - consume(inputs, market_path, reference_path, &contracts, start, end) - }) + let selected = + |inputs: &ResearchSegmentValidationReport, market_path: &Path, reference_path: &Path| { + let mut contracts = discover(market_path, start, end, &market_ids)?; + enrich_metadata(reference_path, &mut contracts)?; + consume(inputs, market_path, reference_path, &contracts, start, end) + }; + if event_local { + with_event_local_validated_research_segments(&config.segments, selected) + } else { + with_validated_research_segments(&config.segments, selected) + } +} + +pub(crate) fn with_event_local_selected_research_contracts( + config: &ResearchSelectionConfig, + consume: impl FnOnce( + &ResearchSegmentValidationReport, + &Path, + &Path, + &BTreeMap, + DateTime, + DateTime, + ) -> Result, +) -> Result { + with_selected_research_contracts_policy(config, true, consume) } pub fn select_research_contracts( config: &ResearchSelectionConfig, ) -> Result { - with_selected_research_contracts(config, |inputs, _, _, contracts, start, end| { + with_selected_research_contracts_policy(config, false, |inputs, _, _, contracts, start, end| { let market_ids = contracts.keys().cloned().collect(); let symbols = contracts .values()