Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion deployment/aliyun/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -358,7 +358,7 @@ advancing. `priority_trade_backlog` must be zero for the shadow gate to accept a
health sample, while `deferred_trade_markets` makes bounded historical backfill
explicit rather than silently claiming full-cycle trade coverage. Every Data API
request, including a second pagination request for the same market, passes through a
shared start-time pacer with at least 100ms between request starts. Up to four requests
shared start-time pacer with at least 125ms between request starts. Up to four requests
may remain in flight, and each processing chunk retains at most four market responses,
so slow I/O overlaps without creating an unbounded request or memory fan-out. An
absolute 180-second cycle deadline cancels stalled network work and fails closed.
Expand Down
4 changes: 2 additions & 2 deletions deployment/aliyun/polymarket-reference-collector.service
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,8 @@ Type=simple
User=hftcollector
Group=hftcollector
# --max-trade-polls-per-cycle 200: the 180-second cycle watchdog and the
# 100ms minimum Data API request spacing bound one cycle to 1800 request
# starts; 200 polls need at least 20 seconds of pacing, leaving ~160 seconds
# 125ms minimum Data API request spacing bounds one cycle to 1440 request
# starts; 200 polls need at least 25 seconds of pacing, leaving ~155 seconds
# of headroom for metadata and settlement work inside the watchdog. Poll
# concurrency stays at the default 4, so the 672MiB/768MiB memory calibration
# is unchanged while historical trade backfill throughput rises from the
Expand Down
2 changes: 1 addition & 1 deletion deployment/aliyun/polymarket-rust-health-policy.jq
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ and (.market_detail_deferred == (.market_detail_eligible - .market_detail_select
and (.market_detail_priority_deferred == ([.market_detail_priority - .market_detail_budget, 0] | max))
and (.trade_poll_budget_after_market_details | type == "number" and floor == . and . >= 0)
and (.trade_poll_budget_after_market_details == (.trade_poll_budget - .market_detail_selected))
and (.trade_request_spacing_ms | type == "number" and floor == . and . >= 100)
and (.trade_request_spacing_ms | type == "number" and floor == . and . >= 125)
and (.eligible_trade_markets | type == "number" and floor == . and . >= 0)
and (.priority_trade_markets | type == "number" and floor == . and . >= 0)
and (.selected_trade_markets | type == "number" and floor == . and . >= 0)
Expand Down
4 changes: 2 additions & 2 deletions deployment/aliyun/test-polymarket-raw-ops-control-plane.sh
Original file line number Diff line number Diff line change
Expand Up @@ -5082,7 +5082,7 @@ jq -n '{
cycle_started_at:"2026-07-15T00:00:00Z",cycle_duration_ms:1000,
target_markets:120,
missing_target_symbols:[],api_errors:[],malformed_trade_rows:0,
trade_poll_budget:200,trade_poll_concurrency:4,trade_request_spacing_ms:100,
trade_poll_budget:200,trade_poll_concurrency:4,trade_request_spacing_ms:125,
priority_trade_markets_before_market_details:108,
market_detail_budget:4,market_detail_eligible:3,market_detail_priority:2,
market_detail_selected:3,market_detail_deferred:0,market_detail_priority_deferred:0,
Expand Down Expand Up @@ -5151,7 +5151,7 @@ for mutation in \
'.market_detail_priority_deferred = 1' \
'.trade_poll_budget_after_market_details = 196' \
'del(.trade_request_spacing_ms)' \
'.trade_request_spacing_ms = 99' \
'.trade_request_spacing_ms = 124' \
'.priority_trade_markets = 107' \
'.priority_trade_backlog = 1' \
'.selected_trade_markets = 196' \
Expand Down
8 changes: 7 additions & 1 deletion rust_hft/tools/collector/src/bin/polymarket-raw-ops.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ use hft_collector::polymarket_parity::{
use hft_collector::polymarket_raw::{
finalize_reference_tape, run_reference, ReferenceConfig, DEFAULT_MAX_CONCURRENT_TRADE_POLLS,
DEFAULT_MAX_MARKETS_PER_LANE, DEFAULT_MAX_TRADE_POLLS_PER_CYCLE, DEFAULT_TAPE_MAX_BYTES,
DEFAULT_TRADE_REQUEST_SPACING_MS,
};
use hft_collector::polymarket_research_import::{
validate_research_segments, ArtifactTriplet, ResearchSegmentValidationConfig,
Expand Down Expand Up @@ -73,7 +74,7 @@ enum Command {
trade_finalization_lag_secs: i64,
#[arg(long, default_value_t = 3)]
trade_finalization_stable_polls: u64,
#[arg(long, default_value_t = 100)]
#[arg(long, default_value_t = DEFAULT_TRADE_REQUEST_SPACING_MS)]
per_market_delay_ms: u64,
#[arg(long, default_value_t = DEFAULT_TAPE_MAX_BYTES)]
tape_max_bytes: u64,
Expand Down Expand Up @@ -430,6 +431,7 @@ mod tests {
max_markets,
max_trade_polls_per_cycle,
max_concurrent_trade_polls,
per_market_delay_ms,
tape_max_bytes,
..
} = command
Expand All @@ -445,6 +447,10 @@ mod tests {
max_concurrent_trade_polls,
ReferenceConfig::default().max_concurrent_trade_polls
);
assert_eq!(
Duration::from_millis(per_market_delay_ms),
ReferenceConfig::default().per_market_delay
);
assert_eq!(tape_max_bytes, ReferenceConfig::default().tape_max_bytes);
}

Expand Down
7 changes: 6 additions & 1 deletion rust_hft/tools/collector/src/polymarket_raw.rs
Original file line number Diff line number Diff line change
Expand Up @@ -45,14 +45,15 @@ const MAX_TRADE_ROWS_PER_SNAPSHOT: usize = 20_000;
pub const DEFAULT_MAX_MARKETS_PER_LANE: usize = 10_000;
pub const DEFAULT_MAX_TRADE_POLLS_PER_CYCLE: usize = 112;
pub const DEFAULT_MAX_CONCURRENT_TRADE_POLLS: usize = 4;
pub const DEFAULT_TRADE_REQUEST_SPACING_MS: u64 = 125;
// Default cap on one active tape's bytes before fail-closed rotation. At
// tick-level recording a single UTC-hour tape reached 20-25 GiB (#655), and
// upload processing needs ~1.6x the tape size of transient spool disk.
pub const DEFAULT_TAPE_MAX_BYTES: u64 = 4 * 1024 * 1024 * 1024;
// A nonzero cap below one record's order of magnitude would rotate every
// write batch and flood the uploader with trivial tapes.
const MIN_TAPE_MAX_BYTES: u64 = 1024 * 1024;
const MIN_TRADE_REQUEST_SPACING: Duration = Duration::from_millis(100);
const MIN_TRADE_REQUEST_SPACING: Duration = Duration::from_millis(DEFAULT_TRADE_REQUEST_SPACING_MS);
const TARGET_MARKET_WINDOWS_SECS: [usize; 2] = [300, 900];
const SETTLEMENT_PRICE: Decimal = Decimal::from_parts(999, 0, 0, false, 3);
const SETTLEMENT_LOSER_PRICE: Decimal = Decimal::from_parts(1, 0, 0, false, 3);
Expand Down Expand Up @@ -4213,6 +4214,10 @@ mod tests {
default.max_concurrent_trade_polls,
DEFAULT_MAX_CONCURRENT_TRADE_POLLS
);
assert_eq!(
default.per_market_delay,
Duration::from_millis(DEFAULT_TRADE_REQUEST_SPACING_MS)
);
default.validate().unwrap();

let malformed_market_id = ReferenceConfig {
Expand Down
Loading