diff --git a/crates/db/migrations/0013_design_quota_manual.sql b/crates/db/migrations/0013_design_quota_manual.sql new file mode 100644 index 000000000..3b0f3d183 --- /dev/null +++ b/crates/db/migrations/0013_design_quota_manual.sql @@ -0,0 +1,21 @@ +-- Split the design daily run quota by run origin. +-- +-- `design_quota.runs_used` counted *every* sandbox run against a single +-- 10-run/hotkey/day ceiling, and the organizer's own round scheduler charged +-- the same bucket as the miner's `POST /v1/harness`. A full UTC day dispatches +-- `ROUNDS_PER_DAY (10) × PROMPTS_PER_ROUND (3)` = 30 runs to every registered +-- harness, so an honest, fully participating miner exhausted the day's quota +-- after ~3.3 rounds, sat out the remaining rounds, and could not even submit +-- (intake 409s when scheduling fails). +-- +-- `manual_runs_used` isolates the anti-spam ceiling that actually belongs to +-- miner-initiated submissions; organizer-scheduled work is `runs_used - +-- manual_runs_used` and is bounded by a separate cap derived from the live +-- round schedule. Existing rows backfill to 0 manual runs, which only ever +-- widens a live miner's submission budget. + +ALTER TABLE design_quota + ADD COLUMN manual_runs_used INTEGER NOT NULL DEFAULT 0; + +ALTER TABLE design_quota + ADD CONSTRAINT design_quota_manual_runs_nonneg CHECK (manual_runs_used >= 0); diff --git a/crates/design-challenge-task/src/lib.rs b/crates/design-challenge-task/src/lib.rs index b47d30419..8d1a0d600 100644 --- a/crates/design-challenge-task/src/lib.rs +++ b/crates/design-challenge-task/src/lib.rs @@ -49,7 +49,7 @@ pub const ROUNDS_PER_DAY: u64 = 10; /// (`DESIGN_AGENT_RUN_TIMEOUT_SECS` override). pub const AGENT_RUN_TIMEOUT_SECS: u64 = 1_800; -/// Prompts selected per round (~2–3 × harness under daily quota). +/// Prompts selected per round (each becomes one organizer-scheduled sandbox run). /// /// Default only — runtime code must use [`prompts_per_round`] /// (`DESIGN_PROMPTS_PER_ROUND` override). @@ -60,8 +60,19 @@ pub const PROMPTS_PER_ROUND: usize = 3; /// [`SCORING_WINDOW_ROUNDS`] rounds, cheat excluded. pub const SCORING_WINDOW_ROUNDS: u64 = 10; -/// Max sandboxed runs per hotkey per UTC day. -pub const DAILY_RUN_QUOTA: u32 = 10; +/// Anti-spam ceiling on **miner-initiated** sandbox runs per hotkey per UTC +/// day — runs created by `POST /v1/harness`. Organizer-scheduled round runs +/// draw on [`scheduled_daily_run_cap`] instead, so a harness that participates +/// in every round of the day is never blocked from submitting. +/// +/// Default only — runtime code must use [`manual_daily_run_quota`] +/// (`DESIGN_MANUAL_DAILY_RUN_QUOTA` override). +pub const MANUAL_DAILY_RUN_QUOTA: u32 = 10; + +/// Headroom multiplier on the derived organizer-scheduled daily run cap. The +/// cap is a runaway-scheduler guard, not a participation limit, so it must +/// stay comfortably above the full-day schedule volume. +pub const SCHEDULED_DAILY_RUN_HEADROOM: u32 = 2; /// Minimum annotations required per pair before Elo consume. pub const MIN_ANNOTATIONS_PER_PAIR: u32 = 3; @@ -107,6 +118,61 @@ pub fn prompts_per_round() -> usize { .unwrap_or(PROMPTS_PER_ROUND) } +/// Rounds in a UTC day under the effective [`round_secs`]. Equals +/// [`ROUNDS_PER_DAY`] in production; staging compresses rounds, so scheduling +/// limits must derive from this and never from the constant. +#[must_use] +pub fn rounds_per_day_effective() -> u64 { + (86_400 / round_secs()).max(1) +} + +/// Sandbox runs the organizer dispatches to one harness across a full UTC day +/// (`rounds/day × prompts/round`). This is the volume an honest, fully +/// participating harness must be allowed to execute. +#[must_use] +pub fn scheduled_runs_per_day() -> u32 { + let per_round = u64::try_from(prompts_per_round()).unwrap_or(PROMPTS_PER_ROUND as u64); + u32::try_from(rounds_per_day_effective().saturating_mul(per_round)).unwrap_or(u32::MAX) +} + +/// Effective anti-spam ceiling on miner-initiated runs per hotkey per UTC day +/// (`DESIGN_MANUAL_DAILY_RUN_QUOTA` override; default +/// [`MANUAL_DAILY_RUN_QUOTA`]). +#[must_use] +pub fn manual_daily_run_quota() -> u32 { + u32::try_from(env_u64( + "DESIGN_MANUAL_DAILY_RUN_QUOTA", + u64::from(MANUAL_DAILY_RUN_QUOTA), + 1, + )) + .unwrap_or(MANUAL_DAILY_RUN_QUOTA) +} + +/// Effective ceiling on organizer-scheduled runs per hotkey per UTC day +/// (`DESIGN_SCHEDULED_DAILY_RUN_CAP` override; default +/// [`scheduled_runs_per_day`] × [`SCHEDULED_DAILY_RUN_HEADROOM`]). +/// +/// The floor is [`scheduled_runs_per_day`]: an operator override can never sit +/// below the day's own schedule, which is the bug this cap replaced. +#[must_use] +pub fn scheduled_daily_run_cap() -> u32 { + let floor = scheduled_runs_per_day(); + let default = floor.saturating_mul(SCHEDULED_DAILY_RUN_HEADROOM); + u32::try_from(env_u64( + "DESIGN_SCHEDULED_DAILY_RUN_CAP", + u64::from(default), + u64::from(floor), + )) + .unwrap_or(default) +} + +/// Total sandbox runs one hotkey may accumulate in a UTC day across both +/// origins (display / dashboard value; enforcement is per-origin). +#[must_use] +pub fn daily_run_quota() -> u32 { + manual_daily_run_quota().saturating_add(scheduled_daily_run_cap()) +} + /// Compute round id from unix seconds under an explicit round length. #[must_use] pub const fn round_id_at_with(unix_secs: u64, round_secs: u64) -> u64 { @@ -160,6 +226,31 @@ mod tests { assert_eq!(round_secs(), ROUND_SECS); assert_eq!(agent_run_timeout_secs(), AGENT_RUN_TIMEOUT_SECS); assert_eq!(prompts_per_round(), PROMPTS_PER_ROUND); + assert_eq!(manual_daily_run_quota(), MANUAL_DAILY_RUN_QUOTA); + } + + #[test] + fn scheduled_cap_covers_a_full_day_of_rounds() { + // The bug this replaced: a 10-run/day cap against a 10-round × 3-prompt + // schedule locked an honest harness out after ~3.3 rounds. + assert_eq!(rounds_per_day_effective(), ROUNDS_PER_DAY); + assert_eq!( + scheduled_runs_per_day(), + u32::try_from(ROUNDS_PER_DAY).unwrap() * u32::try_from(PROMPTS_PER_ROUND).unwrap() + ); + assert_eq!(scheduled_runs_per_day(), 30); + assert!(scheduled_daily_run_cap() >= scheduled_runs_per_day()); + assert_eq!( + scheduled_daily_run_cap(), + scheduled_runs_per_day() * SCHEDULED_DAILY_RUN_HEADROOM + ); + assert!(daily_run_quota() > scheduled_runs_per_day()); + // An operator override below the schedule clamps back up to it, so no + // env value can re-create the lockout. (Own var name: env is global.) + let floor = u64::from(scheduled_runs_per_day()); + std::env::set_var("BASE_SCHEDULED_CAP_CLAMP_TEST", "1"); + assert_eq!(env_u64("BASE_SCHEDULED_CAP_CLAMP_TEST", 60, floor), floor); + std::env::remove_var("BASE_SCHEDULED_CAP_CLAMP_TEST"); } #[test] diff --git a/crates/design-challenge/src/corpus.rs b/crates/design-challenge/src/corpus.rs new file mode 100644 index 000000000..e43426245 --- /dev/null +++ b/crates/design-challenge/src/corpus.rs @@ -0,0 +1,182 @@ +//! Shared anti-cheat corpus: other hotkeys' prior art only. +//! +//! Same-hotkey revisions are never comparison material. Review victims must be +//! strictly earlier than the candidate. Gate + review share this module so they +//! cannot drift. Pass the candidate row explicitly (not via recent-list lookup). + +use challenge_agentic::{CorpusEntry, GateCorpusEntry}; +use design_store::HarnessRow; + +const BASELINE_AGENT: &str = + include_str!("../../../docs/external-miner/examples/design-baseline/agent.py"); + +fn other_miners<'a>( + candidate: &'a HarnessRow, + recent: &'a [HarnessRow], +) -> impl Iterator { + let miner = candidate.miner_hotkey.to_ascii_lowercase(); + recent + .iter() + .filter(move |h| h.id != candidate.id && h.miner_hotkey.to_ascii_lowercase() != miner) +} + +fn corpus_id(h: &HarnessRow) -> String { + format!("harness:{}", h.id) +} + +/// Pre-LLM copy-gate corpus (`created_at_ms` kept for gate ordering). +#[must_use] +pub fn gate_corpus(candidate: &HarnessRow, recent: &[HarnessRow]) -> Vec { + other_miners(candidate, recent) + .map(|h| GateCorpusEntry { + id: corpus_id(h), + source: h.agent_py.clone(), + created_at_ms: h.created_at_ms, + }) + .collect() +} + +/// Reviewer corpus: baseline + other hotkeys' earlier harnesses. +/// Untimestamped rows are dropped; a legacy candidate (`created_at_ms == 0`) +/// keeps every timestamped other-hotkey row so the corpus cannot go empty. +#[must_use] +pub fn review_corpus(candidate: &HarnessRow, recent: &[HarnessRow]) -> Vec { + let mut corpus = vec![CorpusEntry { + id: "baseline".into(), + source: BASELINE_AGENT.to_owned(), + }]; + let cand_ts = candidate.created_at_ms; + corpus.extend(other_miners(candidate, recent).filter_map(|h| { + if h.created_at_ms == 0 || (cand_ts > 0 && h.created_at_ms >= cand_ts) { + return None; + } + Some(CorpusEntry { + id: corpus_id(h), + source: h.agent_py.clone(), + }) + })); + corpus +} + +#[cfg(test)] +mod tests { + use super::*; + + fn harness(id: &str, miner: &str, source: &str, created_at_ms: u64) -> HarnessRow { + HarnessRow { + id: id.into(), + miner_hotkey: miner.into(), + agent_py: source.into(), + pyproject_toml: "[project]\nname='x'\nversion='0.1.0'\n".into(), + extra_files: std::collections::BTreeMap::new(), + active: true, + eliminated_until_round: 0, + created_at_ms, + } + } + + const AA: &str = "aa"; + const BB: &str = "bb"; + + fn ids(entries: &[CorpusEntry]) -> Vec<&str> { + entries.iter().map(|e| e.id.as_str()).collect() + } + + fn gate_ids(entries: &[GateCorpusEntry]) -> Vec<&str> { + entries.iter().map(|e| e.id.as_str()).collect() + } + + #[test] + fn own_previous_version_is_never_compared_against() { + let v1 = harness("h1", AA, "def run(t):\n pass\n", 1_000); + let v2 = harness("h2", AA, "def run(t):\n pass\n", 2_000); + let recent = vec![v2.clone(), v1]; + + assert!( + gate_corpus(&v2, &recent).is_empty(), + "a miner's own v1 must not be a copy victim for their v2" + ); + assert_eq!( + ids(&review_corpus(&v2, &recent)), + vec!["baseline"], + "self-revision must not reach the LLM corpus either" + ); + } + + #[test] + fn hotkey_match_is_case_insensitive() { + let mine_old = harness("h1", "AABB", "old\n", 1_000); + let mine_new = harness("h2", "aabb", "new\n", 2_000); + let recent = vec![mine_new.clone(), mine_old]; + assert!(gate_corpus(&mine_new, &recent).is_empty()); + assert_eq!(ids(&review_corpus(&mine_new, &recent)), vec!["baseline"]); + } + + #[test] + fn other_miner_prior_art_stays_in_both_corpora() { + let victim = harness("h1", BB, "def run(t):\n pass\n", 1_000); + let copier = harness("h2", AA, "def run(t):\n pass\n", 2_000); + let recent = vec![copier.clone(), victim]; + + assert_eq!(gate_ids(&gate_corpus(&copier, &recent)), vec!["harness:h1"]); + assert_eq!( + ids(&review_corpus(&copier, &recent)), + vec!["baseline", "harness:h1"] + ); + } + + #[test] + fn candidate_outside_the_recent_window_still_excludes_itself() { + // The candidate is deliberately absent from `recent` (aged out): the + // rules must come from the candidate row, not from a lookup. + let mine_old = harness("h1", AA, "old\n", 1_000); + let theirs = harness("h3", BB, "theirs\n", 1_500); + let mine_new = harness("h2", AA, "new\n", 2_000); + let recent = vec![theirs, mine_old]; + + assert_eq!( + gate_ids(&gate_corpus(&mine_new, &recent)), + vec!["harness:h3"] + ); + assert_eq!( + ids(&review_corpus(&mine_new, &recent)), + vec!["baseline", "harness:h3"] + ); + } + + #[test] + fn review_corpus_holds_prior_art_only() { + let candidate = harness("h1", AA, "mine\n", 1_000); + let later = harness("h2", BB, "later\n", 5_000); + let unknown = harness("h3", BB, "legacy\n", 0); + let recent = vec![later, unknown]; + + // A later copycat must never make the original look like the copier. + assert_eq!(ids(&review_corpus(&candidate, &recent)), vec!["baseline"]); + // The gate keeps both and orders them itself. + assert_eq!(gate_corpus(&candidate, &recent).len(), 2); + } + + #[test] + fn legacy_candidate_keeps_timestamped_other_hotkeys() { + let legacy = harness("h0", AA, "legacy\n", 0); + let prior = harness("h1", BB, "prior\n", 1_000); + let recent = vec![prior]; + assert_eq!( + ids(&review_corpus(&legacy, &recent)), + vec!["baseline", "harness:h1"], + "unknown candidate timestamp must not empty the review corpus" + ); + } + + #[test] + fn baseline_is_always_available_to_the_reviewer() { + let candidate = harness("h1", AA, "mine\n", 1_000); + let corpus = review_corpus(&candidate, &[]); + assert_eq!(ids(&corpus), vec!["baseline"]); + assert!( + corpus[0].source.contains("def run("), + "baseline agent source" + ); + } +} diff --git a/crates/design-challenge/src/lib.rs b/crates/design-challenge/src/lib.rs index 20e3f4381..3e98a2166 100644 --- a/crates/design-challenge/src/lib.rs +++ b/crates/design-challenge/src/lib.rs @@ -5,19 +5,19 @@ //! `challenge_id = "design"`. #![forbid(unsafe_code)] -#![allow(clippy::missing_errors_doc)] -#![allow(clippy::doc_markdown)] -#![allow(clippy::cast_possible_truncation)] -#![allow(clippy::cast_sign_loss)] -#![allow(clippy::too_many_lines)] -#![allow(clippy::duration_suboptimal_units)] -#![allow(clippy::map_unwrap_or)] -#![allow(clippy::cast_possible_wrap)] -#![allow(clippy::result_large_err)] +#![allow( + clippy::missing_errors_doc, + clippy::doc_markdown, + clippy::too_many_lines +)] +#![allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] +#![allow(clippy::duration_suboptimal_units, clippy::map_unwrap_or)] +#![allow(clippy::cast_possible_wrap, clippy::result_large_err)] #![allow(clippy::case_sensitive_file_extension_comparisons)] #![allow(clippy::struct_field_names)] pub mod backfill; +pub mod corpus; pub mod host_sim; mod orchestrator; pub mod score; @@ -28,8 +28,10 @@ pub use challenge_common::{ GatewayClient, GatewayClientConfig, LeafEmitError, }; pub use design_challenge_task::{ - agent_run_timeout_secs, prompts_per_round, round_id_at, round_secs, CHALLENGE_ID, - CHALLENGE_ID_BYTES, DAILY_RUN_QUOTA, PROMPTS_PER_ROUND, ROUND_SECS, SCORE_MAX, SCORING_VERSION, + agent_run_timeout_secs, daily_run_quota, manual_daily_run_quota, prompts_per_round, + round_id_at, round_secs, rounds_per_day_effective, scheduled_daily_run_cap, + scheduled_runs_per_day, CHALLENGE_ID, CHALLENGE_ID_BYTES, MANUAL_DAILY_RUN_QUOTA, + PROMPTS_PER_ROUND, ROUNDS_PER_DAY, ROUND_SECS, SCORE_MAX, SCORING_VERSION, SCORING_WINDOW_ROUNDS, }; pub use design_http::{ diff --git a/crates/design-challenge/src/orchestrator.rs b/crates/design-challenge/src/orchestrator.rs index c3aaa58ed..b174caf1a 100644 --- a/crates/design-challenge/src/orchestrator.rs +++ b/crates/design-challenge/src/orchestrator.rs @@ -8,8 +8,7 @@ use std::time::Duration; use async_trait::async_trait; use chain::ChainClient; use challenge_agentic::{ - copy_gate, AgenticBackend, AgenticError, AgenticVerdict, CorpusEntry, GateCorpusEntry, - ReviewRequest, VerdictKind, + copy_gate, AgenticBackend, AgenticError, AgenticVerdict, ReviewRequest, VerdictKind, }; use challenge_common::{ emit_signed_leaf_set, expected_set_at_chain, submit_signed_leaf_set, ExpectedSet, @@ -22,13 +21,14 @@ use design_prompts::{prompt_set_digest, select_prompts_for_round}; use design_sandbox::{SandboxBackend, SandboxError}; use design_sanitize::sanitize_bundle; use design_store::{ - DesignStore, FinalScore, RatingRow, RoundRow, RunStage, StageEvent, StorePatch, + DesignStore, FinalScore, RatingRow, RoundRow, RunOrigin, RunStage, StageEvent, StorePatch, }; use serde_json::json; use submission_gating::{GatingState, GatingStore}; use tokio::time::sleep; use tracing::{info, warn}; +use crate::corpus; use crate::score::{not_attempted, score_window, to_leaf, window_start, WindowScorePlan}; use crate::screenshot::{capture_full_page_png, png_artifact_tuple}; use crate::CHALLENGE_ID; @@ -277,8 +277,15 @@ impl Orchestrator { .map(|s| chain::current_epoch_pre_run_coinbase(&s, s.current_block)) .unwrap_or(0); for h in harnesses { - match schedule_harness_for_round(self.store.as_ref(), &h, rid, self.cfg.netuid, epoch) - .await + match schedule_harness_for_round( + self.store.as_ref(), + &h, + rid, + self.cfg.netuid, + epoch, + RunOrigin::Scheduled, + ) + .await { Ok(ids) if !ids.is_empty() => { info!( @@ -291,7 +298,7 @@ impl Orchestrator { } Ok(_) => {} Err(e) => { - // Quota / elimination are expected; do not abort the loop. + // Elimination cooldown is expected; do not abort the loop. warn!( harness_id = %h.id, miner = %h.miner_hotkey, @@ -789,11 +796,13 @@ impl Orchestrator { .map_err(|e| RunFailure::new(ErrorClass::AstInfra, e.to_string()))?; let report = serde_json::to_value(&sanitized.report).unwrap_or_default(); - // Pre-LLM copy gate: byte/AST copy of an *earlier* harness → terminal - // `rejected` without spending the LLM review. - let gate_corpus = self - .gate_corpus(&run.harness_id, &harness.miner_hotkey) - .await; + // Pre-LLM copy gate → terminal `rejected` (one fetch for gate + review). + let recent = self + .store + .list_recent_harnesses(64) + .await + .map_err(|e| RunFailure::new(ErrorClass::AstInfra, e.to_string()))?; + let gate_corpus = corpus::gate_corpus(&harness, &recent); if let Some(hit) = copy_gate(&harness.agent_py, harness.created_at_ms, &gate_corpus) { warn!( run_id = %run.id, @@ -870,7 +879,7 @@ impl Orchestrator { self.pause_stage().await; let verdict = self - .run_agentic_review(run, &harness.agent_py, &pages, &report) + .run_agentic_review(run, &harness, &recent, &pages, &report) .await?; let verdict_json = serde_json::to_value(&verdict).unwrap_or_default(); match verdict.verdict { @@ -924,33 +933,11 @@ impl Orchestrator { Ok(()) } - /// Corpus for the pre-LLM copy gate (recent harnesses minus the candidate - /// and any prior revisions from the same miner hotkey). - async fn gate_corpus( - &self, - exclude_harness_id: &str, - exclude_miner_hotkey: &str, - ) -> Vec { - let miner = exclude_miner_hotkey.to_ascii_lowercase(); - self.store - .list_recent_harnesses(64) - .await - .unwrap_or_default() - .into_iter() - .filter(|h| h.id != exclude_harness_id) - .filter(|h| h.miner_hotkey.to_ascii_lowercase() != miner) - .map(|h| GateCorpusEntry { - id: format!("harness:{}", h.id), - source: h.agent_py, - created_at_ms: h.created_at_ms, - }) - .collect() - } - async fn run_agentic_review( &self, run: &design_store::RunState, - agent_py: &str, + harness: &design_store::HarnessRow, + recent: &[design_store::HarnessRow], pages: &[(String, String, String, String, u32)], sanitize_report: &serde_json::Value, ) -> Result { @@ -958,7 +945,7 @@ impl Orchestrator { let _ = std::fs::remove_dir_all(&work); std::fs::create_dir_all(work.join("pages")) .map_err(|e| RunFailure::new(ErrorClass::AstInfra, e.to_string()))?; - std::fs::write(work.join("agent.py"), agent_py) + std::fs::write(work.join("agent.py"), &harness.agent_py) .map_err(|e| RunFailure::new(ErrorClass::AstInfra, e.to_string()))?; for (path, sanitized, _, _, _) in pages { let name = path.rsplit('/').next().unwrap_or(path.as_str()); @@ -971,47 +958,10 @@ impl Orchestrator { ) .map_err(|e| RunFailure::new(ErrorClass::AstInfra, e.to_string()))?; - let recent = self - .store - .list_recent_harnesses(64) - .await - .map_err(|e| RunFailure::new(ErrorClass::AstInfra, e.to_string()))?; - let cand = recent.iter().find(|h| h.id == run.harness_id); - let cand_created = cand.map(|h| h.created_at_ms).unwrap_or(0); - let cand_miner = cand - .map(|h| h.miner_hotkey.to_ascii_lowercase()) - .unwrap_or_default(); - let mut corpus: Vec = vec![CorpusEntry { - // The published baseline is always in the corpus (same as prism): - // it anchors originality judgments and keeps an empty recent-set - // from producing incoherent verdicts. - id: "baseline".into(), - source: include_str!("../../../docs/external-miner/examples/design-baseline/agent.py") - .to_owned(), - }]; - corpus.extend( - recent - .into_iter() - .filter(|h| h.id != run.harness_id) - // Same-hotkey revisions are self-improvement, not copying. - .filter(|h| { - cand_miner.is_empty() || h.miner_hotkey.to_ascii_lowercase() != cand_miner - }) - // Prior art only (created_at ordered like the pre-LLM gate): a - // later byte-copy must never poison the original's review. - .filter(|h| { - cand_created == 0 || (h.created_at_ms > 0 && h.created_at_ms < cand_created) - }) - .map(|h| CorpusEntry { - id: format!("harness:{}", h.id), - source: h.agent_py, - }), - ); - let req = ReviewRequest { workdir: work.clone(), primary_relpaths: vec!["agent.py".into()], - corpus, + corpus: corpus::review_corpus(harness, recent), metrics_relpath: None, pages_relpath: Some("pages".into()), sanitize_report_relpath: Some("sanitize_report.json".into()), diff --git a/crates/design-challenge/tests/cheat_fixtures.rs b/crates/design-challenge/tests/cheat_fixtures.rs index 765497571..9d2a447f2 100644 --- a/crates/design-challenge/tests/cheat_fixtures.rs +++ b/crates/design-challenge/tests/cheat_fixtures.rs @@ -13,12 +13,13 @@ use challenge_agentic::{ copy_gate, AgenticBackend, CheatCode, CorpusEntry, GateCorpusEntry, ReviewRequest, SimAgent, VerdictKind, }; +use design_challenge::corpus; use design_challenge::score::{round_win_delta, score_window, ScorePlan, WindowScorePlan}; use design_challenge::{host_sim_allowed, require_host_sim_for_force, SCORE_MAX}; use design_harness::{validate_bundle, HarnessBundle}; use design_sandbox::{SandboxBackend, SimSandbox}; use design_sanitize::sanitize_bundle; -use design_store::FinalScore; +use design_store::{FinalScore, HarnessRow}; use tempfile::tempdir; const BASELINE_AGENT: &str = @@ -74,6 +75,90 @@ fn window_plan(wins: &[(&str, u32)], participants: &[&str], cheat: &[&str]) -> W } } +fn harness_row(id: &str, miner: &str, agent_py: &str, created_at_ms: u64) -> HarnessRow { + HarnessRow { + id: id.into(), + miner_hotkey: miner.into(), + agent_py: agent_py.into(), + pyproject_toml: BASELINE_PYPROJECT.into(), + extra_files: BTreeMap::new(), + active: true, + eliminated_until_round: 0, + created_at_ms, + } +} + +/// A miner republishing their own harness must pass both anti-cheat stages, +/// while the identical bytes coming from a *different* hotkey are rejected. +/// Same inputs, same corpus builder — only the owning hotkey differs. +#[tokio::test] +async fn self_revision_is_clean_but_cross_hotkey_copy_is_a_copy() { + let miner_a = "aa".repeat(32); + let miner_b = "bb".repeat(32); + let original = harness_row("h-orig", &miner_a, BASELINE_AGENT, 1_000); + + // Same hotkey, later revision of its own (here byte-identical) harness. + let own_revision = harness_row("h-rev", &miner_a, BASELINE_AGENT, 2_000); + // Different hotkey, same bytes: a copy of someone else's prior art. + let foreign_copy = harness_row("h-copy", &miner_b, BASELINE_AGENT, 2_000); + + let recent = vec![foreign_copy.clone(), own_revision.clone(), original.clone()]; + + // Pre-LLM gate: the corpus for a self-revision has no victim to hit. + let own_gate = corpus::gate_corpus(&own_revision, &recent); + assert!( + copy_gate( + &own_revision.agent_py, + own_revision.created_at_ms, + &own_gate + ) + .is_none(), + "a miner's own earlier harness must never trip the copy gate" + ); + let foreign_gate = corpus::gate_corpus(&foreign_copy, &recent); + let hit = copy_gate( + &foreign_copy.agent_py, + foreign_copy.created_at_ms, + &foreign_gate, + ) + .expect("cross-hotkey byte copy must still be rejected"); + assert_eq!(hit.nearest_id, "harness:h-orig"); + assert!(hit.byte_identical); + + // LLM review: same asymmetry in the corpus handed to the reviewer. + let pages: &[(&str, &str)] = &[("index.html", "")]; + let own_dir = tempdir().unwrap(); + let own_verdict = SimAgent::new() + .review(&review_req( + own_dir.path(), + &own_revision.agent_py, + corpus::review_corpus(&own_revision, &recent), + Some(pages), + )) + .await + .unwrap(); + assert_eq!( + own_verdict.verdict, + VerdictKind::Clean, + "iterating on your own harness must not read as copying: {own_verdict:?}" + ); + + let copy_dir = tempdir().unwrap(); + let copy_verdict = SimAgent::new() + .review(&review_req( + copy_dir.path(), + &foreign_copy.agent_py, + corpus::review_corpus(&foreign_copy, &recent), + Some(pages), + )) + .await + .unwrap(); + assert_eq!(copy_verdict.verdict, VerdictKind::Cheat); + assert!(copy_verdict + .cheat_codes + .contains(&CheatCode::NearIdenticalHarnessCopy)); +} + #[tokio::test] async fn byte_level_harness_copy_is_cheat_score_zero() { let dir = tempdir().unwrap(); diff --git a/crates/design-db/src/lib.rs b/crates/design-db/src/lib.rs index 31df2266b..87dad3578 100644 --- a/crates/design-db/src/lib.rs +++ b/crates/design-db/src/lib.rs @@ -887,7 +887,7 @@ pub async fn design_scores_for_epoch( Ok(rows) } -/// Get / bump daily quota. Returns `runs_used` after bump (0 bump = read). +/// Bump daily quota. Returns `(runs_used, manual_runs_used)` after the write. /// /// # Errors /// SQL error. @@ -896,18 +896,24 @@ pub async fn design_quota_bump( miner_hotkey: &str, day: &str, bump: i32, -) -> Result { - let row: (i32,) = sqlx::query_as( - "INSERT INTO design_quota (miner_hotkey, day, runs_used) VALUES ($1, $2::date, $3) \ - ON CONFLICT (miner_hotkey, day) DO UPDATE SET runs_used = design_quota.runs_used + $3 \ - RETURNING runs_used", + manual: bool, +) -> Result<(i32, i32), DbError> { + let manual_bump = if manual { bump } else { 0 }; + let row: (i32, i32) = sqlx::query_as( + "INSERT INTO design_quota (miner_hotkey, day, runs_used, manual_runs_used) \ + VALUES ($1, $2::date, $3, $4) \ + ON CONFLICT (miner_hotkey, day) DO UPDATE SET \ + runs_used = design_quota.runs_used + $3, \ + manual_runs_used = design_quota.manual_runs_used + $4 \ + RETURNING runs_used, manual_runs_used", ) .bind(miner_hotkey) .bind(day) .bind(bump) + .bind(manual_bump) .fetch_one(pool) .await?; - Ok(row.0) + Ok(row) } /// Read quota without bump. @@ -918,15 +924,16 @@ pub async fn design_quota_get( pool: &PgPool, miner_hotkey: &str, day: &str, -) -> Result { - let n: Option = sqlx::query_scalar( - "SELECT runs_used FROM design_quota WHERE miner_hotkey = $1 AND day = $2::date", +) -> Result<(i32, i32), DbError> { + let row: Option<(i32, i32)> = sqlx::query_as( + "SELECT runs_used, manual_runs_used FROM design_quota \ + WHERE miner_hotkey = $1 AND day = $2::date", ) .bind(miner_hotkey) .bind(day) .fetch_optional(pool) .await?; - Ok(n.unwrap_or(0)) + Ok(row.unwrap_or((0, 0))) } /// Append stage event. diff --git a/crates/design-http/src/api.rs b/crates/design-http/src/api.rs index b7020e82c..bf4f6e8e4 100644 --- a/crates/design-http/src/api.rs +++ b/crates/design-http/src/api.rs @@ -16,7 +16,8 @@ use design_harness::{ }; use design_prompts::{load_prompt_set, prompt_set_digest, select_prompts_for_round}; use design_store::{ - DesignStore, HarnessRow, RoundAward, RunStage, RunState, StageEvent, StoreError, StorePatch, + DesignStore, HarnessRow, RoundAward, RunOrigin, RunStage, RunState, StageEvent, StoreError, + StorePatch, }; use serde::Deserialize; use serde_json::{json, Value}; @@ -24,7 +25,8 @@ use sha2::{Digest, Sha256}; use submission_gating::{GatingState, GatingStore, MetagraphCache}; use design_challenge_task::{ - round_id_at, round_secs, CHALLENGE_ID, DAILY_RUN_QUOTA, MIN_ANNOTATIONS_PER_PAIR, RUN_ID_DOMAIN, + manual_daily_run_quota, round_id_at, round_secs, scheduled_daily_run_cap, CHALLENGE_ID, + MIN_ANNOTATIONS_PER_PAIR, RUN_ID_DOMAIN, }; /// Hook invoked after admin persists winners (score + leaf emit). @@ -396,6 +398,7 @@ async fn post_harness( rid, st.netuid, epoch, + RunOrigin::Manual, ) .await { @@ -441,14 +444,21 @@ async fn post_harness( /// Schedule an active harness into `rid` (create missing queued runs). /// /// Idempotent: if runs for `(harness, round)` already exist, returns those ids. -/// Honours daily quota and elimination cooldown. Used by submit (next round) -/// and by the orchestrator (every open round). +/// Honours the elimination cooldown and the daily ceiling **for `origin`**. +/// Used by submit ([`RunOrigin::Manual`], next round) and by the orchestrator +/// ([`RunOrigin::Scheduled`], every open round). +/// +/// The two origins never share a budget: the organizer dispatches +/// `rounds/day × prompts/round` runs to every registered harness, so charging +/// that volume to the miner's anti-spam quota would lock an honest harness out +/// of most of its own day. pub async fn schedule_harness_for_round( store: &dyn DesignStore, harness: &HarnessRow, rid: u64, netuid: u16, epoch: u64, + origin: RunOrigin, ) -> Result, String> { if !harness.active { return Ok(vec![]); @@ -473,7 +483,8 @@ pub async fn schedule_harness_for_round( let used = store .quota_get(&harness.miner_hotkey, &day) .await - .map_err(|e| e.to_string())?; + .map_err(|e| e.to_string())? + .used(origin); if store .get_round(rid) .await @@ -496,12 +507,18 @@ pub async fn schedule_harness_for_round( } let prompts = select_prompts_for_round(rid).map_err(|e| e.to_string())?; let mut run_ids: Vec = Vec::new(); - if used >= DAILY_RUN_QUOTA { - return Err("daily quota exceeded".into()); + let cap = origin_daily_cap(origin); + // All-or-nothing: a round that can only afford part of its prompt set would + // leave the harness permanently short for that round (re-scheduling is a + // no-op once any run exists), so refuse instead of degrading it. + let needed = u32::try_from(prompts.len()).unwrap_or(u32::MAX); + if used.saturating_add(needed) > cap { + return Err(format!( + "daily {} run quota exceeded ({used}+{needed}/{cap})", + origin.as_str() + )); } - let remaining = DAILY_RUN_QUOTA.saturating_sub(used); - let n = (prompts.len() as u32).min(remaining); - for p in prompts.into_iter().take(n as usize) { + for p in prompts { let run_id = make_run_id(rid, &harness.id, &p.id); if store .get_run(&run_id) @@ -539,12 +556,22 @@ pub async fn schedule_harness_for_round( }), ) .await; - let _ = store.quota_bump(&harness.miner_hotkey, &day, 1).await; + let _ = store + .quota_bump(&harness.miner_hotkey, &day, 1, origin) + .await; run_ids.push(run_id); } Ok(run_ids) } +/// Daily ceiling that applies to `origin`. +fn origin_daily_cap(origin: RunOrigin) -> u32 { + match origin { + RunOrigin::Manual => manual_daily_run_quota(), + RunOrigin::Scheduled => scheduled_daily_run_cap(), + } +} + fn make_run_id(round_id: u64, harness_id: &str, prompt_id: &str) -> String { let mut h = Sha256::new(); h.update(RUN_ID_DOMAIN); @@ -593,17 +620,36 @@ async fn list_harness(State(st): State>, Query(q): Query>, Path(hotkey): Path) -> Response { let day = utc_day(now_secs()); match st.store.quota_get(&hotkey, &day).await { - Ok(used) => Json(json!({ - "miner_hotkey": hotkey, - "day": day, - "runs_used": used, - "limit": DAILY_RUN_QUOTA, - })) - .into_response(), + Ok(used) => Json(quota_json(&hotkey, &day, used)).into_response(), Err(e) => json_err(StatusCode::INTERNAL_SERVER_ERROR, "store", &e.to_string()), } } +/// Per-origin quota view shared by `/v1/quota/{hotkey}` and `/v1/miners`. +/// +/// `runs_used` / `limit` stay the whole-day totals for existing clients; +/// enforcement is the per-origin pair below them. +pub(crate) fn quota_json(hotkey: &str, day: &str, used: design_store::QuotaUsage) -> Value { + let manual_limit = manual_daily_run_quota(); + let scheduled_limit = scheduled_daily_run_cap(); + json!({ + "miner_hotkey": hotkey, + "day": day, + "runs_used": used.total, + "limit": manual_limit.saturating_add(scheduled_limit), + "manual": { + "runs_used": used.manual, + "limit": manual_limit, + "remaining": manual_limit.saturating_sub(used.manual), + }, + "scheduled": { + "runs_used": used.scheduled(), + "limit": scheduled_limit, + "remaining": scheduled_limit.saturating_sub(used.scheduled()), + }, + }) +} + async fn get_prompts() -> Response { match load_prompt_set() { Ok(set) => Json(json!({ @@ -943,9 +989,10 @@ struct WinnersBody { /// Operator escape hatch when a round opened with no/few runs (e.g. challenge /// restart). Idempotent for the current round: `schedule_harness_for_round` /// returns the existing run ids for a `(harness, round)` pair that already has -/// runs, so a repeated call creates nothing and consumes no quota. Harnesses -/// that fail scheduling (daily quota) are reported under `skipped`; one bad -/// harness never blocks the rest. +/// runs, so a repeated call creates nothing and consumes no quota. This is +/// organizer work, so it draws on the scheduled cap, never on the miner's +/// submission quota. Harnesses that fail scheduling are reported under +/// `skipped`; one bad harness never blocks the rest. async fn admin_requeue_current(State(st): State>, headers: HeaderMap) -> Response { if let Err(r) = check_admin(&st, &headers) { return r; @@ -959,7 +1006,16 @@ async fn admin_requeue_current(State(st): State>, headers: HeaderM let mut scheduled = Vec::new(); let mut skipped = Vec::new(); for harness in &harnesses { - match schedule_harness_for_round(st.store.as_ref(), harness, rid, st.netuid, epoch).await { + match schedule_harness_for_round( + st.store.as_ref(), + harness, + rid, + st.netuid, + epoch, + RunOrigin::Scheduled, + ) + .await + { Ok(run_ids) => scheduled.push(json!({ "harness_id": harness.id, "miner_hotkey": harness.miner_hotkey, @@ -1156,6 +1212,115 @@ mod tests { .await } + async fn schedule( + store: &dyn DesignStore, + harness: &HarnessRow, + rid: u64, + origin: RunOrigin, + ) -> Result, String> { + schedule_harness_for_round(store, harness, rid, 100, 0, origin).await + } + + fn harness_row(hotkey: &str, marker: &str) -> HarnessRow { + HarnessRow { + id: format!("harness-{marker}"), + miner_hotkey: hotkey.to_owned(), + agent_py: format!("def run(task, llm, out):\n pass # {marker}\n"), + pyproject_toml: "[project]\nname='x'\nversion='0.1.0'\n".into(), + extra_files: BTreeMap::new(), + active: true, + eliminated_until_round: 0, + created_at_ms: 0, + } + } + + /// A harness that participates in every round of a UTC day must never be + /// quota-blocked: the organizer dispatches `rounds/day × prompts/round` + /// runs, which used to overrun a shared 10-run/day ceiling after ~3 rounds. + #[tokio::test] + async fn full_day_of_scheduled_rounds_is_never_quota_blocked() { + let (st, _g) = app_state(None); + let harness = harness_row(&hk(0xAA), "day"); + st.store.insert_harness(&harness).await.unwrap(); + + let per_round = design_challenge_task::prompts_per_round(); + let rounds = design_challenge_task::rounds_per_day_effective(); + let base = round_id_at(now_secs()); + let mut total = 0usize; + for i in 0..rounds { + let ids = schedule(st.store.as_ref(), &harness, base + i, RunOrigin::Scheduled) + .await + .unwrap_or_else(|e| panic!("round {i} of {rounds} refused: {e}")); + assert_eq!(ids.len(), per_round, "round {i} scheduled a partial set"); + total += ids.len(); + } + assert_eq!( + total, + usize::try_from(rounds).unwrap() * per_round, + "every round of the day must run all its prompts" + ); + + let used = st + .store + .quota_get(&hk(0xAA), &utc_day(now_secs())) + .await + .unwrap(); + assert_eq!(usize::try_from(used.total).unwrap(), total); + assert_eq!(used.manual, 0, "organizer work is not miner spend"); + assert!(used.scheduled() <= scheduled_daily_run_cap()); + } + + /// The miner-facing intake keeps a hard anti-spam ceiling, and burning it + /// leaves organizer-scheduled rounds untouched. + #[tokio::test] + async fn manual_submissions_stay_rate_limited() { + let (st, _g) = app_state(None); + let per_round = u32::try_from(design_challenge_task::prompts_per_round()).unwrap(); + let cap = manual_daily_run_quota(); + let base = round_id_at(now_secs()); + + let mut manual_runs = 0u32; + let mut blocked = None; + for i in 0..(cap / per_round + 2) { + // A fresh digest per attempt: the worst case for intake spam. + let harness = harness_row(&hk(0xAA), &format!("spam{i}")); + st.store.insert_harness(&harness).await.unwrap(); + match schedule( + st.store.as_ref(), + &harness, + base + u64::from(i), + RunOrigin::Manual, + ) + .await + { + Ok(ids) => manual_runs += u32::try_from(ids.len()).unwrap(), + Err(e) => { + blocked = Some(e); + break; + } + } + } + let err = blocked.expect("manual submissions must hit the anti-spam ceiling"); + assert!(err.contains("manual"), "{err}"); + assert!( + manual_runs <= cap, + "{manual_runs} manual runs exceeded {cap}" + ); + + // The spent manual budget must not touch the organizer's schedule. + let harness = harness_row(&hk(0xAA), "day"); + st.store.insert_harness(&harness).await.unwrap(); + let ids = schedule( + st.store.as_ref(), + &harness, + base + 100, + RunOrigin::Scheduled, + ) + .await + .expect("scheduled rounds must survive an exhausted manual quota"); + assert_eq!(ids.len(), design_challenge_task::prompts_per_round()); + } + #[tokio::test] async fn accepted_runs_target_next_round() { let (st, _g) = app_state(None); @@ -1224,11 +1389,11 @@ mod tests { // Memory store has no delete — schedule a *later* round via the public helper. let later = rid + 1; let harness = st.store.get_harness(&harness_id).await.unwrap().unwrap(); - let ids = schedule_harness_for_round(st.store.as_ref(), &harness, later, 100, 0) + let ids = schedule(st.store.as_ref(), &harness, later, RunOrigin::Scheduled) .await .unwrap(); assert_eq!(ids.len(), design_challenge_task::prompts_per_round()); - let again = schedule_harness_for_round(st.store.as_ref(), &harness, later, 100, 0) + let again = schedule(st.store.as_ref(), &harness, later, RunOrigin::Scheduled) .await .unwrap(); assert_eq!(again, ids, "idempotent for the same round"); @@ -1415,10 +1580,15 @@ mod tests { let day = utc_day(now_secs()); let used = st.store.quota_get(&hk(0xAA), &day).await.unwrap(); assert_eq!( - usize::try_from(used).unwrap(), + usize::try_from(used.total).unwrap(), 2 * design_challenge_task::prompts_per_round(), "next-round + current-round schedule only" ); + assert_eq!( + usize::try_from(used.manual).unwrap(), + design_challenge_task::prompts_per_round(), + "only the miner's own submission is charged to the manual quota" + ); } #[tokio::test] diff --git a/crates/design-http/src/stats.rs b/crates/design-http/src/stats.rs index 72105e7fb..2024cb948 100644 --- a/crates/design-http/src/stats.rs +++ b/crates/design-http/src/stats.rs @@ -7,7 +7,10 @@ use axum::extract::{Path, State}; use axum::http::StatusCode; use axum::response::{IntoResponse, Response}; use axum::Json; -use design_challenge_task::{round_id_at, round_secs, CHALLENGE_ID, DAILY_RUN_QUOTA}; +use design_challenge_task::{ + daily_run_quota, manual_daily_run_quota, round_id_at, round_secs, scheduled_daily_run_cap, + CHALLENGE_ID, +}; use design_prompts::prompt_set_digest; use design_store::{DesignStore, RunStage}; use serde_json::{json, Value}; @@ -92,7 +95,9 @@ pub async fn get_stats(State(st): State>) -> Response { "ratings_count": ratings.len(), "elimination_signal_count": elim, "agents": st.store.count_harness_miners().await.unwrap_or(0), - "daily_run_quota": DAILY_RUN_QUOTA, + "daily_run_quota": daily_run_quota(), + "manual_daily_run_quota": manual_daily_run_quota(), + "scheduled_daily_run_cap": scheduled_daily_run_cap(), })) .into_response() } @@ -172,7 +177,9 @@ pub async fn get_dashboard(State(st): State>) -> Response { "previous_ratings": lb(&prev), }, "recent_runs": jobs, - "daily_run_quota": DAILY_RUN_QUOTA, + "daily_run_quota": daily_run_quota(), + "manual_daily_run_quota": manual_daily_run_quota(), + "scheduled_daily_run_cap": scheduled_daily_run_cap(), "poll_hint_ms": 1000, })) .into_response() @@ -188,7 +195,7 @@ pub async fn get_miner(State(st): State>, Path(hotkey): Path h, Err(e) => return json_err(StatusCode::INTERNAL_SERVER_ERROR, "store", &e.to_string()), }; - let used = st.store.quota_get(&hk, &day).await.unwrap_or(0); + let used = st.store.quota_get(&hk, &day).await.unwrap_or_default(); // All recent generations for this miner (not only the current round) so // public agent history can list every design run the harness produced. let hid: std::collections::HashSet<_> = harnesses.iter().map(|h| h.id.clone()).collect(); @@ -225,12 +232,7 @@ pub async fn get_miner(State(st): State>, Path(hotkey): Path Result { - Ok(u32::try_from( + async fn quota_get(&self, miner: &str, day: &str) -> Result { + Ok(usage_from( dbs::design_quota_get(&self.pool, miner, day) .await .map_err(map_db)?, - ) - .unwrap_or(0)) + )) } - async fn quota_bump(&self, miner: &str, day: &str, bump: u32) -> Result { - Ok(u32::try_from( - dbs::design_quota_bump(&self.pool, miner, day, i32::try_from(bump).unwrap_or(0)) - .await - .map_err(map_db)?, - ) - .unwrap_or(0)) + async fn quota_bump( + &self, + miner: &str, + day: &str, + bump: u32, + origin: RunOrigin, + ) -> Result { + Ok(usage_from( + dbs::design_quota_bump( + &self.pool, + miner, + day, + i32::try_from(bump).unwrap_or(0), + origin == RunOrigin::Manual, + ) + .await + .map_err(map_db)?, + )) + } +} + +fn usage_from((total, manual): (i32, i32)) -> QuotaUsage { + QuotaUsage { + total: u32::try_from(total).unwrap_or(0), + manual: u32::try_from(manual).unwrap_or(0), } } diff --git a/crates/design-store/src/lib.rs b/crates/design-store/src/lib.rs index 79625e96f..62a1f860d 100644 --- a/crates/design-store/src/lib.rs +++ b/crates/design-store/src/lib.rs @@ -14,6 +14,7 @@ mod store; pub use dbstore::DbDesignStore; pub use store::{ - ArtifactPage, DesignStore, FinalScore, HarnessRow, MemoryDesignStore, PairRow, RatingRow, - RoundAward, RoundRow, RunStage, RunState, StageEvent, StoreError, StorePatch, + ArtifactPage, DesignStore, FinalScore, HarnessRow, MemoryDesignStore, PairRow, QuotaUsage, + RatingRow, RoundAward, RoundRow, RunOrigin, RunStage, RunState, StageEvent, StoreError, + StorePatch, }; diff --git a/crates/design-store/src/store.rs b/crates/design-store/src/store.rs index aeabe6d73..bc5a64e71 100644 --- a/crates/design-store/src/store.rs +++ b/crates/design-store/src/store.rs @@ -17,6 +17,56 @@ pub enum FinalScore { NoScore(u8), } +/// Who asked for a run to exist. +/// +/// The two origins draw on separate daily ceilings: a miner cannot spam the +/// intake, and the organizer's own round schedule can never exhaust the +/// miner's submission budget. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RunOrigin { + /// Created by the miner's `POST /v1/harness`. + Manual, + /// Created by the organizer's round scheduler (or an operator requeue). + Scheduled, +} + +impl RunOrigin { + /// Label for errors / logs. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Manual => "manual", + Self::Scheduled => "scheduled", + } + } +} + +/// Daily sandbox-run counters for one `(miner hotkey, UTC day)`. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct QuotaUsage { + /// Runs created across both origins. + pub total: u32, + /// Subset created by the miner's own submissions. + pub manual: u32, +} + +impl QuotaUsage { + /// Runs the organizer's scheduler created. + #[must_use] + pub const fn scheduled(self) -> u32 { + self.total.saturating_sub(self.manual) + } + + /// Runs already charged to `origin`. + #[must_use] + pub const fn used(self, origin: RunOrigin) -> u32 { + match origin { + RunOrigin::Manual => self.manual, + RunOrigin::Scheduled => self.scheduled(), + } + } +} + /// Run lifecycle (DB CHECK mirrors this list). #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] @@ -394,10 +444,16 @@ pub trait DesignStore: Send + Sync + std::fmt::Debug { to_round: u64, ) -> Result, StoreError>; - /// Quota get. - async fn quota_get(&self, miner: &str, day: &str) -> Result; - /// Quota bump; returns new total. - async fn quota_bump(&self, miner: &str, day: &str, bump: u32) -> Result; + /// Daily run counters for `(miner, UTC day)`. + async fn quota_get(&self, miner: &str, day: &str) -> Result; + /// Charge `bump` runs of `origin`; returns the new counters. + async fn quota_bump( + &self, + miner: &str, + day: &str, + bump: u32, + origin: RunOrigin, + ) -> Result; } /// In-memory store. @@ -414,7 +470,7 @@ pub struct MemoryDesignStore { annotations: Mutex>, ratings: Mutex>, awards: Mutex>, - quota: Mutex>, + quota: Mutex>, } impl MemoryDesignStore { @@ -1003,22 +1059,34 @@ impl DesignStore for MemoryDesignStore { Ok(out) } - async fn quota_get(&self, miner: &str, day: &str) -> Result { - Ok(*self + async fn quota_get(&self, miner: &str, day: &str) -> Result { + Ok(self .quota .lock() .map_err(|_| StoreError::Backend("poison".into()))? .get(&(miner.to_owned(), day.to_owned())) - .unwrap_or(&0)) + .copied() + .unwrap_or_default()) } - async fn quota_bump(&self, miner: &str, day: &str, bump: u32) -> Result { + async fn quota_bump( + &self, + miner: &str, + day: &str, + bump: u32, + origin: RunOrigin, + ) -> Result { let mut m = self .quota .lock() .map_err(|_| StoreError::Backend("poison".into()))?; - let e = m.entry((miner.to_owned(), day.to_owned())).or_insert(0); - *e = e.saturating_add(bump); + let e = m + .entry((miner.to_owned(), day.to_owned())) + .or_insert_with(QuotaUsage::default); + e.total = e.total.saturating_add(bump); + if origin == RunOrigin::Manual { + e.manual = e.manual.saturating_add(bump); + } Ok(*e) } } @@ -1055,6 +1123,34 @@ mod tests { assert!(s.claim_next_run(1).await.unwrap().is_none()); let future = s.claim_next_run(5).await.unwrap().unwrap(); assert_eq!(future.id, "r2"); - assert_eq!(s.quota_bump("aa", "2026-08-04", 1).await.unwrap(), 1); + } + + #[tokio::test] + async fn quota_counts_origins_separately() { + let s = MemoryDesignStore::new(); + let day = "2026-08-04"; + assert_eq!(s.quota_get("aa", day).await.unwrap(), QuotaUsage::default()); + + let after_manual = s.quota_bump("aa", day, 1, RunOrigin::Manual).await.unwrap(); + assert_eq!(after_manual.total, 1); + assert_eq!(after_manual.manual, 1); + assert_eq!(after_manual.scheduled(), 0); + + let after_sched = s + .quota_bump("aa", day, 3, RunOrigin::Scheduled) + .await + .unwrap(); + assert_eq!(after_sched.total, 4); + assert_eq!(after_sched.manual, 1, "scheduled work is not miner spend"); + assert_eq!(after_sched.scheduled(), 3); + assert_eq!(after_sched.used(RunOrigin::Manual), 1); + assert_eq!(after_sched.used(RunOrigin::Scheduled), 3); + + // Buckets are per (hotkey, day). + assert_eq!(s.quota_get("bb", day).await.unwrap(), QuotaUsage::default()); + assert_eq!( + s.quota_get("aa", "2026-08-05").await.unwrap(), + QuotaUsage::default() + ); } } diff --git a/docs/DESIGN_CHALLENGE.md b/docs/DESIGN_CHALLENGE.md index b571630bc..22c136395 100644 --- a/docs/DESIGN_CHALLENGE.md +++ b/docs/DESIGN_CHALLENGE.md @@ -317,7 +317,18 @@ docker compose -f docker-compose.yml -f deploy/compose/role-master.yml \ deterministic weighted draw `SHA256(domain || round_id || bank_digest)` → 3 prompts per round; identical for every harness in that round. -- **Quota**: **10** runs/day/hotkey (`DAILY_RUN_QUOTA = 10`; ~2–3 prompts/round × harness). +- **Quota (two buckets, per hotkey per UTC day)** — an honest harness that runs + every round must never be locked out, so organizer-scheduled work does not + draw on the miner's anti-spam allowance: + + | Bucket | Charged by | Cap | Override | + |--------|-----------|-----|----------| + | Manual | `POST /v1/harness` (miner-initiated) | `MANUAL_DAILY_RUN_QUOTA = 10` | `DESIGN_MANUAL_DAILY_RUN_QUOTA` | + | Scheduled | Round dispatch + `admin/rounds/current/requeue` | `rounds/day × prompts/round × SCHEDULED_DAILY_RUN_HEADROOM` (= **60** at 10 rounds × 3 prompts) | `DESIGN_SCHEDULED_DAILY_RUN_CAP` (clamped ≥ the day's own schedule) | + + Enforcement is **per bucket**: exhausting manual submissions never stops the + round scheduler, and the scheduled cap is a runaway-scheduler guard, not a + participation limit. `GET /v1/quota/{hotkey}` reports both. - **Auto-retry**: infra-class failures (`install` / `ast_infra` / `llm_infra`) requeue up to 3 times (`DESIGN_AUTO_RETRY_MAX`), then terminal `NoScore(ChallengeInternal)` + gating `blocked`. @@ -361,6 +372,14 @@ LLM review is **skipped**. Unknown timestamps (baseline, legacy rows) fall through to the LLM. Starting from the published miner **baseline** is never a cheat signal (baseline-zeroing fix); copying another *miner's* harness is. +**Corpus rule (both the gate and the LLM review):** the comparison corpus is +**other hotkeys' prior art only** — entries owned by the candidate's own +`miner_hotkey` are excluded, and so is anything created at or after the +candidate. A miner iterating on their own harness is therefore never scored +against their own previous version. Selection lives in one place, +[`crates/design-challenge/src/corpus.rs`](../crates/design-challenge/src/corpus.rs), +so the gate and the review can never disagree. + ### Allowed inspiration Internet + PyPI via egress, external APIs / MCP servers at run time, Mobbin / diff --git a/docs/DESIGN_CHALLENGE_CHECKLIST.md b/docs/DESIGN_CHALLENGE_CHECKLIST.md index 09751cf95..1972a6b80 100644 --- a/docs/DESIGN_CHALLENGE_CHECKLIST.md +++ b/docs/DESIGN_CHALLENGE_CHECKLIST.md @@ -18,7 +18,7 @@ pins without bumping `challenge_scoring_version`. | (S) | Sandbox hardening | ## 4. Sandbox hardening | `## 4. Sandbox hardening` | | (Z) | Sanitize rules | ## 5. Sanitize rules | `## 5. Sanitize rules` | | (V) | Viewer headers / CSP sandbox | ## 6. Viewer headers and CSP | `## 6. Viewer headers and CSP` | -| (R) | Rounds 8_640s (10/day) + bank_v1 auto + quota 10/day | ## 7. Rounds and quotas | `## 7. Rounds and quotas` | +| (R) | Rounds 8_640s (10/day) + bank_v1 auto + split manual/scheduled quota | ## 7. Rounds and quotas | `## 7. Rounds and quotas` | | (L) | Admin winners 1\|2 + rolling 10-round points share + AgenticReview | ## 8. Admin winners + agentic anti-cheat | `## 8. Admin winners + agentic anti-cheat` | | (X) | Elimination bottom 20% + 10-round cooldown | ## 9. Elimination | `## 9. Elimination` | | (D) | D24 exact-E participant set | ## 10. Declared participant set and `NoScore` reasons (D24) | `## 10. Declared participant set and` | @@ -44,7 +44,9 @@ pins without bumping `challenge_scoring_version`. | rounds_per_day | `10 rounds` | | agent_run_timeout | `AGENT_RUN_TIMEOUT_SECS = 1_800` | | scoring_window | `SCORING_WINDOW_ROUNDS = 10` | -| daily_quota | `DAILY_RUN_QUOTA = 10` | +| daily_quota | `MANUAL_DAILY_RUN_QUOTA = 10` | +| scheduled_quota | `DESIGN_SCHEDULED_DAILY_RUN_CAP` | +| selfsim_excluded | `other hotkeys' prior art only` | | prompts_per_round | `3 prompts` | | bank_v1 | `bank_v1.json` | | agent_py | `agent.py` | diff --git a/docs/external-miner/design.md b/docs/external-miner/design.md index 48484900d..44e5d0515 100644 --- a/docs/external-miner/design.md +++ b/docs/external-miner/design.md @@ -126,13 +126,20 @@ Minimal `harness.json` shape: - An accepted harness **waits for the next round**: runs are scheduled into `round_id + 1` and start when that round opens, never mid-round. - Sandbox **run** timeout is **30 minutes** (`AGENT_RUN_TIMEOUT_SECS = 1800`). -- **10** sandboxed runs per hotkey per UTC day. - Each round picks **3** prompts via deterministic weighted draw for all harnesses. +- Daily run quota is **split by origin**, so participating in every round can + never lock you out: + - **Manual** — **10** runs/day, charged only by your own `POST /v1/harness`. + This is anti-spam on resubmission, not a cap on participation. + - **Scheduled** — organizer round dispatch, capped well above the full day's + schedule (10 rounds × 3 prompts = **30** runs; cap **60**). You never spend + manual quota by being scheduled. - Infra failures (package install, review/LLM infra) **auto-retry up to 3 times**; cheat / rejected verdicts are terminal. Manual retry: `POST /v1/runs/{id}/retry`. -Check quota: `GET /v1/quota/{hotkey}`. +Check quota: `GET /v1/quota/{hotkey}` — `manual` and `scheduled` objects +(`runs_used` / `limit` / `remaining`) alongside the whole-day `runs_used`. ## Scoring (summary) @@ -140,7 +147,10 @@ After sanitize, master-side **agentic anti-cheat** runs in a containerized reviewer. A pre-LLM **copy gate** rejects a byte/AST copy of an *earlier* harness outright (`rejected`, `Score(0)`, no LLM call); `cheat` / `suspicious` from the LLM review → `Score(0)`. Starting from the published **baseline** is -fine — copying another *miner's* harness is not. +fine — copying another *miner's* harness is not. Both the copy gate and the LLM +review compare you against **other hotkeys' earlier harnesses only**: your own +previous versions are excluded from the corpus, so iterating on your own +harness is never read as self-copying. Clean runs await **admin winners** (1 or 2 harnesses per round); each round win is one **point**. Rewards are **not** winner-take-all on a single round: the diff --git a/docs/external-miner/troubleshoot.md b/docs/external-miner/troubleshoot.md index 4c14f768b..5c86efa73 100644 --- a/docs/external-miner/troubleshoot.md +++ b/docs/external-miner/troubleshoot.md @@ -9,7 +9,7 @@ | Symptom | Likely cause | What to check | |---------|--------------|---------------| | `400` on `POST /v1/harness` | Invalid bundle | `agent.py` defines `run`, `pyproject.toml` non-empty, size limits | -| Quota exhausted | Daily cap 10 | `GET /v1/quota/{hotkey}`; wait until next UTC day | +| `409 schedule` "daily manual run quota exceeded" | Manual anti-spam cap (10/day) — being scheduled into rounds does **not** spend it | `GET /v1/quota/{hotkey}` → `manual.remaining`; wait until next UTC day | | `auto_retry` events, class `install` | Dep won't install (bad name/version, heavy source build) | `GET /v1/runs/{id}/logs` phase `install`; fix `pyproject.toml` deps | | Run `failed` / Score 0 | Missing pages, timeout, crash | `GET /v1/runs/{id}/events`; ensure three required HTML pages | | External call refused (`403`) | Target is internal-blocklisted (metadata IP, loopback, RFC1918/VPC, control plane) | Call public endpoints only; egress is otherwise open | diff --git a/xtask/src/design_check.rs b/xtask/src/design_check.rs index b06e68033..f499c6185 100644 --- a/xtask/src/design_check.rs +++ b/xtask/src/design_check.rs @@ -37,7 +37,9 @@ const CONTENT_PINS: &[(&str, &str)] = &[ ("rounds_per_day", "10 rounds"), ("agent_run_timeout", "AGENT_RUN_TIMEOUT_SECS = 1_800"), ("scoring_window", "SCORING_WINDOW_ROUNDS = 10"), - ("daily_quota", "DAILY_RUN_QUOTA = 10"), + ("daily_quota", "MANUAL_DAILY_RUN_QUOTA = 10"), + ("scheduled_quota", "DESIGN_SCHEDULED_DAILY_RUN_CAP"), + ("selfsim_excluded", "other hotkeys' prior art only"), ("prompts_per_round", "3 prompts"), ("bank_v1", "bank_v1.json"), ("agent_py", "agent.py"),