diff --git a/bins/prism-challenge/src/main.rs b/bins/prism-challenge/src/main.rs index 13618ee31..a750d055a 100644 --- a/bins/prism-challenge/src/main.rs +++ b/bins/prism-challenge/src/main.rs @@ -428,7 +428,8 @@ async fn cmd_serve(cli: Cli) -> Result<(), String> { emit_poll: Duration::from_secs(15), max_attempts: MAX_ATTEMPTS, similarity_corpus_limit: 6, - stuck_grace_secs: 7 * 3600, + // > wait_running(15m) + train(6h) + ssh margin(~65m) ≈ 7h20m. + stuck_grace_secs: 10 * 3600, stage_delay, auto_retry_max: cli.auto_retry_max, }; diff --git a/crates/prism-challenge/src/orchestrator.rs b/crates/prism-challenge/src/orchestrator.rs index f332d8564..ec1161361 100644 --- a/crates/prism-challenge/src/orchestrator.rs +++ b/crates/prism-challenge/src/orchestrator.rs @@ -58,7 +58,11 @@ pub struct OrchestratorConfig { pub max_attempts: u32, /// Similarity corpus size (recent submissions + baseline). pub similarity_corpus_limit: u32, - /// Stuck sweep grace (seconds). + /// Stuck sweep grace (seconds). Must exceed max healthy wall-clock of a + /// live worker hold: `PRISM_SSH_RUNNING_TIMEOUT` (≤15m) + train cap (6h) + + /// harness SSH margin (~65m) ≈ 7h20m. A prior 7h grace false-positive + /// swept a healthy ~7h19m train (`swept: stuck beyond grace`) with no + /// log harvest. Default 10h. pub stuck_grace_secs: u64, /// Local/e2e only: pause after each published stage so mid-flight is /// photographable. Zero in production (default). @@ -80,7 +84,7 @@ impl Default for OrchestratorConfig { emit_poll: Duration::from_secs(15), max_attempts: 2, similarity_corpus_limit: 6, - stuck_grace_secs: 7 * 3600, + stuck_grace_secs: 10 * 3600, stage_delay: Duration::ZERO, auto_retry_max: 3, } @@ -197,28 +201,32 @@ impl Orchestrator { .await .map_err(|e| e.to_string())?; for row in stuck { + // Harvest on-pod harness log **before** reclaim — otherwise the + // costly long attempt leaves only `swept: stuck beyond grace`. + let harvested = if let Some(pod) = row.pod_id.as_deref() { + self.backend.harvest_logs(pod).await.unwrap_or_default() + } else { + String::new() + }; if let Some(pod) = row.pod_id.clone() { let _ = self.backend.terminate(&pod).await; let _ = self.backend.verify_terminated(&pod).await; } - let id = row.id.clone(); - let _ = self - .store - .apply( - &id, - &StatePatch { - status: Some(Stage::Failed), - error_detail: Some("swept: stuck beyond grace".into()), - retry_bump: 1, - ..StatePatch::default() - }, - Some(&StageEvent { - stage: Stage::Failed, - detail: Some(serde_json::json!({"reason": "stuck-sweep"})), - at_ms: 0, - }), + let msg = if harvested.trim().is_empty() { + "swept: stuck beyond grace".into() + } else { + format!( + "swept: stuck beyond grace; harvested: {}", + prism_lium::truncate_tail(&harvested, prism_lium::HARNESS_LOG_RETAIN_BYTES) ) - .await; + }; + // Infra-class: auto-retry while budget remains (do **not** burn a + // retry_bump without requeue — that previously exhausted manual + // retry while leaving gating `registered`). + if self.maybe_auto_retry(&row, "install", &msg).await { + continue; + } + self.fail_terminal(&row, "install", &msg).await; } Ok(()) } diff --git a/crates/prism-lium/src/client.rs b/crates/prism-lium/src/client.rs index ea9168299..3d77785a0 100644 --- a/crates/prism-lium/src/client.rs +++ b/crates/prism-lium/src/client.rs @@ -10,9 +10,11 @@ use tokio::time::sleep; use tracing::{debug, info, warn}; use crate::error::{CostGuardrailError, LiumError}; -use crate::ssh::{parse_ssh_target, resolve_private_key, ssh_exec, ssh_exec_allow_fail, SshTarget}; +use crate::ssh::{ + parse_ssh_target, resolve_private_key, ssh_exec, ssh_exec_allow_fail, truncate_tail, SshTarget, +}; use crate::types::{GpuPreference, Instance, InstanceSpec, LiumSshConfig, Offer, RemoteExecResult}; -use crate::{EvalJobBackend, LIUM_API_BASE_URL, MIN_LIFETIME_HOURS}; +use crate::{EvalJobBackend, HARNESS_LOG_RETAIN_BYTES, LIUM_API_BASE_URL, MIN_LIFETIME_HOURS}; /// Pod image: Lium-owned DinD variant pulses its own dockerd init and never /// Only `daturaai/*-dind` pods deliver a reachable sshd on this marketplace @@ -433,12 +435,21 @@ echo '{harness_b64}' | base64 -d > /tmp/prism_eval/prism_harness.py echo '{arch_b64}' | base64 -d > /tmp/prism_eval/architecture.py echo '{train_b64}' | base64 -d > /tmp/prism_eval/training.py cd /tmp/prism_eval +# Persist full harness output on-pod so timeout / stuck-sweep can harvest the +# fatal tail even when the long-lived SSH session is killed without pipes. +set +e PRISM_DATASET_URL='{dataset_url}' \ PRISM_DATASET_SHA256='{dataset_sha}' \ PRISM_MAX_TRAIN_STEPS='{steps}' \ PRISM_TRAIN_HOURS_CAP='{train_hours}' \ PRISM_GPU_TYPE='{gpu_type}' \ -{test_env}timeout --kill-after=60 {timeout_secs} python3 prism_harness.py\n", +{test_env}timeout --kill-after=60 {timeout_secs} python3 prism_harness.py \ + > /tmp/prism_eval/harness.log 2>&1 +ec=$? +set -e +# Surface the log tail on the SSH channel for the happy path + failure parse. +tail -c 524288 /tmp/prism_eval/harness.log || true +exit $ec\n", harness_b64 = harness_b64, arch_b64 = arch_b64, train_b64 = train_b64, @@ -451,7 +462,7 @@ PRISM_GPU_TYPE='{gpu_type}' \ timeout_secs = train_cap_secs.saturating_add(3600), ); - let out = ssh_exec_allow_fail( + let out = match ssh_exec_allow_fail( &target, &key, &remote, @@ -460,12 +471,36 @@ PRISM_GPU_TYPE='{gpu_type}' \ train_cap_secs.saturating_add(3900), ) .await - .map_err(|e| LiumError::Exec(format!("harness transport: {e}")))?; + { + Ok(o) => o, + Err(e) => { + // Session timed out / dropped — second SSH pulls the on-pod log. + let harvested = self + .harvest_logs_inner(instance_id) + .await + .unwrap_or_default(); + return Err(LiumError::Exec(format!( + "harness transport: {e}; harvested: {}", + truncate_tail(&harvested, HARNESS_LOG_RETAIN_BYTES) + ))); + } + }; if !out.stdout.contains("EVAL_OK") { + let mut detail = out.stdout.clone(); + if !out.stderr.is_empty() { + detail.push_str("\n--- stderr ---\n"); + detail.push_str(&out.stderr); + } + if detail.trim().is_empty() { + detail = self + .harvest_logs_inner(instance_id) + .await + .unwrap_or_default(); + } return Err(LiumError::Exec(format!( "harness failed (code {}): {}", out.returncode, - truncate(&out.stderr, 4000) + truncate_tail(&detail, HARNESS_LOG_RETAIN_BYTES) ))); } let line = out @@ -484,6 +519,17 @@ PRISM_GPU_TYPE='{gpu_type}' \ Ok(v) } + /// SSH-fetch the on-pod harness log tail (empty when missing / unreachable). + async fn harvest_logs_inner(&self, instance_id: &str) -> Result { + let target = self.resolve_ssh_target(instance_id).await?; + let key = resolve_private_key(self.ssh.private_key_path.as_deref())?; + let cmd = format!( + "tail -c {HARNESS_LOG_RETAIN_BYTES} /tmp/prism_eval/harness.log 2>/dev/null || true" + ); + let out = ssh_exec_allow_fail(&target, &key, &cmd, 1, self.ssh.ssh_retry_secs, 45).await?; + Ok(truncate_tail(&out.stdout, HARNESS_LOG_RETAIN_BYTES)) + } + async fn gpu_smoke(&self, target: &SshTarget, key: &Path) -> Result { let smoke = ssh_exec( target, @@ -843,6 +889,10 @@ impl EvalJobBackend for LiumClient { self.exec_eval_live(instance_id, architecture_py, training_py) .await } + + async fn harvest_logs(&self, instance_id: &str) -> Result { + self.harvest_logs_inner(instance_id).await + } } #[cfg(test)] diff --git a/crates/prism-lium/src/lib.rs b/crates/prism-lium/src/lib.rs index e36248e8e..f544f2277 100644 --- a/crates/prism-lium/src/lib.rs +++ b/crates/prism-lium/src/lib.rs @@ -39,7 +39,7 @@ pub use client::LiumClient; pub use error::{CostGuardrailError, LiumError}; pub use receipt::{EvalReceipt, NoScoreGate}; pub use sim::SimLiumBackend; -pub use ssh::{parse_ssh_target, resolve_private_key, SshTarget}; +pub use ssh::{parse_ssh_target, resolve_private_key, truncate_tail, SshTarget}; pub use types::{ EvalTelemetry, GpuPreference, Instance, InstanceSpec, LiumSshConfig, Offer, RemoteExecResult, TelemetryPoint, @@ -72,8 +72,24 @@ pub trait EvalJobBackend: Send + Sync { architecture_py: &str, training_py: &str, ) -> Result; + + /// Best-effort tail of the on-pod harness log (before terminate/reclaim). + /// + /// Default is empty — Sim has nothing to fetch. Live backends SSH + /// `tail` of `/tmp/prism_eval/harness.log` so stuck-sweep / timeout + /// paths retain the fatal end of a multi-hour train instead of a blank + /// `swept: stuck beyond grace`. + async fn harvest_logs(&self, _instance_id: &str) -> Result { + Ok(String::new()) + } } +/// Bytes retained when surfacing harness stderr / harvested logs into +/// `error_detail` / stage events. Prefer the **tail** (fatals land at the +/// end); a prior 4 KiB head cap ate inductor autotune spam and dropped the +/// real traceback (~4054 chars stored). +pub const HARNESS_LOG_RETAIN_BYTES: usize = 32_768; + /// Default Lium API base URL. pub const LIUM_API_BASE_URL: &str = "https://lium.io/api"; diff --git a/crates/prism-lium/src/ssh.rs b/crates/prism-lium/src/ssh.rs index f14d5a872..bb548f464 100644 --- a/crates/prism-lium/src/ssh.rs +++ b/crates/prism-lium/src/ssh.rs @@ -195,13 +195,13 @@ pub async fn ssh_exec( return Ok(SshExecOutput { returncode: out.status.code().unwrap_or(0), stdout, - stderr: truncate_str(&stderr, 4000), + stderr: truncate_tail(&stderr, crate::HARNESS_LOG_RETAIN_BYTES), }); } last_err = format!( "ssh exit {:?}: {}", out.status.code(), - truncate_str(&stderr, 200) + truncate_tail(&stderr, 200) ); } Ok(Err(e)) => { @@ -266,7 +266,7 @@ pub async fn ssh_exec_allow_fail( return Ok(SshExecOutput { returncode: out.status.code().unwrap_or(-1), stdout, - stderr: truncate_str(&stderr, 4000), + stderr: truncate_tail(&stderr, crate::HARNESS_LOG_RETAIN_BYTES), }); } Ok(Err(e)) => { @@ -292,12 +292,17 @@ pub struct SshExecOutput { pub stderr: String, } -fn truncate_str(s: &str, n: usize) -> String { +/// Keep the **tail** of a log (fatals / tracebacks), UTF-8 safe. +#[must_use] +pub fn truncate_tail(s: &str, n: usize) -> String { if s.len() <= n { - s.to_owned() - } else { - format!("{}…", &s[..n]) + return s.to_owned(); + } + let mut start = s.len().saturating_sub(n); + while start < s.len() && !s.is_char_boundary(start) { + start += 1; } + format!("…{}", &s[start..]) } #[cfg(test)] @@ -305,6 +310,15 @@ mod tests { use super::*; use serde_json::json; + #[test] + fn truncate_tail_keeps_suffix() { + let s = format!("{}FATAL_TRACEBACK", "x".repeat(100)); + let t = truncate_tail(&s, 20); + assert!(t.starts_with('…')); + assert!(t.ends_with("FATAL_TRACEBACK")); + assert!(t.ends_with(&s[s.len() - 20..])); + } + #[test] fn parse_ssh_connect_cmd() { let raw = json!({}); diff --git a/docs/COMPLETENESS.md b/docs/COMPLETENESS.md index 965ca9e2f..85f869085 100644 --- a/docs/COMPLETENESS.md +++ b/docs/COMPLETENESS.md @@ -104,7 +104,7 @@ Agent/operator contracts: root [`AGENTS.md`](../AGENTS.md), [`deploy/AGENTS.md`] | design rating / elimination | done | Integer Elo (K=32), bottom 20% / 4-round cooldown, exact-E leaves. | | design API | done | Harness/quota/runs/viewer/annotate/ops on `:8093`. | | prism Lium backend | done | `PRISM_FORCE_SIM=false` in staging; the binary logs `eval_backend=lium`. API key is mounted from a file so it never appears in `docker inspect`. | -| prism orchestration | done | DB-backed claim/execute/review/similarity/score state machine (`prism_submission` + append-only `prism_stage_event`), pre-pod screens (copy gate + static cheat + AST similarity) before Lium rent, sweeper (7h grace), boot recovery, epoch-close batched D24 leaf emission with **WTA** (`prism-emit` outbox: `emitted_epoch` watermark + `prism_emit_cursor` + positive-score carry + `apply_wta`, migration 0012). `PRISM_MAX_CONCURRENT_EVALS` default/prod = 8. | +| prism orchestration | done | DB-backed claim/execute/review/similarity/score state machine (`prism_submission` + append-only `prism_stage_event`), pre-pod screens (copy gate + static cheat + AST similarity) before Lium rent, sweeper (10h grace + pre-reclaim log harvest), boot recovery, epoch-close batched D24 leaf emission with **WTA** (`prism-emit` outbox: `emitted_epoch` watermark + `prism_emit_cursor` + positive-score carry + `apply_wta`, migration 0012). `PRISM_MAX_CONCURRENT_EVALS` default/prod = 8. | | prism recipe v1 | done | `prism-recipe` contract, fineweb-edu pinned shard (URL + SHA-256, harness re-verifies), 6h train / 7h pod caps, baseline sources, recipe pin hex on the API. | | prism LLM review | done | `prism-review` quality + similarity prompts (versioned), OpenRouter client (key file only, never env), deterministic sim fallback; anti-copy forces `Copied`/`Suspicious` → Score 0. | | prism API | done | Full status surface: submissions list/detail/events/status/jobs/recipe/baseline, idempotent accept. | diff --git a/docs/PRISM.md b/docs/PRISM.md index 6497a9a30..d51e74e31 100644 --- a/docs/PRISM.md +++ b/docs/PRISM.md @@ -59,9 +59,11 @@ stateDiagram-v2 ``` All transitions are append-only events in `prism_stage_event`; the row state -lives in `prism_submission`. The sweeper fails rows stuck past the 7h grace -as `ChallengeInternal`, and `recover_on_boot` cleans pods referenced by -interrupted rows. +lives in `prism_submission`. The sweeper fails rows stuck past the **10h** +grace (aligned above wait-RUNNING + 6h train + SSH margin; a prior 7h grace +false-positive swept healthy ~7h19m trains) as `ChallengeInternal` after +harvesting the on-pod harness log tail, and `recover_on_boot` cleans pods +referenced by interrupted rows. Evaluation (Lium / Sim, review, agentic, leaf emit) is **master-only**. Validators never run `prism-challenge` — they fetch sealed weights only.