From e86f5b20d0245e963b92d38933ef2b36e35c72f3 Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Sat, 12 Sep 2026 04:46:48 +0000
Subject: [PATCH] fix(collector): fail stuck recovery running jobs instead of
 succeeding

Recovery oneshot TimeoutStartSec is now 7200s so a hung drain receives
TERM. An unfinished *.running job is marked failed rather than leaving
systemd Result=success. Failed jobs still require an identity-bearing
resume; this does not auto-retry them.

Co-authored-by: Wild Card <proerror77@users.noreply.github.com>
---
 .../binance-lob-archiver-recovery@.service    |  3 ++-
 .../aliyun/host-rust-lob-recovery-queue.sh    |  7 +++++-
 .../aliyun/test-rust-lob-recovery-queue.sh    | 23 +++++++++++++++++++
 3 files changed, 31 insertions(+), 2 deletions(-)

diff --git a/deployment/aliyun/binance-lob-archiver-recovery@.service b/deployment/aliyun/binance-lob-archiver-recovery@.service
index 1a9329fe4..343344f95 100644
--- a/deployment/aliyun/binance-lob-archiver-recovery@.service
+++ b/deployment/aliyun/binance-lob-archiver-recovery@.service
@@ -8,7 +8,8 @@ AssertPathIsMountPoint=/data
 [Service]
 Type=oneshot
 ExecStart=/opt/monday/bin/monday-rust-lob-recovery-queue drain %i
-TimeoutStartSec=0
+TimeoutStartSec=7200
+TimeoutStopSec=120
 Nice=10
 IOSchedulingClass=best-effort
 IOSchedulingPriority=7
diff --git a/deployment/aliyun/host-rust-lob-recovery-queue.sh b/deployment/aliyun/host-rust-lob-recovery-queue.sh
index 535333e7d..b0ebed204 100755
--- a/deployment/aliyun/host-rust-lob-recovery-queue.sh
+++ b/deployment/aliyun/host-rust-lob-recovery-queue.sh
@@ -1368,7 +1368,12 @@ drain_market() {
     if finalize_passed_running "$running_dir"; then
       exit 0
     fi
-    fail "unfinished running recovery job requires manual intervention: $running_dir"
+    CURRENT_RUNNING_DIR=$running_dir
+    CURRENT_STEP=unfinished-running
+    mark_failed "$running_dir" "$CURRENT_STEP" \
+      "unfinished running recovery job requires manual resume: $running_dir"
+    CURRENT_RUNNING_DIR=
+    exit 1
   fi
   ready_dir=$(oldest_ready_dir)
   [[ -n $ready_dir ]] || exit 0
diff --git a/deployment/aliyun/test-rust-lob-recovery-queue.sh b/deployment/aliyun/test-rust-lob-recovery-queue.sh
index e8b87e0a0..29e5957bd 100755
--- a/deployment/aliyun/test-rust-lob-recovery-queue.sh
+++ b/deployment/aliyun/test-rust-lob-recovery-queue.sh
@@ -14,6 +14,12 @@ fi
 grep -Fq "active V2 controller is required" "$RECOVERY"
 grep -Fq 'monday.rust_lob_controller_release.v2' "$RECOVERY"
 grep -Fq 'monday.rust_lob_controller_release.v2' "$RECOVERY"
+grep -Fxq 'TimeoutStartSec=7200' "$SCRIPT_DIR/binance-lob-archiver-recovery@.service"
+grep -Fxq 'TimeoutStopSec=120' "$SCRIPT_DIR/binance-lob-archiver-recovery@.service"
+if grep -Fxq 'TimeoutStartSec=0' "$SCRIPT_DIR/binance-lob-archiver-recovery@.service"; then
+  printf 'recovery oneshot still has unbounded TimeoutStartSec\n' >&2
+  exit 1
+fi
 projection_contract='"$ACTIVE_CONTROLLER/deployment/binance-lob-archiver-production-$MARKET.env"'
 resolved_contract='secure_regular_file "$installed_env" 0'
 obsolete_contract='secure_regular_file "$ENV_FILE" 0'
@@ -542,4 +548,21 @@ expect_rejected missing-remote-object fixture_drain
 unset FIXTURE_OSS_FAIL
 [[ $(jq -r .result "$attempt/result.json") == failed ]]
 [[ $(sha256sum "$EVIDENCE_ROOT/$RESUME_JOB_ID/result.json" | awk '{print $1}') == "$fixture_old_result_sha" ]]
+
+# An unfinished *.running job with the active identity is marked failed, not
+# left as a successful drain. Resume still requires exact identities.
+fixture_job 109 running
+rm -f "$fixture/payload.calls"
+saved_active=$fixture_active_c
+fixture_active_c=$fixture_old_c
+expect_rejected unfinished-running fixture_drain
+fixture_active_c=$saved_active
+[[ -d $QUEUE_MARKET_ROOT/$RESUME_JOB_ID.failed ]]
+[[ ! -e $QUEUE_MARKET_ROOT/$RESUME_JOB_ID.running ]]
+[[ $(jq -r '.result == "failed" and .step == "unfinished-running"' \
+  "$EVIDENCE_ROOT/$RESUME_JOB_ID/result.json") == true ]]
+if grep -Fq -- '--upload-only' "$fixture/payload.calls" 2>/dev/null; then
+  printf 'unfinished running drain started payload work\n' >&2
+  exit 1
+fi
 printf 'Explicit recovery adoption, historical readback, mixed drain and interruption behavior passed\n'