Skip to content

Commit e483aae

Browse files
committed
fix(web): unlock composer when a follow-up never terminates
A persisted FOLLOWUP_STARTED without COMPLETED/FAILED left followupPending true across reload, so ask/writeup/resolve stayed disabled. Synthesize a terminal failure from the standby wrapper and on fresh SSE replay, and clear the leftover lock on run finish/reopen.
1 parent db296d2 commit e483aae

6 files changed

Lines changed: 146 additions & 2 deletions

File tree

apps/web/run_manager.py

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3161,10 +3161,26 @@ def _ensure_standby(self, run_id: str, cmd: dict[str, Any]) -> bool:
31613161
driver = build_standby_driver(cmd, mgr=self)
31623162

31633163
async def _go() -> None:
3164+
followup_terminal = False
3165+
3166+
async def _note_followup_terminal(ev: Event) -> None:
3167+
nonlocal followup_terminal
3168+
if ev.event_type not in {
3169+
EventType.FOLLOWUP_COMPLETED, EventType.FOLLOWUP_FAILED,
3170+
}:
3171+
return
3172+
wanted = str(cmd.get("followup_id") or "")
3173+
ev_id = str((ev.payload or {}).get("followup_id") or "")
3174+
if not wanted or ev_id == wanted:
3175+
followup_terminal = True
3176+
31643177
async def _emit_followup_failed(detail: str) -> None:
3165-
if action not in {"ask", "writeup"}:
3178+
nonlocal followup_terminal
3179+
if action not in {"ask", "writeup"} or followup_terminal:
31663180
return
31673181
try:
3182+
if bool(getattr(run.bus, "_closed", False)):
3183+
self._fresh_bus(run)
31683184
await run.bus.emit(Event(
31693185
event_type=EventType.FOLLOWUP_FAILED,
31703186
run_id=run_id,
@@ -3174,9 +3190,11 @@ async def _emit_followup_failed(detail: str) -> None:
31743190
"detail": detail,
31753191
},
31763192
))
3193+
followup_terminal = True
31773194
except Exception:
31783195
pass
31793196

3197+
run.bus.add_sink(_note_followup_terminal)
31803198
try:
31813199
LOG.info("standby worker starting for %s action=%s",
31823200
run_id, cmd.get("action"))
@@ -3210,6 +3228,11 @@ async def _emit_followup_failed(detail: str) -> None:
32103228
except Exception:
32113229
pass
32123230
finally:
3231+
run.bus.remove_sink(_note_followup_terminal)
3232+
if action in {"ask", "writeup"} and not followup_terminal:
3233+
# Crash, kill, or a failed start/complete emit must not leave
3234+
# the finished composer locked on FOLLOWUP_STARTED.
3235+
await _emit_followup_failed("后续操作已中断")
32133236
# Do not close the bus; retain the completed task as an observable
32143237
# receipt. `_ensure_standby` checks `.done()` and replaces it on the
32153238
# next command, so this does not block subsequent follow-ups.

apps/web/server.py

Lines changed: 31 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1482,6 +1482,9 @@ async def gen():
14821482
replayed_seq = 0
14831483
replayed_count = 0
14841484
last_lifecycle = ""
1485+
last_followup = ""
1486+
last_followup_id = ""
1487+
last_followup_kind = "ask"
14851488
async for ev in run.store.replay_monotonic(run_id, after_seq=last_id):
14861489
replayed_seq = ev.seq
14871490
replayed_count += 1
@@ -1490,6 +1493,14 @@ async def gen():
14901493
EventType.RUN_FINISHED,
14911494
EventType.RUN_REOPENED):
14921495
last_lifecycle = ev.event_type.value
1496+
if ev.event_type in (EventType.FOLLOWUP_STARTED,
1497+
EventType.FOLLOWUP_COMPLETED,
1498+
EventType.FOLLOWUP_FAILED):
1499+
last_followup = ev.event_type.value
1500+
last_followup_id = str(
1501+
(ev.payload or {}).get("followup_id") or "")
1502+
last_followup_kind = str(
1503+
(ev.payload or {}).get("kind") or "ask")
14931504
yield {
14941505
"id": str(ev.seq),
14951506
"event": ev.event_type.value,
@@ -1524,7 +1535,26 @@ async def gen():
15241535
"event": synth.event_type.value,
15251536
"data": synth.model_dump_json(),
15261537
}
1527-
# live tail: everything after what we just replayed (or after the
1538+
# Ghost-followup guard: a persisted FOLLOWUP_STARTED without a
1539+
# terminal event and no live standby would lock ask/writeup/resolve
1540+
# across reload. Synthesize FOLLOWUP_FAILED so replay can settle.
1541+
if (fresh and not manager._standby_busy(run)
1542+
and last_followup == EventType.FOLLOWUP_STARTED.value):
1543+
replayed_seq = max(replayed_seq, run.store.last_stream_seq(run_id)) + 1
1544+
synth = Event(
1545+
event_type=EventType.FOLLOWUP_FAILED, run_id=run_id,
1546+
seq=replayed_seq,
1547+
payload={
1548+
"followup_id": last_followup_id,
1549+
"kind": last_followup_kind,
1550+
"detail": "后续操作已中断",
1551+
})
1552+
yield {
1553+
"id": str(replayed_seq),
1554+
"event": synth.event_type.value,
1555+
"data": synth.model_dump_json(),
1556+
}
1557+
# live tail: everything after what we just replayed (or after the)
15281558
# client's Last-Event-ID on a reconnect). A finished run's bus is
15291559
# closed, so subscribe() returns after backlog replay. Do NOT let the
15301560
# HTTP response EOF: browser EventSource treats EOF as an error and

apps/web/ui/lib/events.ts

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2579,6 +2579,9 @@ export function reduce(prev: DeckState, ev: MutekiEvent): DeckState {
25792579
}
25802580
s.finished = true;
25812581
s.preparing = false;
2582+
// A generation terminal is not a follow-up terminal. Drop a leftover
2583+
// ask/writeup lock so a later finished composer cannot stay disabled.
2584+
s.followupPending = false;
25822585
// A hard stop can kill the coordinator before it ever emits race_concluded
25832586
// (operator stop cancels the whole run task mid-race). Clear the pill here so
25842587
// the UI never sticks on "racing" past the terminal event.
@@ -2791,6 +2794,7 @@ export function reduce(prev: DeckState, ev: MutekiEvent): DeckState {
27912794
// false-positive flag invalidation. Keep the operator copy precise.
27922795
s.finished = false;
27932796
s.preparing = false;
2797+
s.followupPending = false;
27942798
s.solved = false;
27952799
s.acceptedOnly = false;
27962800
s.finishedAt = undefined;

tests/test_standby_hitl.py

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -869,6 +869,52 @@ async def _run():
869869
assert lifecycle[-1].payload["detail"] == "后续操作已取消"
870870

871871

872+
def test_standby_wrapper_synthesizes_failed_when_start_has_no_terminal(
873+
tmp_path, monkeypatch):
874+
from apps.web import run_manager as rm
875+
from muteki.core.events import Event, EventType
876+
import apps.web.drivers as drivers
877+
878+
async def _started_then_exit(run):
879+
await run.bus.emit(Event(
880+
event_type=EventType.FOLLOWUP_STARTED,
881+
run_id=run.run_id,
882+
payload={
883+
"followup_id": "followup-orphan", "kind": "ask",
884+
"question": "证据来源是什么?",
885+
},
886+
))
887+
888+
monkeypatch.setattr(
889+
drivers, "build_standby_driver", lambda cmd, mgr=None: _started_then_exit,
890+
)
891+
892+
async def _run():
893+
mgr = rm.RunManager(sessions_root=tmp_path / "sessions")
894+
run = mgr.create("run-x")
895+
run.started = True
896+
run.finished = True
897+
run.solved = True
898+
assert mgr._ensure_standby(run.run_id, {
899+
"action": "ask", "text": "证据来源是什么?",
900+
"followup_id": "followup-orphan",
901+
})
902+
await asyncio.gather(run.standby_task, return_exceptions=True)
903+
return [event async for event in run.store.replay(run.run_id)]
904+
905+
events = asyncio.run(_run())
906+
lifecycle = [
907+
event for event in events if event.event_type in {
908+
EventType.FOLLOWUP_STARTED, EventType.FOLLOWUP_FAILED,
909+
}
910+
]
911+
assert [event.event_type for event in lifecycle] == [
912+
EventType.FOLLOWUP_STARTED, EventType.FOLLOWUP_FAILED,
913+
]
914+
assert lifecycle[-1].payload["followup_id"] == "followup-orphan"
915+
assert lifecycle[-1].payload["detail"] == "后续操作已中断"
916+
917+
872918
def test_standby_final_cancel_log_redacts_callback_exception(
873919
tmp_path, monkeypatch, caplog):
874920
from apps.web import run_manager as rm

tests/test_web_deck_ux.py

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -461,6 +461,21 @@ def test_followup_failure_replaces_its_correlated_pending_status():
461461
assert(s.chat.filter((m) => m.followupId === "F1"
462462
&& m.content.includes("后续操作已取消")).length === 1,
463463
"terminal row remains idempotent after replay");
464+
465+
s = lib.reduce(s, {{ event_type: lib.EventType.FOLLOWUP_STARTED,
466+
run_id: "run-followup", ts: 4, payload: {{ followup_id: "F2",
467+
kind: "ask", question: "还卡着吗?" }} }});
468+
assert(s.followupPending, "a new follow-up can pend again");
469+
s = lib.reduce(s, {{ event_type: lib.EventType.RUN_FINISHED,
470+
run_id: "run-followup", ts: 5, payload: {{ solved: true }} }});
471+
assert(!s.followupPending, "run.finished clears a leftover follow-up lock");
472+
s = lib.reduce(s, {{ event_type: lib.EventType.FOLLOWUP_STARTED,
473+
run_id: "run-followup", ts: 6, payload: {{ followup_id: "F3",
474+
kind: "writeup" }} }});
475+
assert(s.followupPending, "writeup can pend after finish");
476+
s = lib.reduce(s, {{ event_type: lib.EventType.RUN_REOPENED,
477+
run_id: "run-followup", ts: 7, payload: {{ reason: "resolve" }} }});
478+
assert(!s.followupPending, "run.reopened clears a leftover follow-up lock");
464479
"""
465480
)
466481
_run_ui_node(script)

tests/test_web_server.py

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -731,6 +731,32 @@ async def test_events_injects_run_finished_for_ghost_run(server_mgr) -> None:
731731
assert EventType.RUN_FINISHED.value in seen # the synthetic terminator
732732

733733

734+
async def test_events_injects_followup_failed_for_orphan_start(server_mgr) -> None:
735+
from muteki.core.events import Event
736+
s, mgr = server_mgr
737+
rid = "ghost-followup-1"
738+
run = mgr.create(rid)
739+
await run.bus.emit(Event(event_type=EventType.RUN_STARTED, run_id=rid,
740+
payload={"challenge": {"name": "x"}}))
741+
await run.bus.emit(Event(event_type=EventType.RUN_FINISHED, run_id=rid,
742+
payload={"solved": True}))
743+
await run.bus.emit(Event(
744+
event_type=EventType.FOLLOWUP_STARTED, run_id=rid,
745+
payload={"followup_id": "F-orphan", "kind": "ask", "question": "why?"},
746+
))
747+
run.started = True
748+
run.finished = True
749+
run.task = None
750+
run.standby_task = None
751+
async with httpx.AsyncClient(base_url=s.base, timeout=30, trust_env=False) as client:
752+
seen: set = set()
753+
await asyncio.wait_for(
754+
_collect_sse(client, rid, seen, EventType.FOLLOWUP_FAILED.value),
755+
timeout=15)
756+
assert EventType.FOLLOWUP_STARTED.value in seen
757+
assert EventType.FOLLOWUP_FAILED.value in seen
758+
759+
734760
async def test_events_do_not_inject_run_finished_for_protocol2_ghost(server_mgr) -> None:
735761
from muteki.core.events import Event
736762

0 commit comments

Comments
 (0)