Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Agent.md
Original file line numberDiff line numberDiff line change
Expand Up@@ -93,7 +93,7 @@ Community needs voiced in HN agent-UI discussions map directly to EMRG's design:
pkill -f "emrg.server"; rm -f ~/.emrg/emrgd.port; python -m emrg
```

Python: `uv run pytest tests/ -v` (636) — import check: `uv run python -c "from emrg.client.app import run_client"`
Python: `uv run pytest tests/ -v` (639) — import check: `uv run python -c "from emrg.client.app import run_client"`
GUI: `cd emrg/gui && npm test` (96: 22 daemon_client + 22 app-commands + 27 renderer smoke + 15 i18n + 7 integration + 3 commands) — syntax: `node --check main.js preload.js daemon_client.js renderer/js/*.js`
CI: `uv run pytest` + GUI tests + **actionlint workflow lint** (`rhysd/actionlint@v1.7.12` gate, #444 — workflow 解析错误在 PR CI 即失败,如 `if:` secrets 上下文)
Re-trigger: `scripts/re-trigger-ci.sh [branch]` (workflow_dispatch, #527 — 替代空 commit 重触发:Actions outage 会整段丢弃 push 事件,dispatch 走 API 路径不受影响)
Expand Down
2 changes: 1 addition & 1 deletion README.cn.md
Original file line numberDiff line numberDiff line change
Expand Up@@ -274,7 +274,7 @@ EMRG 不只是追赶——它自己追上来。
git clone https://github.com/argszero/emrg.git
cd emrg
uv sync # 安装依赖
uv run pytest tests/ -v # 跑测试(当前 636 项)
uv run pytest tests/ -v # 跑测试(当前 639 项)
uv run python -m emrg # 启动 TUI
# CI 含 actionlint workflow 门禁(#444):workflow 解析错误在 PR 即失败

Expand Down
2 changes: 1 addition & 1 deletion README.md
Original file line numberDiff line numberDiff line change
Expand Up@@ -273,7 +273,7 @@ EMRG doesn't just keep up — it catches up on its own.
git clone https://github.com/argszero/emrg.git
cd emrg
uv sync # install deps
uv run pytest tests/ -v # run tests (currently 636 items)
uv run pytest tests/ -v # run tests (currently 639 items)
uv run python -m emrg # launch TUI
# CI includes actionlint workflow gate (#444): workflow parse errors fail PR CI

Expand Down
63 changes: 40 additions & 23 deletions emrg/server/scheduler.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -133,12 +133,14 @@ def __init__(
self._logs_dir.mkdir(parents=True, exist_ok=True)
self.evolutions: list[EvolutionLog] = []

# ── Saturation halt — stop burning tokens on empty cycles ───
# ── Saturation — slow down, never stop (rant 2026-08-09T09:35:55) ──
# Track consecutive cycles where git HEAD didn't advance (NTE).
# After _IDLE_HALT_THRESHOLD empty cycles, switch to trigger-only:
# - Scheduled runs are skipped
# - Only manual trigger (/trigger) resumes the cycle
# - Counter resets on trigger or when git HEAD advances
# After _IDLE_HALT_THRESHOLD empty cycles, switch to low-frequency
# heartbeat full cycles instead of the old complete halt:
# - Scheduled runs continue at heartbeat interval (never skipped)
# - heartbeat = max(interval, min(interval*8, 8h)) — 60s task → 8min
# - Manual trigger (/trigger) or upstream git HEAD advance restores
# the normal frequency immediately (counter reset to 0)
#
# Counter is persisted to disk to survive daemon restarts.
self._IDLE_HALT_THRESHOLD = 30
Expand DownExpand Up@@ -509,13 +511,21 @@ async def run(self) -> None:
)

while self._running:
# Saturation → wait at the heartbeat interval (low-frequency full
# cycle, rant 2026-08-09T09:35:55); otherwise the normal interval.
# Manual trigger wakes immediately either way. Never skip a cycle.
wait_timeout = (
self._heartbeat_interval()
if self._saturation_heartbeat_active()
else self.interval
)
# Wait for interval or manual trigger (interruptible)
self._next_run_at = time.time() + self.interval
self._next_run_at = time.time() + wait_timeout
manual_trigger = False
try:
await asyncio.wait_for(
self._trigger_event.wait(),
timeout=self.interval,
timeout=wait_timeout,
)
# Manual trigger fired — clear and proceed
self._trigger_event.clear()
Expand All@@ -527,19 +537,17 @@ async def run(self) -> None:
# Normal scheduled run
pass

# Saturation halt: if too many empty cycles, skip scheduled runs.
# Manual triggers always bypass the halt and reset the counter.
# Manual triggers always reset the saturation counter; otherwise
# saturated ticks keep running full cycles at heartbeat cadence.
if manual_trigger:
if self._empty_cycles >= self._IDLE_HALT_THRESHOLD:
logger.info(
"EvolutionHandler[%s]: resumed via manual trigger "
"(was halted at %d empty cycles)",
"(was in saturation at %d empty cycles)",
self.name, self._empty_cycles,
)
self._empty_cycles = 0
self._save_saturation_state()
elif self._saturation_halt_active():
continue

logger.debug("EvolutionHandler[%s] tick", self.name)
self._cycle_running = True
Expand DownExpand Up@@ -637,29 +645,38 @@ def _remote_advanced(self) -> bool:
except Exception:
return False

def _saturation_halt_active(self) -> bool:
"""Whether a scheduled tick should be skipped due to saturation halt.
def _heartbeat_interval(self) -> int:
"""Low-frequency heartbeat interval (rant 2026-08-09T09:35:55):
heartbeat = max(task_interval, min(task_interval * 8, 8 hours)).
Protection = slow down, never stop. Long-interval tasks (>= 8h)
keep their original cadence (the 8x/8h caps don't apply).
"""
return max(self.interval, min(self.interval * 8, 8 * 3600))

def _saturation_heartbeat_active(self) -> bool:
"""Whether this tick should run at the low-frequency heartbeat interval
instead of the normal interval.

Extracted from the run loop so the halt decision is testable:
at/above the threshold the tick is skipped UNLESS the upstream
remote advanced (auto-resume: reset the counter and run the cycle,
so a halted handler does not miss new work forever).
Replaces the old complete saturation halt (rant 2026-08-09T09:35:55):
at/above the empty-cycle threshold the handler keeps running full
cycles, just at a reduced cadence — never skipping. Upstream advance
auto-resumes (counter reset, normal frequency), so a saturated handler
does not miss new work forever.
"""
if self._empty_cycles < self._IDLE_HALT_THRESHOLD:
return False
if self._remote_advanced():
logger.info(
"EvolutionHandler[%s]: upstream advanced — resuming from saturation halt",
"EvolutionHandler[%s]: upstream advanced — resuming normal frequency from saturation",
self.name,
)
self._empty_cycles = 0
self._save_saturation_state()
return False
logger.info(
"EvolutionHandler[%s]: saturation halt — "
"skipping scheduled run (%d empty cycles). "
"Use /trigger to resume.",
self.name, self._empty_cycles,
"EvolutionHandler[%s]: saturation (%d empty cycles) — "
"running full cycle at heartbeat interval (%ds) — never halting",
self.name, self._empty_cycles, self._heartbeat_interval(),
)
return True

Expand Down
93 changes: 76 additions & 17 deletions tests/test_scheduler.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -1218,14 +1218,30 @@ def _original_connect_to_server():
return importlib.import_module("emrg.connect").connect_to_server


# ── Saturation halt auto-resume on upstream advance ───────────────
# The halt skips scheduled runs entirely, so a halted handler can never
# detect a HEAD change itself (only /trigger could resume it). If every
# instance halted during an idle stretch, new upstream work would go
# unnoticed — the halt must auto-resume when origin/master advances.

def test_saturation_halt_active_true_when_remote_unchanged(tmp_path):
"""At/above threshold + unchanged remote → tick skipped (halt stays)."""
# ── Saturation heartbeat: slow down, never stop (rant 2026-08-09T09:35:55) ─
# The old complete halt (skipping scheduled runs) is replaced by
# low-frequency full cycles: saturated ticks still run, just at the heartbeat
# interval. Upstream advance auto-resumes (counter reset, normal frequency).

def test_heartbeat_interval_formula(tmp_path):
"""heartbeat = max(interval, min(interval*8, 8h)); long intervals unchanged."""
from emrg.server import scheduler as mod
handler = _make_handler(tmp_path, project="", path=str(tmp_path))
for interval, expected in [
(1, 8), # min*8 floor below the 8h cap
(60, 480), # emrg-task: 8 minutes
(600, 4800), # 10-min task: 80 minutes
(3600, 28800), # 1h task: 8h (cap)
(14400, 28800), # 4h task: min(115200, 28800) = 8h (cap)
(28800, 28800), # 8h task: unchanged (max keeps original)
(86400, 86400), # 24h task: unchanged (8x beyond cap → original)
]:
handler.interval = interval
assert handler._heartbeat_interval() == expected, (interval, expected)


def test_saturation_heartbeat_active_true_when_remote_unchanged(tmp_path):
"""At/above threshold + unchanged remote → heartbeat cadence (not skip)."""
from emrg.server import scheduler as mod

handler = _make_handler(tmp_path, project="", path=str(tmp_path))
Expand All@@ -1234,14 +1250,36 @@ def test_saturation_halt_active_true_when_remote_unchanged(tmp_path):
orig_run = mod.subprocess.run
mod.subprocess.run = fake
try:
assert handler._saturation_halt_active() is True
assert handler._saturation_heartbeat_active() is True
assert handler._empty_cycles == 30 # counter untouched
assert handler._heartbeat_interval() == 480 # 60s task → 8 min
finally:
mod.subprocess.run = orig_run


def test_saturation_heartbeat_log_message_no_skip(tmp_path, caplog):
"""Saturation log must say heartbeat, never 'skipping scheduled run'."""
import logging
from emrg.server import scheduler as mod

handler = _make_handler(tmp_path, project="", path=str(tmp_path))
handler._empty_cycles = 30
fake = FakeGitRun(remote_head="abc123")
orig_run = mod.subprocess.run
mod.subprocess.run = fake
try:
with caplog.at_level(logging.INFO, logger="emrg.server.scheduler"):
assert handler._saturation_heartbeat_active() is True
msgs = " ".join(r.message for r in caplog.records)
assert "skipping scheduled run" not in msgs, \
"old complete-halt log must not appear (rant 09:35:55)"
assert "heartbeat" in msgs and "never halting" in msgs, msgs
finally:
mod.subprocess.run = orig_run


def test_saturation_halt_resumes_and_resets_when_remote_advanced(tmp_path):
"""At/above threshold + remote advanced → resume, counter reset to 0."""
def test_saturation_heartbeat_resumes_and_resets_when_remote_advanced(tmp_path):
"""At/above threshold + remote advanced → normal frequency, counter reset."""
from emrg.server import scheduler as mod

handler = _make_handler(tmp_path, project="", path=str(tmp_path))
Expand All@@ -1250,22 +1288,43 @@ def test_saturation_halt_resumes_and_resets_when_remote_advanced(tmp_path):
orig_run = mod.subprocess.run
mod.subprocess.run = fake
try:
assert handler._saturation_halt_active() is False
assert handler._empty_cycles == 0 # reset → scheduled runs resume
assert handler._saturation_heartbeat_active() is False
assert handler._empty_cycles == 0 # reset → normal frequency resumes
finally:
mod.subprocess.run = orig_run


def test_saturation_halt_active_false_below_threshold(tmp_path):
"""Below threshold → never halt (remote state irrelevant)."""
def test_saturation_heartbeat_false_below_threshold(tmp_path):
"""Below threshold → normal interval (remote state irrelevant)."""
handler = _make_handler(tmp_path, project="", path=str(tmp_path))
handler._empty_cycles = 10
assert handler._saturation_halt_active() is False
assert handler._saturation_heartbeat_active() is False
assert handler._empty_cycles == 10


def test_saturated_tick_still_runs_full_cycle(tmp_path):
"""Saturated handler runs a full cycle (never skipped) at heartbeat."""
from emrg.server import scheduler as mod

handler, captured = _make_cycle_handler(tmp_path, frames=[
{"request_id": "r1", "content": "Done", "done": True,
"delta": False, "session_id": "s"},
])
handler._empty_cycles = 30 # saturated
fake = FakeGitRun(remote_head="abc123") # unchanged → stay saturated
orig_run = mod.subprocess.run
mod.subprocess.run = fake
try:
asyncio.run(handler._run_evolution_cycle())
finally:
mod.subprocess.run = orig_run
assert "log" in captured, "saturated tick must still run a full cycle"
assert handler._empty_cycles == 31, \
"NTE cycle during saturation keeps incrementing (heartbeat continues)"


def test_remote_advanced_false_without_git_repo(tmp_path):
"""Not a git repo / ls-remote fails → False (stay halted, no crash)."""
"""Not a git repo / ls-remote fails → False (stay saturated, no crash)."""
from emrg.server import scheduler as mod

handler = _make_handler(tmp_path, project="", path=str(tmp_path))
Expand Down
Loading