Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Agent.md
Original file line numberDiff line numberDiff line change
Expand Up@@ -118,7 +118,7 @@ Community needs voiced in HN agent-UI discussions map directly to EMRG's design:
pkill -f "emrg.server"; rm -f ~/.emrg/emrgd.token; python -m emrg
```

Python: `uv run pytest tests/ -v` (1052) — import check: `uv run python -c "from emrg.client.app import run_client"`
Python: `uv run pytest tests/ -v` (1054) — import check: `uv run python -c "from emrg.client.app import run_client"`
GUI: `cd emrg/gui && npm test` (259: 45 daemon_client + 20 conn-manager + 22 app-commands + 129 renderer smoke + 15 i18n + 8 integration + 3 commands + 8 build-config + 7 gui-state + 2 tool-group) — syntax: `node --check main.js preload.js daemon_client.js renderer/js/*.js`
CI: `uv run pytest` (ubuntu + **windows-2025 matrix** — Windows pytest 回归在 PR CI 即失败,v0.2.29 教训 #725) + GUI tests + **actionlint workflow lint** (`rhysd/actionlint@v1.7.12` gate, #444 — workflow 解析错误在 PR CI 即失败,如 `if:` secrets 上下文)
Re-trigger: `scripts/re-trigger-ci.sh [branch]` (workflow_dispatch, #527 — 替代空 commit 重触发:Actions outage 会整段丢弃 push 事件,dispatch 走 API 路径不受影响)
Expand Down
69 changes: 64 additions & 5 deletions emrg/server/__main__.py
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,12 @@
"""Entry point for EMRG daemon: python -m emrg.server"""
import asyncio
import logging
import sys
import traceback
from logging.handlers import RotatingFileHandler
from pathlib import Path

from emrg.server.daemon import DaemonExit, run_server
from emrg.server.logcontext import session_label


Expand DownExpand Up@@ -47,12 +50,20 @@ def _configure_logging() -> None:
encoding="utf-8", # rant 2026-08-07T14:00Z: default locale code page (GBK on zh-CN Windows) mojibakes CJK log lines
)
file_handler.setFormatter(_fmt)
stream_handler = logging.StreamHandler() # also to stderr (visible when run directly)
stream_handler.setFormatter(_fmt)
handlers = [file_handler]
# StreamHandler only for interactive runs: daemon_manager spawns emrgd
# with stderr=DEVNULL, so the stream handler is pure waste there — and
# since _redirect_std_streams() later points sys.stderr at the crash log,
# a stream handler would duplicate the whole log into it (rant
# 2026-08-25T09:25:32 wants an *independent* crash log).
if sys.stderr.isatty():
stream_handler = logging.StreamHandler() # visible when run directly
stream_handler.setFormatter(_fmt)
handlers.append(stream_handler)

logging.basicConfig(
level=logging.DEBUG,
handlers=[file_handler, stream_handler],
handlers=handlers,
)
# Suppress noisy httpcore/httpx DEBUG logs (rant #24)
logging.getLogger("httpcore").setLevel(logging.WARNING)
Expand All@@ -63,13 +74,61 @@ def _configure_logging() -> None:
logging.getLogger("websockets").setLevel(logging.INFO)


def _redirect_std_streams() -> None:
"""Redirect sys.stdout/sys.stderr to ~/.emrg/emrgd-crash.log (rant
2026-08-25T09:25:32 — daemon silent death).

daemon_manager spawns emrgd with stdout/stderr=DEVNULL, so anything that
bypasses logging — asyncio's default exception handler output, C-level
abort/assert traces, faulthandler dumps — vanished silently. This gives
the daemon a second, independent sink for such output, and enables
faulthandler so even a fatal-signal death (SIGSEGV etc., where no Python
code runs) leaves a stack dump behind.
"""
crash_log = Path.home() / ".emrg" / "emrgd-crash.log"
try:
stream = open(str(crash_log), "a", encoding="utf-8", buffering=1)
except OSError:
return # best-effort: keep the original DEVNULL sinks
sys.stdout = stream
sys.stderr = stream
try:
import faulthandler

faulthandler.enable(file=stream)
except Exception:
pass # best-effort: faulthandler may reject the stream (no fileno)


def main() -> None:
from emrg.config import load_config
from emrg.server.daemon import run_server

_configure_logging()
_redirect_std_streams()
config = load_config()
asyncio.run(run_server(config.llm))
try:
result = asyncio.run(run_server(config.llm))
except KeyboardInterrupt:
# SIGINT delivered at the event-loop poll point escapes the main
# coroutine — record it here so no stop path is ever silent.
result = DaemonExit("sigint", 130, None)
logging.getLogger("emrg.server").info(
"daemon terminated by SIGINT outside the event loop"
)
except SystemExit:
# POSIX SIGTERM handler (daemon._sigterm_handler) → SystemExit that
# escaped the loop.
result = DaemonExit("sigterm", 143, None)
logging.getLogger("emrg.server").info(
"daemon terminated by SIGTERM outside the event loop"
)
except BaseException:
result = DaemonExit("crash", 1, traceback.format_exc())
logging.getLogger("emrg.server").critical(
"daemon crashed outside the event loop", exc_info=True
)
result.write_record()
sys.exit(result.exit_code)


if __name__ == "__main__":
Expand Down
124 changes: 121 additions & 3 deletions emrg/server/daemon.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -24,6 +24,7 @@
import subprocess
import sys
import time
import traceback
from datetime import datetime
from pathlib import Path
from typing import Optional
Expand DownExpand Up@@ -416,6 +417,10 @@ async def serve(self) -> None:
logger.info("daemon serve cancelled (asyncio.CancelledError) — cleanup started")
except Exception:
self._stop_reason = "crash"
# Rant 2026-08-25T09:25:32: keep the traceback so run_server can
# include it in the durable exit record (serve() returns normally
# after this handler runs — the exception is not re-raised).
self._crash_traceback = traceback.format_exc()
logger.error("daemon serve crashed — cleanup started", exc_info=True)
finally:
await self._shutdown_all()
Expand DownExpand Up@@ -4059,13 +4064,126 @@ async def _consolidate_session_memories(
logger.debug("memory consolidation failed", exc_info=True)


async def run_server(llm_config: LlmConfig) -> None:
"""Run the EMRG server until interrupted."""
_EXIT_RECORD_PATH = Path.home() / ".emrg" / "emrgd-exit.log"


class DaemonExit:
"""Exit metadata produced by run_server (rant 2026-08-25T09:25:32).

The caller (emrg.server.__main__.main) writes the durable exit record
once, on every stop path — normal or abnormal — so a silent daemon
death is attributable next time.
"""

__slots__ = ("reason", "exit_code", "traceback")

def __init__(self, reason: str, exit_code: int, traceback: str | None) -> None:
self.reason = reason
self.exit_code = exit_code
self.traceback = traceback

def write_record(self) -> None:
_write_exit_record(self.reason, self.exit_code, self.traceback)


def _write_exit_record(reason: str, exit_code: int, traceback_text: str | None) -> None:
"""Append a one-line JSON exit record to ~/.emrg/emrgd-exit.log and mirror
it into emrgd.log (rant 2026-08-25T09:25:32 — daemon silent death).

The dedicated file is append-only and survives emrgd.log rotation
(10MB x 3), so every daemon stop — timestamp, reason, exit code,
traceback — stays queryable long after the fact.
"""
record = {
"timestamp": datetime.now().astimezone().isoformat(),
"pid": os.getpid(),
"reason": reason,
"exit_code": exit_code,
"traceback": traceback_text,
}
line = json.dumps(record, ensure_ascii=False)
logger.info("daemon exit record: %s", line)
try:
with open(_EXIT_RECORD_PATH, "a", encoding="utf-8") as fh:
fh.write(line + "\n")
except OSError as exc:
logger.warning("failed to write exit record to %s: %s", _EXIT_RECORD_PATH, exc)


def _asyncio_exception_handler(loop, context) -> None:
"""Route background-task crashes into emrgd.log (rant 2026-08-25T09:25:32).

asyncio's default exception handler writes to stderr — which
daemon_manager points at DEVNULL — so an unhandled exception in a
TaskHandler / websockets callback / any background task used to vanish
without a trace (the daemon "silent death" cases). The loop handler is
installed by run_server; this one logs through the RotatingFileHandler
instead, keeping the task identity and the exception itself.
"""
exc = context.get("exception")
logger.error(
"unhandled asyncio exception (message=%r, task=%r, future=%r)",
context.get("message"), context.get("task"), context.get("future"),
exc_info=exc,
)


def _sigterm_handler(signum, frame) -> None:
"""POSIX SIGTERM → SystemExit so main() records a sigterm exit record.

Windows does not support signal.signal(SIGTERM) (daemon_manager hard-
kills there via TerminateProcess, which no Python code can survive) —
this handler is only installed where signal.signal accepts it.
"""
raise SystemExit(f"SIGTERM ({signum}) received")


async def run_server(llm_config: LlmConfig) -> DaemonExit:
"""Run the EMRG server until interrupted; return exit metadata.

Rant 2026-08-25T09:25:32 (daemon silent death): every stop path —
normal return, serve_forever crash, SIGINT, SIGTERM, cancellation —
produces a DaemonExit (reason / exit code / traceback) that the caller
persists as the durable exit record. Also installs the asyncio loop
exception handler so background task crashes reach emrgd.log instead of
vanishing into stderr=DEVNULL.
"""
loop = asyncio.get_running_loop()
loop.set_exception_handler(_asyncio_exception_handler)
try:
signal.signal(signal.SIGTERM, _sigterm_handler)
except (ValueError, OSError, AttributeError):
pass # Windows: signal.signal(SIGTERM) unsupported

server = EmrgServer(llm_config)
try:
await server.serve()
reason = server._stop_reason or "normal"
exit_code = 1 if reason == "crash" else 0
traceback_text = getattr(server, "_crash_traceback", None)
except asyncio.CancelledError:
reason = "cancel"
exit_code = 0
traceback_text = None
server._stop_reason = reason
logger.info("daemon serve cancelled — cleanup started")
except KeyboardInterrupt:
# Rant 2026-08-19T14:02:37 — attribute the stop: serve()'s teardown
# already logged the full cleanup; this line identifies the trigger.
server._stop_reason = "sigint"
reason = "sigint"
exit_code = 130
traceback_text = None
server._stop_reason = reason
logger.info("shutdown signal received (SIGINT), cleanup started")
except BaseException as exc:
# Safety net: anything that escapes serve() (a signal SystemExit, a
# hard crash) is recorded with its traceback, never silently.
reason = "sigterm" if isinstance(exc, SystemExit) else "crash"
exit_code = 143 if isinstance(exc, SystemExit) else 1
traceback_text = "".join(
traceback.format_exception(type(exc), exc, exc.__traceback__)
)
logger.critical(
"daemon crashed (%s: %s)", type(exc).__name__, exc, exc_info=True
)
return DaemonExit(reason, exit_code, traceback_text)
11 changes: 7 additions & 4 deletions emrg/server/promote_prompt.md
Original file line numberDiff line numberDiff line change
Expand Up@@ -109,7 +109,7 @@ Read the full config of `{{ project.name }}` from `~/.emrg/projects.yml` (path,
| **Lobsters** | strict rules — read the community guide first | |
| **Tech forums/communities** | V2EX, Stack Overflow relevant tags, etc. | join discussions and provide value, end with a natural link |
| **Discord/Slack** | relevant tech channels | mention naturally when helping people solve problems |
| **Blogs (blogger.com / Dev.to / Medium)** | own blog as home turf: long-form output of design philosophy, architecture decisions, latest progress (see Blog Publishing section) | deep content, not ads; project link at the end; low cadence (≤1 post/week) |
| **Blogs (blogger.com / Dev.to / Medium)** | own blog as home turf: long-form output of design philosophy, architecture decisions, latest progress (see Blog Publishing section) | deep content, not ads; project link at the end; cadence 1-3 days per post (see Blog Publishing) |

#### Secondary channels (one-off)

Expand DownExpand Up@@ -160,9 +160,12 @@ long-form output on your own turf.
give value first, project mention natural (this is a home turf, but still not a hard ad).
- **Fact-checking**: any claim about project capabilities/versions/mechanisms MUST be
verified via §0.4 first (latest commit/release); cite the latest commit/release.
- **Cadence**: low frequency, high quality — default ≤1 post/week; a new release or
major progress may add an immediate post. §0.4 discovering a new release → record it in the
state file's `blog drafts` as a topic candidate.
- **Cadence**: 1-3 days per post — at least 2 posts/week (rant 2026-08-25T10:01:20 — the old
≤1 post/week was too slow: publish-ready drafts piled up while the project ships ~8
releases/2 days, and a postmortem draft waited a full week, its window slipping to 08-27).
Publish within 1-3 days whenever a draft is ready; a new release or major progress may add
an immediate post. §0.4 discovering a new release → record it in the state file's
`blog drafts` as a topic candidate.
- **Distribution**: publish on your own blog (blogger etc.); optionally cross-post to
Dev.to/Medium (same content, note the original source link).
- **State file**: `blog posts` field (title + platform + link + publish time + topic) to
Expand Down
53 changes: 53 additions & 0 deletions tests/test_daemon.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -2135,3 +2135,56 @@ def test_tool_content_for_llm_missing_image(tmp_path):
out = server._tool_content_for_llm(ref)
assert isinstance(out, str)
assert "Image unavailable" in out


# ── Daemon observability (rant 2026-08-25T09:25:32 — silent death) ──


def test_write_exit_record(tmp_path, monkeypatch):
"""Exit records are durable one-line JSON in ~/.emrg/emrgd-exit.log —
every daemon stop, normal or abnormal, must be attributable."""
from emrg.server.daemon import _write_exit_record

target = tmp_path / "emrgd-exit.log"
monkeypatch.setattr("emrg.server.daemon._EXIT_RECORD_PATH", target)

_write_exit_record("crash", 1, "Traceback (most recent call last):\n boom")
_write_exit_record("normal", 0, None)

lines = target.read_text(encoding="utf-8").splitlines()
assert len(lines) == 2, "one JSON line per exit record"
crash = json.loads(lines[0])
assert crash["reason"] == "crash"
assert crash["exit_code"] == 1
assert "Traceback" in crash["traceback"]
normal = json.loads(lines[1])
assert normal["reason"] == "normal"
assert normal["exit_code"] == 0
assert normal["traceback"] is None
assert normal["timestamp"] and normal["pid"]


def test_asyncio_exception_handler_routes_to_logger(caplog):
"""Background-task crashes must reach the logger (emrgd.log), not die in
asyncio's default handler → stderr=DEVNULL (rant 2026-08-25T09:25:32)."""
import logging

from emrg.server.daemon import _asyncio_exception_handler

with caplog.at_level(logging.ERROR, logger="emrg.server.daemon"):
try:
raise ValueError("boom in background task")
except ValueError as exc:
_asyncio_exception_handler(None, {
"message": "Exception in Task-7",
"task": "Task-7",
"exception": exc,
})
assert any(
"unhandled asyncio exception" in r.getMessage() for r in caplog.records
)
# exc_info is not part of getMessage(); the formatted traceback (with the
# exception type name) lands in the record text / caplog.text instead.
assert any(
"ValueError" in (r.exc_text or "") for r in caplog.records
) or "ValueError" in caplog.text
Loading