From cc96e14e6f6ce021c1149160d93783630764d2d3 Mon Sep 17 00:00:00 2001 From: EMRG Evolution Date: Tue, 25 Aug 2026 10:26:47 +0800 Subject: [PATCH 1/2] emrg: daemon observability: durable exit records + stderr crash log (rant 2026-08-25T09:25:32) --- Agent.md | 2 +- emrg/server/__main__.py | 69 ++++++++++++++++++++-- emrg/server/daemon.py | 124 +++++++++++++++++++++++++++++++++++++++- tests/test_daemon.py | 53 +++++++++++++++++ 4 files changed, 239 insertions(+), 9 deletions(-) diff --git a/Agent.md b/Agent.md index 82f90e7f..c803e7e9 100644 --- a/Agent.md +++ b/Agent.md @@ -118,7 +118,7 @@ Community needs voiced in HN agent-UI discussions map directly to EMRG's design: pkill -f "emrg.server"; rm -f ~/.emrg/emrgd.token; python -m emrg ``` -Python: `uv run pytest tests/ -v` (1052) — import check: `uv run python -c "from emrg.client.app import run_client"` +Python: `uv run pytest tests/ -v` (1054) — import check: `uv run python -c "from emrg.client.app import run_client"` GUI: `cd emrg/gui && npm test` (259: 45 daemon_client + 20 conn-manager + 22 app-commands + 129 renderer smoke + 15 i18n + 8 integration + 3 commands + 8 build-config + 7 gui-state + 2 tool-group) — syntax: `node --check main.js preload.js daemon_client.js renderer/js/*.js` CI: `uv run pytest` (ubuntu + **windows-2025 matrix** — Windows pytest 回归在 PR CI 即失败，v0.2.29 教训 #725) + GUI tests + **actionlint workflow lint** (`rhysd/actionlint@v1.7.12` gate, #444 — workflow 解析错误在 PR CI 即失败，如 `if:` secrets 上下文) Re-trigger: `scripts/re-trigger-ci.sh [branch]` (workflow_dispatch, #527 — 替代空 commit 重触发：Actions outage 会整段丢弃 push 事件，dispatch 走 API 路径不受影响) diff --git a/emrg/server/__main__.py b/emrg/server/__main__.py index ada001f0..c480227a 100644 --- a/emrg/server/__main__.py +++ b/emrg/server/__main__.py @@ -1,9 +1,12 @@ """Entry point for EMRG daemon: python -m emrg.server""" import asyncio import logging +import sys +import traceback from logging.handlers import RotatingFileHandler from pathlib import Path +from emrg.server.daemon import DaemonExit, run_server from emrg.server.logcontext import session_label @@ -47,12 +50,20 @@ def _configure_logging() -> None: encoding="utf-8", # rant 2026-08-07T14:00Z: default locale code page (GBK on zh-CN Windows) mojibakes CJK log lines ) file_handler.setFormatter(_fmt) - stream_handler = logging.StreamHandler() # also to stderr (visible when run directly) - stream_handler.setFormatter(_fmt) + handlers = [file_handler] + # StreamHandler only for interactive runs: daemon_manager spawns emrgd + # with stderr=DEVNULL, so the stream handler is pure waste there — and + # since _redirect_std_streams() later points sys.stderr at the crash log, + # a stream handler would duplicate the whole log into it (rant + # 2026-08-25T09:25:32 wants an *independent* crash log). + if sys.stderr.isatty(): + stream_handler = logging.StreamHandler() # visible when run directly + stream_handler.setFormatter(_fmt) + handlers.append(stream_handler) logging.basicConfig( level=logging.DEBUG, - handlers=[file_handler, stream_handler], + handlers=handlers, ) # Suppress noisy httpcore/httpx DEBUG logs (rant #24) logging.getLogger("httpcore").setLevel(logging.WARNING) @@ -63,13 +74,61 @@ def _configure_logging() -> None: logging.getLogger("websockets").setLevel(logging.INFO) +def _redirect_std_streams() -> None: + """Redirect sys.stdout/sys.stderr to ~/.emrg/emrgd-crash.log (rant + 2026-08-25T09:25:32 — daemon silent death). + + daemon_manager spawns emrgd with stdout/stderr=DEVNULL, so anything that + bypasses logging — asyncio's default exception handler output, C-level + abort/assert traces, faulthandler dumps — vanished silently. This gives + the daemon a second, independent sink for such output, and enables + faulthandler so even a fatal-signal death (SIGSEGV etc., where no Python + code runs) leaves a stack dump behind. + """ + crash_log = Path.home() / ".emrg" / "emrgd-crash.log" + try: + stream = open(str(crash_log), "a", encoding="utf-8", buffering=1) + except OSError: + return # best-effort: keep the original DEVNULL sinks + sys.stdout = stream + sys.stderr = stream + try: + import faulthandler + + faulthandler.enable(file=stream) + except Exception: + pass # best-effort: faulthandler may reject the stream (no fileno) + + def main() -> None: from emrg.config import load_config - from emrg.server.daemon import run_server _configure_logging() + _redirect_std_streams() config = load_config() - asyncio.run(run_server(config.llm)) + try: + result = asyncio.run(run_server(config.llm)) + except KeyboardInterrupt: + # SIGINT delivered at the event-loop poll point escapes the main + # coroutine — record it here so no stop path is ever silent. + result = DaemonExit("sigint", 130, None) + logging.getLogger("emrg.server").info( + "daemon terminated by SIGINT outside the event loop" + ) + except SystemExit: + # POSIX SIGTERM handler (daemon._sigterm_handler) → SystemExit that + # escaped the loop. + result = DaemonExit("sigterm", 143, None) + logging.getLogger("emrg.server").info( + "daemon terminated by SIGTERM outside the event loop" + ) + except BaseException: + result = DaemonExit("crash", 1, traceback.format_exc()) + logging.getLogger("emrg.server").critical( + "daemon crashed outside the event loop", exc_info=True + ) + result.write_record() + sys.exit(result.exit_code) if __name__ == "__main__": diff --git a/emrg/server/daemon.py b/emrg/server/daemon.py index 6194aadd..34661340 100644 --- a/emrg/server/daemon.py +++ b/emrg/server/daemon.py @@ -24,6 +24,7 @@ import subprocess import sys import time +import traceback from datetime import datetime from pathlib import Path from typing import Optional @@ -416,6 +417,10 @@ async def serve(self) -> None: logger.info("daemon serve cancelled (asyncio.CancelledError) — cleanup started") except Exception: self._stop_reason = "crash" + # Rant 2026-08-25T09:25:32: keep the traceback so run_server can + # include it in the durable exit record (serve() returns normally + # after this handler runs — the exception is not re-raised). + self._crash_traceback = traceback.format_exc() logger.error("daemon serve crashed — cleanup started", exc_info=True) finally: await self._shutdown_all() @@ -4059,13 +4064,126 @@ async def _consolidate_session_memories( logger.debug("memory consolidation failed", exc_info=True) -async def run_server(llm_config: LlmConfig) -> None: - """Run the EMRG server until interrupted.""" +_EXIT_RECORD_PATH = Path.home() / ".emrg" / "emrgd-exit.log" + + +class DaemonExit: + """Exit metadata produced by run_server (rant 2026-08-25T09:25:32). + + The caller (emrg.server.__main__.main) writes the durable exit record + once, on every stop path — normal or abnormal — so a silent daemon + death is attributable next time. + """ + + __slots__ = ("reason", "exit_code", "traceback") + + def __init__(self, reason: str, exit_code: int, traceback: str | None) -> None: + self.reason = reason + self.exit_code = exit_code + self.traceback = traceback + + def write_record(self) -> None: + _write_exit_record(self.reason, self.exit_code, self.traceback) + + +def _write_exit_record(reason: str, exit_code: int, traceback_text: str | None) -> None: + """Append a one-line JSON exit record to ~/.emrg/emrgd-exit.log and mirror + it into emrgd.log (rant 2026-08-25T09:25:32 — daemon silent death). + + The dedicated file is append-only and survives emrgd.log rotation + (10MB x 3), so every daemon stop — timestamp, reason, exit code, + traceback — stays queryable long after the fact. + """ + record = { + "timestamp": datetime.now().astimezone().isoformat(), + "pid": os.getpid(), + "reason": reason, + "exit_code": exit_code, + "traceback": traceback_text, + } + line = json.dumps(record, ensure_ascii=False) + logger.info("daemon exit record: %s", line) + try: + with open(_EXIT_RECORD_PATH, "a", encoding="utf-8") as fh: + fh.write(line + "\n") + except OSError as exc: + logger.warning("failed to write exit record to %s: %s", _EXIT_RECORD_PATH, exc) + + +def _asyncio_exception_handler(loop, context) -> None: + """Route background-task crashes into emrgd.log (rant 2026-08-25T09:25:32). + + asyncio's default exception handler writes to stderr — which + daemon_manager points at DEVNULL — so an unhandled exception in a + TaskHandler / websockets callback / any background task used to vanish + without a trace (the daemon "silent death" cases). The loop handler is + installed by run_server; this one logs through the RotatingFileHandler + instead, keeping the task identity and the exception itself. + """ + exc = context.get("exception") + logger.error( + "unhandled asyncio exception (message=%r, task=%r, future=%r)", + context.get("message"), context.get("task"), context.get("future"), + exc_info=exc, + ) + + +def _sigterm_handler(signum, frame) -> None: + """POSIX SIGTERM → SystemExit so main() records a sigterm exit record. + + Windows does not support signal.signal(SIGTERM) (daemon_manager hard- + kills there via TerminateProcess, which no Python code can survive) — + this handler is only installed where signal.signal accepts it. + """ + raise SystemExit(f"SIGTERM ({signum}) received") + + +async def run_server(llm_config: LlmConfig) -> DaemonExit: + """Run the EMRG server until interrupted; return exit metadata. + + Rant 2026-08-25T09:25:32 (daemon silent death): every stop path — + normal return, serve_forever crash, SIGINT, SIGTERM, cancellation — + produces a DaemonExit (reason / exit code / traceback) that the caller + persists as the durable exit record. Also installs the asyncio loop + exception handler so background task crashes reach emrgd.log instead of + vanishing into stderr=DEVNULL. + """ + loop = asyncio.get_running_loop() + loop.set_exception_handler(_asyncio_exception_handler) + try: + signal.signal(signal.SIGTERM, _sigterm_handler) + except (ValueError, OSError, AttributeError): + pass # Windows: signal.signal(SIGTERM) unsupported + server = EmrgServer(llm_config) try: await server.serve() + reason = server._stop_reason or "normal" + exit_code = 1 if reason == "crash" else 0 + traceback_text = getattr(server, "_crash_traceback", None) + except asyncio.CancelledError: + reason = "cancel" + exit_code = 0 + traceback_text = None + server._stop_reason = reason + logger.info("daemon serve cancelled — cleanup started") except KeyboardInterrupt: # Rant 2026-08-19T14:02:37 — attribute the stop: serve()'s teardown # already logged the full cleanup; this line identifies the trigger. - server._stop_reason = "sigint" + reason = "sigint" + exit_code = 130 + traceback_text = None + server._stop_reason = reason logger.info("shutdown signal received (SIGINT), cleanup started") + except BaseException as exc: + # Safety net: anything that escapes serve() (a signal SystemExit, a + # hard crash) is recorded with its traceback, never silently. + reason = "sigterm" if isinstance(exc, SystemExit) else "crash" + exit_code = 143 if isinstance(exc, SystemExit) else 1 + traceback_text = "".join( + traceback.format_exception(type(exc), exc, exc.__traceback__) + ) + logger.critical( + "daemon crashed (%s: %s)", type(exc).__name__, exc, exc_info=True + ) + return DaemonExit(reason, exit_code, traceback_text) diff --git a/tests/test_daemon.py b/tests/test_daemon.py index 051bb4c4..f8fc496e 100644 --- a/tests/test_daemon.py +++ b/tests/test_daemon.py @@ -2135,3 +2135,56 @@ def test_tool_content_for_llm_missing_image(tmp_path): out = server._tool_content_for_llm(ref) assert isinstance(out, str) assert "Image unavailable" in out + + +# ── Daemon observability (rant 2026-08-25T09:25:32 — silent death) ── + + +def test_write_exit_record(tmp_path, monkeypatch): + """Exit records are durable one-line JSON in ~/.emrg/emrgd-exit.log — + every daemon stop, normal or abnormal, must be attributable.""" + from emrg.server.daemon import _write_exit_record + + target = tmp_path / "emrgd-exit.log" + monkeypatch.setattr("emrg.server.daemon._EXIT_RECORD_PATH", target) + + _write_exit_record("crash", 1, "Traceback (most recent call last):\n boom") + _write_exit_record("normal", 0, None) + + lines = target.read_text(encoding="utf-8").splitlines() + assert len(lines) == 2, "one JSON line per exit record" + crash = json.loads(lines[0]) + assert crash["reason"] == "crash" + assert crash["exit_code"] == 1 + assert "Traceback" in crash["traceback"] + normal = json.loads(lines[1]) + assert normal["reason"] == "normal" + assert normal["exit_code"] == 0 + assert normal["traceback"] is None + assert normal["timestamp"] and normal["pid"] + + +def test_asyncio_exception_handler_routes_to_logger(caplog): + """Background-task crashes must reach the logger (emrgd.log), not die in + asyncio's default handler → stderr=DEVNULL (rant 2026-08-25T09:25:32).""" + import logging + + from emrg.server.daemon import _asyncio_exception_handler + + with caplog.at_level(logging.ERROR, logger="emrg.server.daemon"): + try: + raise ValueError("boom in background task") + except ValueError as exc: + _asyncio_exception_handler(None, { + "message": "Exception in Task-7", + "task": "Task-7", + "exception": exc, + }) + assert any( + "unhandled asyncio exception" in r.getMessage() for r in caplog.records + ) + # exc_info is not part of getMessage(); the formatted traceback (with the + # exception type name) lands in the record text / caplog.text instead. + assert any( + "ValueError" in (r.exc_text or "") for r in caplog.records + ) or "ValueError" in caplog.text From bba65b58c1472a8800f382eaa1b27ff4a1b6f2ea Mon Sep 17 00:00:00 2001 From: EMRG Evolution Date: Tue, 25 Aug 2026 10:26:50 +0800 Subject: [PATCH 2/2] emrg: promote prompt: blog cadence 1-3 days per post, not <=1/week (rant 2026-08-25T10:01:20) --- emrg/server/promote_prompt.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/emrg/server/promote_prompt.md b/emrg/server/promote_prompt.md index 86f7ef1e..38446d5c 100644 --- a/emrg/server/promote_prompt.md +++ b/emrg/server/promote_prompt.md @@ -109,7 +109,7 @@ Read the full config of `{{ project.name }}` from `~/.emrg/projects.yml` (path, | **Lobsters** | strict rules — read the community guide first | | | **Tech forums/communities** | V2EX, Stack Overflow relevant tags, etc. | join discussions and provide value, end with a natural link | | **Discord/Slack** | relevant tech channels | mention naturally when helping people solve problems | -| **Blogs (blogger.com / Dev.to / Medium)** | own blog as home turf: long-form output of design philosophy, architecture decisions, latest progress (see Blog Publishing section) | deep content, not ads; project link at the end; low cadence (≤1 post/week) | +| **Blogs (blogger.com / Dev.to / Medium)** | own blog as home turf: long-form output of design philosophy, architecture decisions, latest progress (see Blog Publishing section) | deep content, not ads; project link at the end; cadence 1-3 days per post (see Blog Publishing) | #### Secondary channels (one-off) @@ -160,9 +160,12 @@ long-form output on your own turf. give value first, project mention natural (this is a home turf, but still not a hard ad). - **Fact-checking**: any claim about project capabilities/versions/mechanisms MUST be verified via §0.4 first (latest commit/release); cite the latest commit/release. -- **Cadence**: low frequency, high quality — default ≤1 post/week; a new release or - major progress may add an immediate post. §0.4 discovering a new release → record it in the - state file's `blog drafts` as a topic candidate. +- **Cadence**: 1-3 days per post — at least 2 posts/week (rant 2026-08-25T10:01:20 — the old + ≤1 post/week was too slow: publish-ready drafts piled up while the project ships ~8 + releases/2 days, and a postmortem draft waited a full week, its window slipping to 08-27). + Publish within 1-3 days whenever a draft is ready; a new release or major progress may add + an immediate post. §0.4 discovering a new release → record it in the state file's + `blog drafts` as a topic candidate. - **Distribution**: publish on your own blog (blogger etc.); optionally cross-post to Dev.to/Medium (same content, note the original source link). - **State file**: `blog posts` field (title + platform + link + publish time + topic) to