diff --git a/Agent.md b/Agent.md index cbdb188e..2c363c9e 100644 --- a/Agent.md +++ b/Agent.md @@ -118,10 +118,11 @@ Community needs voiced in HN agent-UI discussions map directly to EMRG's design: pkill -f "emrg.server"; rm -f ~/.emrg/emrgd.token; python -m emrg ``` -Python: `uv run pytest tests/ -v` (1082) — import check: `uv run python -c "from emrg.client.app import run_client"` +Python: `uv run pytest tests/ -v` (1087) — import check: `uv run python -c "from emrg.client.app import run_client"` GUI: `cd emrg/gui && npm test` (262: 45 daemon_client + 20 conn-manager + 22 app-commands + 132 renderer smoke + 15 i18n + 8 integration + 3 commands + 8 build-config + 7 gui-state + 2 tool-group) — syntax: `node --check main.js preload.js daemon_client.js renderer/js/*.js` CI: `uv run pytest` (ubuntu + **windows-2025 matrix** — Windows pytest 回归在 PR CI 即失败,v0.2.29 教训 #725) + GUI tests + **actionlint workflow lint** (`rhysd/actionlint@v1.7.12` gate, #444 — workflow 解析错误在 PR CI 即失败,如 `if:` secrets 上下文) Re-trigger: `scripts/re-trigger-ci.sh [branch]` (workflow_dispatch, #527 — 替代空 commit 重触发:Actions outage 会整段丢弃 push 事件,dispatch 走 API 路径不受影响) +Git-over-https 兜底: `python scripts/sync-master-from-api.py [--repo owner/name] [--ref master]` — 受限网络下 github.com:443 不可达而 api.github.com 可达时,用 Git Data API 的 verification payload + signature 字节级重建上游 commit(含 web-flow GPG 签名 squash merge,reconstruct_commit 经 hermetic 测试验证 sha 一致)并推进本地 refs;内容对象缺失时 fail-loud 提示改用 git fetch(10+ 周期实证的恢复路径) ## Packaging diff --git a/scripts/sync-master-from-api.py b/scripts/sync-master-from-api.py new file mode 100644 index 00000000..53a8c9b4 --- /dev/null +++ b/scripts/sync-master-from-api.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python3 +"""Advance local git refs via the GitHub REST API when git-over-https is down. + +EMRG has repeatedly hit github.com:443 being unreachable while api.github.com +stays up (10+ documented cycles, e.g. 08-22..08-26). The usual fallback flow: +the local repo already contains the content (a branch pushed via the Git Data +API that later got squash-merged upstream), so advancing local refs only needs +the missing *commit* objects — reconstructed byte-exact from the API's +verification payload + signature, including web-flow GPG-signed squash merges. + +Usage: + python scripts/sync-master-from-api.py [--repo owner/name] [--ref master] + +Behavior: + * resolves repo from --repo or `git remote get-url origin` + * walks the remote commit chain from head down to the first commit + already present locally, writing each missing commit object via + `git hash-object -t commit -w` (byte-exact, GPG signature preserved) + * verifies the root tree sha matches the remote (fail-loud if content + objects are missing locally — prefer `git fetch` when https returns) + * updates refs/heads/ and refs/remotes/origin/ + +Requirements: git on PATH; api.github.com reachable. Auth: optional for public +repos (GH_TOKEN or gh CLI used if available, higher rate limit). +""" +from __future__ import annotations + +import argparse +import base64 +import json +import os +import re +import subprocess +import sys +import urllib.request + +API = "https://api.github.com" + + +def api_get(url: str) -> dict: + """GET a GitHub API URL, using GH_TOKEN or gh CLI auth when available.""" + headers = {"User-Agent": "emrg-sync-master-from-api", "Accept": "application/vnd.github+json"} + token = os.environ.get("GH_TOKEN") or os.environ.get("GITHUB_TOKEN") + if token: + headers["Authorization"] = "Bearer " + token + req = urllib.request.Request(url, headers=headers) + try: + with urllib.request.urlopen(req, timeout=20) as resp: + return json.load(resp) + except urllib.error.HTTPError as e: + if e.code == 403 and not token: + # anonymous rate-limited; try gh CLI which uses keyring auth + out = subprocess.run(["gh", "api", url.replace(API, ""), "--jq", "."], + capture_output=True, text=True, timeout=30) + if out.returncode == 0 and out.stdout.strip(): + return json.loads(out.stdout) + raise + + +def reconstruct_commit(payload: str, signature: str | None, message: str) -> bytes: + """Rebuild the raw commit object bytes from the API's signed payload. + + The verification payload is exactly the content that was GPG-signed: + header block + blank line + message. The raw object additionally embeds + the `gpgsig` header between the committer line and the blank line, with + every continuation line prefixed by a single space. Unsigned commits have + no signature — the raw object equals the payload as-is. + """ + if signature: + idx = payload.index("\n\n") + header = payload[:idx] + msg = payload[idx + 2 :] + sig_lines = signature.split("\n") + gpgsig = ["gpgsig " + sig_lines[0]] + [" " + l for l in sig_lines[1:]] + raw = header + "\n" + "\n".join(gpgsig) + "\n\n" + msg + if message and message not in raw: + raise ValueError("payload/message mismatch: reconstructed object does not contain the API message") + return raw.encode("utf-8") + raw = payload.encode("utf-8") + if message and message.encode("utf-8") not in raw: + raise ValueError("payload/message mismatch: unsigned payload does not contain the API message") + return raw + + +def write_commit_object(raw: bytes) -> str: + """Write a raw commit object into the local store, returning its sha.""" + r = subprocess.run(["git", "hash-object", "-t", "commit", "-w", "--stdin"], + input=raw, capture_output=True) + if r.returncode != 0: + raise RuntimeError("git hash-object failed: " + r.stderr.decode(errors="replace")) + return r.stdout.decode().strip() + + +def has_object(sha: str) -> bool: + return subprocess.run(["git", "cat-file", "-e", sha + "^{commit}"], + capture_output=True).returncode == 0 + + +def rev_parse(ref: str) -> str: + r = subprocess.run(["git", "rev-parse", "-q", "--verify", ref], capture_output=True) + return r.stdout.decode().strip() if r.returncode == 0 else "" + + +def repo_from_origin() -> str: + r = subprocess.run(["git", "remote", "get-url", "origin"], capture_output=True, text=True) + url = r.stdout.strip() + m = re.search(r"(?:github\.com[:/])([^/]+)/([^/.]+)", url) + if not m: + raise SystemExit("cannot infer owner/repo from origin URL: " + url) + return m.group(1) + "/" + m.group(2) + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--repo", help="owner/name (default: inferred from origin URL)") + ap.add_argument("--ref", default="master", help="branch name to sync (default: master)") + args = ap.parse_args() + + repo = args.repo or repo_from_origin() + head = api_get(f"{API}/repos/{repo}/commits/{args.ref}")["sha"] + print(f"remote {repo} {args.ref} head: {head[:7]}") + + # Walk the commit chain, reconstructing missing commits until a known one. + sha = head + created = 0 + while sha and not has_object(sha): + c = api_get(f"{API}/repos/{repo}/commits/{sha}") + body = c["commit"] + payload = body["verification"]["payload"] + signature = body["verification"]["signature"] or None + raw = reconstruct_commit(payload, signature, body["message"]) + got = write_commit_object(raw) + if got != sha: + raise RuntimeError(f"reconstruction mismatch: want {sha}, got {got} — aborting (no refs touched)") + created += 1 + print(f" + {sha[:7]} ({body['author']['name']}, {body['message'].splitlines()[0][:60]})") + parents = [p["sha"] for p in c["parents"]] + sha = parents[0] if len(parents) == 1 else None # merge commits: stop, local must have them + if created == 0: + print(f" (head already present locally: {sha[:7]})") + + # Verify the root tree matches (fail-loud if content objects are missing). + tree = api_get(f"{API}/repos/{repo}/git/commits/{head}")["tree"]["sha"] + local_tree = subprocess.run(["git", "rev-parse", head + "^{tree}"], + capture_output=True, text=True) + if local_tree.returncode != 0 or local_tree.stdout.strip() != tree: + raise SystemExit(f"tree mismatch or missing objects for {head[:7]} " + f"(want {tree}, got {local_tree.stdout.strip() or 'NONE'}) — " + "run `git fetch` when https returns") + + for ref in (f"refs/heads/{args.ref}", f"refs/remotes/origin/{args.ref}"): + subprocess.run(["git", "update-ref", ref, head], check=True) + print(f"updated refs/heads/{args.ref} and refs/remotes/origin/{args.ref} -> {head[:7]}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_sync_master_from_api.py b/tests/test_sync_master_from_api.py new file mode 100644 index 00000000..02b447e8 --- /dev/null +++ b/tests/test_sync_master_from_api.py @@ -0,0 +1,136 @@ +"""scripts/sync-master-from-api.py 回归测试。 + +背景(rant 驱动,10+ 周期实证):git-over-https (github.com:443) 在受限网络反复 +不可达而 api.github.com 可达。本脚本用 Git Data API 的 verification payload + +signature 字节级重建上游 commit(含 web-flow GPG 签名 squash merge),推进本地 +refs。核心风险 = 重建逻辑产生错误 sha(→ 本地历史与上游分叉): + 1. 文本断言:脚本必须包含签名感知重建 + 树校验 + 失败即止(不触碰 refs)的接线 + 2. 行为断言(hermetic,无网络):在临时 git 仓库里用 git commit-tree 合成 + unsigned / signed 两类 commit,验证 reconstruct_commit() 字节级复现同一 sha +""" +import importlib.util +import os +import subprocess + +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +SCRIPT = REPO_ROOT / "scripts" / "sync-master-from-api.py" + +EMPTY_TREE = "4b825dc642cb6eb9a060e54bf8d69288fbee4904" +GIT_ENV = { + "GIT_AUTHOR_NAME": "Test Author", + "GIT_AUTHOR_EMAIL": "test@example.com", + "GIT_AUTHOR_DATE": "1700000000 +0800", + "GIT_COMMITTER_NAME": "Test Committer", + "GIT_COMMITTER_EMAIL": "committer@example.com", + "GIT_COMMITTER_DATE": "1700000000 +0800", +} + + +def _load_module(): + spec = importlib.util.spec_from_file_location("sync_master_from_api", SCRIPT) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +def _git(*args, cwd=None, env=None): + e = dict(os.environ) + e.update(env or {}) + return subprocess.run(["git"] + list(args), capture_output=True, cwd=cwd, env=e) + + +def _init_repo(tmp_path: Path) -> Path: + repo = tmp_path / "repo" + repo.mkdir() + _git("init", "-q", cwd=repo) + _git("config", "user.name", "Test Author", cwd=repo) + _git("config", "user.email", "test@example.com", cwd=repo) + return repo + + +def _make_commit(repo: Path, message: str = "test commit"): + """Create a commit via commit-tree; return (raw_bytes, sha).""" + r = _git("commit-tree", EMPTY_TREE, "-m", message, cwd=repo, env=GIT_ENV) + assert r.returncode == 0, r.stderr + sha = r.stdout.decode().strip() + raw = _git("cat-file", "commit", sha, cwd=repo) + return raw.stdout, sha + + +# ---------------------------------------------------------------- text wiring + + +def test_script_reconstructs_gpg_signed_commits(): + content = SCRIPT.read_text(encoding="utf-8") + assert "gpgsig " in content # signature block embedding + assert "verification" in content # payload source + assert "reconstruct_commit" in content # core logic named + + +def test_script_fails_loud_before_touching_refs(): + content = SCRIPT.read_text(encoding="utf-8") + assert "aborting (no refs touched)" in content # mismatch → stop, refs safe + assert "tree mismatch" in content # content-object check + assert "update-ref" in content # refs updated only at the end + + +# ---------------------------------------------------------- byte-exact logic + + +def test_reconstruct_unsigned_commit_is_byte_exact(tmp_path): + mod = _load_module() + repo = _init_repo(tmp_path) + raw, sha = _make_commit(repo, "unsigned msg") + payload = raw.decode("utf-8") # API payload == raw for unsigned commits + rebuilt = mod.reconstruct_commit(payload, None, "unsigned msg") + assert rebuilt == raw + r = subprocess.run(["git", "hash-object", "-t", "commit", "--stdin"], + input=rebuilt, capture_output=True, cwd=repo) + assert r.returncode == 0 + assert r.stdout.decode().strip() == sha # byte-exact sha reproduction + + +def test_reconstruct_signed_commit_is_byte_exact(tmp_path): + """Insert a fake gpgsig block into a synthetic commit, then verify the + signed branch of reconstruct_commit() reproduces the exact raw bytes.""" + mod = _load_module() + repo = _init_repo(tmp_path) + raw_u, sha_u = _make_commit(repo, "signed msg") + + idx = raw_u.index(b"\n\n") + header, msg = raw_u[:idx], raw_u[idx + 2:] + sig = ("-----BEGIN PGP SIGNATURE-----\n" + "\n" + "wsFcBAABCAAQBQJabcdeCRC1aQ7uu5UhlAAARDgQACxKc\n" + "KDO6GweASekxICOQVyQPEatLzNCjKyEEth8Z6TfQ97s\n" + "=/aNl\n" + "-----END PGP SIGNATURE-----") + # raw signed object: header + newline + gpgsig block (continuation lines + # space-prefixed) + blank line + message + sig_lines = sig.split("\n") + gpgsig = ["gpgsig " + sig_lines[0]] + [" " + l for l in sig_lines[1:]] + raw_s = header + b"\n" + "\n".join(gpgsig).encode("utf-8") + b"\n\n" + msg + + r = subprocess.run(["git", "hash-object", "-t", "commit", "-w", "--stdin"], + input=raw_s, capture_output=True, cwd=repo) + assert r.returncode == 0 + sha_s = r.stdout.decode().strip() + + payload_s = (header + b"\n\n" + msg).decode("utf-8") # what the API stores + rebuilt = mod.reconstruct_commit(payload_s, sig, "signed msg") + assert rebuilt == raw_s + r2 = subprocess.run(["git", "hash-object", "-t", "commit", "--stdin"], + input=rebuilt, capture_output=True, cwd=repo) + assert r2.stdout.decode().strip() == sha_s + + +def test_reconstruct_commit_mismatch_raises(tmp_path): + mod = _load_module() + repo = _init_repo(tmp_path) + raw, _ = _make_commit(repo, "real msg") + import pytest + + with pytest.raises(ValueError): + mod.reconstruct_commit(raw.decode("utf-8"), None, "different msg")