emrg: scripts/sync-master-from-api.py — advance local refs via Git Data API when git-over-https is down

Agent.md

@@ -118,10 +118,11 @@ Community needs voiced in HN agent-UI discussions map directly to EMRG's design:
 pkill -f "emrg.server"; rm -f ~/.emrg/emrgd.token; python -m emrg
 ```
 
-Python: `uv run pytest tests/ -v` (1082) — import check: `uv run python -c "from emrg.client.app import run_client"`
+Python: `uv run pytest tests/ -v` (1087) — import check: `uv run python -c "from emrg.client.app import run_client"`
 GUI: `cd emrg/gui && npm test` (262: 45 daemon_client + 20 conn-manager + 22 app-commands + 132 renderer smoke + 15 i18n + 8 integration + 3 commands + 8 build-config + 7 gui-state + 2 tool-group) — syntax: `node --check main.js preload.js daemon_client.js renderer/js/*.js`
 CI: `uv run pytest` (ubuntu + **windows-2025 matrix** — Windows pytest 回归在 PR CI 即失败，v0.2.29 教训 #725) + GUI tests + **actionlint workflow lint** (`rhysd/actionlint@v1.7.12` gate, #444 — workflow 解析错误在 PR CI 即失败，如 `if:` secrets 上下文)
 Re-trigger: `scripts/re-trigger-ci.sh [branch]` (workflow_dispatch, #527 — 替代空 commit 重触发：Actions outage 会整段丢弃 push 事件，dispatch 走 API 路径不受影响)
+Git-over-https 兜底: `python scripts/sync-master-from-api.py [--repo owner/name] [--ref master]` — 受限网络下 github.com:443 不可达而 api.github.com 可达时，用 Git Data API 的 verification payload + signature 字节级重建上游 commit（含 web-flow GPG 签名 squash merge，reconstruct_commit 经 hermetic 测试验证 sha 一致）并推进本地 refs；内容对象缺失时 fail-loud 提示改用 git fetch（10+ 周期实证的恢复路径）
 
 ## Packaging
 

scripts/sync-master-from-api.py

@@ -0,0 +1,158 @@
+#!/usr/bin/env python3
+"""Advance local git refs via the GitHub REST API when git-over-https is down.
+
+EMRG has repeatedly hit github.com:443 being unreachable while api.github.com
+stays up (10+ documented cycles, e.g. 08-22..08-26). The usual fallback flow:
+the local repo already contains the content (a branch pushed via the Git Data
+API that later got squash-merged upstream), so advancing local refs only needs
+the missing *commit* objects — reconstructed byte-exact from the API's
+verification payload + signature, including web-flow GPG-signed squash merges.
+
+Usage:
+ python scripts/sync-master-from-api.py [--repo owner/name] [--ref master]
+
+Behavior:
+ * resolves repo from --repo or `git remote get-url origin`
+ * walks the remote commit chain from <ref> head down to the first commit
+ already present locally, writing each missing commit object via
+ `git hash-object -t commit -w` (byte-exact, GPG signature preserved)
+ * verifies the root tree sha matches the remote (fail-loud if content
+ objects are missing locally — prefer `git fetch` when https returns)
+ * updates refs/heads/<ref> and refs/remotes/origin/<ref>
+
+Requirements: git on PATH; api.github.com reachable. Auth: optional for public
+repos (GH_TOKEN or gh CLI used if available, higher rate limit).
+"""
+from __future__ import annotations
+
+import argparse
+import base64
+import json
+import os
+import re
+import subprocess
+import sys
+import urllib.request
+
+API = "https://api.github.com"
+
+
+def api_get(url: str) -> dict:
+ """GET a GitHub API URL, using GH_TOKEN or gh CLI auth when available."""
+ headers = {"User-Agent": "emrg-sync-master-from-api", "Accept": "application/vnd.github+json"}
+ token = os.environ.get("GH_TOKEN") or os.environ.get("GITHUB_TOKEN")
+ if token:
+ headers["Authorization"] = "Bearer " + token
+ req = urllib.request.Request(url, headers=headers)
+ try:
+ with urllib.request.urlopen(req, timeout=20) as resp:
+ return json.load(resp)
+ except urllib.error.HTTPError as e:
+ if e.code == 403 and not token:
+ # anonymous rate-limited; try gh CLI which uses keyring auth
+ out = subprocess.run(["gh", "api", url.replace(API, ""), "--jq", "."],
+ capture_output=True, text=True, timeout=30)
+ if out.returncode == 0 and out.stdout.strip():
+ return json.loads(out.stdout)
+ raise
+
+
+def reconstruct_commit(payload: str, signature: str | None, message: str) -> bytes:
+ """Rebuild the raw commit object bytes from the API's signed payload.
+
+ The verification payload is exactly the content that was GPG-signed:
+ header block + blank line + message. The raw object additionally embeds
+ the `gpgsig` header between the committer line and the blank line, with
+ every continuation line prefixed by a single space. Unsigned commits have
+ no signature — the raw object equals the payload as-is.
+ """
+ if signature:
+ idx = payload.index("\n\n")
+ header = payload[:idx]
+ msg = payload[idx + 2 :]
+ sig_lines = signature.split("\n")
+ gpgsig = ["gpgsig " + sig_lines[0]] + [" " + l for l in sig_lines[1:]]
+ raw = header + "\n" + "\n".join(gpgsig) + "\n\n" + msg
+ if message and message not in raw:
+ raise ValueError("payload/message mismatch: reconstructed object does not contain the API message")
+ return raw.encode("utf-8")
+ raw = payload.encode("utf-8")
+ if message and message.encode("utf-8") not in raw:
+ raise ValueError("payload/message mismatch: unsigned payload does not contain the API message")
+ return raw
+
+
+def write_commit_object(raw: bytes) -> str:
+ """Write a raw commit object into the local store, returning its sha."""
+ r = subprocess.run(["git", "hash-object", "-t", "commit", "-w", "--stdin"],
+ input=raw, capture_output=True)
+ if r.returncode != 0:
+ raise RuntimeError("git hash-object failed: " + r.stderr.decode(errors="replace"))
+ return r.stdout.decode().strip()
+
+
+def has_object(sha: str) -> bool:
+ return subprocess.run(["git", "cat-file", "-e", sha + "^{commit}"],
+ capture_output=True).returncode == 0
+
+
+def rev_parse(ref: str) -> str:
+ r = subprocess.run(["git", "rev-parse", "-q", "--verify", ref], capture_output=True)
+ return r.stdout.decode().strip() if r.returncode == 0 else ""
+
+
+def repo_from_origin() -> str:
+ r = subprocess.run(["git", "remote", "get-url", "origin"], capture_output=True, text=True)
+ url = r.stdout.strip()
+ m = re.search(r"(?:github\.com[:/])([^/]+)/([^/.]+)", url)
+ if not m:
+ raise SystemExit("cannot infer owner/repo from origin URL: " + url)
+ return m.group(1) + "/" + m.group(2)
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser(description=__doc__)
+ ap.add_argument("--repo", help="owner/name (default: inferred from origin URL)")
+ ap.add_argument("--ref", default="master", help="branch name to sync (default: master)")
+ args = ap.parse_args()
+
+ repo = args.repo or repo_from_origin()
+ head = api_get(f"{API}/repos/{repo}/commits/{args.ref}")["sha"]
+ print(f"remote {repo} {args.ref} head: {head[:7]}")
+
+ # Walk the commit chain, reconstructing missing commits until a known one.
+ sha = head
+ created = 0
+ while sha and not has_object(sha):
+ c = api_get(f"{API}/repos/{repo}/commits/{sha}")
+ body = c["commit"]
+ payload = body["verification"]["payload"]
+ signature = body["verification"]["signature"] or None
+ raw = reconstruct_commit(payload, signature, body["message"])
+ got = write_commit_object(raw)
+ if got != sha:
+ raise RuntimeError(f"reconstruction mismatch: want {sha}, got {got} — aborting (no refs touched)")
+ created += 1
+ print(f" + {sha[:7]} ({body['author']['name']}, {body['message'].splitlines()[0][:60]})")
+ parents = [p["sha"] for p in c["parents"]]
+ sha = parents[0] if len(parents) == 1 else None # merge commits: stop, local must have them
+ if created == 0:
+ print(f" (head already present locally: {sha[:7]})")
+
+ # Verify the root tree matches (fail-loud if content objects are missing).
+ tree = api_get(f"{API}/repos/{repo}/git/commits/{head}")["tree"]["sha"]
+ local_tree = subprocess.run(["git", "rev-parse", head + "^{tree}"],
+ capture_output=True, text=True)
+ if local_tree.returncode != 0 or local_tree.stdout.strip() != tree:
+ raise SystemExit(f"tree mismatch or missing objects for {head[:7]} "
+ f"(want {tree}, got {local_tree.stdout.strip() or 'NONE'}) — "
+ "run `git fetch` when https returns")
+
+ for ref in (f"refs/heads/{args.ref}", f"refs/remotes/origin/{args.ref}"):
+ subprocess.run(["git", "update-ref", ref, head], check=True)
+ print(f"updated refs/heads/{args.ref} and refs/remotes/origin/{args.ref} -> {head[:7]}")
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())

tests/test_sync_master_from_api.py

@@ -0,0 +1,136 @@
+"""scripts/sync-master-from-api.py 回归测试。
+
+背景（rant 驱动，10+ 周期实证）：git-over-https (github.com:443) 在受限网络反复
+不可达而 api.github.com 可达。本脚本用 Git Data API 的 verification payload +
+signature 字节级重建上游 commit（含 web-flow GPG 签名 squash merge），推进本地
+refs。核心风险 = 重建逻辑产生错误 sha（→ 本地历史与上游分叉）：
+ 1. 文本断言：脚本必须包含签名感知重建 + 树校验 + 失败即止（不触碰 refs）的接线
+ 2. 行为断言（hermetic，无网络）：在临时 git 仓库里用 git commit-tree 合成
+ unsigned / signed 两类 commit，验证 reconstruct_commit() 字节级复现同一 sha
+"""
+import importlib.util
+import os
+import subprocess
+
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+SCRIPT = REPO_ROOT / "scripts" / "sync-master-from-api.py"
+
+EMPTY_TREE = "4b825dc642cb6eb9a060e54bf8d69288fbee4904"
+GIT_ENV = {
+ "GIT_AUTHOR_NAME": "Test Author",
+ "GIT_AUTHOR_EMAIL": "test@example.com",
+ "GIT_AUTHOR_DATE": "1700000000 +0800",
+ "GIT_COMMITTER_NAME": "Test Committer",
+ "GIT_COMMITTER_EMAIL": "committer@example.com",
+ "GIT_COMMITTER_DATE": "1700000000 +0800",
+}
+
+
+def _load_module():
+ spec = importlib.util.spec_from_file_location("sync_master_from_api", SCRIPT)
+ mod = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(mod)
+ return mod
+
+
+def _git(*args, cwd=None, env=None):
+ e = dict(os.environ)
+ e.update(env or {})
+ return subprocess.run(["git"] + list(args), capture_output=True, cwd=cwd, env=e)
+
+
+def _init_repo(tmp_path: Path) -> Path:
+ repo = tmp_path / "repo"
+ repo.mkdir()
+ _git("init", "-q", cwd=repo)
+ _git("config", "user.name", "Test Author", cwd=repo)
+ _git("config", "user.email", "test@example.com", cwd=repo)
+ return repo
+
+
+def _make_commit(repo: Path, message: str = "test commit"):
+ """Create a commit via commit-tree; return (raw_bytes, sha)."""
+ r = _git("commit-tree", EMPTY_TREE, "-m", message, cwd=repo, env=GIT_ENV)
+ assert r.returncode == 0, r.stderr
+ sha = r.stdout.decode().strip()
+ raw = _git("cat-file", "commit", sha, cwd=repo)
+ return raw.stdout, sha
+
+
+# ---------------------------------------------------------------- text wiring
+
+
+def test_script_reconstructs_gpg_signed_commits():
+ content = SCRIPT.read_text(encoding="utf-8")
+ assert "gpgsig " in content # signature block embedding
+ assert "verification" in content # payload source
+ assert "reconstruct_commit" in content # core logic named
+
+
+def test_script_fails_loud_before_touching_refs():
+ content = SCRIPT.read_text(encoding="utf-8")
+ assert "aborting (no refs touched)" in content # mismatch → stop, refs safe
+ assert "tree mismatch" in content # content-object check
+ assert "update-ref" in content # refs updated only at the end
+
+
+# ---------------------------------------------------------- byte-exact logic
+
+
+def test_reconstruct_unsigned_commit_is_byte_exact(tmp_path):
+ mod = _load_module()
+ repo = _init_repo(tmp_path)
+ raw, sha = _make_commit(repo, "unsigned msg")
+ payload = raw.decode("utf-8") # API payload == raw for unsigned commits
+ rebuilt = mod.reconstruct_commit(payload, None, "unsigned msg")
+ assert rebuilt == raw
+ r = subprocess.run(["git", "hash-object", "-t", "commit", "--stdin"],
+ input=rebuilt, capture_output=True, cwd=repo)
+ assert r.returncode == 0
+ assert r.stdout.decode().strip() == sha # byte-exact sha reproduction
+
+
+def test_reconstruct_signed_commit_is_byte_exact(tmp_path):
+ """Insert a fake gpgsig block into a synthetic commit, then verify the
+ signed branch of reconstruct_commit() reproduces the exact raw bytes."""
+ mod = _load_module()
+ repo = _init_repo(tmp_path)
+ raw_u, sha_u = _make_commit(repo, "signed msg")
+
+ idx = raw_u.index(b"\n\n")
+ header, msg = raw_u[:idx], raw_u[idx + 2:]
+ sig = ("-----BEGIN PGP SIGNATURE-----\n"
+ "\n"
+ "wsFcBAABCAAQBQJabcdeCRC1aQ7uu5UhlAAARDgQACxKc\n"
+ "KDO6GweASekxICOQVyQPEatLzNCjKyEEth8Z6TfQ97s\n"
+ "=/aNl\n"
+ "-----END PGP SIGNATURE-----")
+ # raw signed object: header + newline + gpgsig block (continuation lines
+ # space-prefixed) + blank line + message
+ sig_lines = sig.split("\n")
+ gpgsig = ["gpgsig " + sig_lines[0]] + [" " + l for l in sig_lines[1:]]
+ raw_s = header + b"\n" + "\n".join(gpgsig).encode("utf-8") + b"\n\n" + msg
+
+ r = subprocess.run(["git", "hash-object", "-t", "commit", "-w", "--stdin"],
+ input=raw_s, capture_output=True, cwd=repo)
+ assert r.returncode == 0
+ sha_s = r.stdout.decode().strip()
+
+ payload_s = (header + b"\n\n" + msg).decode("utf-8") # what the API stores
+ rebuilt = mod.reconstruct_commit(payload_s, sig, "signed msg")
+ assert rebuilt == raw_s
+ r2 = subprocess.run(["git", "hash-object", "-t", "commit", "--stdin"],
+ input=rebuilt, capture_output=True, cwd=repo)
+ assert r2.stdout.decode().strip() == sha_s
+
+
+def test_reconstruct_commit_mismatch_raises(tmp_path):
+ mod = _load_module()
+ repo = _init_repo(tmp_path)
+ raw, _ = _make_commit(repo, "real msg")
+ import pytest
+
+ with pytest.raises(ValueError):
+ mod.reconstruct_commit(raw.decode("utf-8"), None, "different msg")