emrg: scripts/sync-master-from-api.py — auto-fetch missing blobs/trees via Git Data API

Agent.md

@@ -119,7 +119,7 @@ Community needs voiced in HN agent-UI discussions map directly to EMRG's design:
 pkill -f "emrg.server"; rm -f ~/.emrg/emrgd.token; python -m emrg
 ```
 
-Python: `uv run pytest tests/ -v` (1099) — import check: `uv run python -c "from emrg.client.app import run_client"`
+Python: `uv run pytest tests/ -v` (1102) — import check: `uv run python -c "from emrg.client.app import run_client"`
 GUI: `cd emrg/gui && npm test` (265: 45 daemon_client + 20 conn-manager + 22 app-commands + 132 renderer smoke + 15 i18n + 8 integration + 3 commands + 8 build-config + 7 gui-state + 2 tool-group + 3 preload-api) — syntax: `node --check main.js preload.js daemon_client.js renderer/js/*.js`; renderer React suite: `cd emrg/gui/renderer && npm run typecheck && npm test` (151 vitest: 5 snapshot-store + 9 utils + 3 ErrorBoundary + 2 App smoke + 11 commands + 4 copywriting + 11 i18n + 11 markdown + 15 transcript + 7 TranscriptView + 15 history + 22 composer + 14 Composer + 12 sidebar + 10 Sidebar) + `npm run build` → `renderer/dist/`
 CI: `uv run pytest` (ubuntu + **windows-2025 matrix** — Windows pytest 回归在 PR CI 即失败，v0.2.29 教训 #725) + GUI tests + **actionlint workflow lint** (`rhysd/actionlint@v1.7.12` gate, #444 — workflow 解析错误在 PR CI 即失败，如 `if:` secrets 上下文)
 Re-trigger: `scripts/re-trigger-ci.sh [branch]` (workflow_dispatch, #527 — 替代空 commit 重触发：Actions outage 会整段丢弃 push 事件，dispatch 走 API 路径不受影响)

scripts/sync-master-from-api.py

@@ -8,16 +8,26 @@
 the missing *commit* objects — reconstructed byte-exact from the API's
 verification payload + signature, including web-flow GPG-signed squash merges.
 
+
+When the head commit's *content* objects (blobs/trees) are also missing locally
+(e.g. a parallel PR introduced files this repo never had — first hit in cycle
+cyc20260826-154904 with #994's GUI assets), the script now auto-fetches them via
+the Git Data API: blobs via `git/blobs/{sha}` + `git hash-object -w`, trees via
+`git/trees/{sha}` + `git mktree` (canonical ordering), recursing bottom-up. The
+previous behavior failed loud with "run git fetch when https returns" and forced
+a manual gh-api + mktree recovery dance.
+
 Usage:
- python scripts/sync-master-from-api.py [--repo owner/name] [--ref master]
+ python scripts/sync-master-from-api.py [--repo owner/name] [--ref master] [--repo owner/name] [--ref master]
 
 Behavior:
  * resolves repo from --repo or `git remote get-url origin`
  * walks the remote commit chain from <ref> head down to the first commit
  already present locally, writing each missing commit object via
  `git hash-object -t commit -w` (byte-exact, GPG signature preserved)
- * verifies the root tree sha matches the remote (fail-loud if content
- objects are missing locally — prefer `git fetch` when https returns)
+ * verifies the root tree sha matches the remote; if content objects are
+ missing, fetches missing blobs/trees via the Git Data API (disable with
+ --no-fetch-objects) and re-verifies — fail-loud only if still mismatched
  * updates refs/heads/<ref> and refs/remotes/origin/<ref>
 
 Requirements: git on PATH; api.github.com reachable. Auth: optional for public
@@ -121,6 +131,52 @@ def has_object(sha: str) -> bool:
  capture_output=True).returncode == 0
 
 
+
+def _object_exists(sha: str) -> bool:
+ """Any object (blob/tree/commit) present locally by sha."""
+ return subprocess.run(["git", "cat-file", "-e", sha],
+ capture_output=True).returncode == 0
+
+
+def _fetch_blob(repo: str, blob_sha: str) -> None:
+ """Fetch one missing blob via the Git Data API, writing it byte-exact."""
+ if _object_exists(blob_sha):
+ return
+ b = api_get(f"{API}/repos/{repo}/git/blobs/{blob_sha}")
+ if b.get("encoding") == "base64":
+ content = base64.b64decode(b["content"])
+ else: # utf-8 text blobs are returned raw
+ content = b["content"].encode("utf-8")
+ r = subprocess.run(["git", "hash-object", "-w", "--stdin"], input=content,
+ capture_output=True)
+ if r.returncode != 0 or r.stdout.decode().strip() != blob_sha:
+ raise RuntimeError(f"blob materialization mismatch for {blob_sha[:7]} "
+ f"(got {r.stdout.decode().strip()[:7] or 'NONE'})")
+
+
+def _fetch_tree(repo: str, tree_sha: str) -> None:
+ """Recursively materialize a missing tree: blobs via hash-object, subtrees
+ via git mktree (git canonical ordering), bottom-up. Idempotent."""
+ if _object_exists(tree_sha):
+ return
+ t = api_get(f"{API}/repos/{repo}/git/trees/{tree_sha}")
+ if t.get("truncated"):
+ raise RuntimeError(f"tree {tree_sha[:7]} truncated by API (>100k entries)")
+ entries = t.get("tree", [])
+ for e in entries:
+ if e["type"] == "blob":
+ _fetch_blob(repo, e["sha"])
+ elif e["type"] == "tree":
+ _fetch_tree(repo, e["sha"])
+ # commit entries (submodules): leave to git fetch — rare in this repo
+ lines = [f"{e['mode']} {e['type']} {e['sha']}\t{e['path']}" for e in entries]
+ r = subprocess.run(["git", "mktree"], input=("\n".join(lines) + "\n").encode("utf-8"),
+ capture_output=True)
+ if r.returncode != 0 or r.stdout.decode().strip() != tree_sha:
+ raise RuntimeError(f"tree materialization mismatch for {tree_sha[:7]} "
+ f"(got {r.stdout.decode().strip()[:7] or 'NONE'})")
+
+
 def rev_parse(ref: str) -> str:
  r = subprocess.run(["git", "rev-parse", "-q", "--verify", ref], capture_output=True)
  return r.stdout.decode().strip() if r.returncode == 0 else ""
@@ -139,6 +195,8 @@ def main() -> int:
  ap = argparse.ArgumentParser(description=__doc__)
  ap.add_argument("--repo", help="owner/name (default: inferred from origin URL)")
  ap.add_argument("--ref", default="master", help="branch name to sync (default: master)")
+ ap.add_argument("--no-fetch-objects", action="store_true",
+ help="fail loud on missing content objects instead of fetching them")
  args = ap.parse_args()
 
  repo = args.repo or repo_from_origin()
@@ -169,9 +227,17 @@ def main() -> int:
  local_tree = subprocess.run(["git", "rev-parse", head + "^{tree}"],
  capture_output=True, text=True)
  if local_tree.returncode != 0 or local_tree.stdout.strip() != tree:
- raise SystemExit(f"tree mismatch or missing objects for {head[:7]} "
- f"(want {tree}, got {local_tree.stdout.strip() or 'NONE'}) — "
- "run `git fetch` when https returns")
+ if not args.no_fetch_objects:
+ print(f" root tree {tree[:7]} missing locally — fetching blobs/trees via Git Data API")
+ _fetch_tree(repo, tree)
+ local_tree = subprocess.run(["git", "rev-parse", head + "^{tree}"],
+ capture_output=True, text=True)
+ if local_tree.returncode == 0 and local_tree.stdout.strip() == tree:
+ print(f" materialized root tree {tree[:7]} (blobs + subtrees) OK")
+ if local_tree.returncode != 0 or local_tree.stdout.strip() != tree:
+ raise SystemExit(f"tree mismatch or missing objects for {head[:7]} "
+ f"(want {tree}, got {local_tree.stdout.strip() or 'NONE'}) — "
+ "run `git fetch` when https returns")
 
  for ref in (f"refs/heads/{args.ref}", f"refs/remotes/origin/{args.ref}"):
  subprocess.run(["git", "update-ref", ref, head], check=True)

tests/test_sync_master_from_api.py

@@ -8,6 +8,7 @@
  2. 行为断言（hermetic，无网络）：在临时 git 仓库里用 git commit-tree 合成
  unsigned / signed 两类 commit，验证 reconstruct_commit() 字节级复现同一 sha
 """
+import base64
 import importlib.util
 import os
 import subprocess
@@ -144,3 +145,134 @@ def test_reconstruct_commit_mismatch_raises(tmp_path):
 
  with pytest.raises(ValueError):
  mod.reconstruct_commit(raw.decode("utf-8"), None, "different msg")
+
+
+# ------------------------------------------------- content-object auto-fetch
+
+
+def test_script_auto_fetches_missing_content_objects():
+ """cyc20260826-154904 教训：head commit 存在但其 blobs/trees 本地缺失时，
+ 脚本应经 Git Data API 自动补全（blob hash-object + tree mktree），而非直接
+ fail-loud 要求 git fetch。"""
+ content = SCRIPT.read_text(encoding="utf-8")
+ assert "git/blobs" in content # blob fetch path
+ assert "mktree" in content # tree rebuild path
+ assert "no-fetch-objects" in content # opt-out flag exists
+
+
+def _tree_entries(repo: Path, tree_sha: str) -> list[dict]:
+ """Parse `git ls-tree` of a tree into GitHub tree-API entry dicts."""
+ r = _git("ls-tree", tree_sha, cwd=repo)
+ assert r.returncode == 0, r.stderr
+ entries = []
+ for line in r.stdout.decode().splitlines():
+ mode, typ, sha, path = line.split(None, 3)
+ entries.append({"path": path, "mode": mode, "type": typ, "sha": sha})
+ return entries
+
+
+def test_fetch_missing_tree_and_blobs_hermetic(tmp_path, monkeypatch):
+ """在空仓库中，用假 API 补全 root tree → sub tree → blobs 全链路；
+ 验证对象落库且 root tree sha 与源仓库一致（递归 + mktree 排序正确）。"""
+ mod = _load_module()
+
+ # 源仓库：a.txt + sub/b.txt 两个 blob、一个子树
+ src = tmp_path / "src"
+ src.mkdir()
+ _git("init", "-q", cwd=src)
+ _git("config", "user.name", "Test Author", cwd=src)
+ _git("config", "user.email", "test@example.com", cwd=src)
+ (src / "a.txt").write_text("hello alpha\n", encoding="utf-8")
+ (src / "sub").mkdir()
+ (src / "sub" / "b.txt").write_text("beta bytes\n", encoding="utf-8")
+ _git("add", ".", cwd=src)
+ r = _git("commit", "-m", "content commit", cwd=src, env=GIT_ENV)
+ assert r.returncode == 0, r.stderr
+
+ root = _git("rev-parse", "HEAD^{tree}", cwd=src).stdout.decode().strip()
+ entries = _tree_entries(src, root)
+ assert len(entries) == 2 # a.txt + sub/
+ sub_tree = next(e["sha"] for e in entries if e["type"] == "tree")
+ sub_entries = _tree_entries(src, sub_tree)
+ assert len(sub_entries) == 1 and sub_entries[0]["path"] == "b.txt"
+ blob_a = next(e["sha"] for e in entries if e["type"] == "blob")
+ blob_b = sub_entries[0]["sha"]
+
+ # 假 API：按 sha 提供 tree（非递归）与 blob（base64）
+ trees = {root: entries, sub_tree: sub_entries}
+ blobs = {
+ blob_a: _git("cat-file", "blob", blob_a, cwd=src).stdout,
+ blob_b: _git("cat-file", "blob", blob_b, cwd=src).stdout,
+ }
+
+ def fake_get(url: str) -> dict:
+ if "/git/blobs/" in url:
+ sha = url.rsplit("/", 1)[1]
+ return {"content": base64.b64encode(blobs[sha]).decode("ascii"),
+ "encoding": "base64"}
+ if "/git/trees/" in url:
+ sha = url.rsplit("/", 1)[1]
+ return {"tree": trees[sha], "truncated": False}
+ raise AssertionError(f"unexpected API call: {url}")
+
+ monkeypatch.setattr(mod, "api_get", fake_get)
+
+ # 目标：全新空仓库（无任何对象）——模拟本地缺失 blobs/trees 的场景
+ target = tmp_path / "target"
+ target.mkdir()
+ _git("init", "-q", cwd=target)
+ monkeypatch.chdir(target)
+
+ mod._fetch_tree("owner/repo", root)
+
+ # 全部对象落库，root tree 可解析且 sha 一致
+ assert _git("cat-file", "-e", root, cwd=target).returncode == 0
+ assert _git("cat-file", "-e", sub_tree, cwd=target).returncode == 0
+ assert _git("cat-file", "-e", blob_a, cwd=target).returncode == 0
+ assert _git("cat-file", "-e", blob_b, cwd=target).returncode == 0
+ r = _git("rev-parse", root, cwd=target)
+ assert r.returncode == 0 and r.stdout.decode().strip() == root
+
+
+def test_fetch_missing_objects_idempotent(tmp_path, monkeypatch):
+ """已存在的对象不再请求 API（幂等），且 blob/tree 均可安全重入。"""
+ mod = _load_module()
+
+ src = tmp_path / "src"
+ src.mkdir()
+ _git("init", "-q", cwd=src)
+ _git("config", "user.name", "Test Author", cwd=src)
+ _git("config", "user.email", "test@example.com", cwd=src)
+ (src / "x.txt").write_text("x\n", encoding="utf-8")
+ _git("add", ".", cwd=src)
+ r = _git("commit", "-m", "x", cwd=src, env=GIT_ENV)
+ assert r.returncode == 0, r.stderr
+
+ root = _git("rev-parse", "HEAD^{tree}", cwd=src).stdout.decode().strip()
+ entries = _tree_entries(src, root)
+ blob = next(e["sha"] for e in entries if e["type"] == "blob")
+
+ target = tmp_path / "target"
+ target.mkdir()
+ _git("init", "-q", cwd=target)
+ monkeypatch.chdir(target)
+
+ calls = {"n": 0}
+
+ def fake_get(url: str) -> dict:
+ calls["n"] += 1
+ if "/git/blobs/" in url:
+ return {"content": base64.b64encode(b"x\n").decode("ascii"),
+ "encoding": "base64"}
+ if "/git/trees/" in url:
+ return {"tree": entries, "truncated": False}
+ raise AssertionError(f"unexpected API call: {url}")
+
+ monkeypatch.setattr(mod, "api_get", fake_get)
+ mod._fetch_tree("owner/repo", root)
+ n1 = calls["n"]
+ assert n1 >= 1
+ # 第二遍：全部已存在 → 零 API 调用
+ mod._fetch_tree("owner/repo", root)
+ assert calls["n"] == n1
+ assert _git("cat-file", "-e", blob, cwd=target).returncode == 0