From 55c836059f816bb16ae531d9b34eceeceb788a04 Mon Sep 17 00:00:00 2001 From: EMRG Evolution Date: Wed, 26 Aug 2026 17:40:08 +0800 Subject: [PATCH] =?UTF-8?q?emrg:=20scripts/sync-master-from-api.py=20?= =?UTF-8?q?=E2=80=94=20auto-fetch=20missing=20blobs/trees=20via=20Git=20Da?= =?UTF-8?q?ta=20API?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Agent.md | 2 +- scripts/sync-master-from-api.py | 78 +++++++++++++++-- tests/test_sync_master_from_api.py | 132 +++++++++++++++++++++++++++++ 3 files changed, 205 insertions(+), 7 deletions(-) diff --git a/Agent.md b/Agent.md index 3cf6abb8..f9a1c4a8 100644 --- a/Agent.md +++ b/Agent.md @@ -119,7 +119,7 @@ Community needs voiced in HN agent-UI discussions map directly to EMRG's design: pkill -f "emrg.server"; rm -f ~/.emrg/emrgd.token; python -m emrg ``` -Python: `uv run pytest tests/ -v` (1099) — import check: `uv run python -c "from emrg.client.app import run_client"` +Python: `uv run pytest tests/ -v` (1102) — import check: `uv run python -c "from emrg.client.app import run_client"` GUI: `cd emrg/gui && npm test` (265: 45 daemon_client + 20 conn-manager + 22 app-commands + 132 renderer smoke + 15 i18n + 8 integration + 3 commands + 8 build-config + 7 gui-state + 2 tool-group + 3 preload-api) — syntax: `node --check main.js preload.js daemon_client.js renderer/js/*.js`; renderer React suite: `cd emrg/gui/renderer && npm run typecheck && npm test` (78 vitest: 5 snapshot-store + 9 utils + 3 ErrorBoundary + 2 App smoke + 11 commands + 4 copywriting + 11 i18n + 11 markdown + 15 transcript + 7 TranscriptView) + `npm run build` → `renderer/dist/` CI: `uv run pytest` (ubuntu + **windows-2025 matrix** — Windows pytest 回归在 PR CI 即失败，v0.2.29 教训 #725) + GUI tests + **actionlint workflow lint** (`rhysd/actionlint@v1.7.12` gate, #444 — workflow 解析错误在 PR CI 即失败，如 `if:` secrets 上下文) Re-trigger: `scripts/re-trigger-ci.sh [branch]` (workflow_dispatch, #527 — 替代空 commit 重触发：Actions outage 会整段丢弃 push 事件，dispatch 走 API 路径不受影响) diff --git a/scripts/sync-master-from-api.py b/scripts/sync-master-from-api.py index 82f6f422..4ad0ca69 100644 --- a/scripts/sync-master-from-api.py +++ b/scripts/sync-master-from-api.py @@ -8,16 +8,26 @@ the missing *commit* objects — reconstructed byte-exact from the API's verification payload + signature, including web-flow GPG-signed squash merges. + +When the head commit's *content* objects (blobs/trees) are also missing locally +(e.g. a parallel PR introduced files this repo never had — first hit in cycle +cyc20260826-154904 with #994's GUI assets), the script now auto-fetches them via +the Git Data API: blobs via `git/blobs/{sha}` + `git hash-object -w`, trees via +`git/trees/{sha}` + `git mktree` (canonical ordering), recursing bottom-up. The +previous behavior failed loud with "run git fetch when https returns" and forced +a manual gh-api + mktree recovery dance. + Usage: - python scripts/sync-master-from-api.py [--repo owner/name] [--ref master] + python scripts/sync-master-from-api.py [--repo owner/name] [--ref master] [--repo owner/name] [--ref master] Behavior: * resolves repo from --repo or `git remote get-url origin` * walks the remote commit chain from head down to the first commit already present locally, writing each missing commit object via `git hash-object -t commit -w` (byte-exact, GPG signature preserved) - * verifies the root tree sha matches the remote (fail-loud if content - objects are missing locally — prefer `git fetch` when https returns) + * verifies the root tree sha matches the remote; if content objects are + missing, fetches missing blobs/trees via the Git Data API (disable with + --no-fetch-objects) and re-verifies — fail-loud only if still mismatched * updates refs/heads/ and refs/remotes/origin/ Requirements: git on PATH; api.github.com reachable. Auth: optional for public @@ -121,6 +131,52 @@ def has_object(sha: str) -> bool: capture_output=True).returncode == 0 + +def _object_exists(sha: str) -> bool: + """Any object (blob/tree/commit) present locally by sha.""" + return subprocess.run(["git", "cat-file", "-e", sha], + capture_output=True).returncode == 0 + + +def _fetch_blob(repo: str, blob_sha: str) -> None: + """Fetch one missing blob via the Git Data API, writing it byte-exact.""" + if _object_exists(blob_sha): + return + b = api_get(f"{API}/repos/{repo}/git/blobs/{blob_sha}") + if b.get("encoding") == "base64": + content = base64.b64decode(b["content"]) + else: # utf-8 text blobs are returned raw + content = b["content"].encode("utf-8") + r = subprocess.run(["git", "hash-object", "-w", "--stdin"], input=content, + capture_output=True) + if r.returncode != 0 or r.stdout.decode().strip() != blob_sha: + raise RuntimeError(f"blob materialization mismatch for {blob_sha[:7]} " + f"(got {r.stdout.decode().strip()[:7] or 'NONE'})") + + +def _fetch_tree(repo: str, tree_sha: str) -> None: + """Recursively materialize a missing tree: blobs via hash-object, subtrees + via git mktree (git canonical ordering), bottom-up. Idempotent.""" + if _object_exists(tree_sha): + return + t = api_get(f"{API}/repos/{repo}/git/trees/{tree_sha}") + if t.get("truncated"): + raise RuntimeError(f"tree {tree_sha[:7]} truncated by API (>100k entries)") + entries = t.get("tree", []) + for e in entries: + if e["type"] == "blob": + _fetch_blob(repo, e["sha"]) + elif e["type"] == "tree": + _fetch_tree(repo, e["sha"]) + # commit entries (submodules): leave to git fetch — rare in this repo + lines = [f"{e['mode']} {e['type']} {e['sha']}\t{e['path']}" for e in entries] + r = subprocess.run(["git", "mktree"], input=("\n".join(lines) + "\n").encode("utf-8"), + capture_output=True) + if r.returncode != 0 or r.stdout.decode().strip() != tree_sha: + raise RuntimeError(f"tree materialization mismatch for {tree_sha[:7]} " + f"(got {r.stdout.decode().strip()[:7] or 'NONE'})") + + def rev_parse(ref: str) -> str: r = subprocess.run(["git", "rev-parse", "-q", "--verify", ref], capture_output=True) return r.stdout.decode().strip() if r.returncode == 0 else "" @@ -139,6 +195,8 @@ def main() -> int: ap = argparse.ArgumentParser(description=__doc__) ap.add_argument("--repo", help="owner/name (default: inferred from origin URL)") ap.add_argument("--ref", default="master", help="branch name to sync (default: master)") + ap.add_argument("--no-fetch-objects", action="store_true", + help="fail loud on missing content objects instead of fetching them") args = ap.parse_args() repo = args.repo or repo_from_origin() @@ -169,9 +227,17 @@ def main() -> int: local_tree = subprocess.run(["git", "rev-parse", head + "^{tree}"], capture_output=True, text=True) if local_tree.returncode != 0 or local_tree.stdout.strip() != tree: - raise SystemExit(f"tree mismatch or missing objects for {head[:7]} " - f"(want {tree}, got {local_tree.stdout.strip() or 'NONE'}) — " - "run `git fetch` when https returns") + if not args.no_fetch_objects: + print(f" root tree {tree[:7]} missing locally — fetching blobs/trees via Git Data API") + _fetch_tree(repo, tree) + local_tree = subprocess.run(["git", "rev-parse", head + "^{tree}"], + capture_output=True, text=True) + if local_tree.returncode == 0 and local_tree.stdout.strip() == tree: + print(f" materialized root tree {tree[:7]} (blobs + subtrees) OK") + if local_tree.returncode != 0 or local_tree.stdout.strip() != tree: + raise SystemExit(f"tree mismatch or missing objects for {head[:7]} " + f"(want {tree}, got {local_tree.stdout.strip() or 'NONE'}) — " + "run `git fetch` when https returns") for ref in (f"refs/heads/{args.ref}", f"refs/remotes/origin/{args.ref}"): subprocess.run(["git", "update-ref", ref, head], check=True) diff --git a/tests/test_sync_master_from_api.py b/tests/test_sync_master_from_api.py index 8e228563..b333bec4 100644 --- a/tests/test_sync_master_from_api.py +++ b/tests/test_sync_master_from_api.py @@ -8,6 +8,7 @@ 2. 行为断言（hermetic，无网络）：在临时 git 仓库里用 git commit-tree 合成 unsigned / signed 两类 commit，验证 reconstruct_commit() 字节级复现同一 sha """ +import base64 import importlib.util import os import subprocess @@ -144,3 +145,134 @@ def test_reconstruct_commit_mismatch_raises(tmp_path): with pytest.raises(ValueError): mod.reconstruct_commit(raw.decode("utf-8"), None, "different msg") + + +# ------------------------------------------------- content-object auto-fetch + + +def test_script_auto_fetches_missing_content_objects(): + """cyc20260826-154904 教训：head commit 存在但其 blobs/trees 本地缺失时， + 脚本应经 Git Data API 自动补全（blob hash-object + tree mktree），而非直接 + fail-loud 要求 git fetch。""" + content = SCRIPT.read_text(encoding="utf-8") + assert "git/blobs" in content # blob fetch path + assert "mktree" in content # tree rebuild path + assert "no-fetch-objects" in content # opt-out flag exists + + +def _tree_entries(repo: Path, tree_sha: str) -> list[dict]: + """Parse `git ls-tree` of a tree into GitHub tree-API entry dicts.""" + r = _git("ls-tree", tree_sha, cwd=repo) + assert r.returncode == 0, r.stderr + entries = [] + for line in r.stdout.decode().splitlines(): + mode, typ, sha, path = line.split(None, 3) + entries.append({"path": path, "mode": mode, "type": typ, "sha": sha}) + return entries + + +def test_fetch_missing_tree_and_blobs_hermetic(tmp_path, monkeypatch): + """在空仓库中，用假 API 补全 root tree → sub tree → blobs 全链路； + 验证对象落库且 root tree sha 与源仓库一致（递归 + mktree 排序正确）。""" + mod = _load_module() + + # 源仓库：a.txt + sub/b.txt 两个 blob、一个子树 + src = tmp_path / "src" + src.mkdir() + _git("init", "-q", cwd=src) + _git("config", "user.name", "Test Author", cwd=src) + _git("config", "user.email", "test@example.com", cwd=src) + (src / "a.txt").write_text("hello alpha\n", encoding="utf-8") + (src / "sub").mkdir() + (src / "sub" / "b.txt").write_text("beta bytes\n", encoding="utf-8") + _git("add", ".", cwd=src) + r = _git("commit", "-m", "content commit", cwd=src, env=GIT_ENV) + assert r.returncode == 0, r.stderr + + root = _git("rev-parse", "HEAD^{tree}", cwd=src).stdout.decode().strip() + entries = _tree_entries(src, root) + assert len(entries) == 2 # a.txt + sub/ + sub_tree = next(e["sha"] for e in entries if e["type"] == "tree") + sub_entries = _tree_entries(src, sub_tree) + assert len(sub_entries) == 1 and sub_entries[0]["path"] == "b.txt" + blob_a = next(e["sha"] for e in entries if e["type"] == "blob") + blob_b = sub_entries[0]["sha"] + + # 假 API：按 sha 提供 tree（非递归）与 blob（base64） + trees = {root: entries, sub_tree: sub_entries} + blobs = { + blob_a: _git("cat-file", "blob", blob_a, cwd=src).stdout, + blob_b: _git("cat-file", "blob", blob_b, cwd=src).stdout, + } + + def fake_get(url: str) -> dict: + if "/git/blobs/" in url: + sha = url.rsplit("/", 1)[1] + return {"content": base64.b64encode(blobs[sha]).decode("ascii"), + "encoding": "base64"} + if "/git/trees/" in url: + sha = url.rsplit("/", 1)[1] + return {"tree": trees[sha], "truncated": False} + raise AssertionError(f"unexpected API call: {url}") + + monkeypatch.setattr(mod, "api_get", fake_get) + + # 目标：全新空仓库（无任何对象）——模拟本地缺失 blobs/trees 的场景 + target = tmp_path / "target" + target.mkdir() + _git("init", "-q", cwd=target) + monkeypatch.chdir(target) + + mod._fetch_tree("owner/repo", root) + + # 全部对象落库，root tree 可解析且 sha 一致 + assert _git("cat-file", "-e", root, cwd=target).returncode == 0 + assert _git("cat-file", "-e", sub_tree, cwd=target).returncode == 0 + assert _git("cat-file", "-e", blob_a, cwd=target).returncode == 0 + assert _git("cat-file", "-e", blob_b, cwd=target).returncode == 0 + r = _git("rev-parse", root, cwd=target) + assert r.returncode == 0 and r.stdout.decode().strip() == root + + +def test_fetch_missing_objects_idempotent(tmp_path, monkeypatch): + """已存在的对象不再请求 API（幂等），且 blob/tree 均可安全重入。""" + mod = _load_module() + + src = tmp_path / "src" + src.mkdir() + _git("init", "-q", cwd=src) + _git("config", "user.name", "Test Author", cwd=src) + _git("config", "user.email", "test@example.com", cwd=src) + (src / "x.txt").write_text("x\n", encoding="utf-8") + _git("add", ".", cwd=src) + r = _git("commit", "-m", "x", cwd=src, env=GIT_ENV) + assert r.returncode == 0, r.stderr + + root = _git("rev-parse", "HEAD^{tree}", cwd=src).stdout.decode().strip() + entries = _tree_entries(src, root) + blob = next(e["sha"] for e in entries if e["type"] == "blob") + + target = tmp_path / "target" + target.mkdir() + _git("init", "-q", cwd=target) + monkeypatch.chdir(target) + + calls = {"n": 0} + + def fake_get(url: str) -> dict: + calls["n"] += 1 + if "/git/blobs/" in url: + return {"content": base64.b64encode(b"x\n").decode("ascii"), + "encoding": "base64"} + if "/git/trees/" in url: + return {"tree": entries, "truncated": False} + raise AssertionError(f"unexpected API call: {url}") + + monkeypatch.setattr(mod, "api_get", fake_get) + mod._fetch_tree("owner/repo", root) + n1 = calls["n"] + assert n1 >= 1 + # 第二遍：全部已存在 → 零 API 调用 + mod._fetch_tree("owner/repo", root) + assert calls["n"] == n1 + assert _git("cat-file", "-e", blob, cwd=target).returncode == 0