From a0b94ff7127f9e276237d10aa88d6ef96f241ce8 Mon Sep 17 00:00:00 2001 From: Rahul Krishna Date: Wed, 19 Aug 2026 15:37:19 -0400 Subject: [PATCH] fix(pycg): canonicalize the builtins module spelling on PyCG edges (#132) PyCG spells the builtins module ``; Jedi spells it `builtins`. Nothing normalized the two, so `_home_external_symbols` minted a separate `can://.../@external//` home per spelling and one builtin ended up with two identities. Measured on the `requests` fixture at -a 2: 29 ids under `/` against 14 under `builtins/`, with 12 names present under both (len, isinstance, getattr, sorted, ...). Two things follow from that. A consumer asking "who calls len" gets two disjoint answers, neither complete. And provenance can never merge for a builtin: `merge_edges` coalesces on (src, dst), so differing dst ids keep the two backends' edges apart -- in the same run 198 non-builtin edges do carry `prov: ["jedi", "pycg"]`, while builtins are structurally excluded from it. Canonicalization happens at `build_call_graph_edges`' single exit, so every shard strategy is covered, and before `merge_edges` runs in core.py -- doing it at id-minting time would leave two already-merged edges with identical endpoints and split provenance, moving the symptom rather than removing it. Endpoints that collide once rewritten are coalesced with summed weight and unioned provenance, matching merge_edges' semantics. That coalescing is done locally rather than through `_coalesce_edges`, which raises on its duplicate branch (#133). PyCG only ever emits the bare `` module, so an exact-match alias suffices; the dotted forms (`builtins.str`, `builtins.dict`) are Jedi's and are already canonical. --- .../semantic_analysis/pycg/pycg_analysis.py | 39 +++++++++++++ test/test_pycg_builtin_canonicalization.py | 56 +++++++++++++++++++ 2 files changed, 95 insertions(+) create mode 100644 test/test_pycg_builtin_canonicalization.py diff --git a/codeanalyzer/semantic_analysis/pycg/pycg_analysis.py b/codeanalyzer/semantic_analysis/pycg/pycg_analysis.py index 3a19915..d953cad 100644 --- a/codeanalyzer/semantic_analysis/pycg/pycg_analysis.py +++ b/codeanalyzer/semantic_analysis/pycg/pycg_analysis.py @@ -88,6 +88,44 @@ def _handler(signum: int, frame: object) -> None: from codeanalyzer.utils import ProgressBar, logger +# PyCG spells the builtins module ````; Jedi spells it ``builtins``. Left +# unnormalized, one builtin gets two ``@external`` ``can://`` homes and the two +# backends' edges can never coalesce, so a call both resolvers agree on can never +# reach ``prov: ["jedi", "pycg"]`` (#132). PyCG only ever emits the bare module, so +# an exact-match alias is enough -- the dotted forms (``builtins.str`` etc.) are +# Jedi's and are already canonical. +_PYCG_MODULE_ALIASES = {"": "builtins"} + + +def _canonical_endpoint(sig: str) -> str: + """Rewrite a PyCG endpoint's module segment to the canonical spelling.""" + module, dot, name = sig.rpartition(".") + if dot and module in _PYCG_MODULE_ALIASES: + return f"{_PYCG_MODULE_ALIASES[module]}.{name}" + return sig + + +def _canonicalize_edges(edges: List[PyCallEdge]) -> List[PyCallEdge]: + """Canonicalize endpoint spellings, coalescing pairs that collide as a result. + + Two spellings of one target are one edge: weights sum and provenance unions, + matching ``call_graph.merge_edges``. Deliberately does not route through + ``_coalesce_edges``, which raises on its duplicate branch (#133). + """ + merged: Dict[Tuple[str, str], PyCallEdge] = {} + for edge in edges: + src = _canonical_endpoint(edge.src) + dst = _canonical_endpoint(edge.dst) + key = (src, dst) + current = merged.get(key) + if current is None: + merged[key] = edge.model_copy(update={"src": src, "dst": dst}) + else: + current.weight += edge.weight + current.prov = sorted(set(current.prov) | set(edge.prov)) + return list(merged.values()) + + def _shard_root_path(files: List[str], project_dir: Path) -> Path: """Content-derived mini-project root for a shard: same project + same file set → same path on every run (determinism, issue #99).""" @@ -1110,6 +1148,7 @@ def build_call_graph_edges( with _shard_symlink_root(entry_points, self.project_dir) as (root, eps): edges = self._run_pycg_batch(eps, root, resolver, prefix="") + edges = _canonicalize_edges(edges) elapsed = time.perf_counter() - t0 logger.info("✅ PyCG: %d edges in %.1fs", len(edges), elapsed) return edges diff --git a/test/test_pycg_builtin_canonicalization.py b/test/test_pycg_builtin_canonicalization.py new file mode 100644 index 0000000..8c10e5c --- /dev/null +++ b/test/test_pycg_builtin_canonicalization.py @@ -0,0 +1,56 @@ +"""PyCG builtin module spelling is canonicalized before edges leave the backend (#132). + +PyCG spells the builtins module ````; Jedi spells it ``builtins``. Left +unnormalized, one builtin gets two ``@external`` ``can://`` homes and the two +backends' edges can never coalesce into ``prov: ["jedi", "pycg"]``. +""" +from codeanalyzer.schema.py_schema import PyCallEdge +from codeanalyzer.semantic_analysis.call_graph import merge_edges +from codeanalyzer.semantic_analysis.pycg.pycg_analysis import ( + _canonical_endpoint, + _canonicalize_edges, +) + + +def test_builtin_module_is_rewritten(): + assert _canonical_endpoint(".isinstance") == "builtins.isinstance" + assert _canonical_endpoint(".len") == "builtins.len" + + +def test_already_canonical_and_unrelated_names_are_untouched(): + for sig in ( + "builtins.isinstance", # Jedi's spelling, already canonical + "builtins.str.format", # dotted builtin type -- module is `builtins.str` + "requests.api.get", # ordinary first-party signature + "isinstance", # no module segment at all + "", # bare, no dot -> not an endpoint we rewrite + ): + assert _canonical_endpoint(sig) == sig + + +def test_colliding_spellings_coalesce_with_summed_weight(): + edges = [ + PyCallEdge(src="a.f", dst=".len", weight=3, prov=["pycg"]), + PyCallEdge(src="a.f", dst="builtins.len", weight=2, prov=["pycg"]), + ] + out = _canonicalize_edges(edges) + assert len(out) == 1 + assert (out[0].src, out[0].dst) == ("a.f", "builtins.len") + assert out[0].weight == 5 + + +def test_canonicalization_lets_provenance_merge_across_backends(): + """The point of #132: without this, a builtin can never reach prov=[jedi,pycg].""" + pycg = _canonicalize_edges( + [PyCallEdge(src="a.f", dst=".len", weight=1, prov=["pycg"])] + ) + jedi = [PyCallEdge(src="a.f", dst="builtins.len", weight=1, prov=["jedi"])] + merged = merge_edges(jedi, pycg) + assert len(merged) == 1 + assert merged[0].prov == ["jedi", "pycg"] + + +def test_source_edges_are_not_mutated(): + original = PyCallEdge(src="a.f", dst=".len", weight=1, prov=["pycg"]) + _canonicalize_edges([original]) + assert original.dst == ".len"