From c90ba8142778fd5a11211b4325b500503a32472f Mon Sep 17 00:00:00 2001 From: Rahul Krishna Date: Sat, 27 Jun 2026 15:23:13 -0400 Subject: [PATCH 1/4] feat(python): add bulk/projected accessors to avoid N+1 reconstruction Add set-at-a-time, field-projected reads to the Python facade so callers can enumerate the application in one round-trip instead of paying the per-entity reconstruction get_all_methods_in_application() does (tens of thousands of Bolt round-trips on large apps via the Neo4j backend). New `PyCallableOverview` projection model and three accessors on the PythonAnalysisBackend ABC, both backends, and the facade: - get_callables_overview() -> List[PyCallableOverview]: every callable (methods, module-level and nested functions) as a lightweight projection. - get_method_bodies(signatures) -> Dict[str, str]: batch source-body fetch. - get_decorated_callables(markers) -> List[PyCallableOverview]: overviews filtered by decorator (fills the get_methods_with_decorators gap). The in-process and Neo4j backends enumerate the same callable set (a "method" is one a class declares directly, mirroring PY_HAS_METHOD). Offline unit tests cover the in-process walk; the Neo4j test module asserts byte-for-byte parity against it when a server is reachable. Refs #180 --- cldk/analysis/python/backend.py | 20 +++ .../python/codeanalyzer/codeanalyzer.py | 68 ++++++++- cldk/analysis/python/neo4j/neo4j_backend.py | 38 +++++ cldk/analysis/python/neo4j/reconstruct.py | 21 +++ cldk/analysis/python/python_analysis.py | 49 +++++++ cldk/models/python/__init__.py | 3 + cldk/models/python/projections.py | 58 ++++++++ .../python/test_python_bulk_accessors.py | 133 ++++++++++++++++++ .../python/test_python_neo4j_backend.py | 24 ++++ 9 files changed, 413 insertions(+), 1 deletion(-) create mode 100644 cldk/models/python/projections.py create mode 100644 tests/analysis/python/test_python_bulk_accessors.py diff --git a/cldk/analysis/python/backend.py b/cldk/analysis/python/backend.py index 46c6262..2a1048d 100644 --- a/cldk/analysis/python/backend.py +++ b/cldk/analysis/python/backend.py @@ -37,6 +37,7 @@ from cldk.models.python import ( PyApplication, PyCallable, + PyCallableOverview, PyClass, PyClassAttribute, PyModule, @@ -139,3 +140,22 @@ def get_all_constructors(self, qualified_class_name: str) -> Dict[str, PyCallabl @abstractmethod def get_all_fields(self, qualified_class_name: str) -> List[PyClassAttribute]: """The attributes/fields of a class.""" + + # -----[ bulk / projected accessors ]----- + # Set-at-a-time, field-projected reads — one round-trip on the Neo4j backend, one symbol-table + # walk in-process — for callers that enumerate the whole application and would otherwise pay the + # per-entity reconstruction of get_all_methods_in_application. + @abstractmethod + def get_callables_overview(self) -> List[PyCallableOverview]: + """A lightweight projection of every callable in the application (methods, module-level and + nested functions), without the full :class:`PyCallable` reconstruction.""" + + @abstractmethod + def get_method_bodies(self, signatures: List[str]) -> Dict[str, str]: + """Source bodies for the given callable signatures, keyed by signature. Signatures with no + matching callable are omitted.""" + + @abstractmethod + def get_decorated_callables(self, markers: List[str]) -> List[PyCallableOverview]: + """Overviews of callables decorated with any of ``markers`` (matched against the decorator + names).""" diff --git a/cldk/analysis/python/codeanalyzer/codeanalyzer.py b/cldk/analysis/python/codeanalyzer/codeanalyzer.py index c5fd4f3..a05f96b 100644 --- a/cldk/analysis/python/codeanalyzer/codeanalyzer.py +++ b/cldk/analysis/python/codeanalyzer/codeanalyzer.py @@ -51,7 +51,7 @@ import logging from pathlib import Path -from typing import Dict, List, Tuple, Union +from typing import Dict, Iterator, List, Tuple, Union import networkx as nx @@ -66,6 +66,7 @@ PyApplication, PyCallEdge, PyCallable, + PyCallableOverview, PyClass, PyClassAttribute, PyComment, @@ -75,6 +76,20 @@ logger = logging.getLogger(__name__) +def _overview(c: PyCallable, class_signature: str | None, kind: str) -> PyCallableOverview: + """Project a :class:`PyCallable` into a lightweight :class:`PyCallableOverview`.""" + return PyCallableOverview( + signature=c.signature, + name=c.name, + class_signature=class_signature, + kind=kind, + path=c.path, + start_line=c.start_line, + end_line=c.end_line, + decorators=list(c.decorators or []), + ) + + class PyCodeanalyzer(PythonAnalysisBackend): """In-process driver for the ``codeanalyzer-python`` analysis backend. @@ -523,6 +538,57 @@ def get_all_fields(self, qualified_class_name: str) -> List[PyClassAttribute]: cls = self.get_class(qualified_class_name) return list(cls.attributes.values()) if cls else [] + # ----------------------------------------------------------- bulk / projected accessors + def _iter_callables(self) -> Iterator[Tuple[PyCallable, "str | None", str]]: + """Yield ``(callable, class_signature, kind)`` for every callable in the application. + + Walks the in-memory symbol table the same way the Neo4j backend's ``MATCH (c:PyCallable)`` + sees nodes: a callable is a ``"method"`` only when a class declares it directly (mirroring + ``PY_HAS_METHOD``); module-level functions and functions nested inside a callable are + ``"function"`` with a ``None`` class signature. The two backends therefore enumerate the + same set. + """ + + def from_callable(c: PyCallable): + for inner in c.inner_callables.values(): + yield inner, None, "function" + yield from from_callable(inner) + for inner_cls in c.inner_classes.values(): + yield from from_class(inner_cls) + + def from_class(cls: PyClass): + for m in cls.methods.values(): + yield m, cls.signature, "method" + yield from from_callable(m) + for inner_cls in cls.inner_classes.values(): + yield from from_class(inner_cls) + + for module in self.application.symbol_table.values(): + for cls in module.classes.values(): + yield from from_class(cls) + for fn in module.functions.values(): + yield fn, None, "function" + yield from from_callable(fn) + + def get_callables_overview(self) -> List[PyCallableOverview]: + """Return a lightweight overview of every callable in the application (see + :meth:`PythonAnalysisBackend.get_callables_overview`).""" + return [_overview(c, class_sig, kind) for c, class_sig, kind in self._iter_callables()] + + def get_method_bodies(self, signatures: List[str]) -> Dict[str, str]: + """Return ``{signature: code}`` for the requested signatures that exist.""" + wanted = set(signatures) + return {c.signature: c.code for c, _, _ in self._iter_callables() if c.signature in wanted} + + def get_decorated_callables(self, markers: List[str]) -> List[PyCallableOverview]: + """Return overviews of callables decorated with any of ``markers``.""" + marker_set = set(markers) + return [ + _overview(c, class_sig, kind) + for c, class_sig, kind in self._iter_callables() + if marker_set.intersection(c.decorators or []) + ] + # ----------------------------------------------------------- callers/callees def get_all_callers(self, target_class_name: str, target_method_declaration: str) -> Dict: """Return all methods that call a specific target method. diff --git a/cldk/analysis/python/neo4j/neo4j_backend.py b/cldk/analysis/python/neo4j/neo4j_backend.py index f17d2fb..9546dda 100644 --- a/cldk/analysis/python/neo4j/neo4j_backend.py +++ b/cldk/analysis/python/neo4j/neo4j_backend.py @@ -82,6 +82,7 @@ PyApplication, PyCallEdge, PyCallable, + PyCallableOverview, PyClass, PyClassAttribute, PyModule, @@ -428,3 +429,40 @@ def get_all_constructors(self, qualified_class_name: str) -> Dict[str, PyCallabl def get_all_fields(self, qualified_class_name: str) -> List[PyClassAttribute]: cls = self.get_class(qualified_class_name) return list(cls.attributes.values()) if cls else [] + + # ===================================================================================== + # PythonAnalysisBackend — bulk / projected accessors (one round-trip each) + # ===================================================================================== + # Field-projected RETURNs that sidestep the per-entity reconstruction fan-out: each is a single + # Cypher statement, not the N+1 walk get_symbol_table()/get_all_methods_in_application() pays. + _OVERVIEW_PROJECTION = ( + "OPTIONAL MATCH (owner:PyClass)-[:PY_HAS_METHOD]->(c) " + "RETURN c.signature AS signature, c.name AS name, c.decorators AS decorators, " + "c.path AS path, c.start_line AS start_line, c.end_line AS end_line, " + "owner.signature AS class_signature" + ) + + def get_callables_overview(self) -> List[PyCallableOverview]: + rows = self._run( + "MATCH (c:PyCallable) WHERE c._module IN $mods " + self._OVERVIEW_PROJECTION, + mods=self._modules, + ) + return [R.overview(r) for r in rows] + + def get_method_bodies(self, signatures: List[str]) -> Dict[str, str]: + rows = self._run( + "MATCH (c:PyCallable) WHERE c._module IN $mods AND c.signature IN $sigs " + "RETURN c.signature AS signature, c.code AS code", + mods=self._modules, + sigs=list(signatures), + ) + return {r["signature"]: r.get("code") for r in rows} + + def get_decorated_callables(self, markers: List[str]) -> List[PyCallableOverview]: + rows = self._run( + "MATCH (c:PyCallable) WHERE c._module IN $mods " + "AND any(d IN c.decorators WHERE d IN $markers) " + self._OVERVIEW_PROJECTION, + mods=self._modules, + markers=list(markers), + ) + return [R.overview(r) for r in rows] diff --git a/cldk/analysis/python/neo4j/reconstruct.py b/cldk/analysis/python/neo4j/reconstruct.py index c5d22b7..6c68176 100644 --- a/cldk/analysis/python/neo4j/reconstruct.py +++ b/cldk/analysis/python/neo4j/reconstruct.py @@ -42,6 +42,7 @@ from cldk.models.python import ( PyCallable, + PyCallableOverview, PyClass, PyClassAttribute, PyComment, @@ -132,6 +133,26 @@ def import_(module: str, name: str, alias: str | None = None) -> PyImport: return PyImport(module=module, name=name, alias=alias) +def overview(row: Props) -> PyCallableOverview: + """Build a :class:`PyCallableOverview` from a projected callable row. + + ``row`` is a flat ``RETURN`` projection (not a node's ``properties()``): ``signature``, ``name``, + ``decorators``, ``path``, ``start_line``, ``end_line``, and ``class_signature`` (the owning + class via ``PY_HAS_METHOD``, or ``None`` for a module-level / nested function). + """ + class_sig = row.get("class_signature") + return PyCallableOverview( + signature=row.get("signature", ""), + name=row.get("name", ""), + class_signature=class_sig, + kind="method" if class_sig else "function", + path=row.get("path", ""), + start_line=row.get("start_line", -1), + end_line=row.get("end_line", -1), + decorators=list(row.get("decorators", []) or []), + ) + + # -----[ declarations ]----- def callable_( props: Props, diff --git a/cldk/analysis/python/python_analysis.py b/cldk/analysis/python/python_analysis.py index 6c3190b..dbfbb7c 100644 --- a/cldk/analysis/python/python_analysis.py +++ b/cldk/analysis/python/python_analysis.py @@ -61,6 +61,7 @@ from cldk.models.python import ( PyApplication, PyCallable, + PyCallableOverview, PyClass, PyClassAttribute, PyComment, @@ -524,6 +525,54 @@ def get_methods(self) -> Dict[str, Dict[str, PyCallable]]: """ return self.backend.get_all_methods_in_application() + def get_callables_overview(self) -> List[PyCallableOverview]: + """Return a lightweight overview of every callable in the project, in one bulk read. + + A field-projected alternative to :meth:`get_methods` for enumeration: each + :class:`~cldk.models.python.PyCallableOverview` carries the callable's signature, owning + class (if any), kind, location, and decorators — but not the full reconstruction (call + sites, inner callables, locals). On the Neo4j backend this is a single Cypher query instead + of the per-entity fan-out :meth:`get_methods` pays. Body-inspect the few you need afterwards + via :meth:`get_method` or :meth:`get_method_bodies`. + + Returns: + A flat list of :class:`~cldk.models.python.PyCallableOverview`, one per callable + (methods, module-level functions, and nested functions). + + See Also: + :meth:`get_decorated_callables`: The same projection filtered by decorator. + :meth:`get_method_bodies`: Bulk source-body fetch for chosen signatures. + """ + return self.backend.get_callables_overview() + + def get_method_bodies(self, signatures: List[str]) -> Dict[str, str]: + """Return source bodies for the given callable signatures, in one bulk read. + + Args: + signatures: Callable signatures to fetch bodies for (e.g. from + :meth:`get_callables_overview`). + + Returns: + A dict mapping each signature to its source body. Signatures with no matching callable + are omitted. + """ + return self.backend.get_method_bodies(signatures) + + def get_decorated_callables(self, markers: List[str]) -> List[PyCallableOverview]: + """Return overviews of callables decorated with any of the given markers, in one bulk read. + + Args: + markers: Decorator names to match (e.g. ``["staticmethod", "app.route"]``). + + Returns: + A list of :class:`~cldk.models.python.PyCallableOverview` for every callable carrying at + least one of ``markers`` as a decorator. + + See Also: + :meth:`get_callables_overview`: The unfiltered projection. + """ + return self.backend.get_decorated_callables(markers) + def get_methods_in_class(self, qualified_class_name: str) -> Dict[str, PyCallable]: """Return all methods defined in a specific class. diff --git a/cldk/models/python/__init__.py b/cldk/models/python/__init__.py index 0ce861c..6814335 100644 --- a/cldk/models/python/__init__.py +++ b/cldk/models/python/__init__.py @@ -36,10 +36,13 @@ PyVariableDeclaration, ) +from .projections import PyCallableOverview + __all__ = [ "PyApplication", "PyCallEdge", "PyCallable", + "PyCallableOverview", "PyCallableParameter", "PyCallsite", "PyClass", diff --git a/cldk/models/python/projections.py b/cldk/models/python/projections.py new file mode 100644 index 0000000..eaff7cc --- /dev/null +++ b/cldk/models/python/projections.py @@ -0,0 +1,58 @@ +################################################################################ +# Copyright IBM Corporation 2026 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +"""CLDK-defined projection models for the Python facade. + +Unlike the rest of :mod:`cldk.models.python`, these are **not** part of the ``codeanalyzer-python`` +schema — they are lightweight, field-projected views CLDK exposes so callers can enumerate the +application set-at-a-time without paying for the full per-callable reconstruction. They map cleanly +to a single Cypher ``RETURN`` on the Neo4j backend and to one symbol-table walk in-process. +""" + +from __future__ import annotations + +from typing import List, Optional + +from pydantic import BaseModel + + +class PyCallableOverview(BaseModel): + """A lightweight projection of one callable — enough to enumerate and filter without the full + :class:`~cldk.models.python.PyCallable` reconstruction (call-sites, inner callables, locals). + + Returned set-at-a-time by :meth:`PythonAnalysis.get_callables_overview` / + :meth:`PythonAnalysis.get_decorated_callables`. Body-inspect only the few you need afterwards + via :meth:`PythonAnalysis.get_method`/:meth:`PythonAnalysis.get_method_bodies`. + + Attributes: + signature: The callable's unique signature (the key the call graph references). + name: The callable's short name. + class_signature: Signature of the class that declares this callable as a method, or ``None`` + for a module-level or nested function. + kind: ``"method"`` when ``class_signature`` is set, else ``"function"``. + path: Project-relative path of the declaring module. + start_line / end_line: The callable's line span. + decorators: The decorator names applied to the callable. + """ + + signature: str + name: str + class_signature: Optional[str] = None + kind: str + path: str + start_line: int + end_line: int + decorators: List[str] = [] diff --git a/tests/analysis/python/test_python_bulk_accessors.py b/tests/analysis/python/test_python_bulk_accessors.py new file mode 100644 index 0000000..4a36c9a --- /dev/null +++ b/tests/analysis/python/test_python_bulk_accessors.py @@ -0,0 +1,133 @@ +################################################################################ +# Copyright IBM Corporation 2026 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +"""Offline unit tests for the in-process bulk/projected accessors. + +These build a small in-memory ``PyApplication`` and attach it to a bare ``PyCodeanalyzer`` (no +analyzer run, no Neo4j), so they exercise ``get_callables_overview`` / ``get_method_bodies`` / +``get_decorated_callables`` and the ``_iter_callables`` walk without any external dependency. The +Neo4j backend is checked for byte-for-byte parity against this same logic in +``test_python_neo4j_backend.py`` when a server is available. +""" + +from codeanalyzer.schema.py_schema import PyApplication, PyCallable, PyClass, PyModule + +from cldk.analysis.python.codeanalyzer.codeanalyzer import PyCodeanalyzer + + +def _callable(name, signature, *, code="", decorators=None, inner_callables=None, inner_classes=None): + return PyCallable( + name=name, + path="pkg/models.py", + signature=signature, + code=code, + decorators=decorators or [], + inner_callables=inner_callables or {}, + inner_classes=inner_classes or {}, + ) + + +def _class(name, signature, *, methods=None, inner_classes=None): + return PyClass(name=name, signature=signature, methods=methods or {}, inner_classes=inner_classes or {}) + + +def _backend(): + """A PyCodeanalyzer wired to a hand-built application, bypassing the analyzer run.""" + decorate = _callable("_decorate", "pkg.models.greet.._decorate", code="return s.upper()") + greet = _callable( + "greet", + "pkg.models.greet", + code="def greet(who): ...", + decorators=["app.route"], + inner_callables={"_decorate": decorate}, + ) + meta = _class( + "Meta", + "pkg.models.Entity.Meta", + methods={"m": _callable("m", "pkg.models.Entity.Meta.m", code="return 1")}, + ) + entity = _class( + "Entity", + "pkg.models.Entity", + methods={ + "__init__": _callable("__init__", "pkg.models.Entity.__init__", code="self.x = 1"), + "describe": _callable("describe", "pkg.models.Entity.describe", code="return self.x", decorators=["property"]), + }, + inner_classes={"pkg.models.Entity.Meta": meta}, + ) + module = PyModule( + file_path="pkg/models.py", + module_name="pkg.models", + classes={"pkg.models.Entity": entity}, + functions={"greet": greet}, + ) + app = PyApplication(symbol_table={"pkg/models.py": module}) + + backend = object.__new__(PyCodeanalyzer) + backend.application = app + return backend + + +def test_callables_overview_enumerates_all_callables(): + overviews = {o.signature: o for o in _backend().get_callables_overview()} + # methods, the module function, the inner class method, and the nested function are all present + assert set(overviews) == { + "pkg.models.Entity.__init__", + "pkg.models.Entity.describe", + "pkg.models.Entity.Meta.m", + "pkg.models.greet", + "pkg.models.greet.._decorate", + } + + +def test_overview_kind_and_owning_class(): + overviews = {o.signature: o for o in _backend().get_callables_overview()} + + describe = overviews["pkg.models.Entity.describe"] + assert describe.kind == "method" + assert describe.class_signature == "pkg.models.Entity" + assert describe.decorators == ["property"] + + inner_method = overviews["pkg.models.Entity.Meta.m"] + assert inner_method.kind == "method" + assert inner_method.class_signature == "pkg.models.Entity.Meta" + + greet = overviews["pkg.models.greet"] + assert greet.kind == "function" + assert greet.class_signature is None + + nested = overviews["pkg.models.greet.._decorate"] + assert nested.kind == "function" + assert nested.class_signature is None + + +def test_method_bodies_returns_only_requested_existing(): + bodies = _backend().get_method_bodies(["pkg.models.greet", "pkg.models.Entity.describe", "does.not.exist"]) + assert bodies == { + "pkg.models.greet": "def greet(who): ...", + "pkg.models.Entity.describe": "return self.x", + } + + +def test_decorated_callables_filters_by_marker(): + backend = _backend() + routed = backend.get_decorated_callables(["app.route"]) + assert [o.signature for o in routed] == ["pkg.models.greet"] + + props = backend.get_decorated_callables(["property"]) + assert [o.signature for o in props] == ["pkg.models.Entity.describe"] + + assert backend.get_decorated_callables(["nonexistent"]) == [] diff --git a/tests/analysis/python/test_python_neo4j_backend.py b/tests/analysis/python/test_python_neo4j_backend.py index f51b95f..ad5981d 100644 --- a/tests/analysis/python/test_python_neo4j_backend.py +++ b/tests/analysis/python/test_python_neo4j_backend.py @@ -208,6 +208,30 @@ def test_methods_and_fields_parity(backends): assert _norm(ref.get_all_fields(sig)) == _norm(neo.get_all_fields(sig)) +def test_bulk_accessors_parity(backends): + ref, neo = backends + + # get_callables_overview: same set of callables, identical projection per signature. + ov_ref = {o.signature: o.model_dump() for o in ref.get_callables_overview()} + ov_neo = {o.signature: o.model_dump() for o in neo.get_callables_overview()} + assert set(ov_ref) == set(ov_neo) + for sig in ov_ref: + assert ov_ref[sig] == ov_neo[sig], f"overview for {sig} differs" + + # get_method_bodies: identical bodies for the whole frontier, and missing sigs omitted on both. + sigs = list(ov_ref) + assert ref.get_method_bodies(sigs) == neo.get_method_bodies(sigs) + assert ref.get_method_bodies(["nope.not.here"]) == neo.get_method_bodies(["nope.not.here"]) == {} + + # get_decorated_callables: parity for whatever decorators the project actually uses. + markers = sorted({d for o in ov_ref.values() for d in o["decorators"]}) + if markers: + dec_ref = {o.signature: o.model_dump() for o in ref.get_decorated_callables(markers)} + dec_neo = {o.signature: o.model_dump() for o in neo.get_decorated_callables(markers)} + assert dec_ref == dec_neo + assert ref.get_decorated_callables(["__no_such_decorator__"]) == neo.get_decorated_callables(["__no_such_decorator__"]) == [] + + def test_call_graph_parity(backends): ref, neo = backends g_ref, g_neo = ref.get_call_graph(), neo.get_call_graph() From b38f595e1a7b11b1253938ddb2ab6fd8bcfffb6d Mon Sep 17 00:00:00 2001 From: Rahul Krishna Date: Sat, 27 Jun 2026 15:23:40 -0400 Subject: [PATCH 2/4] perf(python): reuse one Neo4j read session instead of one per query PyNeo4jBackend._run opened a fresh driver session on every call, so the N+1 reconstruction fan-out (get_symbol_table / get_all_methods_in_application) paid session-acquisition overhead on each of its tens of thousands of queries. Reuse a single lazily-opened session for the backend's lifetime, dropping it on error so the next call reopens cleanly. Closed in close(). Refs #180 --- cldk/analysis/python/neo4j/neo4j_backend.py | 33 ++++++++++++++++++--- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/cldk/analysis/python/neo4j/neo4j_backend.py b/cldk/analysis/python/neo4j/neo4j_backend.py index 9546dda..2b4afbe 100644 --- a/cldk/analysis/python/neo4j/neo4j_backend.py +++ b/cldk/analysis/python/neo4j/neo4j_backend.py @@ -128,6 +128,9 @@ def __init__( self.application_name = application_name self._database = neo4j_database self._driver = GraphDatabase.driver(neo4j_uri, auth=(neo4j_username, neo4j_password)) + # One long-lived read session reused across queries (see _run). Reconstruction is an N+1 + # fan-out, so reopening a session per query added real per-call overhead. Created lazily. + self._session_obj: Any | None = None # The application's module file_keys, used to scope every query to this app. self._modules: List[str] = self._load_module_keys() @@ -136,19 +139,41 @@ def __init__( # -----[ lifecycle ]----- def close(self) -> None: - """Close the underlying Neo4j driver.""" + """Close the reused session (if any) and the underlying Neo4j driver.""" + self._close_session() self._driver.close() + def _close_session(self) -> None: + if self._session_obj is not None: + try: + self._session_obj.close() + except Exception: # noqa: BLE001 - best-effort cleanup + pass + self._session_obj = None + def __enter__(self) -> "PyNeo4jBackend": return self def __exit__(self, *exc: Any) -> None: self.close() + def _session(self) -> Any: + """The reused read session, opened lazily on first use.""" + if self._session_obj is None: + self._session_obj = self._driver.session(database=self._database) + return self._session_obj + def _run(self, query: str, **params: Any) -> List[Dict[str, Any]]: - """Run a Cypher statement and return the records as plain dicts (nodes/rels → prop maps).""" - with self._driver.session(database=self._database) as session: - return [record.data() for record in session.run(query, **params)] + """Run a Cypher statement and return the records as plain dicts (nodes/rels → prop maps). + + Reuses one long-lived session across calls. If a query fails the session may be left in a + bad state, so it is dropped before re-raising and the next call reopens a fresh one. + """ + try: + return [record.data() for record in self._session().run(query, **params)] + except Exception: + self._close_session() + raise def _load_module_keys(self) -> List[str]: """The application's module ``file_key``s — the scope key for every other query.""" From 675efa2016359e7ddfdb3750c0fce87ed9799173 Mon Sep 17 00:00:00 2001 From: Rahul Krishna Date: Sat, 27 Jun 2026 15:31:02 -0400 Subject: [PATCH 3/4] chore: sync uv.lock with codeanalyzer-typescript 0.4.3 The lockfile still pinned codeanalyzer-typescript 0.4.0 while pyproject was bumped to 0.4.3 (#179); regenerate so the two agree. No runtime deps added (black stays in the test dependency group, not runtime). --- uv.lock | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/uv.lock b/uv.lock index 269f450..63281ee 100644 --- a/uv.lock +++ b/uv.lock @@ -347,7 +347,7 @@ test = [ requires-dist = [ { name = "clang", specifier = "==17.0.6" }, { name = "codeanalyzer-python", specifier = "==0.2.0" }, - { name = "codeanalyzer-typescript", specifier = "==0.4.0" }, + { name = "codeanalyzer-typescript", specifier = "==0.4.3" }, { name = "libclang", specifier = "==17.0.6" }, { name = "neo4j", marker = "extra == 'neo4j'", specifier = ">=5.14,<7" }, { name = "networkx", specifier = ">=3.4.2,<4" }, @@ -419,14 +419,14 @@ wheels = [ [[package]] name = "codeanalyzer-typescript" -version = "0.4.0" +version = "0.4.3" source = { registry = "https://pypi.org/simple" } wheels = [ - { url = "https://files.pythonhosted.org/packages/b0/81/da7f318dc2465fe0a37c072746cabc6f44ae6f4a541898390055285e6db1/codeanalyzer_typescript-0.4.0-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:01bd9ff546302b105656a6b38c4039bab1a61f9c3c98f087d53990d3fc750a50", size = 31038787, upload-time = "2026-06-19T23:05:39.331Z" }, - { url = "https://files.pythonhosted.org/packages/5f/8e/c02efa6d5bbdc2ce3bf21f2e312c6152dd60c2b53924e9160c3ae87ec23d/codeanalyzer_typescript-0.4.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:a5e23461301f3206ebe5f71a04ab5e3f7e1e38a447660e6de1cd667ec65e69c7", size = 28679376, upload-time = "2026-06-19T23:05:42.093Z" }, - { url = "https://files.pythonhosted.org/packages/4e/f7/10ccfb06a19e608b0745e6195a71f3728c0abbec8718eed5f8411544ed8e/codeanalyzer_typescript-0.4.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:1ee5c678c7d19b91665e03af3008d091a7fd89112fa72c12937d8508c95fba17", size = 40232939, upload-time = "2026-06-19T23:05:44.987Z" }, - { url = "https://files.pythonhosted.org/packages/00/bb/68292380194671e674d3b6dbc15659f0bb90d2bb6270fdd159e93a661a4a/codeanalyzer_typescript-0.4.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:25b6596e18483e807e157494ceb692b8a348610304f1e6658024fe99c56338ff", size = 40502175, upload-time = "2026-06-19T23:05:47.89Z" }, - { url = "https://files.pythonhosted.org/packages/ab/17/c619bbfb9db1b1b8612a4bb5a2d21dfbe863ddf7ec1f7ba9e1c6f06e4e13/codeanalyzer_typescript-0.4.0-py3-none-win_amd64.whl", hash = "sha256:95d4d228c17b5bd44f83a4dd15e7e33cbf362d980b17fe9d740e1cd2ab68a9c3", size = 42889400, upload-time = "2026-06-19T23:05:50.857Z" }, + { url = "https://files.pythonhosted.org/packages/f9/ff/be99765bd13613eb518184df08141a44b5d55a96306c9c177dba725b9310/codeanalyzer_typescript-0.4.3-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:f0d60a457c30b94ac52140701f89d20cb552118fc63146d7d69199ae226ca853", size = 31040212, upload-time = "2026-06-27T18:50:27.121Z" }, + { url = "https://files.pythonhosted.org/packages/ed/df/3c2eaf131bc1c8c1333fc33c0157d5230e347390d982013f71d87e010177/codeanalyzer_typescript-0.4.3-py3-none-macosx_11_0_arm64.whl", hash = "sha256:48098a7cc38a8acc74a10a64c19f79a549e919503e6d89b25200935d910424db", size = 28680981, upload-time = "2026-06-27T18:50:30.005Z" }, + { url = "https://files.pythonhosted.org/packages/f5/98/19e27a40be65b76853a8e21713cb9ffa3126956481c65d748e3727918ac0/codeanalyzer_typescript-0.4.3-py3-none-manylinux2014_aarch64.whl", hash = "sha256:5cd3b76040719914463d9aa0bf87bcb346e7f4c568e7d62085bb88c03f555ab8", size = 40234190, upload-time = "2026-06-27T18:50:32.951Z" }, + { url = "https://files.pythonhosted.org/packages/14/61/fa7e886b66c67dd19fc16c8da032a304ff6ad8b06d1232b716fcc55e1f16/codeanalyzer_typescript-0.4.3-py3-none-manylinux2014_x86_64.whl", hash = "sha256:6f87e9e2fdc2b6d926c536c5517661c70aee3720a5803c91fcf4af54d52320e6", size = 40503499, upload-time = "2026-06-27T18:50:36.031Z" }, + { url = "https://files.pythonhosted.org/packages/de/bf/06f1a820ec8ded7721ce61057d67c2758079f5c31f0e70933c3c29dd78c4/codeanalyzer_typescript-0.4.3-py3-none-win_amd64.whl", hash = "sha256:1af81d1ec28c503790d7d3dc745bb5c5418f87b69c427535b76f7c112a297359", size = 42890644, upload-time = "2026-06-27T18:50:38.974Z" }, ] [[package]] From 8ccdd83372509524106bb49975ae5165ada3d88f Mon Sep 17 00:00:00 2001 From: Rahul Krishna Date: Sat, 27 Jun 2026 15:33:54 -0400 Subject: [PATCH 4/4] feat(python): add get_callsites_for batch accessor (#180 item 3) Batch call-site fetch keyed by owning signature, the last of the four bulk accessors from #180. One projected Cypher statement on the Neo4j backend (an OPTIONAL MATCH over PY_HAS_CALLSITE so an existing callable with no call sites still gets an empty-list entry, matching the in-process backend); one symbol-table walk in-process. Added to the ABC, both backends, and the facade, with offline and Neo4j-parity test coverage. Refs #180 --- cldk/analysis/python/backend.py | 7 ++++++ .../python/codeanalyzer/codeanalyzer.py | 6 +++++ cldk/analysis/python/neo4j/neo4j_backend.py | 20 ++++++++++++++++ cldk/analysis/python/python_analysis.py | 17 +++++++++++++ .../python/test_python_bulk_accessors.py | 24 ++++++++++++++++--- .../python/test_python_neo4j_backend.py | 8 +++++++ 6 files changed, 79 insertions(+), 3 deletions(-) diff --git a/cldk/analysis/python/backend.py b/cldk/analysis/python/backend.py index 2a1048d..b80948e 100644 --- a/cldk/analysis/python/backend.py +++ b/cldk/analysis/python/backend.py @@ -38,6 +38,7 @@ PyApplication, PyCallable, PyCallableOverview, + PyCallsite, PyClass, PyClassAttribute, PyModule, @@ -159,3 +160,9 @@ def get_method_bodies(self, signatures: List[str]) -> Dict[str, str]: def get_decorated_callables(self, markers: List[str]) -> List[PyCallableOverview]: """Overviews of callables decorated with any of ``markers`` (matched against the decorator names).""" + + @abstractmethod + def get_callsites_for(self, signatures: List[str]) -> Dict[str, List[PyCallsite]]: + """Call sites of the given callable signatures, keyed by owning signature. Each existing + signature gets an entry (an empty list if it has no call sites); signatures with no matching + callable are omitted.""" diff --git a/cldk/analysis/python/codeanalyzer/codeanalyzer.py b/cldk/analysis/python/codeanalyzer/codeanalyzer.py index a05f96b..684897c 100644 --- a/cldk/analysis/python/codeanalyzer/codeanalyzer.py +++ b/cldk/analysis/python/codeanalyzer/codeanalyzer.py @@ -67,6 +67,7 @@ PyCallEdge, PyCallable, PyCallableOverview, + PyCallsite, PyClass, PyClassAttribute, PyComment, @@ -589,6 +590,11 @@ def get_decorated_callables(self, markers: List[str]) -> List[PyCallableOverview if marker_set.intersection(c.decorators or []) ] + def get_callsites_for(self, signatures: List[str]) -> Dict[str, List[PyCallsite]]: + """Return ``{signature: call_sites}`` for the requested signatures that exist.""" + wanted = set(signatures) + return {c.signature: list(c.call_sites) for c, _, _ in self._iter_callables() if c.signature in wanted} + # ----------------------------------------------------------- callers/callees def get_all_callers(self, target_class_name: str, target_method_declaration: str) -> Dict: """Return all methods that call a specific target method. diff --git a/cldk/analysis/python/neo4j/neo4j_backend.py b/cldk/analysis/python/neo4j/neo4j_backend.py index 2b4afbe..260d355 100644 --- a/cldk/analysis/python/neo4j/neo4j_backend.py +++ b/cldk/analysis/python/neo4j/neo4j_backend.py @@ -83,6 +83,7 @@ PyCallEdge, PyCallable, PyCallableOverview, + PyCallsite, PyClass, PyClassAttribute, PyModule, @@ -491,3 +492,22 @@ def get_decorated_callables(self, markers: List[str]) -> List[PyCallableOverview markers=list(markers), ) return [R.overview(r) for r in rows] + + def get_callsites_for(self, signatures: List[str]) -> Dict[str, List[PyCallsite]]: + # OPTIONAL MATCH so a requested callable with no call sites still yields a row (p is null), + # giving it an empty-list entry — parity with the in-process backend, which keys every + # existing signature. ORDER mirrors _callable_full's call-site ordering. + rows = self._run( + "MATCH (c:PyCallable) WHERE c._module IN $mods AND c.signature IN $sigs " + "OPTIONAL MATCH (c)-[:PY_HAS_CALLSITE]->(s:PyCallSite) " + "RETURN c.signature AS owner, properties(s) AS p " + "ORDER BY s.start_line, s.start_column", + mods=self._modules, + sigs=list(signatures), + ) + out: Dict[str, List[PyCallsite]] = {} + for r in rows: + sites = out.setdefault(r["owner"], []) + if r["p"] is not None: + sites.append(R.callsite(r["p"])) + return out diff --git a/cldk/analysis/python/python_analysis.py b/cldk/analysis/python/python_analysis.py index dbfbb7c..5324f8f 100644 --- a/cldk/analysis/python/python_analysis.py +++ b/cldk/analysis/python/python_analysis.py @@ -62,6 +62,7 @@ PyApplication, PyCallable, PyCallableOverview, + PyCallsite, PyClass, PyClassAttribute, PyComment, @@ -573,6 +574,22 @@ def get_decorated_callables(self, markers: List[str]) -> List[PyCallableOverview """ return self.backend.get_decorated_callables(markers) + def get_callsites_for(self, signatures: List[str]) -> Dict[str, List[PyCallsite]]: + """Return the call sites of the given callables, keyed by signature, in one bulk read. + + Avoids the per-callable reconstruction fan-out when you need call sites for a specific + frontier (e.g. dispatch-edge synthesis or external-reader detection). + + Args: + signatures: Callable signatures to fetch call sites for. + + Returns: + A dict mapping each existing signature to its list of + :class:`~cldk.models.python.PyCallsite` (empty if the callable has no call sites). + Signatures with no matching callable are omitted. + """ + return self.backend.get_callsites_for(signatures) + def get_methods_in_class(self, qualified_class_name: str) -> Dict[str, PyCallable]: """Return all methods defined in a specific class. diff --git a/tests/analysis/python/test_python_bulk_accessors.py b/tests/analysis/python/test_python_bulk_accessors.py index 4a36c9a..fb4542a 100644 --- a/tests/analysis/python/test_python_bulk_accessors.py +++ b/tests/analysis/python/test_python_bulk_accessors.py @@ -23,12 +23,12 @@ ``test_python_neo4j_backend.py`` when a server is available. """ -from codeanalyzer.schema.py_schema import PyApplication, PyCallable, PyClass, PyModule +from codeanalyzer.schema.py_schema import PyApplication, PyCallable, PyCallsite, PyClass, PyModule from cldk.analysis.python.codeanalyzer.codeanalyzer import PyCodeanalyzer -def _callable(name, signature, *, code="", decorators=None, inner_callables=None, inner_classes=None): +def _callable(name, signature, *, code="", decorators=None, inner_callables=None, inner_classes=None, call_sites=None): return PyCallable( name=name, path="pkg/models.py", @@ -37,6 +37,7 @@ def _callable(name, signature, *, code="", decorators=None, inner_callables=None decorators=decorators or [], inner_callables=inner_callables or {}, inner_classes=inner_classes or {}, + call_sites=call_sites or [], ) @@ -64,7 +65,13 @@ def _backend(): "pkg.models.Entity", methods={ "__init__": _callable("__init__", "pkg.models.Entity.__init__", code="self.x = 1"), - "describe": _callable("describe", "pkg.models.Entity.describe", code="return self.x", decorators=["property"]), + "describe": _callable( + "describe", + "pkg.models.Entity.describe", + code="return self.x", + decorators=["property"], + call_sites=[PyCallsite(method_name="greet", start_line=7, start_column=4)], + ), }, inner_classes={"pkg.models.Entity.Meta": meta}, ) @@ -131,3 +138,14 @@ def test_decorated_callables_filters_by_marker(): assert [o.signature for o in props] == ["pkg.models.Entity.describe"] assert backend.get_decorated_callables(["nonexistent"]) == [] + + +def test_callsites_for_keys_existing_signatures_only(): + backend = _backend() + sites = backend.get_callsites_for( + ["pkg.models.Entity.describe", "pkg.models.greet", "does.not.exist"] + ) + # both existing callables get a key; the one with no call sites maps to an empty list + assert set(sites) == {"pkg.models.Entity.describe", "pkg.models.greet"} + assert [s.method_name for s in sites["pkg.models.Entity.describe"]] == ["greet"] + assert sites["pkg.models.greet"] == [] diff --git a/tests/analysis/python/test_python_neo4j_backend.py b/tests/analysis/python/test_python_neo4j_backend.py index ad5981d..640aff9 100644 --- a/tests/analysis/python/test_python_neo4j_backend.py +++ b/tests/analysis/python/test_python_neo4j_backend.py @@ -231,6 +231,14 @@ def test_bulk_accessors_parity(backends): assert dec_ref == dec_neo assert ref.get_decorated_callables(["__no_such_decorator__"]) == neo.get_decorated_callables(["__no_such_decorator__"]) == [] + # get_callsites_for: same keys (every existing signature) and identical, identically-ordered sites. + cs_ref = ref.get_callsites_for(sigs) + cs_neo = neo.get_callsites_for(sigs) + assert set(cs_ref) == set(cs_neo) + for sig in cs_ref: + assert [_norm(s) for s in cs_ref[sig]] == [_norm(s) for s in cs_neo[sig]], f"call sites for {sig} differ" + assert ref.get_callsites_for(["nope.not.here"]) == neo.get_callsites_for(["nope.not.here"]) == {} + def test_call_graph_parity(backends): ref, neo = backends