feat(python): bulk/projected accessors for the Python facade (#180)

cldk/analysis/python/backend.py

@@ -37,6 +37,8 @@
 from cldk.models.python import (
  PyApplication,
  PyCallable,
+ PyCallableOverview,
+ PyCallsite,
  PyClass,
  PyClassAttribute,
  PyModule,
@@ -139,3 +141,28 @@ def get_all_constructors(self, qualified_class_name: str) -> Dict[str, PyCallabl
  @abstractmethod
  def get_all_fields(self, qualified_class_name: str) -> List[PyClassAttribute]:
  """The attributes/fields of a class."""
+
+ # -----[ bulk / projected accessors ]-----
+ # Set-at-a-time, field-projected reads — one round-trip on the Neo4j backend, one symbol-table
+ # walk in-process — for callers that enumerate the whole application and would otherwise pay the
+ # per-entity reconstruction of get_all_methods_in_application.
+ @abstractmethod
+ def get_callables_overview(self) -> List[PyCallableOverview]:
+ """A lightweight projection of every callable in the application (methods, module-level and
+ nested functions), without the full :class:`PyCallable` reconstruction."""
+
+ @abstractmethod
+ def get_method_bodies(self, signatures: List[str]) -> Dict[str, str]:
+ """Source bodies for the given callable signatures, keyed by signature. Signatures with no
+ matching callable are omitted."""
+
+ @abstractmethod
+ def get_decorated_callables(self, markers: List[str]) -> List[PyCallableOverview]:
+ """Overviews of callables decorated with any of ``markers`` (matched against the decorator
+ names)."""
+
+ @abstractmethod
+ def get_callsites_for(self, signatures: List[str]) -> Dict[str, List[PyCallsite]]:
+ """Call sites of the given callable signatures, keyed by owning signature. Each existing
+ signature gets an entry (an empty list if it has no call sites); signatures with no matching
+ callable are omitted."""

cldk/analysis/python/codeanalyzer/codeanalyzer.py

@@ -51,7 +51,7 @@
 
 import logging
 from pathlib import Path
-from typing import Dict, List, Tuple, Union
+from typing import Dict, Iterator, List, Tuple, Union
 
 import networkx as nx
 
@@ -66,6 +66,8 @@
  PyApplication,
  PyCallEdge,
  PyCallable,
+ PyCallableOverview,
+ PyCallsite,
  PyClass,
  PyClassAttribute,
  PyComment,
@@ -75,6 +77,20 @@
 logger = logging.getLogger(__name__)
 
 
+def _overview(c: PyCallable, class_signature: str | None, kind: str) -> PyCallableOverview:
+ """Project a :class:`PyCallable` into a lightweight :class:`PyCallableOverview`."""
+ return PyCallableOverview(
+ signature=c.signature,
+ name=c.name,
+ class_signature=class_signature,
+ kind=kind,
+ path=c.path,
+ start_line=c.start_line,
+ end_line=c.end_line,
+ decorators=list(c.decorators or []),
+ )
+
+
 class PyCodeanalyzer(PythonAnalysisBackend):
  """In-process driver for the ``codeanalyzer-python`` analysis backend.
 
@@ -523,6 +539,62 @@ def get_all_fields(self, qualified_class_name: str) -> List[PyClassAttribute]:
  cls = self.get_class(qualified_class_name)
  return list(cls.attributes.values()) if cls else []
 
+ # ----------------------------------------------------------- bulk / projected accessors
+ def _iter_callables(self) -> Iterator[Tuple[PyCallable, "str | None", str]]:
+ """Yield ``(callable, class_signature, kind)`` for every callable in the application.
+
+ Walks the in-memory symbol table the same way the Neo4j backend's ``MATCH (c:PyCallable)``
+ sees nodes: a callable is a ``"method"`` only when a class declares it directly (mirroring
+ ``PY_HAS_METHOD``); module-level functions and functions nested inside a callable are
+ ``"function"`` with a ``None`` class signature. The two backends therefore enumerate the
+ same set.
+ """
+
+ def from_callable(c: PyCallable):
+ for inner in c.inner_callables.values():
+ yield inner, None, "function"
+ yield from from_callable(inner)
+ for inner_cls in c.inner_classes.values():
+ yield from from_class(inner_cls)
+
+ def from_class(cls: PyClass):
+ for m in cls.methods.values():
+ yield m, cls.signature, "method"
+ yield from from_callable(m)
+ for inner_cls in cls.inner_classes.values():
+ yield from from_class(inner_cls)
+
+ for module in self.application.symbol_table.values():
+ for cls in module.classes.values():
+ yield from from_class(cls)
+ for fn in module.functions.values():
+ yield fn, None, "function"
+ yield from from_callable(fn)
+
+ def get_callables_overview(self) -> List[PyCallableOverview]:
+ """Return a lightweight overview of every callable in the application (see
+ :meth:`PythonAnalysisBackend.get_callables_overview`)."""
+ return [_overview(c, class_sig, kind) for c, class_sig, kind in self._iter_callables()]
+
+ def get_method_bodies(self, signatures: List[str]) -> Dict[str, str]:
+ """Return ``{signature: code}`` for the requested signatures that exist."""
+ wanted = set(signatures)
+ return {c.signature: c.code for c, _, _ in self._iter_callables() if c.signature in wanted}
+
+ def get_decorated_callables(self, markers: List[str]) -> List[PyCallableOverview]:
+ """Return overviews of callables decorated with any of ``markers``."""
+ marker_set = set(markers)
+ return [
+ _overview(c, class_sig, kind)
+ for c, class_sig, kind in self._iter_callables()
+ if marker_set.intersection(c.decorators or [])
+ ]
+
+ def get_callsites_for(self, signatures: List[str]) -> Dict[str, List[PyCallsite]]:
+ """Return ``{signature: call_sites}`` for the requested signatures that exist."""
+ wanted = set(signatures)
+ return {c.signature: list(c.call_sites) for c, _, _ in self._iter_callables() if c.signature in wanted}
+
  # ----------------------------------------------------------- callers/callees
  def get_all_callers(self, target_class_name: str, target_method_declaration: str) -> Dict:
  """Return all methods that call a specific target method.

cldk/analysis/python/neo4j/neo4j_backend.py

@@ -82,6 +82,8 @@
  PyApplication,
  PyCallEdge,
  PyCallable,
+ PyCallableOverview,
+ PyCallsite,
  PyClass,
  PyClassAttribute,
  PyModule,
@@ -127,6 +129,9 @@ def __init__(
  self.application_name = application_name
  self._database = neo4j_database
  self._driver = GraphDatabase.driver(neo4j_uri, auth=(neo4j_username, neo4j_password))
+ # One long-lived read session reused across queries (see _run). Reconstruction is an N+1
+ # fan-out, so reopening a session per query added real per-call overhead. Created lazily.
+ self._session_obj: Any | None = None
 
  # The application's module file_keys, used to scope every query to this app.
  self._modules: List[str] = self._load_module_keys()
@@ -135,19 +140,41 @@ def __init__(
 
  # -----[ lifecycle ]-----
  def close(self) -> None:
- """Close the underlying Neo4j driver."""
+ """Close the reused session (if any) and the underlying Neo4j driver."""
+ self._close_session()
  self._driver.close()
 
+ def _close_session(self) -> None:
+ if self._session_obj is not None:
+ try:
+ self._session_obj.close()
+ except Exception: # noqa: BLE001 - best-effort cleanup
+ pass
+ self._session_obj = None
+
  def __enter__(self) -> "PyNeo4jBackend":
  return self
 
  def __exit__(self, *exc: Any) -> None:
  self.close()
 
+ def _session(self) -> Any:
+ """The reused read session, opened lazily on first use."""
+ if self._session_obj is None:
+ self._session_obj = self._driver.session(database=self._database)
+ return self._session_obj
+
  def _run(self, query: str, **params: Any) -> List[Dict[str, Any]]:
- """Run a Cypher statement and return the records as plain dicts (nodes/rels → prop maps)."""
- with self._driver.session(database=self._database) as session:
- return [record.data() for record in session.run(query, **params)]
+ """Run a Cypher statement and return the records as plain dicts (nodes/rels → prop maps).
+
+ Reuses one long-lived session across calls. If a query fails the session may be left in a
+ bad state, so it is dropped before re-raising and the next call reopens a fresh one.
+ """
+ try:
+ return [record.data() for record in self._session().run(query, **params)]
+ except Exception:
+ self._close_session()
+ raise
 
  def _load_module_keys(self) -> List[str]:
  """The application's module ``file_key``s — the scope key for every other query."""
@@ -428,3 +455,59 @@ def get_all_constructors(self, qualified_class_name: str) -> Dict[str, PyCallabl
  def get_all_fields(self, qualified_class_name: str) -> List[PyClassAttribute]:
  cls = self.get_class(qualified_class_name)
  return list(cls.attributes.values()) if cls else []
+
+ # =====================================================================================
+ # PythonAnalysisBackend — bulk / projected accessors (one round-trip each)
+ # =====================================================================================
+ # Field-projected RETURNs that sidestep the per-entity reconstruction fan-out: each is a single
+ # Cypher statement, not the N+1 walk get_symbol_table()/get_all_methods_in_application() pays.
+ _OVERVIEW_PROJECTION = (
+ "OPTIONAL MATCH (owner:PyClass)-[:PY_HAS_METHOD]->(c) "
+ "RETURN c.signature AS signature, c.name AS name, c.decorators AS decorators, "
+ "c.path AS path, c.start_line AS start_line, c.end_line AS end_line, "
+ "owner.signature AS class_signature"
+ )
+
+ def get_callables_overview(self) -> List[PyCallableOverview]:
+ rows = self._run(
+ "MATCH (c:PyCallable) WHERE c._module IN $mods " + self._OVERVIEW_PROJECTION,
+ mods=self._modules,
+ )
+ return [R.overview(r) for r in rows]
+
+ def get_method_bodies(self, signatures: List[str]) -> Dict[str, str]:
+ rows = self._run(
+ "MATCH (c:PyCallable) WHERE c._module IN $mods AND c.signature IN $sigs "
+ "RETURN c.signature AS signature, c.code AS code",
+ mods=self._modules,
+ sigs=list(signatures),
+ )
+ return {r["signature"]: r.get("code") for r in rows}
+
+ def get_decorated_callables(self, markers: List[str]) -> List[PyCallableOverview]:
+ rows = self._run(
+ "MATCH (c:PyCallable) WHERE c._module IN $mods "
+ "AND any(d IN c.decorators WHERE d IN $markers) " + self._OVERVIEW_PROJECTION,
+ mods=self._modules,
+ markers=list(markers),
+ )
+ return [R.overview(r) for r in rows]
+
+ def get_callsites_for(self, signatures: List[str]) -> Dict[str, List[PyCallsite]]:
+ # OPTIONAL MATCH so a requested callable with no call sites still yields a row (p is null),
+ # giving it an empty-list entry — parity with the in-process backend, which keys every
+ # existing signature. ORDER mirrors _callable_full's call-site ordering.
+ rows = self._run(
+ "MATCH (c:PyCallable) WHERE c._module IN $mods AND c.signature IN $sigs "
+ "OPTIONAL MATCH (c)-[:PY_HAS_CALLSITE]->(s:PyCallSite) "
+ "RETURN c.signature AS owner, properties(s) AS p "
+ "ORDER BY s.start_line, s.start_column",
+ mods=self._modules,
+ sigs=list(signatures),
+ )
+ out: Dict[str, List[PyCallsite]] = {}
+ for r in rows:
+ sites = out.setdefault(r["owner"], [])
+ if r["p"] is not None:
+ sites.append(R.callsite(r["p"]))
+ return out

cldk/analysis/python/neo4j/reconstruct.py

@@ -42,6 +42,7 @@
 
 from cldk.models.python import (
  PyCallable,
+ PyCallableOverview,
  PyClass,
  PyClassAttribute,
  PyComment,
@@ -132,6 +133,26 @@ def import_(module: str, name: str, alias: str | None = None) -> PyImport:
  return PyImport(module=module, name=name, alias=alias)
 
 
+def overview(row: Props) -> PyCallableOverview:
+ """Build a :class:`PyCallableOverview` from a projected callable row.
+
+ ``row`` is a flat ``RETURN`` projection (not a node's ``properties()``): ``signature``, ``name``,
+ ``decorators``, ``path``, ``start_line``, ``end_line``, and ``class_signature`` (the owning
+ class via ``PY_HAS_METHOD``, or ``None`` for a module-level / nested function).
+ """
+ class_sig = row.get("class_signature")
+ return PyCallableOverview(
+ signature=row.get("signature", ""),
+ name=row.get("name", ""),
+ class_signature=class_sig,
+ kind="method" if class_sig else "function",
+ path=row.get("path", ""),
+ start_line=row.get("start_line", -1),
+ end_line=row.get("end_line", -1),
+ decorators=list(row.get("decorators", []) or []),
+ )
+
+
 # -----[ declarations ]-----
 def callable_(
  props: Props,

cldk/analysis/python/python_analysis.py

@@ -61,6 +61,8 @@
 from cldk.models.python import (
  PyApplication,
  PyCallable,
+ PyCallableOverview,
+ PyCallsite,
  PyClass,
  PyClassAttribute,
  PyComment,
@@ -524,6 +526,70 @@ def get_methods(self) -> Dict[str, Dict[str, PyCallable]]:
  """
  return self.backend.get_all_methods_in_application()
 
+ def get_callables_overview(self) -> List[PyCallableOverview]:
+ """Return a lightweight overview of every callable in the project, in one bulk read.
+
+ A field-projected alternative to :meth:`get_methods` for enumeration: each
+ :class:`~cldk.models.python.PyCallableOverview` carries the callable's signature, owning
+ class (if any), kind, location, and decorators — but not the full reconstruction (call
+ sites, inner callables, locals). On the Neo4j backend this is a single Cypher query instead
+ of the per-entity fan-out :meth:`get_methods` pays. Body-inspect the few you need afterwards
+ via :meth:`get_method` or :meth:`get_method_bodies`.
+
+ Returns:
+ A flat list of :class:`~cldk.models.python.PyCallableOverview`, one per callable
+ (methods, module-level functions, and nested functions).
+
+ See Also:
+ :meth:`get_decorated_callables`: The same projection filtered by decorator.
+ :meth:`get_method_bodies`: Bulk source-body fetch for chosen signatures.
+ """
+ return self.backend.get_callables_overview()
+
+ def get_method_bodies(self, signatures: List[str]) -> Dict[str, str]:
+ """Return source bodies for the given callable signatures, in one bulk read.
+
+ Args:
+ signatures: Callable signatures to fetch bodies for (e.g. from
+ :meth:`get_callables_overview`).
+
+ Returns:
+ A dict mapping each signature to its source body. Signatures with no matching callable
+ are omitted.
+ """
+ return self.backend.get_method_bodies(signatures)
+
+ def get_decorated_callables(self, markers: List[str]) -> List[PyCallableOverview]:
+ """Return overviews of callables decorated with any of the given markers, in one bulk read.
+
+ Args:
+ markers: Decorator names to match (e.g. ``["staticmethod", "app.route"]``).
+
+ Returns:
+ A list of :class:`~cldk.models.python.PyCallableOverview` for every callable carrying at
+ least one of ``markers`` as a decorator.
+
+ See Also:
+ :meth:`get_callables_overview`: The unfiltered projection.
+ """
+ return self.backend.get_decorated_callables(markers)
+
+ def get_callsites_for(self, signatures: List[str]) -> Dict[str, List[PyCallsite]]:
+ """Return the call sites of the given callables, keyed by signature, in one bulk read.
+
+ Avoids the per-callable reconstruction fan-out when you need call sites for a specific
+ frontier (e.g. dispatch-edge synthesis or external-reader detection).
+
+ Args:
+ signatures: Callable signatures to fetch call sites for.
+
+ Returns:
+ A dict mapping each existing signature to its list of
+ :class:`~cldk.models.python.PyCallsite` (empty if the callable has no call sites).
+ Signatures with no matching callable are omitted.
+ """
+ return self.backend.get_callsites_for(signatures)
+
  def get_methods_in_class(self, qualified_class_name: str) -> Dict[str, PyCallable]:
  """Return all methods defined in a specific class.
 

cldk/models/python/__init__.py

@@ -36,10 +36,13 @@
  PyVariableDeclaration,
 )
 
+from .projections import PyCallableOverview
+
 __all__ = [
  "PyApplication",
  "PyCallEdge",
  "PyCallable",
+ "PyCallableOverview",
  "PyCallableParameter",
  "PyCallsite",
  "PyClass",