diff --git a/codeanalyzer/neo4j/project.py b/codeanalyzer/neo4j/project.py index 672ee80..f7b01cd 100644 --- a/codeanalyzer/neo4j/project.py +++ b/codeanalyzer/neo4j/project.py @@ -47,7 +47,7 @@ PyModule, PyVariableDeclaration, ) -from codeanalyzer.schema.py_schema import PyCallsite +from codeanalyzer.schema.py_schema import PyCallsite, PyDecorator def project(app: PyApplication, app_name: str, sig_to_id: dict, @@ -369,6 +369,9 @@ def _project_class( ) b.edge(parent_rel, parent, ref) + for d in cl.decorators or []: + _project_decorator(b, ref, d) + for base in cl.base_classes or []: if base: b.edge_to_symbol("PY_EXTENDS", ref, _symbol_ref(base, externals, sig_to_id)) @@ -439,9 +442,36 @@ def _project_variable( b.edge("PY_DECLARES_VAR", owner, ref) -def _project_decorator(b: RowBuilder, on: NodeRef, decorator: str) -> None: - dec = b.node(["PyDecorator"], "name", decorator, {"name": decorator}) - b.edge("PY_DECORATED_BY", on, dec) +def _project_decorator(b: RowBuilder, on: NodeRef, decorator: PyDecorator) -> None: + """Project one decorator application (#128). + + The merge key is the resolved ``qualified_name`` when Jedi supplies one, so + ``@lru_cache`` and ``@lru_cache(maxsize=128)`` land on one node instead of two, + and two spellings of one decorator stop being separate nodes. Unresolved + decorators fall back to the written spelling. Per-application facts (the + arguments) ride on the relationship, not the shared node -- ``:PyDecorator`` + has no ``_module`` and is never pruned, so anything application-specific on it + would accumulate across every project in the database. + """ + key = decorator.qualified_name or decorator.name + dec = b.node( + ["PyDecorator"], + "name", + key, + {"name": key, "qualified_name": decorator.qualified_name or ""}, + ) + b.edge( + "PY_DECORATED_BY", + on, + dec, + { + "expression": decorator.expression or "", + "positional_arguments": list(decorator.positional_arguments or []), + "keyword_arguments_json": json.dumps( + dict(decorator.keyword_arguments or {}), sort_keys=True + ), + }, + ) # ---------------------------------------------------------------------------------------------- @@ -482,6 +512,7 @@ def _class_props(cl: PyClass, file_key: str, source: str) -> Props: "name": cl.name, "code": _span_code(source, cl.span), "base_classes": list(cl.base_classes or []), + "decorators": [d.qualified_name or d.name for d in (cl.decorators or [])], "docstring": _docstring_of(cl.comments), "start_line": cl.start_line, "end_line": cl.end_line, @@ -504,7 +535,7 @@ def _callable_props(c: PyCallable, file_key: str, source: str) -> Props: "start_line": c.start_line, "end_line": c.end_line, "docstring": _docstring_of(c.comments), - "decorators": list(c.decorators or []), + "decorators": [d.qualified_name or d.name for d in (c.decorators or [])], "parameters_json": _stringify_if(c.parameters), "accessed_symbols_json": _stringify_if(c.accessed_symbols), "_module": file_key, diff --git a/codeanalyzer/neo4j/schema.py b/codeanalyzer/neo4j/schema.py index dcb1861..0216357 100644 --- a/codeanalyzer/neo4j/schema.py +++ b/codeanalyzer/neo4j/schema.py @@ -101,6 +101,7 @@ class RelType: "name": "string", "code": "string", "base_classes": "string[]", + "decorators": "string[]", "docstring": "string", **_SPAN, "_module": "string", @@ -138,7 +139,7 @@ class RelType: "PyDecorator", "PyDecorator", "name", - {"name": "string"}, + {"name": "string", "qualified_name": "string"}, ), NodeLabel( "PyCallSite", @@ -234,7 +235,16 @@ class RelType: ["PyModule", "PyPackage"], {"spellings": "string[]", "imported_names": "string[]", "aliases": "string[]"}, ), - RelType("PY_DECORATED_BY", ["PyCallable"], ["PyDecorator"]), + RelType( + "PY_DECORATED_BY", + ["PyCallable", "PyClass"], + ["PyDecorator"], + { + "expression": "string", + "positional_arguments": "string[]", + "keyword_arguments_json": "string", + }, + ), # Level-3 CPG overlay (-a 3 only): the cross-language dataflow vocabulary, # PY_-namespaced so per-language SDK backends can scope their queries. RelType("PY_HAS_CFG_NODE", ["PyCallable"], ["PyCFGNode"]), diff --git a/codeanalyzer/schema/py_schema.py b/codeanalyzer/schema/py_schema.py index 2d2ee98..fc22f6d 100644 --- a/codeanalyzer/schema/py_schema.py +++ b/codeanalyzer/schema/py_schema.py @@ -217,6 +217,24 @@ class PyVariableDeclaration(BaseModel): end_column: int = -1 +@builder +class PyDecorator(BaseModel): + """One decorator application, structured rather than a source string (#128). + + ``name`` is the spelling as written (``lru_cache``, ``builtins.staticmethod``); + ``qualified_name`` is Jedi's resolution of it (``functools.lru_cache``) and is + absent when it cannot be resolved. ``expression`` keeps the full unparsed source + so nothing is lost for decorators too complex to decompose. + """ + + name: str + qualified_name: Optional[str] = None + positional_arguments: List[str] = [] + keyword_arguments: Dict[str, str] = {} + expression: str = "" + span: Optional[Span] = None + + @builder class PyCallableParameter(BaseModel): """Represents a parameter of a Python callable (function/method).""" @@ -224,6 +242,7 @@ class PyCallableParameter(BaseModel): name: str type: Optional[str] = None default_value: Optional[str] = None + decorators: List[PyDecorator] = [] start_line: int = -1 end_line: int = -1 start_column: int = -1 @@ -271,7 +290,7 @@ class PyCallable(BaseModel): kind: str = "function" span: Optional[Span] = None comments: List[PyComment] = [] - decorators: List[str] = [] + decorators: List[PyDecorator] = [] parameters: List[PyCallableParameter] = [] return_type: Optional[str] = None start_line: int = -1 @@ -304,6 +323,7 @@ class PyClassAttribute(BaseModel): type: Optional[str] = None initializer: Optional[str] = None comments: List[PyComment] = [] + decorators: List[PyDecorator] = [] start_line: int = -1 end_line: int = -1 @@ -319,6 +339,7 @@ class PyClass(BaseModel): span: Optional[Span] = None comments: List[PyComment] = [] base_classes: List[str] = [] + decorators: List[PyDecorator] = [] callables: Dict[str, PyCallable] = {} # methods, keystone containment name attributes: Dict[str, PyClassAttribute] = {} types: Dict[str, "PyClass"] = {} # inner classes, keystone containment name diff --git a/codeanalyzer/syntactic_analysis/symbol_table_builder.py b/codeanalyzer/syntactic_analysis/symbol_table_builder.py index bc0ca9f..126b2a1 100644 --- a/codeanalyzer/syntactic_analysis/symbol_table_builder.py +++ b/codeanalyzer/syntactic_analysis/symbol_table_builder.py @@ -16,6 +16,7 @@ PyCallableParameter, PyCallArgument, PyCallsite, + PyDecorator, PyClass, PyClassAttribute, PyComment, @@ -295,6 +296,7 @@ def _add_class(self, node: AST, script: Script, source: str, prefix: str = "") - .name(class_name) .signature(signature) .span(span) + .decorators(self._decorators(child, script, source)) .start_line(start_line) .end_line(end_line) .comments(self._pycomments(child, code)) @@ -331,7 +333,7 @@ def _callables(self, node: AST, script: Script, source: str, prefix: str = "") - getattr(child, "end_lineno", child.lineno), getattr(child, "end_col_offset", child.col_offset)), ) - decorators = [ast.unparse(d) for d in child.decorator_list] + decorators = self._decorators(child, script, source) if prefix: # We're in a nested context - build signature with prefix @@ -590,6 +592,68 @@ def build_param( return params + def _decorators( + self, node: ast.AST, script: Optional[Script], source: str = "" + ) -> List[PyDecorator]: + """Structure each entry of ``node.decorator_list`` (#128). + + ``name`` is the spelling as written and ``qualified_name`` is Jedi's + resolution of it, inferred at the last identifier of the callee so that + ``@a.b.c`` resolves ``c`` rather than ``a``. Resolution is best-effort: + dynamic, conditional and re-exported decorators stay unresolved, and a + failure here must never abort the symbol table. + """ + out: List[PyDecorator] = [] + for dec in getattr(node, "decorator_list", []) or []: + callee = dec.func if isinstance(dec, ast.Call) else dec + positional: List[str] = [] + keyword: Dict[str, str] = {} + if isinstance(dec, ast.Call): + positional = [ast.unparse(a) for a in dec.args] + for kw in dec.keywords: + # ``**kwargs`` has no arg name; keep it addressable rather + # than dropping it. + key = kw.arg if kw.arg is not None else f"**{ast.unparse(kw.value)}" + keyword[key] = ast.unparse(kw.value) + span = Span( + start=(dec.lineno, dec.col_offset), + end=(getattr(dec, "end_lineno", dec.lineno), + getattr(dec, "end_col_offset", dec.col_offset)), + bytes=byte_offsets(source, dec.lineno, dec.col_offset, + getattr(dec, "end_lineno", dec.lineno), + getattr(dec, "end_col_offset", dec.col_offset)), + ) if source else None + out.append( + PyDecorator.builder() + .name(ast.unparse(callee)) + .qualified_name(self._decorator_qualified_name(callee, script)) + .positional_arguments(positional) + .keyword_arguments(keyword) + .expression(ast.unparse(dec)) + .span(span) + .build() + ) + return out + + @staticmethod + def _decorator_qualified_name( + callee: ast.AST, script: Optional[Script] + ) -> Optional[str]: + """Jedi's full name for a decorator's callee, or ``None``.""" + if script is None: + return None + line = getattr(callee, "end_lineno", getattr(callee, "lineno", None)) + col = getattr(callee, "end_col_offset", None) + if line is None or col is None: + return None + try: + d = SymbolTableBuilder._first_definition( + script.infer(line=line, column=max(col - 1, 0)) + ) + except Exception: + return None + return getattr(d, "full_name", None) if d is not None else None + def _accessed_symbols( self, fn_node: ast.FunctionDef, script: Script ) -> List[PySymbol]: diff --git a/schema.neo4j.json b/schema.neo4j.json index 32f8422..f3035d7 100644 --- a/schema.neo4j.json +++ b/schema.neo4j.json @@ -41,6 +41,7 @@ "name": "string", "code": "string", "base_classes": "string[]", + "decorators": "string[]", "docstring": "string", "start_line": "integer", "end_line": "integer", @@ -92,7 +93,8 @@ "merge_label": "PyDecorator", "key": "name", "properties": { - "name": "string" + "name": "string", + "qualified_name": "string" } }, { @@ -280,12 +282,17 @@ { "type": "PY_DECORATED_BY", "from": [ - "PyCallable" + "PyCallable", + "PyClass" ], "to": [ "PyDecorator" ], - "properties": {} + "properties": { + "expression": "string", + "positional_arguments": "string[]", + "keyword_arguments_json": "string" + } }, { "type": "PY_HAS_CFG_NODE", diff --git a/test/test_decorators_structured.py b/test/test_decorators_structured.py new file mode 100644 index 0000000..52af81d --- /dev/null +++ b/test/test_decorators_structured.py @@ -0,0 +1,113 @@ +"""Decorators are structured records, not source strings (#128). + +Covers the four things the flat-string shape could not express: the callee is +separable from its arguments, the callee resolves to a qualified name, classes +carry decorators at all, and the span locates the decorator in the source. +""" +import ast +from pathlib import Path + +import pytest + +jedi = pytest.importorskip("jedi") + +from codeanalyzer.syntactic_analysis.symbol_table_builder import SymbolTableBuilder + +SRC = '''\ +import builtins +from dataclasses import dataclass +from functools import lru_cache + + +def audit(fn): + return fn + + +@dataclass(frozen=True) +class Point: + x: int + + @builtins.staticmethod + def make(a, b): + return a + b + + +@audit +@lru_cache(maxsize=128) +def risky(cmd): + return cmd + + +def plain(x): + return x +''' + + +@pytest.fixture +def decorated(tmp_path: Path): + f = tmp_path / "app.py" + f.write_text(SRC) + script = jedi.Script(path=str(f)) + tree = ast.parse(SRC) + by_name = {n.name: n for n in tree.body if hasattr(n, "name")} + by_name["make"] = tree.body[4].body[1] # method inside Point + builder = SymbolTableBuilder.__new__(SymbolTableBuilder) + return builder, script, by_name + + +def _one(builder, script, node): + return builder._decorators(node, script, SRC) + + +def test_class_decorator_is_captured_with_arguments(decorated): + builder, script, nodes = decorated + (dec,) = _one(builder, script, nodes["Point"]) + assert dec.name == "dataclass" + assert dec.qualified_name == "dataclasses.dataclass" + assert dec.keyword_arguments == {"frozen": "True"} + assert dec.positional_arguments == [] + + +def test_callee_separates_from_arguments(decorated): + builder, script, nodes = decorated + decs = _one(builder, script, nodes["risky"]) + assert [d.name for d in decs] == ["audit", "lru_cache"] + # The whole point: @lru_cache and @lru_cache(maxsize=128) share a callee. + assert decs[1].keyword_arguments == {"maxsize": "128"} + assert decs[1].expression == "lru_cache(maxsize=128)" + + +def test_local_and_library_callees_both_resolve(decorated): + builder, script, nodes = decorated + decs = _one(builder, script, nodes["risky"]) + assert decs[0].qualified_name == "app.audit" + assert decs[1].qualified_name == "functools.lru_cache" + + +def test_dotted_spelling_resolves_to_the_same_builtin(decorated): + """@builtins.staticmethod and @staticmethod are one decorator (see #135).""" + builder, script, nodes = decorated + (dec,) = _one(builder, script, nodes["make"]) + assert dec.name == "builtins.staticmethod" + assert dec.qualified_name == "builtins.staticmethod" + + +def test_span_locates_the_decorator_in_source(decorated): + builder, script, nodes = decorated + (dec,) = _one(builder, script, nodes["Point"]) + lo, hi = dec.span.bytes + assert SRC.encode()[lo:hi].decode() == "dataclass(frozen=True)" + + +def test_undecorated_is_empty_not_missing(decorated): + builder, script, nodes = decorated + assert _one(builder, script, nodes["plain"]) == [] + + +def test_unresolvable_decorator_yields_none_and_does_not_raise(): + src = "@thing.not_real\ndef f():\n pass\n" + builder = SymbolTableBuilder.__new__(SymbolTableBuilder) + node = ast.parse(src).body[0] + (dec,) = builder._decorators(node, None, src) + assert dec.name == "thing.not_real" + assert dec.qualified_name is None