feat(schema)!: structured decorators on callables and classes

codeanalyzer/neo4j/project.py

@@ -47,7 +47,7 @@
  PyModule,
  PyVariableDeclaration,
 )
-from codeanalyzer.schema.py_schema import PyCallsite
+from codeanalyzer.schema.py_schema import PyCallsite, PyDecorator
 
 
 def project(app: PyApplication, app_name: str, sig_to_id: dict,
@@ -369,6 +369,9 @@ def _project_class(
  )
  b.edge(parent_rel, parent, ref)
 
+ for d in cl.decorators or []:
+ _project_decorator(b, ref, d)
+
  for base in cl.base_classes or []:
  if base:
  b.edge_to_symbol("PY_EXTENDS", ref, _symbol_ref(base, externals, sig_to_id))
@@ -439,9 +442,36 @@ def _project_variable(
  b.edge("PY_DECLARES_VAR", owner, ref)
 
 
-def _project_decorator(b: RowBuilder, on: NodeRef, decorator: str) -> None:
- dec = b.node(["PyDecorator"], "name", decorator, {"name": decorator})
- b.edge("PY_DECORATED_BY", on, dec)
+def _project_decorator(b: RowBuilder, on: NodeRef, decorator: PyDecorator) -> None:
+ """Project one decorator application (#128).
+
+ The merge key is the resolved ``qualified_name`` when Jedi supplies one, so
+ ``@lru_cache`` and ``@lru_cache(maxsize=128)`` land on one node instead of two,
+ and two spellings of one decorator stop being separate nodes. Unresolved
+ decorators fall back to the written spelling. Per-application facts (the
+ arguments) ride on the relationship, not the shared node -- ``:PyDecorator``
+ has no ``_module`` and is never pruned, so anything application-specific on it
+ would accumulate across every project in the database.
+ """
+ key = decorator.qualified_name or decorator.name
+ dec = b.node(
+ ["PyDecorator"],
+ "name",
+ key,
+ {"name": key, "qualified_name": decorator.qualified_name or ""},
+ )
+ b.edge(
+ "PY_DECORATED_BY",
+ on,
+ dec,
+ {
+ "expression": decorator.expression or "",
+ "positional_arguments": list(decorator.positional_arguments or []),
+ "keyword_arguments_json": json.dumps(
+ dict(decorator.keyword_arguments or {}), sort_keys=True
+ ),
+ },
+ )
 
 
 # ----------------------------------------------------------------------------------------------
@@ -482,6 +512,7 @@ def _class_props(cl: PyClass, file_key: str, source: str) -> Props:
  "name": cl.name,
  "code": _span_code(source, cl.span),
  "base_classes": list(cl.base_classes or []),
+ "decorators": [d.qualified_name or d.name for d in (cl.decorators or [])],
  "docstring": _docstring_of(cl.comments),
  "start_line": cl.start_line,
  "end_line": cl.end_line,
@@ -504,7 +535,7 @@ def _callable_props(c: PyCallable, file_key: str, source: str) -> Props:
  "start_line": c.start_line,
  "end_line": c.end_line,
  "docstring": _docstring_of(c.comments),
- "decorators": list(c.decorators or []),
+ "decorators": [d.qualified_name or d.name for d in (c.decorators or [])],
  "parameters_json": _stringify_if(c.parameters),
  "accessed_symbols_json": _stringify_if(c.accessed_symbols),
  "_module": file_key,

codeanalyzer/neo4j/schema.py

@@ -101,6 +101,7 @@ class RelType:
  "name": "string",
  "code": "string",
  "base_classes": "string[]",
+ "decorators": "string[]",
  "docstring": "string",
  **_SPAN,
  "_module": "string",
@@ -138,7 +139,7 @@ class RelType:
  "PyDecorator",
  "PyDecorator",
  "name",
- {"name": "string"},
+ {"name": "string", "qualified_name": "string"},
  ),
  NodeLabel(
  "PyCallSite",
@@ -234,7 +235,16 @@ class RelType:
  ["PyModule", "PyPackage"],
  {"spellings": "string[]", "imported_names": "string[]", "aliases": "string[]"},
  ),
- RelType("PY_DECORATED_BY", ["PyCallable"], ["PyDecorator"]),
+ RelType(
+ "PY_DECORATED_BY",
+ ["PyCallable", "PyClass"],
+ ["PyDecorator"],
+ {
+ "expression": "string",
+ "positional_arguments": "string[]",
+ "keyword_arguments_json": "string",
+ },
+ ),
  # Level-3 CPG overlay (-a 3 only): the cross-language dataflow vocabulary,
  # PY_-namespaced so per-language SDK backends can scope their queries.
  RelType("PY_HAS_CFG_NODE", ["PyCallable"], ["PyCFGNode"]),

codeanalyzer/schema/py_schema.py

@@ -217,13 +217,32 @@ class PyVariableDeclaration(BaseModel):
  end_column: int = -1
 
 
+@builder
+class PyDecorator(BaseModel):
+ """One decorator application, structured rather than a source string (#128).
+
+ ``name`` is the spelling as written (``lru_cache``, ``builtins.staticmethod``);
+ ``qualified_name`` is Jedi's resolution of it (``functools.lru_cache``) and is
+ absent when it cannot be resolved. ``expression`` keeps the full unparsed source
+ so nothing is lost for decorators too complex to decompose.
+ """
+
+ name: str
+ qualified_name: Optional[str] = None
+ positional_arguments: List[str] = []
+ keyword_arguments: Dict[str, str] = {}
+ expression: str = ""
+ span: Optional[Span] = None
+
+
 @builder
 class PyCallableParameter(BaseModel):
  """Represents a parameter of a Python callable (function/method)."""
 
  name: str
  type: Optional[str] = None
  default_value: Optional[str] = None
+ decorators: List[PyDecorator] = []
  start_line: int = -1
  end_line: int = -1
  start_column: int = -1
@@ -271,7 +290,7 @@ class PyCallable(BaseModel):
  kind: str = "function"
  span: Optional[Span] = None
  comments: List[PyComment] = []
- decorators: List[str] = []
+ decorators: List[PyDecorator] = []
  parameters: List[PyCallableParameter] = []
  return_type: Optional[str] = None
  start_line: int = -1
@@ -304,6 +323,7 @@ class PyClassAttribute(BaseModel):
  type: Optional[str] = None
  initializer: Optional[str] = None
  comments: List[PyComment] = []
+ decorators: List[PyDecorator] = []
  start_line: int = -1
  end_line: int = -1
 
@@ -319,6 +339,7 @@ class PyClass(BaseModel):
  span: Optional[Span] = None
  comments: List[PyComment] = []
  base_classes: List[str] = []
+ decorators: List[PyDecorator] = []
  callables: Dict[str, PyCallable] = {} # methods, keystone containment name
  attributes: Dict[str, PyClassAttribute] = {}
  types: Dict[str, "PyClass"] = {} # inner classes, keystone containment name

codeanalyzer/syntactic_analysis/symbol_table_builder.py

@@ -16,6 +16,7 @@
  PyCallableParameter,
  PyCallArgument,
  PyCallsite,
+ PyDecorator,
  PyClass,
  PyClassAttribute,
  PyComment,
@@ -295,6 +296,7 @@ def _add_class(self, node: AST, script: Script, source: str, prefix: str = "") -
  .name(class_name)
  .signature(signature)
  .span(span)
+ .decorators(self._decorators(child, script, source))
  .start_line(start_line)
  .end_line(end_line)
  .comments(self._pycomments(child, code))
@@ -331,7 +333,7 @@ def _callables(self, node: AST, script: Script, source: str, prefix: str = "") -
  getattr(child, "end_lineno", child.lineno),
  getattr(child, "end_col_offset", child.col_offset)),
  )
- decorators = [ast.unparse(d) for d in child.decorator_list]
+ decorators = self._decorators(child, script, source)
 
  if prefix:
  # We're in a nested context - build signature with prefix
@@ -590,6 +592,68 @@ def build_param(
 
  return params
 
+ def _decorators(
+ self, node: ast.AST, script: Optional[Script], source: str = ""
+ ) -> List[PyDecorator]:
+ """Structure each entry of ``node.decorator_list`` (#128).
+
+ ``name`` is the spelling as written and ``qualified_name`` is Jedi's
+ resolution of it, inferred at the last identifier of the callee so that
+ ``@a.b.c`` resolves ``c`` rather than ``a``. Resolution is best-effort:
+ dynamic, conditional and re-exported decorators stay unresolved, and a
+ failure here must never abort the symbol table.
+ """
+ out: List[PyDecorator] = []
+ for dec in getattr(node, "decorator_list", []) or []:
+ callee = dec.func if isinstance(dec, ast.Call) else dec
+ positional: List[str] = []
+ keyword: Dict[str, str] = {}
+ if isinstance(dec, ast.Call):
+ positional = [ast.unparse(a) for a in dec.args]
+ for kw in dec.keywords:
+ # ``**kwargs`` has no arg name; keep it addressable rather
+ # than dropping it.
+ key = kw.arg if kw.arg is not None else f"**{ast.unparse(kw.value)}"
+ keyword[key] = ast.unparse(kw.value)
+ span = Span(
+ start=(dec.lineno, dec.col_offset),
+ end=(getattr(dec, "end_lineno", dec.lineno),
+ getattr(dec, "end_col_offset", dec.col_offset)),
+ bytes=byte_offsets(source, dec.lineno, dec.col_offset,
+ getattr(dec, "end_lineno", dec.lineno),
+ getattr(dec, "end_col_offset", dec.col_offset)),
+ ) if source else None
+ out.append(
+ PyDecorator.builder()
+ .name(ast.unparse(callee))
+ .qualified_name(self._decorator_qualified_name(callee, script))
+ .positional_arguments(positional)
+ .keyword_arguments(keyword)
+ .expression(ast.unparse(dec))
+ .span(span)
+ .build()
+ )
+ return out
+
+ @staticmethod
+ def _decorator_qualified_name(
+ callee: ast.AST, script: Optional[Script]
+ ) -> Optional[str]:
+ """Jedi's full name for a decorator's callee, or ``None``."""
+ if script is None:
+ return None
+ line = getattr(callee, "end_lineno", getattr(callee, "lineno", None))
+ col = getattr(callee, "end_col_offset", None)
+ if line is None or col is None:
+ return None
+ try:
+ d = SymbolTableBuilder._first_definition(
+ script.infer(line=line, column=max(col - 1, 0))
+ )
+ except Exception:
+ return None
+ return getattr(d, "full_name", None) if d is not None else None
+
  def _accessed_symbols(
  self, fn_node: ast.FunctionDef, script: Script
  ) -> List[PySymbol]:

schema.neo4j.json

@@ -41,6 +41,7 @@
  "name": "string",
  "code": "string",
  "base_classes": "string[]",
+ "decorators": "string[]",
  "docstring": "string",
  "start_line": "integer",
  "end_line": "integer",
@@ -92,7 +93,8 @@
  "merge_label": "PyDecorator",
  "key": "name",
  "properties": {
- "name": "string"
+ "name": "string",
+ "qualified_name": "string"
  }
  },
  {
@@ -280,12 +282,17 @@
  {
  "type": "PY_DECORATED_BY",
  "from": [
- "PyCallable"
+ "PyCallable",
+ "PyClass"
  ],
  "to": [
  "PyDecorator"
  ],
- "properties": {}
+ "properties": {
+ "expression": "string",
+ "positional_arguments": "string[]",
+ "keyword_arguments_json": "string"
+ }
  },
  {
  "type": "PY_HAS_CFG_NODE",