From c2c9f23408c31cc0de42308b13fc1271a7c0edc5 Mon Sep 17 00:00:00 2001 From: Rahul Krishna Date: Wed, 19 Aug 2026 12:03:38 -0400 Subject: [PATCH 1/3] docs(design): spec anonymous-callable materialization (schema 2.1.0) --- .../anonymous-callable-materialization.md | 290 ++++++++++++++++++ 1 file changed, 290 insertions(+) create mode 100644 docs/design/specs/anonymous-callable-materialization.md diff --git a/docs/design/specs/anonymous-callable-materialization.md b/docs/design/specs/anonymous-callable-materialization.md new file mode 100644 index 0000000..9ed3fb0 --- /dev/null +++ b/docs/design/specs/anonymous-callable-materialization.md @@ -0,0 +1,290 @@ +# Anonymous callables become first-class callables (schema 2.1.0) + +- **Status:** accepted, not yet implemented +- **Scope:** `codeanalyzer-typescript` only; TypeScript-local production pending Group A ratification +- **Schema version:** 2.0.0 → 2.1.0 (MINOR) +- **Supersedes:** the endpoint-plaque approach introduced by #13 + +## Problem + +An unnamed function-like node — an arrow or function expression that is not a +variable initializer — is not modelled as a callable. Two independent facts +combine so that its contents are invisible to every analysis level above L1. + +Worked example, the Express route handler idiom: + +```ts +export function login () { + return (req: Request, res: Response, next: NextFunction) => { + models.sequelize.query( + `SELECT * FROM Users WHERE email = '${req.body.email || ''}' …`) + } +} +``` + +**1. The handler is never indexed as a callable.** `computeSignatureForDecl` +(`src/schema/signatures.ts:46`) returns `null` for it, documented at +`signatures.ts:43-44` as "not a nameable declaration (e.g. an anonymous inline +callback)". Consequently `indexCallableDecls` (`src/dataflow/extract.ts:52`) +never sees it, so no CFG, CDG or DDG is ever built for it. + +Note the asymmetry already present in that file: `isCallableDecl` +(`signatures.ts:29-40`) *does* list `ArrowFunction` and `FunctionExpression`; +`contributorName` (`signatures.ts:11-27`) has no case for either. That gap is +the entire change surface. + +**2. Its call sites are attributed to the enclosing function.** `walkBody` +(`src/syntactic_analysis/builders.ts:344`) treats only *named* nested callables +as boundaries via `namedBoundary` (`builders.ts:330-335`), so `query()` is +recorded as a call site of `login`. Def-use over `login`'s single `return` +statement then runs `captureScan` (`src/dataflow/defuse.ts:305`), which by +design skips identifiers declared inside the nested node — and `req` is a +parameter of the arrow, so it is skipped. + +The result is that no `req.body.email` fact exists anywhere on the native DDG. +The handler is not coarsely modelled; it is absent. + +**3. The Jelly-side node is a plaque, not a callable.** When the flow analyzer +resolves an anonymous callback as an edge endpoint, `homeSynthesized` +(`src/schema/v2/emit.ts:317-333`) mints a node for it in a flat, application-scope +`synthesized_callables` map. That node is typed `V2Node`, not `V2Callable`: it +has no `body`, no `cfg`/`cdg`/`ddg`, no nested `callables`, and its span is +emitted as `bytes: [0, 0]`, so it cannot even slice its own source text. It also +receives an *ordinal* id, `@:`. + +TypeScript therefore holds two contradictory positions at once: `walkBody` +attributes the arrow's calls to the enclosing function, while +`synthesized_callables` simultaneously mints a separate node for that same +arrow. The plaque was introduced by #13 to stop Jelly edges dangling. It was an +endpoint patch, never a modelled decision. + +Measurements from the RULES.md experiment (EXP-001) on OWASP Juice Shop: 24 +`req`-rooted `TS_DDG` edges application-wide, none on `routes/login.ts:34`; 883 +anonymous Express handlers carrying empty `code` on the base graph. + +## Contract-impact triage + +**Does this change schema v2 output?** Yes, on three counts against the keystone: + +- The keystone places the identity boundary at the callable leaf line — durable + `can://` ids at callable depth and above. An anonymous arrow *is* a callable, + so it is owed a durable id. `emit.ts:322` gives it an ordinal one. +- The keystone grammar is + `can://////`. No production exists + for a callable with no signature. +- The keystone requires no dangling endpoints: every `src` and `dst` must + resolve to a node **in the tree**. `synthesized_callables` is a flat sibling + map at application scope, not tree containment. Issue #75 is that invariant + failing in the Neo4j projection — `:TSAnonymousCallable` is reachable only by + `TS_RESOLVES_TO`/`TS_CALLS`, so the snapshot wipe's containment traversal + never reaches it and re-import leaves orphans. + +**Repos touched.** Every language has unnameable callables, so canonical v2 is +affected in principle: `codeanalyzer-{typescript,python,java,clang}`, +`python-sdk`, and the keystone docs. This spec deliberately scopes to +TypeScript only — see *Scope boundary*. + +## What the record already settles + +`.claude/SCHEMA_DECISIONS.md:72` (L10, Closures) already rules that "nested +callables get their own graphs; their reads of outer state are *capture uses* +attributed to the declaring statement in the enclosing CFG". The implementation +honours that ruling for named nested callables only. Half of this change is +therefore a conformance gap against an existing decision, not new design. What +L10 never addressed — and what this spec decides — is identity and containment +for the unnamed ones. + +## Prior art + +Both mature reference analyzers take the *opposite* position, deliberately. + +**Python.** `codeanalyzer-python/codeanalyzer/syntactic_analysis/symbol_table_builder.py:619`: +"Lambdas, comprehensions and inline conditionals don't get their own +`PyCallable` so their internals stay attributed to the enclosing function." + +**Java.** No lambda materialization; `LambdaExpr` appears only in a native-image +reflection config. Java's one adjacent precedent is `JCallable.is_implicit` +(`python-sdk/cldk/models/java/models.py:327`, set at `:533`), which materializes +a callable the source never wrote — but only for *named* constructs such as +default constructors. This repo already mirrors that at +`.claude/SCHEMA_DECISIONS.md:36`. + +**Why TypeScript diverges.** A Python lambda is a single expression: no +statements, no branches, no control flow worth building. Folding it into the +enclosing function loses almost nothing. A JavaScript arrow is a full function +body, and in the Express/Angular idioms it is the *dominant* form of the unit of +behaviour — the Juice Shop measurement above puts 883 of them in one +application. Folding those loses the application. The divergence is a genuine +language-structure difference, not a preference. + +## Decisions + +### D1 — An unnamed arrow or function expression is a callable node + +It is materialized as a `V2Callable` and tree-contained in its enclosing +callable's `callables{}` map (`src/schema/v2/model.ts:116`, already present and +documented as "nested callables (closures) — syntactic containment"). It gets +its own `body{}`, `cfg`, `cdg`, `ddg`, `@entry`/`@exit`, and at L4 its +parameters become `@formal_in:N` vertices. + +`V2Callable.kind` already admits `"arrow"` and `"function_expression"` +(`model.ts:113`), so no new node kind is introduced. + +Consequence: `namedBoundary` (`builders.ts:330-335`) must treat unnamed +function-like nodes as boundaries, `indexCallableDecls` must index them, and +`captureScan`'s boundary follows automatically. + +### D2 — Identity: `contributorName` contributes `` + +TypeScript signatures are dot-joined member chains with no parameter lists +(`src/schema/schema.ts:419`), unlike the Python analyzer's `name(params)` form +(`codeanalyzer-python/codeanalyzer/schema/ids.py`). The change is therefore one +new segment contributed by `contributorName` (`signatures.ts:11-27`): + +``` +routes/login.login. + +can://typescript/juice-shop/routes/login.ts/login. +``` + +Angle brackets mark the segment synthetic, following the JVM ``/`` +convention. Two properties make position the right discriminator: + +- **Byte-identical from both providers with no coordination.** `signatures.ts:2-5` + requires the caller-side and callee-side ids to be byte-identical. Both the + compiler resolver and Jelly know source positions; neither counts declaration + ordinals. Jelly's existing v1 signature is already `:` + (`emit.ts:314`), so position is the de-facto interop key today. +- **Durable tier, no collision.** The segment joins the dotted containment chain, + so it lives in the durable tier as the keystone requires, and it cannot + collide with the `@line:col` ordinal namespace that statements and synthetic + vertices use within a callable. + +This decision changes the id **suffix**, under the enclosing callable. Issue #91 +changes the id **prefix** (adding a `` segment). The two are +orthogonal and do not conflict. + +### D3 — Schema 2.1.0, MINOR, with both legacy names retained + +`src/build/neo4j/schema.ts:19` defines the versioning rule: MAJOR on a renamed +or removed label, relationship or key; MINOR on additive change. That rule +governs **schema elements**, not instance data. Re-anchoring a `call_graph` edge +from `login` to the arrow moves instances; no label, relationship type or key is +renamed or removed. The change is MINOR. + +Holding MINOR constrains the design in one specific way: `synthesized_callables` +and `:TSAnonymousCallable` must both survive. They do, with new meanings: + +- **`:TSAnonymousCallable` becomes a second label on the real tree node.** The + materialized arrow carries both `:TSCallable` and `:TSAnonymousCallable`, and + is reached by a normal containment relationship from its enclosing callable. + Existing `MATCH (:TSAnonymousCallable)` queries keep working. This mirrors the + dual-label staging approach epic #64 already used for the TS-prefix migration, + and it closes #75 directly: the wipe traversal now reaches these nodes, + which is exactly the fix that issue proposes. +- **`synthesized_callables` becomes an id index, not a node registry** — a map + from the provider-side signature to the `can://` id of the tree node: + + ```json + "synthesized_callables": { + "routes/login.login:34:10": "can://typescript/juice-shop/routes/login.ts/login." + } + ``` + +Both `SCHEMA_VERSION` constants move in lockstep: `src/schema/v2/emit.ts:33` +(JSON envelope) and `src/build/neo4j/schema.ts:22` (Neo4j projection). The +regenerated `schema.neo4j.json` ships as part of the release artifact; the +release workflow already commits it (`.github/workflows/release.yml:65-70`). + +### D4 — Call sites re-anchor to the arrow; no compensating edge + +After D1, `query()` is a call site of ``, not of `login`. `login`'s +`body{}` loses that call node and the `call_graph` edge's `src` becomes the +arrow. + +No compensating `login → query` edge is emitted. Such an edge was considered and +rejected: `login` does not call `query`, it returns a function that does. The +schema's over-approximate posture licenses imprecision, not facts of the wrong +kind, and a synthetic edge here would make every returned closure a false call +path for taint consumers. + +Emitting the call node in both bodies was also rejected — the keystone makes +containment the single-parent relation, and that is what makes the structure a +tree. + +## Scope boundary + +This spec does **not**: + +- **Fix #57 or #85.** `this.x = fn` inside a constructor function, and named + object-literal methods, are the *named*-but-unmaterialized family. Same + symptom (Jelly names an endpoint the symbol table never built), different root + cause, untouched here. +- **Add argument-level DDG granularity.** Expressing "argument 0 of `query` is a + template whose substitutions are `{p0.body.email, …}`" needs expression-level + nodes and a typed `ddg` edge list; `V2Callable.ddg` is currently `unknown[]` + (`model.ts:118`) and `V2BodyNode` has no argument slot. The keystone already + reserves `[expression, opt]` in its node ladder and specifies `ddg.var` as a + k-limited access path, so this is conformance work belonging to roadmap + candidate 2 (unified body-node model), collision group A. +- **Coin canonical vocabulary.** The `` production is TypeScript-local + and provisional. It is the anonymous-callable side of roadmap candidate 4 + (`can://` grammar conformance, group A) and is expected to be ratified, + amended or replaced when that group convenes. Sibling analyzers should not + adopt it before ratification. +- **Define entrypoints.** An escaping handler's `@formal_in:0` has no incoming + `param_in` edge, because no in-project caller binds it — Express does. That + unbound formal is the natural anchor for an entrypoint/taint-source + definition, which is roadmap candidate 6, collision group B, and issue #72. + +## Release plan + +| Version | Change | +| --- | --- | +| schema 2.1.0 | this spec — `` segment, materialized anonymous callables | +| schema 2.2.0 | #91 — `` segment, `` collapsed | + +This work takes 2.1.0 and #91 moves to 2.2.0. Rationale: EXP-001 and the Juice +Shop dataflow measurement are blocked behind this change, whereas #91's +`` segment is a `cocoa`-facing identity concern with no analysis +blocked behind it. Issue #91 is updated to target 2.2.0. + +Package version: 1.0.0 → 1.1.0. + +## Definition of done + +- `contributorName` returns an `` segment for unnamed `ArrowFunction` + and `FunctionExpression` nodes; `computeSignatureForDecl` no longer returns + `null` for them. +- `namedBoundary`, `indexCallableDecls` and `captureScan` treat unnamed + function-like nodes as callable boundaries. +- Anonymous callables appear in the enclosing callable's `callables{}` with + populated `body`/`cfg`/`cdg`/`ddg` at `-a 3` and `@formal_in:N` at `-a 4`. +- `synthesized_callables` emits an id index; no `bytes: [0, 0]` spans remain. +- Neo4j projection carries `:TSCallable:TSAnonymousCallable` on one node with a + containment relationship from the enclosing callable; the snapshot wipe + reaches it (closes #75). +- Both `SCHEMA_VERSION` constants read `2.1.0`; `schema.neo4j.json` regenerated. +- `test/schema-v2.test.ts` monotonicity gates hold at L1 ⊆ L2 ⊆ L3 ⊆ L4 with + anonymous callables populated. +- Neo4j conformance and container suites pass. +- Juice Shop at `-a 4` yields at least one `req.body.email`-rooted DDG path + reaching the `query` call in `routes/login.ts` — the EXP-001 acceptance check. + +## Caveats and known risks + +- **Callable count grows sharply.** Juice Shop gains on the order of 883 + callables, each with its own CFG/CDG/DDG at `-a 3`. L3/L4 runtime, cache size + and `graph.cypher` size all grow; the per-callable parallel worker pool + absorbs some of it but the artifact grows regardless. Measure before release. +- **Instance-level drift is real even though the version is MINOR.** Any + consumer asking "what does `login` call" gets a different answer. The version + rule classifies this as MINOR because no schema element is removed; the + migration note must state the behavioural change plainly regardless. +- **Position-based ids are not stable across edits.** Inserting a line above the + arrow changes its id. This matches the keystone's existing posture for ordinal + ids (not promised across edits) but is a new property for a *durable*-tier id, + and is the most likely thing Group A amends. +- **Deeply nested closures produce long dotted chains.** A callback inside a + callback inside a handler yields three `` segments. Acceptable, but + worth watching for id length in the Neo4j projection. From 72d629dc86029c3649bea86d1cab29109909c1cb Mon Sep 17 00:00:00 2001 From: Rahul Krishna Date: Wed, 19 Aug 2026 12:20:42 -0400 Subject: [PATCH 2/3] feat(schema): materialize anonymous callables as first-class callables (2.1.0) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An unnamed arrow or function expression was never modelled as a callable: computeSignatureForDecl returned null for it, so indexCallableDecls never saw it and no CFG/CDG/DDG was built, while walkBody attributed its call sites to the callable that merely encloses it. The Jelly side minted a flat application-scope plaque with no body and a bytes: [0, 0] span. For the Express handler idiom no request-rooted fact existed anywhere on the DDG. Unnamed function-like nodes now carry a positional signature segment, , contributed to the dotted chain — durable id tier, disjoint from the @line:col ordinal namespace body nodes use, and computable identically by the resolver and Jelly since both know source positions. They are tree-contained under their enclosing callable and get their own body, cfg, cdg, ddg and formal-in vertices; L3/L4 needed no change, since isFunctionBoundary already treated arrows as boundaries and collectCallables already recursed inner_callables. Also fixes a concise arrow body that is itself a callable (`() => () => x`): walkBody iterated only the body's children, so the inner arrow was skipped and its call sites attributed upward. Behavioural change: call sites re-anchor from the enclosing callable to the arrow, so call_graph edge sources move. No label, relationship type or key is removed, so the bump is MINOR per the rule in neo4j/schema.ts. - synthesized_callables becomes a compatibility index mapping each pre-2.1.0 anonymous-callable id onto the tree id that replaced it; signatures no provider could name are still homed as standalone nodes so nothing dangles. - :TSAnonymousCallable becomes a second label on the real tree node, reached by TS_DECLARES from its enclosing callable, which puts it on the snapshot wipe's containment walk. Measured on this repository: 596 -> 768 callables (+28.9%), artifact +7.0% at -a 3 over src/. Spec: docs/design/specs/anonymous-callable-materialization.md Closes #75 --- schema.neo4j.json | 17 ++- src/build/neo4j/project.ts | 17 ++- src/build/neo4j/schema.ts | 14 ++- src/dataflow/extract.ts | 6 + src/schema/signatures.ts | 24 +++- src/schema/v2/emit.ts | 31 ++++- src/schema/v2/model.ts | 5 +- src/syntactic_analysis/builders.ts | 53 +++++++-- test/anonymous-callables.test.ts | 166 +++++++++++++++++++++++++++ test/fixtures/anon-app/package.json | 1 + test/fixtures/anon-app/src/routes.ts | 21 ++++ test/fixtures/anon-app/tsconfig.json | 1 + test/schema-v2.test.ts | 2 +- 13 files changed, 332 insertions(+), 26 deletions(-) create mode 100644 test/anonymous-callables.test.ts create mode 100644 test/fixtures/anon-app/package.json create mode 100644 test/fixtures/anon-app/src/routes.ts create mode 100644 test/fixtures/anon-app/tsconfig.json diff --git a/schema.neo4j.json b/schema.neo4j.json index dc97524..0ed6245 100644 --- a/schema.neo4j.json +++ b/schema.neo4j.json @@ -1,5 +1,5 @@ { - "schema_version": "2.0.0", + "schema_version": "2.1.0", "generator": "codeanalyzer-typescript", "marker_labels": [], "node_labels": [ @@ -193,10 +193,23 @@ "id": "string", "kind": "string", "_module": "string", + "signature": "string", "name": "string", + "return_type": "string", + "cyclomatic_complexity": "integer", + "accessibility": "string", + "accessor_kind": "string", + "is_static": "boolean", + "is_abstract": "boolean", + "is_async": "boolean", + "is_generator": "boolean", + "is_exported": "boolean", + "is_ambient": "boolean", + "is_implicit": "boolean", "path": "string", + "start_column": "integer", "start_line": "integer", - "start_column": "integer" + "end_line": "integer" } } ], diff --git a/src/build/neo4j/project.ts b/src/build/neo4j/project.ts index 5120636..0f2ab20 100644 --- a/src/build/neo4j/project.ts +++ b/src/build/neo4j/project.ts @@ -70,8 +70,12 @@ export function project(app: V2Application, _appName?: string): GraphRows { for (const ext of Object.values((root.external_symbols ?? {}) as Record)) { b.node([CAN, "TSExternal"], "id", ext.id, prune({ id: ext.id, kind: "external", name: ext.name, module: ext.module })); } - // First-party anonymous callbacks (edge endpoints the tree never names). - for (const sc of Object.values((root.synthesized_callables ?? {}) as Record)) { + // 2.1.0: `synthesized_callables` is mostly a compatibility index (old id → tree id) whose targets + // are already projected as tree nodes. Only the residual fallback entries — a signature no + // provider could name, recognisable because the map key IS the entry's own id — still need a + // standalone node, so call-graph edges pointing at them do not dangle. + for (const [key, sc] of Object.entries((root.synthesized_callables ?? {}) as Record)) { + if (key !== sc.id) continue; b.node([CAN, "TSAnonymousCallable"], "id", sc.id, prune({ id: sc.id, kind: "callable", name: str(sc.name), path: str(sc.path), start_line: spanLine(sc, "start"), start_column: spanCol(sc, "start"), @@ -114,8 +118,15 @@ function projectType(b: RowBuilder, t: V2Type, parent: NodeRef, fileKey: string) for (const f of Object.values(t.fields ?? {})) projectField(b, f, node, fileKey); } +/** An unnamed callable's signature ends with the positional segment `contributorName` gives it. */ +const ANON_SIG = /\.$/; + function projectCallable(b: RowBuilder, c: V2Callable, owner: NodeRef, ownerRel: string, fileKey: string): void { - const node = b.node([CAN, "TSCallable"], "id", c.id, callableProps(c, fileKey)); + // An unnamed callable carries :TSAnonymousCallable alongside :TSCallable — one node, two labels, + // reached by ordinary containment. That is what keeps pre-2.1.0 MATCH (:TSAnonymousCallable) + // queries working and puts these nodes on the snapshot wipe's containment walk (issue #75). + const labels = ANON_SIG.test(c.signature) ? [CAN, "TSCallable", "TSAnonymousCallable"] : [CAN, "TSCallable"]; + const node = b.node(labels, "id", c.id, callableProps(c, fileKey)); b.edge(ownerRel, owner, node); // Body nodes (L1: call sites; L3+: statements + synthetic vertices) + their overlays. diff --git a/src/build/neo4j/schema.ts b/src/build/neo4j/schema.ts index a25165b..d3be823 100644 --- a/src/build/neo4j/schema.ts +++ b/src/build/neo4j/schema.ts @@ -19,7 +19,7 @@ * SCHEMA_VERSION: MAJOR on a breaking change (renamed/removed label, relationship or key), MINOR * on additive. v2 is a MAJOR bump from v1 (keys moved signature→can:// id; labels reshaped). */ -export const SCHEMA_VERSION = "2.0.0"; +export const SCHEMA_VERSION = "2.1.0"; export type PropType = "string" | "integer" | "float" | "boolean" | "string[]" | "integer[]"; @@ -125,10 +125,20 @@ export const NODE_LABELS: NodeLabel[] = [ }, { label: "TSExternal", mergeLabel: CAN, key: "id", properties: { ...COMMON, name: "string", module: "string" } }, { + // 2.1.0: a marker label carried *alongside* :TSCallable by an unnamed arrow / function + // expression, which is now a real tree node reached by TS_DECLARES from its enclosing + // callable. It is no longer a node kind of its own — the property set is TSCallable's — but + // the label is retained so existing MATCH (:TSAnonymousCallable) queries keep resolving. label: "TSAnonymousCallable", mergeLabel: CAN, key: "id", - properties: { ...COMMON, name: "string", path: "string", start_line: "integer", start_column: "integer" }, + properties: { + ...COMMON, signature: "string", name: "string", return_type: "string", cyclomatic_complexity: "integer", + accessibility: "string", accessor_kind: "string", is_static: "boolean", is_abstract: "boolean", + is_async: "boolean", is_generator: "boolean", is_exported: "boolean", is_ambient: "boolean", is_implicit: "boolean", + path: "string", start_column: "integer", + ...SPAN, + }, }, ]; diff --git a/src/dataflow/extract.ts b/src/dataflow/extract.ts index d9a02f9..51d8ea8 100644 --- a/src/dataflow/extract.ts +++ b/src/dataflow/extract.ts @@ -71,6 +71,12 @@ export function indexCallableDecls(project: Project, root: string, onlyFiles?: S const sig = computeSignatureForDecl(n, root); if (sig && !idx.has(sig)) idx.set(sig, init); } + } else if (Node.isArrowFunction(n) || Node.isFunctionExpression(n)) { + // Unnamed: signed and bodied by the same node. The `const f = () => …` case is signed by + // its VariableDeclaration above, and computeSignatureForDecl returns null for the + // initializer itself, so this branch never double-claims it. + const sig = computeSignatureForDecl(n, root); + if (sig && !idx.has(sig)) idx.set(sig, n); } }); } diff --git a/src/schema/signatures.ts b/src/schema/signatures.ts index e26642e..e228fdf 100644 --- a/src/schema/signatures.ts +++ b/src/schema/signatures.ts @@ -23,9 +23,28 @@ export function contributorName(node: Node): string | null { if (init && (Node.isArrowFunction(init) || Node.isFunctionExpression(init))) return node.getName(); return null; } + if (Node.isArrowFunction(node) || Node.isFunctionExpression(node)) return anonName(node); return null; } +/** + * The segment an unnamed function-like node contributes. Position is the only discriminant a + * nameless callable has, and it is the one both the resolver and Jelly can compute independently + * — which is what keeps caller-side and callee-side ids byte-identical. Angle brackets mark the + * segment synthetic (the ``/`` convention). It joins the dotted chain, so an + * anonymous callable lives in the durable id tier and never collides with the `@line:col` + * ordinal namespace that body nodes use. + * + * Returns null when a VariableDeclaration ancestor already names this callable (`const f = () =>`) + * — that case is handled above, and contributing here as well would double the segment. + */ +function anonName(node: Node): string | null { + const parent = node.getParent(); + if (parent && Node.isVariableDeclaration(parent) && parent.getInitializer() === node) return null; + const { line, column } = node.getSourceFile().getLineAndColumnAtPos(node.getStart()); + return ``; +} + export function isCallableDecl(node: Node): boolean { return ( Node.isFunctionDeclaration(node) || @@ -40,8 +59,9 @@ export function isCallableDecl(node: Node): boolean { } /** - * Compute the canonical signature for a declaration node. Returns null when the node is not a - * nameable declaration (e.g. an anonymous inline callback). + * Compute the canonical signature for a declaration node. Unnamed function-like nodes are named + * positionally (see `anonName`), so this returns null only for nodes that are not declarations at + * all. */ export function computeSignatureForDecl(node: Node, root: string): string | null { const sf = node.getSourceFile(); diff --git a/src/schema/v2/emit.ts b/src/schema/v2/emit.ts index 3dfe880..2d9c704 100644 --- a/src/schema/v2/emit.ts +++ b/src/schema/v2/emit.ts @@ -30,7 +30,7 @@ import { applyDataflow } from "./dataflow"; import type { V2Application, V2BodyNode, V2CallEdge, V2Callable, V2External, V2Field, V2Module, V2Node, V2Root, V2Type } from "./model"; const LANGUAGE = "typescript"; -const SCHEMA_VERSION = "2.0.0"; +const SCHEMA_VERSION = "2.1.0"; const ANALYZER_NAME = "codeanalyzer-typescript"; /** Highest analysis level this emitter populates today (L1 tree, L2 call graph, L3/L4 dataflow). */ const MAX_IMPLEMENTED = 4; @@ -310,14 +310,35 @@ function homeExternals(app: TSApplication, appId: string, idBySig: Map:`, so they are addressed *ordinally* under - * the enclosing callable: `@:` (the two-tier identity rule below the - * callable line). + * The compatibility index for anonymous callables (schema 2.1.0). + * + * Anonymous callables are now real nodes in the containment tree, signed positionally + * (`.`) and reachable by containment. This map is no longer a node + * registry: it maps the **pre-2.1.0 id** of each anonymous callable — `@:`, + * derived from the old `:` signature — onto the tree id that replaced it, + * so a consumer holding an old id can still resolve it. + * + * The old host was the nearest enclosing callable the old rules could name, which is recovered by + * stripping the trailing `` chain. An anonymous callable directly under a module had no + * resolvable old id (the old emitter fell back to an opaque `@synthetic/` key that encoded the + * ambiguous `:` signature, which was not unique across files) — those are + * skipped rather than reproduced. + * + * Any signature the call-graph provider still could not name is homed here too, unchanged, so the + * no-dangling rule holds even if a provider reports a function-like node the tree missed. */ function homeSynthesized(app: TSApplication, appId: string, idBySig: Map): Record { const out: Record = {}; + for (const [sig, id] of [...idBySig.entries()]) { + const m = /^(.*?)((?:\.)+)$/.exec(sig); + if (!m) continue; + const host = idBySig.get(m[1] as string); + if (!host) continue; // module-level anonymous callable — no resolvable pre-2.1.0 id + const last = /$/.exec(sig) as RegExpExecArray; + out[`${host}@${last[1]}:${last[2]}`] = { id, kind: "callable" }; + } for (const [sig, sc] of Object.entries(app.synthesized_callables ?? {})) { + if (idBySig.has(sig)) continue; // the tree names it now const m = /^(.*):?$/.exec(sig); const enclosing = m ? idBySig.get(m[1] as string) : undefined; const id = m && enclosing ? `${enclosing}@${m[2]}:${m[3]}` : `${appId}/@synthetic/${encodeURIComponent(sig)}`; diff --git a/src/schema/v2/model.ts b/src/schema/v2/model.ts index ecf04f6..efd5e50 100644 --- a/src/schema/v2/model.ts +++ b/src/schema/v2/model.ts @@ -44,7 +44,10 @@ export interface V2Root { param_out: V2ParamEdge[]; // L4 // TS-additive (parity): edge endpoints outside the containment tree need an id home. external_symbols?: Record; // L2 — imported/library call targets, keyed by id - synthesized_callables?: Record; // L2 — first-party anonymous callbacks, keyed by id + // L2 — 2.1.0 compatibility index: pre-2.1.0 anonymous-callable id → the tree id that replaced + // it. Anonymous callables are real nodes in the tree now; entries whose key equals their own + // `id` are the residual fallback nodes for signatures no provider could name. + synthesized_callables?: Record; } /** A call target outside the project (an imported library member / builtin) — an edge endpoint, not a tree node. */ diff --git a/src/syntactic_analysis/builders.ts b/src/syntactic_analysis/builders.ts index 22952da..f856c13 100644 --- a/src/syntactic_analysis/builders.ts +++ b/src/syntactic_analysis/builders.ts @@ -325,6 +325,9 @@ type Boundary = "callable" | "class" | "skip" | null; function namedBoundary(node: Node): Boundary { if (Node.isFunctionDeclaration(node)) return "callable"; + // Unnamed arrows / function expressions are callables in their own right (they carry a + // positional signature), so their contents must not be attributed to the enclosing callable. + if (Node.isArrowFunction(node) || Node.isFunctionExpression(node)) return "callable"; if (Node.isClassDeclaration(node) || Node.isClassExpression(node)) return "class"; if (Node.isModuleDeclaration(node)) return "skip"; if (Node.isVariableDeclaration(node)) { @@ -357,6 +360,12 @@ function walkBody(body: Node, h: BodyHandlers): void { if (Node.isVariableDeclaration(node)) h.onLocal(node); node.forEachChild(visit); }; + // A concise arrow body can *be* a callable (`() => () => x`). Visiting only the body's children + // would skip it and attribute its call sites to the callable that merely returns it. + if (namedBoundary(body) !== null) { + visit(body); + return; + } body.forEachChild(visit); } @@ -409,6 +418,24 @@ function overloadsOf(fnNode: Node): TSOverloadSignature[] { })); } +/** + * Build a callable from a node `walkBody` reported as a nested boundary. The three shapes are a + * named `function` declaration, a `const f = () => …` (signed by its VariableDeclaration but bodied + * by the initializer), and a bare unnamed arrow / function expression (signed and bodied by itself). + */ +function buildNestedCallable(n: Node, root: string): { sig: string; callable: TSCallable } | null { + if (Node.isVariableDeclaration(n)) { + const init = n.getInitializer(); + if (!init) return null; + const k: TSCallableKind = Node.isArrowFunction(init) ? "arrow" : "function_expression"; + return buildCallable(n, init, k, root); + } + if (Node.isArrowFunction(n) || Node.isFunctionExpression(n)) { + return buildCallable(n, n, Node.isArrowFunction(n) ? "arrow" : "function_expression", root); + } + return buildCallable(n, n, "function", root); +} + export function buildCallable( sigNode: Node, fnNode: Node, @@ -429,16 +456,8 @@ export function buildCallable( onCall: (n) => call_sites.push(buildCallsite(n)), onLocal: (vd) => local_variables.push(buildVariable(vd, "function")), onNestedCallable: (n) => { - if (Node.isVariableDeclaration(n)) { - const init = n.getInitializer(); - if (!init) return; - const k: TSCallableKind = Node.isArrowFunction(init) ? "arrow" : "function_expression"; - const r = buildCallable(n, init, k, root); - if (r) inner_callables[r.sig] = r.callable; - } else { - const r = buildCallable(n, n, "function", root); - if (r) inner_callables[r.sig] = r.callable; - } + const r = buildNestedCallable(n, root); + if (r) inner_callables[r.sig] = r.callable; }, onNestedClass: (n) => { const r = buildClass(n, root); @@ -828,6 +847,20 @@ function buildStatemented(container: Node, root: string, varScope: TSVariableDec } } } + // Bare anonymous callables in top-level expression statements — `app.get('/x', (req, res) => …)` + // is the dominant Express idiom and is reachable through neither getFunctions() nor + // getVariableStatements(), which see declarations only. walkBody stops at every boundary the + // loops above already claimed, so nothing is collected twice. + walkBody(container, { + onCall: () => {}, + onLocal: () => {}, + onNestedCallable: (n) => { + if (!Node.isArrowFunction(n) && !Node.isFunctionExpression(n)) return; // already bucketed + const r = buildCallable(n, n, Node.isArrowFunction(n) ? "arrow" : "function_expression", root); + if (r) functions[r.sig] = r.callable; + }, + onNestedClass: () => {}, + }); const namespaces: Record = {}; for (const ns of c.getModules()) { const r = buildNamespace(ns, root); diff --git a/test/anonymous-callables.test.ts b/test/anonymous-callables.test.ts new file mode 100644 index 0000000..e1a677c --- /dev/null +++ b/test/anonymous-callables.test.ts @@ -0,0 +1,166 @@ +/** + * Issue #92 (schema 2.1.0): an unnamed arrow / function expression is a callable in its own right. + * It is tree-contained under its enclosing callable with a durable positional signature segment + * (``), carries its own body/cfg/cdg/ddg and formal-in vertices, and owns the call + * sites that used to be attributed to the callable that merely encloses it. + * + * The acceptance case is the Express handler idiom: a request-rooted access path must reach the + * sink call on the DDG, which was impossible while the handler had no graph at all. + * + * Spec: docs/design/specs/anonymous-callable-materialization.md + */ +import { describe, expect, test } from "bun:test"; +import * as path from "node:path"; +import { analyze } from "../src/core"; +import type { AnalysisOptions } from "../src/options"; +import { type V2Callable, type V2Module, type V2Node, toV2Detailed } from "../src/schema/v2"; +import { project } from "../src/build/neo4j"; + +const FIXTURE = path.resolve(import.meta.dir, "fixtures/anon-app"); + +function options(level: number): AnalysisOptions { + return { + input: FIXTURE, + output: null, + emit: "json", + appName: "anon-app", + neo4jUri: null, + neo4jUser: "neo4j", + neo4jPassword: "", + neo4jDatabase: null, + analysisLevel: level, + graphs: ["cfg", "dfg", "pdg", "sdg"], + graphFieldDepth: 3, + jobs: 1, + targetFiles: null, + skipTests: true, + eager: true, + noBuild: true, + phantoms: true, + callGraphProvider: "tsc", + cacheDir: null, + verbosity: 0, + } as unknown as AnalysisOptions; +} + +const opts = options(4); +const { application, idBySig, collisions, dangling } = toV2Detailed(await analyze(opts), opts); +const root = application.application; +const mod = root.symbol_table["src/routes.ts"] as V2Module; +const fns = mod.functions as Record; + +const login = fns["login"] as V2Callable; +const handler = (login.callables ?? {})[""] as V2Callable; + +/** Edge lists are typed `unknown[]` on V2Callable until the body-node model lands (roadmap #2). */ +type Edge = { src: string; dst: string; var?: string }; +const edges = (xs: unknown[] | undefined): Edge[] => (xs ?? []) as Edge[]; + +describe("anonymous callables are first-class (issue #92)", () => { + test("a returned arrow is tree-contained under its enclosing callable", () => { + expect(handler).toBeDefined(); + expect(handler.kind).toBe("arrow"); + expect(handler.signature).toBe("src/routes.login."); + expect(handler.id).toBe(`${login.id}/`); + }); + + test("its id is durable-tier and cannot collide with a body node at the same position", () => { + // Statements/synthetics under `login` are addressed `@line:col`; the arrow is a + // containment segment, so the two namespaces stay disjoint even at identical coordinates. + expect(handler.id.startsWith(`${login.id}@`)).toBe(false); + expect(collisions).toEqual([]); + }); + + test("a variable-bound arrow keeps its own name — no segment", () => { + expect((fns["named"] as V2Callable).signature).toBe("src/routes.named"); + expect(Object.keys(fns)).not.toContain(""); + }); + + test("a bare arrow in a module-level expression statement is materialized", () => { + const h = fns[""] as V2Callable; + expect(h).toBeDefined(); + expect(h.kind).toBe("arrow"); + expect(edges(h.ddg).some((e) => e.var === "req.query.probe")).toBe(true); + }); + + test("nested anonymous callables chain their segments", () => { + const outer = fns["outer"] as V2Callable; + const first = Object.values(outer.callables ?? {})[0] as V2Callable; + const second = Object.values(first.callables ?? {})[0] as V2Callable; + expect(second.signature.match(//g)).toHaveLength(2); + expect(second.id.startsWith(first.id)).toBe(true); + }); + + test("call sites re-anchor from the enclosing callable to the arrow", () => { + const calleeOf = (c: V2Callable): unknown[] => + Object.values(c.body ?? {}).filter((n) => n.kind === "call").map((n) => n.callee); + expect(calleeOf(login)).toEqual([]); + expect(calleeOf(handler)).toEqual([`${(fns["query"] as V2Callable).id}`]); + + const srcs = root.call_graph.map((e) => e.src); + expect(srcs).toContain(handler.id); + expect(srcs).not.toContain(login.id); + }); + + test("the handler carries its own graphs and formal-in vertices", () => { + expect(Object.keys(handler.body)).toContain("@formal_in:0"); + expect(handler.body["@formal_in:0"]?.of).toBe("req"); + expect(edges(handler.cfg).length).toBeGreaterThan(0); + expect(edges(handler.cdg).length).toBeGreaterThan(0); + }); + + test("EXP-001: the request-rooted access path reaches the sink call on the DDG", () => { + const ddg = edges(handler.ddg); + const tainted = ddg.find((e) => e.var === "req.body.email"); + expect(tainted).toBeDefined(); + + // …and that definition flows onward to the statement holding the `query(...)` call. + const callKey = Object.entries(handler.body).find(([, n]) => n.kind === "call")?.[0] as string; + const reaches = new Set([tainted?.dst as string]); + for (let i = 0; i < ddg.length; i++) { + for (const e of ddg) if (reaches.has(e.src)) reaches.add(e.dst); + } + expect(reaches.has(callKey)).toBe(true); + }); + + test("no call-graph endpoint dangles", () => { + expect(dangling).toEqual([]); + }); + + test("synthesized_callables is a compatibility index onto the tree", () => { + const index = (root.synthesized_callables ?? {}) as Record; + expect(index[`${login.id}@2:10`]?.id).toBe(handler.id); + // Every entry either points at a tree node or is a residual fallback node keyed by its own id. + for (const [key, entry] of Object.entries(index)) { + expect(key === entry.id || [...idBySig.values()].includes(entry.id)).toBe(true); + } + }); + + test("the envelope declares schema 2.1.0", () => { + expect(application.schema_version).toBe("2.1.0"); + }); +}); + +describe("Neo4j projection of anonymous callables (issues #92, #75)", () => { + const rows = project(application); + const node = rows.nodes.find((n) => n.value === handler.id); + + test("one node carries both :TSCallable and :TSAnonymousCallable", () => { + expect(node?.labels).toContain("TSCallable"); + expect(node?.labels).toContain("TSAnonymousCallable"); + expect(rows.nodes.filter((n) => n.value === handler.id)).toHaveLength(1); + }); + + test("it hangs off its enclosing callable by containment, so the snapshot wipe reaches it", () => { + const decl = rows.edges.find((e) => e.type === "TS_DECLARES" && e.to.value === handler.id); + expect(decl?.from.value).toBe(login.id); + }); + + test("no off-tree :TSAnonymousCallable node remains", () => { + const anon = rows.nodes.filter((n) => n.labels.includes("TSAnonymousCallable")); + expect(anon.length).toBeGreaterThan(0); + for (const n of anon) { + expect(rows.edges.some((e) => e.type === "TS_DECLARES" && e.to.value === n.value)).toBe(true); + } + }); +}); diff --git a/test/fixtures/anon-app/package.json b/test/fixtures/anon-app/package.json new file mode 100644 index 0000000..38dcb9b --- /dev/null +++ b/test/fixtures/anon-app/package.json @@ -0,0 +1 @@ +{ "name": "anon-app", "version": "1.0.0", "private": true } diff --git a/test/fixtures/anon-app/src/routes.ts b/test/fixtures/anon-app/src/routes.ts new file mode 100644 index 0000000..fe40feb --- /dev/null +++ b/test/fixtures/anon-app/src/routes.ts @@ -0,0 +1,21 @@ +export function login() { + return (req: any, res: any) => { + const email = req.body.email; + query(`SELECT * FROM Users WHERE email = '${email}'`); + }; +} + +export function query(sql: string) { + return sql; +} + +const app: any = {}; +app.get("/health", (req: any, res: any) => { + res.send(req.query.probe); +}); + +const named = () => 1; + +export function outer() { + return () => () => named(); +} diff --git a/test/fixtures/anon-app/tsconfig.json b/test/fixtures/anon-app/tsconfig.json new file mode 100644 index 0000000..f31416c --- /dev/null +++ b/test/fixtures/anon-app/tsconfig.json @@ -0,0 +1 @@ +{ "compilerOptions": { "target": "ES2020", "module": "commonjs", "strict": false }, "include": ["src"] } diff --git a/test/schema-v2.test.ts b/test/schema-v2.test.ts index 4924e5b..aef3522 100644 --- a/test/schema-v2.test.ts +++ b/test/schema-v2.test.ts @@ -84,7 +84,7 @@ function allIds(): string[] { describe("schema v2 — L1 envelope", () => { test("root envelope matches the canonical shape", () => { - expect(v2.schema_version).toBe("2.0.0"); + expect(v2.schema_version).toBe("2.1.0"); expect(v2.language).toBe("typescript"); expect(v2.max_level).toBe(1); expect(Object.keys(root).sort()).toEqual(["call_graph", "id", "kind", "param_in", "param_out", "symbol_table"]); From 687406e1826596b3ef76ee868af9a2cf77f395cc Mon Sep 17 00:00:00 2001 From: Rahul Krishna Date: Wed, 19 Aug 2026 12:23:59 -0400 Subject: [PATCH 3/3] test(neo4j): bind the migration assertion to SCHEMA_VERSION The bolt migration test hardcoded '2.0.0' as the post-push schema version, so it broke on the 2.1.0 bump for a reason unrelated to what it covers (wiping 1.1.0 residue). Read the constant instead. --- test/neo4j-bolt.test.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/neo4j-bolt.test.ts b/test/neo4j-bolt.test.ts index ef36c8a..dfbdce0 100644 --- a/test/neo4j-bolt.test.ts +++ b/test/neo4j-bolt.test.ts @@ -12,7 +12,7 @@ import * as os from "node:os"; import * as path from "node:path"; import { Neo4jContainer, type StartedNeo4jContainer } from "@testcontainers/neo4j"; import neo4j, { type Driver } from "neo4j-driver"; -import { type BoltConfig, boltWriter, CONSTRAINTS, INDEXES, project } from "../src/build/neo4j"; +import { type BoltConfig, boltWriter, CONSTRAINTS, INDEXES, project, SCHEMA_VERSION } from "../src/build/neo4j"; import { analyze } from "../src/core"; import type { AnalysisOptions } from "../src/options"; import { toV2 } from "../src/schema/v2"; @@ -164,7 +164,7 @@ containerSuite("neo4j bolt writer", () => { ); test( - "migrates a 1.1.0-shaped graph to 2.0.0, wiping legacy residue (#46)", + "migrates a 1.1.0-shaped graph to the current schema, wiping legacy residue (#46)", async () => { // Seed a minimal schema-1.1.0 graph on a clean store: twin labels, the old // name/file_key/signature keys, and an :Application keyed on `name` (no `id`). @@ -180,7 +180,7 @@ containerSuite("neo4j bolt writer", () => { await seed.close(); } - // A full 2.0.0 push against the same DB must detect the version mismatch and wipe the residue. + // A full current-version push against the same DB must detect the mismatch and wipe the residue. const opts = optsFor(); const rows = project(toV2(await analyze(opts), opts)); await boltWriter(rows, cfg, log, true); @@ -190,7 +190,7 @@ containerSuite("neo4j bolt writer", () => { expect(await num("MATCH (a:Application) RETURN count(a)")).toBe(1); expect( await num( - "MATCH (a:Application) WHERE a.id IS NOT NULL AND a.schema_version = '2.0.0' RETURN count(a)", + `MATCH (a:Application) WHERE a.id IS NOT NULL AND a.schema_version = '${SCHEMA_VERSION}' RETURN count(a)`, ), ).toBe(1);